0

I am trying to run Server - Client example using Hazelcast Jet 0.5.1.

This example is extended from the hazelcast jet following code sample.

hazelcast-jet-0.5.1\code-samples\streaming\map-journal-source\src\main\java\RemoteMapJournalSource.java

Here is a scenario

I have 1 server program & 2 client programs.

Server - Usual hazelcast code puts 100 Integer entries in the Hazelcast map. It has also Event Journal configured for the map.

Client - Connect to hazelcast using client configuration, reads entries from remote journal and drain to IList.

Here are the observations

The Server puts 100 entries to the map, however client 1 gets 200 and the client 2 gets 300 entries (Initial list size is 100)

I was expecting only 100 entries would be available as well entries would get divided between these two client.

Sample Code as follows.

Server program Code

public class RemoteMapJournalSourceSrv1 {

private static final String MAP_NAME = "map";
private static final String SINK_NAME = "list";

public static void main(String[] args) throws Exception {
    System.setProperty("remoteHz.logging.type", "log4j");

    Config hzConfig = getConfig();
    HazelcastInstance remoteHz = startRemoteHzCluster(hzConfig);

    try {

        IMap<Integer, Integer> map = remoteHz.getMap(MAP_NAME);
        System.out.println("*************** Initial Map  address " +  map.size() );
        
        for (int i = 0; i < 100; i++) {
            map.set(i, i);
        
        }
        System.out.println("***************map size "+map.size());
        
        TimeUnit.SECONDS.sleep(300);
        
        
    } finally {
        Hazelcast.shutdownAll();
    }

}

private static String getAddress(HazelcastInstance remoteHz) {
    Address address = remoteHz.getCluster().getLocalMember().getAddress();
    System.out.println("***************Remote address " + address.getHost() + ":" + address.getPort() );
    return address.getHost() + ":" + address.getPort();
}


private static HazelcastInstance startRemoteHzCluster(Config config) {
    HazelcastInstance remoteHz = Hazelcast.newHazelcastInstance(config);
    Hazelcast.newHazelcastInstance(config);
    return remoteHz;
}

private static Config getConfig() {
    Config config = new Config();
    // Add an event journal config for map which has custom capacity of 1000 (default 10_000)
    // and time to live seconds as 10 seconds (default 0 which means infinite)
    config.addEventJournalConfig(new EventJournalConfig().setEnabled(true)
                                                         .setMapName(MAP_NAME)
                                                         .setCapacity(10000)
                                                         .setTimeToLiveSeconds(100));
    return config;
}

Client 1 Program Code

 public class RemoteMapJournalSourceCL1 {

private static final String MAP_NAME = "map";
private static final String SINK_NAME = "list";

public static void main(String[] args) throws Exception {
    System.setProperty("remoteHz.logging.type", "log4j");

      JetInstance localJet = startLocalJetCluster();

    try {
        ClientConfig clientConfig = new ClientConfig();
        GroupConfig groupConfig = new GroupConfig();

        clientConfig.getNetworkConfig().addAddress("localhost:5701");
        clientConfig.setGroupConfig(groupConfig);
        
        IList list1 = localJet.getList(SINK_NAME);
        
        int size1 = list1.size();
        
        System.out.println("***************List Initial size "+size1);

        Pipeline p = Pipeline.create();
        p.drawFrom(Sources.<Integer, Integer, Integer>remoteMapJournal(MAP_NAME, clientConfig,
                e -> e.getType() == EntryEventType.ADDED, EventJournalMapEvent::getNewValue, false))
         .peek()
         .drainTo(Sinks.list(SINK_NAME));
        
        JobConfig jc= new JobConfig();
        jc.setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE);
        localJet.newJob(p,jc);

        TimeUnit.SECONDS.sleep(10);
        
        IList list = localJet.getList(SINK_NAME);
    
    
        int size = list.size();
        
            for(int j =0; j< size; j++ ){
            System.out.println("***************" + list.get(j));
        }
        
        System.out.println("***************Read " + size + " entries from remote map journal.");
    
        
    } finally {
        Hazelcast.shutdownAll();
        Jet.shutdownAll();
    }

}

private static String getAddress(HazelcastInstance remoteHz) {
    Address address = remoteHz.getCluster().getLocalMember().getAddress();
    System.out.println("***************Remote address " + address.getHost() + ":" + address.getPort() );
    return address.getHost() + ":" + address.getPort();
}

private static JetInstance startLocalJetCluster() {
      JetInstance localJet = Jet.newJetInstance();
    return localJet;
}

Client 2 Program code.

public class RemoteMapJournalSourceCL2 {

private static final String MAP_NAME = "map";
private static final String SINK_NAME = "list";

public static void main(String[] args) throws Exception {
    System.setProperty("remoteHz.logging.type", "log4j");

      JetInstance localJet = startLocalJetCluster();

    try {
        ClientConfig clientConfig = new ClientConfig();
        GroupConfig groupConfig = new GroupConfig();

        clientConfig.getNetworkConfig().addAddress("localhost:5701");
        clientConfig.setGroupConfig(groupConfig);
        
        IList list1 = localJet.getList(SINK_NAME);
        int size1 = list1.size();
        System.out.println("***************List Initial size "+size1);

        
        Pipeline p = Pipeline.create();
        p.drawFrom(Sources.<Integer, Integer, Integer>remoteMapJournal(MAP_NAME, clientConfig,
                e -> e.getType() == EntryEventType.ADDED, EventJournalMapEvent::getNewValue, false))
         .peek()
         .drainTo(Sinks.list(SINK_NAME));
        
        JobConfig jc= new JobConfig();
        jc.setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE);
        
        localJet.newJob(p,jc);

        TimeUnit.SECONDS.sleep(10);
        
        IList list = localJet.getList(SINK_NAME);
    
    
        int size = list.size();
        
            for(int j =0; j< size; j++ ){
            System.out.println("***************" + list.get(j));
        }
        
        System.out.println("***************Read " + size + " entries from remote map journal.");
    
        
    } finally {
        Hazelcast.shutdownAll();
        Jet.shutdownAll();
    }

}

private static String getAddress(HazelcastInstance remoteHz) {
    Address address = remoteHz.getCluster().getLocalMember().getAddress();
    System.out.println("***************Remote address " + address.getHost() + ":" + address.getPort() );
    return address.getHost() + ":" + address.getPort();
}

private static JetInstance startLocalJetCluster() {
      JetInstance localJet = Jet.newJetInstance();
    return localJet;
}

Will you please help me to resolve this issue ?

Community
  • 1
  • 1
  • Jet jobs get fully executed on the server, what you're doing is submitting the same job three times: twice from the client and once from the member. As a result the list will have 300 entries because there are three jobs running and consuming the event journal to the list. – Can Gencer Apr 09 '18 at 07:03
  • The Server Program (i.e. RemoteMapJournalSourceSrv1) does not run the job while the client programs (i.e. RemoteMapJournalSourceCL1 & RemoteMapJournalSourceCL2 ) run the job individually. The client programs only consume the event journal to list. – Harshad Murtekar Apr 09 '18 at 07:24
  • 1
    Possible duplicate of [Hazelcast Jet Query](https://stackoverflow.com/questions/49747170/hazelcast-jet-query) – Marko Topolnik Apr 10 '18 at 09:52

0 Answers0