0

I am building an Apache Kafka application where Producer (built on Dotnet Api) successfully published 90,045 messages to topic and Consumer (Built on Node.js) has received all of them. But when consumer is sending those messages to Angular UI through socket.io, following weird things I observe. 1. Angular UI only receives 6921 messages and it does vary every time but never cross 10000 2. In UI Browser console, I could see these 6921 messages multiple times. It creating duplicate messages. 3. In UI there is delay happening to receive these consumer messages.

Consumer (Node.js)

const topic = 'kafka-topic-3'
  const consumer = kafka.consumer({ groupId: 'demo-consumer-group' })
  const run = async () => {
    await consumer.connect()
    await consumer.subscribe({ topic })
    await consumer.run({
      //partitionsConsumedConcurrently:10,
      eachMessage: async ({ topic, partition, message }) => {
        const prefix = `${topic}[${partition} | ${message.offset}] / ${message.timestamp}`
        io.emit('Kafka-message', message.value.toString('utf8'));
        console.log('Received', prefix, message.value.toString('utf8'));
      },
    })
  }

  run().catch(e => console.error(`[example/consumer] ${e.message}`, e))
  module.exports = run;

Angular Service

getLiveDataFromTopic1() { 
    console.log('called getLiveDataFromTopic1');
    let observable = new Observable(observer => {
      this.socket = io.connect(this.url);
      this.socket.on('Kafka-message', (data:any) => {       
        observer.next(data);
      });
      return () => {
        this.socket.off('Kafka-message');
        this.socket.disconnect();
      }
    })
    return observable;
  }

Angular Component

connection: any;
  messages=new Array();
  message: string = "";
  totalMessages: number = 0;

  constructor(private kafkaService: KafkaClientService) {
  }

  ngOnInit(): void {
    console.log('ngonit:,called');
    this.connection = this.kafkaService.getLiveDataFromTopic1().subscribe(msg => {      
      this.messages.push(msg);
      console.log('this.messages',this.messages);
      // this.totalMessages++;
      // this.message = this.message + "<li>" + msg + " </li>";
    })
  }

  ngOnDestroy(): void {
    this.connection.unsubscribe();  
    this.messages.forEach(s=>s.unsubscribe());
    this.messages=[];  
  }
Samal
  • 1
  • 1

0 Answers0