0

I need to read json data from kafka topic. I am not able to consume json data from kafka topic.It will in infinite loop.

So far I suppose I need to:

Implement a custom serializer to convert JSON into byte array implement a custom deserializer to convert byte array into JSON object Produce the message Read the message in Consumer class

package services

import java.util
import java.util.Properties

import constants.ConfigValues
import models.Device
import org.apache.kafka.clients.consumer.KafkaConsumer
import scala.collection.JavaConverters._

class ConsumerService {


  def consumeDeviceData(topic: String) = {
    val props = new Properties()
    props.put("bootstrap.servers", ConfigValues.BOOTSTARP_SERVER)
    props.put("key.deserializer", ConfigValues.KEY_DESERIALIZER)
    //props.put("value.deserializer", ConfigValues.VALUE_DESERIALIZER)
    props.put("value.deserializer", classOf[CustomDeserializer] )
    props.put("auto.offset.reset", "latest")
    props.put("group.id", "device-consumer")
    val consumer = new KafkaConsumer[String, Device](props)
    println(consumer)
    consumer.subscribe(util.Arrays.asList(topic))
    while (true) {
      val record = consumer.poll(1000).asScala
      println("inside")
      println(record)
      for (device <- record.iterator)
        println(device.value())
       // GetDeviceResponse.fromDeviceObject(value.value())
    }
  }
}

CustomDeserializer class:

package services

import java.io.{ByteArrayInputStream, ObjectInputStream}
import java.util

import models.Device
import org.apache.kafka.common.serialization.Deserializer

class CustomDeserializer extends Deserializer[Device] {

  override def configure(configs: util.Map[String, _], isKey: Boolean): Unit = {
  }

  override def deserialize(topic: String, bytes: Array[Byte]): Device = {

    val byteIn = new ByteArrayInputStream(bytes)
    val objIn = new ObjectInputStream(byteIn)
    val obj = objIn.readObject().asInstanceOf[Device]
    println(obj)
    byteIn.close()
    objIn.close()
    obj
  }

  override def close(): Unit = {

  }
}

Device class:

package models


class Device() {
  var DATE: Int=0
  var SOURCETYPE: String=""
  var DEVICEID: String=""
  var EVENTTIME: String=""
  var TIME: Int=0
  var COUNTRYCODE: String=""
  var COUNTRYNAME: String=""
  var COUNTY: String=""
  var DEVICELAT: String=""

  var DEVICELOGCREATEDAT: Double=0

  var DEVICELOGSTATUS: String=""

  var DEVICELONG: String=""

  var DEVICEHPE: String=""

  var DEVICERAWLOCID: String=""

  var DEVICERAWSTATE: String=""

  def this(DATE: Int ,SOURCETYPE: String,DEVICEID: String,EVENTTIME: String, TIME: Int, COUNTRYCODE: String,
  COUNTRYNAME: String, COUNTY: String, DEVICELAT: String, DEVICELOGCREATEDAT: Double, DEVICELOGSTATUS: String, DEVICELONG: String, DEVICEHPE: String, DEVICERAWLOCID: String, DEVICERAWSTATE: String)
{
  this()
  this.DATE=DATE
  this.COUNTRYCODE = COUNTRYCODE
  this.COUNTRYNAME =COUNTRYNAME
  this.DATE=DATE
  this.COUNTY=COUNTY
  this.DEVICEHPE=DEVICEHPE
  this.DEVICEID=DEVICEID
  this.DEVICELAT=DEVICELAT
  this.DEVICELOGCREATEDAT=DEVICELOGCREATEDAT
  this.DEVICELOGSTATUS=DEVICELOGSTATUS
  this.DEVICELONG=DEVICELONG
  this.DEVICERAWLOCID=DEVICERAWLOCID
  this.DEVICERAWSTATE=DEVICERAWSTATE
  this.EVENTTIME=EVENTTIME
  this.SOURCETYPE=SOURCETYPE
  this.TIME=TIME

}
}
  • Where is it blocking? Are you producing data while the consumer is running, since it's starting from the end of the topic? – OneCricketeer Feb 24 '21 at 15:22
  • I already produced data in one topic. After that I am reading data from that topic. When I am running this command bin/kafka-console-consumer.sh --topic DeviceData --from-beginning --bootstrap-server localhost:9092 --partition 0...then its shows d result – rishu nigam Feb 24 '21 at 18:46
  • @OneCricketeer..When I am trying to deseralize the object then i am getting error ::com.fasterxml.jackson.core.JsonParseException: Unexpected character ('>' (code 62)): expected a valid value (JSON String, Number, Array, Object or token 'null', 'true' or 'false') – rishu nigam Feb 25 '21 at 12:34
  • I don't know what serializer you're using, but `<` character isn't valid JSON... Plus, your shown code isn't using Jackson, so you need to edit your post to show what changed – OneCricketeer Mar 01 '21 at 15:31

0 Answers0