I need to read json data from kafka topic. I am not able to consume json data from kafka topic.It will in infinite loop.
So far I suppose I need to:
Implement a custom serializer to convert JSON into byte array implement a custom deserializer to convert byte array into JSON object Produce the message Read the message in Consumer class
package services
import java.util
import java.util.Properties
import constants.ConfigValues
import models.Device
import org.apache.kafka.clients.consumer.KafkaConsumer
import scala.collection.JavaConverters._
class ConsumerService {
def consumeDeviceData(topic: String) = {
val props = new Properties()
props.put("bootstrap.servers", ConfigValues.BOOTSTARP_SERVER)
props.put("key.deserializer", ConfigValues.KEY_DESERIALIZER)
//props.put("value.deserializer", ConfigValues.VALUE_DESERIALIZER)
props.put("value.deserializer", classOf[CustomDeserializer] )
props.put("auto.offset.reset", "latest")
props.put("group.id", "device-consumer")
val consumer = new KafkaConsumer[String, Device](props)
println(consumer)
consumer.subscribe(util.Arrays.asList(topic))
while (true) {
val record = consumer.poll(1000).asScala
println("inside")
println(record)
for (device <- record.iterator)
println(device.value())
// GetDeviceResponse.fromDeviceObject(value.value())
}
}
}
CustomDeserializer class:
package services
import java.io.{ByteArrayInputStream, ObjectInputStream}
import java.util
import models.Device
import org.apache.kafka.common.serialization.Deserializer
class CustomDeserializer extends Deserializer[Device] {
override def configure(configs: util.Map[String, _], isKey: Boolean): Unit = {
}
override def deserialize(topic: String, bytes: Array[Byte]): Device = {
val byteIn = new ByteArrayInputStream(bytes)
val objIn = new ObjectInputStream(byteIn)
val obj = objIn.readObject().asInstanceOf[Device]
println(obj)
byteIn.close()
objIn.close()
obj
}
override def close(): Unit = {
}
}
Device class:
package models
class Device() {
var DATE: Int=0
var SOURCETYPE: String=""
var DEVICEID: String=""
var EVENTTIME: String=""
var TIME: Int=0
var COUNTRYCODE: String=""
var COUNTRYNAME: String=""
var COUNTY: String=""
var DEVICELAT: String=""
var DEVICELOGCREATEDAT: Double=0
var DEVICELOGSTATUS: String=""
var DEVICELONG: String=""
var DEVICEHPE: String=""
var DEVICERAWLOCID: String=""
var DEVICERAWSTATE: String=""
def this(DATE: Int ,SOURCETYPE: String,DEVICEID: String,EVENTTIME: String, TIME: Int, COUNTRYCODE: String,
COUNTRYNAME: String, COUNTY: String, DEVICELAT: String, DEVICELOGCREATEDAT: Double, DEVICELOGSTATUS: String, DEVICELONG: String, DEVICEHPE: String, DEVICERAWLOCID: String, DEVICERAWSTATE: String)
{
this()
this.DATE=DATE
this.COUNTRYCODE = COUNTRYCODE
this.COUNTRYNAME =COUNTRYNAME
this.DATE=DATE
this.COUNTY=COUNTY
this.DEVICEHPE=DEVICEHPE
this.DEVICEID=DEVICEID
this.DEVICELAT=DEVICELAT
this.DEVICELOGCREATEDAT=DEVICELOGCREATEDAT
this.DEVICELOGSTATUS=DEVICELOGSTATUS
this.DEVICELONG=DEVICELONG
this.DEVICERAWLOCID=DEVICERAWLOCID
this.DEVICERAWSTATE=DEVICERAWSTATE
this.EVENTTIME=EVENTTIME
this.SOURCETYPE=SOURCETYPE
this.TIME=TIME
}
}