Trying to load messages from Topic into a silverTable in the WriteStream. But the messages are not loading into silverTable. How to read the messages into silverTable?
var df = spark
.readStream
.format("kafka")
.option("kafka.bootstrap.servers", "10.19.9.4:1111")
.option("subscribe", "testTopic")
.load()
df = df.select($"value",$"topic")
// select the avro encoded value and the topic name from the topic
df.writeStream
.foreachBatch( (batchDF: DataFrame, batchId: Long)=>
{
batchDF.persist()
val topics = batchDF.select("topic").distinct().collect().map(
(row)=>row.getString(0))
topics.foreach((topix)=>{
val silverTable = mappings(topix)
// filter out message for the current topic
var writeDF = batchDF.where(s"topic = '${topix}'")
// decode the avro records to a spark struct
val schemaReg = schemaRegistryMappings(topix)
writeDF = writeDF.withColumn("avroconverted",
from_avro($"value", topix+"-value", schemaReg))
// append to the sliver table
writeDF.write.format("delta").mode("append").saveAsTable("silverTable")
})
}