Is it possible to publish xml in kafka?

Question

Is it possible to publish xml in kafka? If yes, which method can be used to serialize data to be published in kafka?

Answer 1

Kafka accepts any serializable input formats. Xml is just text, so you can use plain string serializer.

Otherwise, if you want additional validation before pushing messages (like checking the content is actually xml), you would need to write your own Serializer/Deserializer implementation

As for apache-kafka-connect, you can refer to this Connect transformer https://github.com/jcustenborder/kafka-connect-transform-xml

Answer 2

you should use a serializer/deserializer for the messages to be published and subscribed. Content on kafka topic looks like a string-xml. here is a c# code sample to serialize and deserialize string-xml

c# code sample to produce-consume kafka messages with confluent.io it supports convenient programming languages

producer

using System;
using Confluent.Kafka;

class Program
{
    public static void Main(string[] args)
    {
        var conf = new ProducerConfig { BootstrapServers = "localhost:9092" };

    Action<DeliveryReport<Null, string>> handler = r => 
        Console.WriteLine(!r.Error.IsError
            ? $"Delivered message to {r.TopicPartitionOffset}"
            : $"Delivery Error: {r.Error.Reason}");

    using (var p = new ProducerBuilder<Null, string>(conf).Build())
    {
     // call XmlSerializeToString(this object objectInstance) from ref#1
     var xmlMessage = "xml object".XmlSerializeToString();
     p.Produce("my-topic", new Message<Null, string> { Value = xmlMessage}, handler);       
        // wait for up to 10 seconds for any inflight messages to be delivered.
      p.Flush(TimeSpan.FromSeconds(10));
    }
}

}

consumer

using System;
using System.Threading;
using Confluent.Kafka;

class Program
{
    public static void Main(string[] args)
    {
        var conf = new ConsumerConfig
        { 
            GroupId = "test-consumer-group",
            BootstrapServers = "localhost:9092",
            // Note: The AutoOffsetReset property determines the start offset in the event
            // there are not yet any committed offsets for the consumer group for the
            // topic/partitions of interest. By default, offsets are committed
            // automatically, so in this example, consumption will only start from the
            // earliest message in the topic 'my-topic' the first time you run the program.
            AutoOffsetReset = AutoOffsetReset.Earliest
        };

        using (var c = new ConsumerBuilder<Ignore, string>(conf).Build())
        {
            c.Subscribe("my-topic");

            CancellationTokenSource cts = new CancellationTokenSource();
            Console.CancelKeyPress += (_, e) => {
                e.Cancel = true; // prevent the process from terminating.
                cts.Cancel();
            };

            try
            {
                while (true)
                {
                    try
                    {
                        var cr = c.Consume(cts.Token);
                        var xmlContent = cr.Value.XmlDeserializeFromString()
                        Console.WriteLine($"Consumed message '{cr.Value}' at: '{cr.TopicPartitionOffset}'.");
                    }
                    catch (ConsumeException e)
                    {
                        Console.WriteLine($"Error occured: {e.Error.Reason}");
                    }
                }
            }
            catch (OperationCanceledException)
            {
                // Ensure the consumer leaves the group cleanly and final offsets are committed.
                c.Close();
            }
        }
    }
}