I have worked out a Kafka concept which is built up in such a way that I have a series of machines which have electric airpressure dustextraction and machinenStatus data. I have set up my Kafka concept in such a way that I create a topic for each machine and each topic has 4 partitions. The partitions are then structured so that airPressureData is placed on partition 0, electricData on partition 1, dustExtractionData on partition 2 and MachineStatus Data on partition 3. I have now written 4 consumers which each subscribe to the individual partitions of the topics using consumer.Assing(). However, I get no data although the producer writes the correct data to the correct partitions. Here is another example consumer:
using System.IO.Compression;
using Confluent.Kafka;
using EnergyMeasurement.Models;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
public class AirPressureConsumerService : BackgroundService
{
private readonly ILogger<AirPressureConsumerService> logger;
private readonly List<string> topics;
private readonly string groupId = "coldPath";
private readonly string bootstrapServers = "localhost:9092";
public AirPressureConsumerService(ILogger<AirPressureConsumerService> logger)
{
this.logger = logger;
this.topics = new List<string> { "thingId1", "thingId2" };
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var config = new ConsumerConfig
{
BootstrapServers = this.bootstrapServers,
GroupId = this.groupId,
AutoOffsetReset = AutoOffsetReset.Earliest
};
using var consumer = new ConsumerBuilder<Ignore, byte[]>(config).Build();
foreach (var topic in this.topics)
{
var topicPartition = new TopicPartition(topic, 0);
consumer.Assign(topicPartition);
while (!stoppingToken.IsCancellationRequested)
{
var consumeResult = consumer.Consume(stoppingToken);
var compressedDto = CompressedDTO.Parser.ParseFrom(consumeResult.Message.Value);
var contentBytes = Decompress(compressedDto.CompressedData.ToByteArray());
var airPressureDTO = AirPressureDTO.Parser.ParseFrom(contentBytes);
Console.WriteLine($"Received message at {consumeResult.TopicPartitionOffset}: {airPressureDTO}");
await Task.Delay(1000, stoppingToken);
}
}
consumer.Close();
}
private byte[] Decompress(byte[] data)
{
using var compressedStream = new MemoryStream(data);
using var decompressStream = new GZipStream(compressedStream, CompressionMode.Decompress);
using var resultStream = new MemoryStream();
decompressStream.CopyTo(resultStream);
return resultStream.ToArray();
}
}
I have checked that the producers write to the correct topics and partitions. I also tried to sign on the whole topic, then I get data. I don't understand why I don't get the data this way.
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
public class Program
{
public static void Main(string[] args)
{
CreateHostBuilder(args).Build().Run();
}
public static IHostBuilder CreateHostBuilder(string[] args) =>
Host.CreateDefaultBuilder(args)
.ConfigureServices((hostContext, services) =>
{
services.AddHostedService<ElectricConsumerService>();
services.AddHostedService<AirPressureConsumerService>();
services.AddHostedService<MachineStatusConsumerService>();
services.AddHostedService<DustExtractionConsumerService>();
services.AddHostedService<TestConsumer>();
});
}