Kafka EOS read-process-write missing some messages

Question

I am currently experimenting with Kafka Exactly-Once Semantics in Python with the Confluent Kafka library. I have 3 programs :

One sending incremental integers in a topic called INPUT_TOPIC with 2 partitions.
The second one implements read-process-write with EOS to copy the messages in an OUTPUT_TOPIC. There is one couple "consumer-producer" per INPUT_TOPIC partition (so 2)
The third one reads messages 2 by 2, and expects a difference of one between the value of the 2 messages. If the difference is not one, that could mean that EOS is not functioning as expected and messages are not forwarded to the OUTPUT_TOPIC.

I am still seeing gaps in the data.

Here is a snippet for the incremental producer :

def run():
    raw_ensure_topic(INPUT_TOPIC, num_partitions=INPUT_TOPIC_PARTITIONS)
    p_conf = get_base_kafka_config()
    p_conf['enable.idempotence'] = True
    p_conf['acks'] = 'all'
    p = Producer(p_conf)
    counter = 0
    while True:
        p.produce(INPUT_TOPIC, f"{counter}", f"{counter}", on_delivery=delivery_report)
        counter += 1
        time.sleep(10.0/1000.0)
        p.poll(0)

Here is the code for the read-process-write:

def __run(consumer_group_id, partition_id):
    raw_ensure_topic(OUTPUT_TOPIC, num_partitions=OUTPUT_TOPIC_PARTITIONS)
    consumer_config = get_consumer_kafka_config(autocommit=False, group_id=consumer_group_id, offset_reset='earliest')
    consumer = Consumer(consumer_config)

    consumer.assign([TopicPartition(INPUT_TOPIC, partition_id)])

    producer_config = get_base_kafka_config()
    producer_config['transactional.id'] = 'eos-transaction-demo'
    producer_config['enable.idempotence'] = True
    producer_config['acks'] = 'all'
    producer = Producer(producer_config)
    producer.init_transactions()
    producer.begin_transaction()
    msg_cnt = 0
    while True:
        msg = consumer.poll(timeout=1.0)
        if msg is None:
            continue

        if msg.error():
            raise msg.error()

        msg_cnt += 1

        if random.randint(0, 1000) > 990:
            print("Generating fake failure...")
            producer.abort_transaction()
            producer.begin_transaction()
            continue

        producer.produce(OUTPUT_TOPIC, msg.key(), msg.value(), on_delivery=delivery_report)
        if msg_cnt % 100 == 0:
            print("=== Committing transaction with {} messages at input offset {} ===".format(
                msg_cnt, msg.offset()))
            # Send the consumer's position to transaction to commit
            # them along with the transaction, committing both
            # input and outputs in the same transaction is what provides EOS.
            producer.send_offsets_to_transaction(
                consumer.position(consumer.assignment()),
                consumer.consumer_group_metadata())

            # Commit the transaction
            producer.commit_transaction()

            # Begin new transaction
            producer.begin_transaction()
            msg_cnt = 0

def run():
    for pid in range(0, INPUT_TOPIC_PARTITIONS):
        __run(f"demo-transfer", pid)

And the program that validates the data:

def run():
    consumer_config = get_consumer_kafka_config(autocommit=True, group_id='validator', offset_reset='earliest')
    print(consumer_config)
    consumer = Consumer(consumer_config)

    missing = []
    last_missing_stats_idx = 0

    consumer.subscribe([OUTPUT_TOPIC])
    while True:
        msgs = consumer.consume(2)
        [v1, v2] = [int(msgs[0].value()), int(msgs[1].value())]
        if v1 in missing:
            missing.remove(v1)
        elif v2 in missing:
            missing.remove(v2)
        elif abs(v1 - v2) != 1:
            missing.extend(range(v1+1, v2))
        if v1 % 100 == 0:
            print(f"Reached {v1}")
        if last_missing_stats_idx+1000 < v1:
            last_missing_stats_idx = v1
            print(f"Missing state {missing}")

Kafka EOS read-process-write missing some messages

0 Answers0