0

I'm calling the consumer.commit() method on each message of consumer running in auto.commit=True mode. When I tried it locally it worked without any data loss or data duplication.

What are the effects the commit() method can give to the consumer?

class KafkaConsumer(object):
    def __init__(self, topics: list[str], **kwargs: Any):
        config = {
            **kwargs,
        }
        self.consumer = DeserializingConsumer(config)
        self.consumer.subscribe(topics=topics)

    def consume(self, poll_timeout_secs: float = 1.0):
        while True:
            msg = self.consumer.poll(timeout=poll_timeout_secs)
            if msg is None:
                continue
            if msg.error():
                raise KafkaException(msg.error())
            else:
                yield msg

kafka_consumer = KafkaConsumer(topics=topics, **kwargs) # passed "enable.auto.commit":True

for message in kafka_consumer.consume():
    event = message.value()
    logger.info(f"message {event}")
    kafka_consumer.commit() # what will be the effect of calling this?
Akash Pagar
  • 637
  • 8
  • 22

0 Answers0