I'm calling the consumer.commit()
method on each message of consumer running in auto.commit=True mode. When I tried it locally it worked without any data loss or data duplication.
What are the effects the commit()
method can give to the consumer?
class KafkaConsumer(object):
def __init__(self, topics: list[str], **kwargs: Any):
config = {
**kwargs,
}
self.consumer = DeserializingConsumer(config)
self.consumer.subscribe(topics=topics)
def consume(self, poll_timeout_secs: float = 1.0):
while True:
msg = self.consumer.poll(timeout=poll_timeout_secs)
if msg is None:
continue
if msg.error():
raise KafkaException(msg.error())
else:
yield msg
kafka_consumer = KafkaConsumer(topics=topics, **kwargs) # passed "enable.auto.commit":True
for message in kafka_consumer.consume():
event = message.value()
logger.info(f"message {event}")
kafka_consumer.commit() # what will be the effect of calling this?