0

when I set msg = consumer.poll(timeout=10.0) the consumer waits 10 seconds and return None as expected, but when I change this to msg = consumer.poll(timeout=3600.0) this consumer just return None immidiately instead of waiting 3600 seconds as expected. Did I miss anything here? here is the full code if needed.

running = True
conf = {'bootstrap.servers': bootstrap_servers,
        'group.id': 'foo',
        'auto.offset.reset': 'earliest',
        'enable.auto.commit': False,
        'on_commit': commit_completed}
consumer = Consumer(conf)


def msg_process(msg):
    print(f"key: {msg.key().decode('utf-8')}, value: {msg.value().decode('utf-8')}")


def basic_consume_loop(consumer, topics):
    try:
        consumer.subscribe(topics)

        msg_count = 0
        while running:
            msg = consumer.poll(timeout=3600.0)
            if msg is None:
                print(f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}: no new message")
                continue

            if msg.error():
                if msg.error().code() == KafkaError._PARTITION_EOF:
                    # End of partition event
                    print('%% %s [%d] reached end at offset %d\n' %
                          (msg.topic(), msg.partition(), msg.offset()))
                elif msg.error():
                    raise KafkaException(msg.error())
            else:
                # consumer.commit(async=False)
                msg_process(msg)
                msg_count += 1
                if msg_count % MIN_COMMIT_COUNT == 0:
                    consumer.commit(async=True)
    finally:
        # Close down consumer to commit final offsets.
        consumer.close()


def shutdown():
    running = False


basic_consume_loop(consumer, [topic_user])
JamesWang
  • 1,175
  • 3
  • 14
  • 32

1 Answers1

0

Probably because fetch.max.wait.ms setting is lower than what is passed in poll().

enjoy
  • 1
  • 1