reprocess maprstream messages using spring integration kafka

Question

This is related to the thread and I am using spring-integration-kafka 2.0 to consume the messages from mapr stream topics. I am facing difficulties to use the KafkaConsumer feature - reprocess maprstream messages - using offset and topic partitions.

If I can integrate seek feature I will be able to reprocess the messages based on offset value.

Can someone please help me to integrate the KafkaConsumer features seek, seekToBegining, seekToEnd in spring integration Kafka? The current consumer configuration is mentioned below:

 <int-kafka:message-driven-channel-adapter
    id="kafkaListener"
    listener-container="container1"
    auto-startup="true"
    phase="100"
    send-timeout="5000"
    channel="inputFromStream"
    error-channel="errorChannel" />

<bean id="container1" class="org.springframework.kafka.listener.KafkaMessageListenerContainer">
    <constructor-arg>
        <bean class="org.springframework.kafka.core.DefaultKafkaConsumerFactory">
            <constructor-arg>
            <map>
                <entry key="bootstrap.servers" value="localhost:9092"/>
                <entry key="group.id" value="siTestGroup1"/>
                <entry key="enable.auto.commit" value="true"/>
                <entry key="auto.commit.interval.ms" value="1000"/>
                <entry key="auto.offset.reset" value="earliest" /> 
                <entry key="max.partition.fetch.bytes" value="3145728"/>
                <entry key="key.deserializer" value="org.apache.kafka.common.serialization.StringDeserializer"/>
                <entry key="value.deserializer" value="org.apache.kafka.common.serialization.StringDeserializer"/> 
            </map>
            </constructor-arg>
        </bean>
    </constructor-arg>
    <constructor-arg>
        <bean class="org.springframework.kafka.listener.config.ContainerProperties">
            <constructor-arg name="topics" value="${maprstream.topicname}" />
        </bean>
    </constructor-arg> 
</bean>

Answer 1

Use a ConsumerAwareRebalanceListener - this is how Spring Cloud Stream does it...

final AtomicBoolean initialAssignment = new AtomicBoolean(true);
if (!"earliest".equals(resetTo) && "!latest".equals(resetTo)) {
    logger.warn("no (or unknown) " + ConsumerConfig.AUTO_OFFSET_RESET_CONFIG +
            " property cannot reset");
    resetOffsets = false;
}
if (groupManagement && resetOffsets) {
    containerProperties.setConsumerRebalanceListener(new ConsumerAwareRebalanceListener() {

        @Override
        public void onPartitionsRevokedBeforeCommit(Consumer<?, ?> consumer, Collection<TopicPartition> tps) {
            // no op
        }

        @Override
        public void onPartitionsRevokedAfterCommit(Consumer<?, ?> consumer, Collection<TopicPartition> tps) {
            // no op
        }

        @Override
        public void onPartitionsAssigned(Consumer<?, ?> consumer, Collection<TopicPartition> tps) {
            if (initialAssignment.getAndSet(false)) {
                if ("earliest".equals(resetTo)) {
                    consumer.seekToBeginning(tps);
                }
                else if ("latest".equals(resetTo)) {
                    consumer.seekToEnd(tps);
                }
            }
        }
    });
}
else if (resetOffsets) {
    Arrays.stream(containerProperties.getTopicPartitions())
            .map(tpio -> new TopicPartitionInitialOffset(tpio.topic(), tpio.partition(),
                    // SK GH-599     "earliest".equals(resetTo) ? SeekPosition.BEGINNING : SeekPosition.END))
                    "earliest".equals(resetTo) ? 0L : Long.MAX_VALUE))
            .collect(Collectors.toList()).toArray(containerProperties.getTopicPartitions());
}