Google Pubsub Java Spring Cloud GCP Reconnectivity Issue

Question

I have a Spring Boot application running on Google Kubernetes Engine connecting to pubsub. Recently we started seeing this problem that app will stop receiving messages from a subscription after a while. If we restart the app, it connects fine and starts picking up the messages.

I've turned on logging to DEBUG and it shows a periodic disconnectivity and reconnection attempts are successfully made and it continues to receive messages after that but at some point it just silently stops receiving messages. In my local, I recreated the scenario by starting the service, it receives messages, then I disconnected the network and after reconnecting the network, the above keepAlive/retry logic does not kick in for some reason. In one scenario it did kick in after couple of hours and everything was fine.

So the question is, why does the retry logic not kick in when there's loss of connection (in this tested case disconnected and reconnected network connection).

I will post same question to Google Discussion Group when I can join it: https://groups.google.com/forum/#!searchin/cloud-pubsub-discuss

It could be something simple as some configuration but I can't seem to find it so far.

My setup to recreate this was as follows:

A barebone Spring Boot app following this tutorial: https://spring.io/guides/gs/messaging-gcp-pubsub/

Using Spring Boot 2.0.4.RELEASE, spring-cloud-gcp-starter-pubsub:1.0.0.RELEASE, and spring-integration-core:5.0.7.RELEASE.

I have multiple subscriptions, as that's our use-case.

The reconnecting error that successfully connects is given below (but this one does not get invoked when network was manually disconnected and reconnected):

DEBUG 59005 --- [pool-2-thread-1] c.g.c.p.v.StreamingSubscriberConnection : stream closed with retryable exception; will reconnect

io.grpc.StatusRuntimeException: UNAVAILABLE: The service was unable to fulfill your request. Please try again. [code=8a75] at io.grpc.Status.asRuntimeException(Status.java:526) ~[grpc-core-1.13.1.jar:1.13.1] at io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onClose(ClientCalls.java:420) [grpc-stub-1.13.1.jar:1.13.1] at io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39) [grpc-core-1.13.1.jar:1.13.1] at io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23) [grpc-core-1.13.1.jar:1.13.1] at io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40) [grpc-core-1.13.1.jar:1.13.1] at io.grpc.internal.CensusStatsModule$StatsClientInterceptor$1$1.onClose(CensusStatsModule.java:684) [grpc-core-1.13.1.jar:1.13.1] at io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39) [grpc-core-1.13.1.jar:1.13.1] at io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23) [grpc-core-1.13.1.jar:1.13.1] at io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40) [grpc-core-1.13.1.jar:1.13.1] at io.grpc.internal.CensusTracingModule$TracingClientInterceptor$1$1.onClose(CensusTracingModule.java:403) [grpc-core-1.13.1.jar:1.13.1] at io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:459) [grpc-core-1.13.1.jar:1.13.1] at io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:63) [grpc-core-1.13.1.jar:1.13.1] at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.close(ClientCallImpl.java:546) [grpc-core-1.13.1.jar:1.13.1] at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.access$600(ClientCallImpl.java:467) [grpc-core-1.13.1.jar:1.13.1] at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:584) [grpc-core-1.13.1.jar:1.13.1] at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) [grpc-core-1.13.1.jar:1.13.1] at io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) [grpc-core-1.13.1.jar:1.13.1] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_161] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_161] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) [na:1.8.0_161] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) [na:1.8.0_161] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_161] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_161] at java.lang.Thread.run(Thread.java:748) [na:1.8.0_161]

import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.gcp.pubsub.core.PubSubTemplate;
import org.springframework.cloud.gcp.pubsub.integration.AckMode;
import org.springframework.cloud.gcp.pubsub.integration.inbound.PubSubInboundChannelAdapter;
import org.springframework.cloud.gcp.pubsub.support.GcpPubSubHeaders;
import org.springframework.context.annotation.Bean;
import org.springframework.integration.annotation.ServiceActivator;
import org.springframework.integration.channel.DirectChannel;
import org.springframework.messaging.MessageChannel;
import org.springframework.messaging.MessageHandler;

import com.google.cloud.pubsub.v1.AckReplyConsumer;

@SpringBootApplication
public class PubsubTestApplication {

  public static void main(String[] args) {
    SpringApplication.run(PubsubTestApplication.class, args);
  }

  @Bean
  public PubSubInboundChannelAdapter messageChannelAdapter(@Qualifier("pubsubInputChannel") MessageChannel inputChannel,
      PubSubTemplate pubSubTemplate) {
    PubSubInboundChannelAdapter adapter = new PubSubInboundChannelAdapter(pubSubTemplate,
        "subscription-1");
    adapter.setOutputChannel(inputChannel);
    adapter.setAckMode(AckMode.MANUAL);

    return adapter;
  }


  @Bean
  public PubSubInboundChannelAdapter messageChannelAdapterA(@Qualifier("pubsubInputChannel") MessageChannel inputChannel,
      PubSubTemplate pubSubTemplate) {
    PubSubInboundChannelAdapter adapter = new PubSubInboundChannelAdapter(pubSubTemplate,
        "subscription-2");
    adapter.setOutputChannel(inputChannel);
    adapter.setAckMode(AckMode.MANUAL);

    return adapter;
  }

  @Bean
  public MessageChannel pubsubInputChannel() {
    return new DirectChannel();
  }

  @Bean
  @ServiceActivator(inputChannel = "pubsubInputChannel")
  public MessageHandler messageReceiver() {
    return message -> {
      System.out.println("Message arrived! Payload: " + new String((byte[])message.getPayload()));
      AckReplyConsumer consumer =
          (AckReplyConsumer) message.getHeaders().get(GcpPubSubHeaders.ACKNOWLEDGEMENT);
      consumer.ack();
    };
  }
}