Application slowness due to Tomcat waiting for readLatch

Question

we are getting issues reported by our clients that the application is slow at times, while checking the slow requests (using Glowroot) we could find quite a lot of threads getting stuck in org.apache.tomcat.util.net.NioEndpoint$KeyAttachment.awaitReadLatch (we are using Tomcat 7), the stack trace is given below. Can someone give us a clue what could be wrong, as we are not able to find out the reason.

java.lang.Thread.run(Thread.java:748)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.run(NioEndpoint.java:1653)
org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:585)
org.apache.coyote.ajp.AjpNioProcessor.process(AjpNioProcessor.java:184)
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:407)
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:118)
org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:929)
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:99)
org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:168)
org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:472)
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:123)
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:222)
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210)
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243)
org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:76)
org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:88)
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210)
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243)
org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:167)
org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:237)
org.springframework.security.web.FilterChainProxy.doFilter(FilterChainProxy.java:149)
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:355)
org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:76)
org.xxx.doFilterInternal(xxx.java:58)
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:355)
org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:76)
org.yyy.doFilterInternal(yyy.java:22)
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:355)
org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:79)
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:355)
org.springframework.security.web.authentication.logout.LogoutFilter.doFilter(LogoutFilter.java:105)
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:355)
org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:76)
org.zzz.doFilterInternal(zzz.java:43)
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:355)
org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter.doFilter(AbstractAuthenticationProcessingFilter.java:200)
org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter.attemptAuthentication(UsernamePasswordAuthenticationFilter.java:72)
org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter.obtainUsername(UsernamePasswordAuthenticationFilter.java:125)
org.apache.catalina.connector.RequestFacade.getParameter(RequestFacade.java:383)
org.apache.catalina.connector.Request.getParameter(Request.java:1148)
org.apache.catalina.connector.Request.parseParameters(Request.java:3075)
org.apache.catalina.connector.Request.readPostBody(Request.java:3122)
org.apache.catalina.connector.CoyoteInputStream.read(CoyoteInputStream.java:200)
org.apache.catalina.connector.InputBuffer.read(InputBuffer.java:315)
org.apache.tomcat.util.buf.ByteChunk.substract(ByteChunk.java:431)
org.apache.catalina.connector.InputBuffer.realReadBytes(InputBuffer.java:290)
org.apache.coyote.Request.doRead(Request.java:422)
org.apache.coyote.ajp.AbstractAjpProcessor$SocketInputBuffer.doRead(AbstractAjpProcessor.java:1064)
org.apache.coyote.ajp.AjpNioProcessor.receive(AjpNioProcessor.java:375)
org.apache.coyote.ajp.AjpNioProcessor.readMessage(AjpNioProcessor.java:406)
org.apache.coyote.ajp.AjpNioProcessor.read(AjpNioProcessor.java:314)
org.apache.coyote.ajp.AjpNioProcessor.readSocket(AjpNioProcessor.java:342)
org.apache.tomcat.util.net.NioSelectorPool.read(NioSelectorPool.java:227)
org.apache.tomcat.util.net.NioSelectorPool.read(NioSelectorPool.java:246)
org.apache.tomcat.util.net.NioBlockingSelector.read(NioBlockingSelector.java:174)
org.apache.tomcat.util.net.NioEndpoint$KeyAttachment.awaitReadLatch(NioEndpoint.java:1554)
org.apache.tomcat.util.net.NioEndpoint$KeyAttachment.awaitLatch(NioEndpoint.java:1552)
java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
sun.misc.Unsafe.park(Native Method)

Tomcat connector configuration:

 <Connector port="5011" protocol="org.apache.coyote.ajp.AjpNioProtocol" redirectPort="8443" keepAliveTimeout="600000" connectionTimeout="300000" maxPostSize="20971520" maxThreads="1000" maxConnections="1000" processorCache="1000" acceptCount="500" acceptorThreadCount="3" />