Facing an issue with UCP connection pool, where the request to obtain connection from the pool is stuck in Runnable state by owner thread. Below is the stack trace of for stuck thread.
From the error trace this looks to be stuck while borrowing connection from UCP pool. We also have set the below timeout properties on oracle.ucp.jdbc.PoolDataSourceFactory in anticipation to resolve this.
connectionWaitTimeout 60
connectionProperties- oracle.net.CONNECT_TIMEOUT 6000 oracle.jdbc.ReadTimeout 300000
Is there any other way/timeout needs to be configured to get the error if client is not able to take connection in sometime rather than keep waiting and thread stuck in Runnable ?
Also verified from JMX Mbean(oracle.uc.admin.UniversalConnectionPoomMBean) that pool has the capacity remaining , its set to 100. That time borrwedConnectionCount was 0 , remainPoolCapacityCount was 100 and pendingRequestsCount continued to be 1.
pub-#74" - Thread t@117
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
at sun.nio.ch.IOUtil.read(IOUtil.java:197)
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380)
- locked <4ed7c65> (a java.lang.Object)
at oracle.net.nt.TimeoutSocketChannel.read(TimeoutSocketChannel.java:144)
at oracle.net.ns.NIOHeader.readHeaderBuffer(NIOHeader.java:82)
at oracle.net.ns.NIOPacket.readFromSocketChannel(NIOPacket.java:139)
at oracle.net.ns.NIOPacket.readFromSocketChannel(NIOPacket.java:101)
at oracle.net.ns.NIONSDataChannel.readDataFromSocketChannel(NIONSDataChannel.java:80)
at oracle.net.ns.NIONSDataChannel.readPayloadDataFromSocketChannel(NIONSDataChannel.java:228)
at oracle.net.ns.NIORedirectPacket.readPayloadBuffer(NIORedirectPacket.java:69)
at oracle.net.ns.NIOPacket.readNIOPacket(NIOPacket.java:282)
at oracle.net.ns.NSProtocolNIO.negotiateConnection(NSProtocolNIO.java:118)
at oracle.net.ns.NSProtocol.connect(NSProtocol.java:317)
at oracle.jdbc.driver.T4CConnection.connect(T4CConnection.java:1438)
at oracle.jdbc.driver.T4CConnection.logon(T4CConnection.java:518)
at oracle.jdbc.driver.PhysicalConnection.connect(PhysicalConnection.java:688)
at oracle.jdbc.driver.T4CDriverExtension.getConnection(T4CDriverExtension.java:39)
at oracle.jdbc.driver.OracleDriver.connect(OracleDriver.java:691)
at oracle.jdbc.pool.OracleDataSource.getPhysicalConnection(OracleDataSource.java:384)
at oracle.jdbc.pool.OracleDataSource.getConnection(OracleDataSource.java:273)
at oracle.jdbc.pool.OracleDataSource$1.build(OracleDataSource.java:1649)
at oracle.jdbc.pool.OracleDataSource$1.build(OracleDataSource.java:1643)
at oracle.ucp.jdbc.oracle.OracleDataSourceConnectionFactoryAdapter.createConnection(OracleDataSourceConnectionFactoryAdapter.java:103)
at oracle.ucp.common.Database.createPooledConnection(Database.java:256)
at oracle.ucp.common.Service.create(Service.java:758)
at oracle.ucp.common.Service.create(Service.java:588)
at oracle.ucp.common.Topology.create(Topology.java:145)
at oracle.ucp.common.Core.growBorrowed(Core.java:699)
at oracle.ucp.common.UniversalConnectionPoolImpl.borrowConnectionWithoutCountingRequests(UniversalConnectionPoolImpl.java:205)
at oracle.ucp.common.UniversalConnectionPoolImpl.borrowConnectionAndValidate(UniversalConnectionPoolImpl.java:143)
at oracle.ucp.common.UniversalConnectionPoolImpl.borrowConnection(UniversalConnectionPoolImpl.java:122)
at oracle.ucp.jdbc.JDBCConnectionPool.borrowConnection(JDBCConnectionPool.java:172)
at oracle.ucp.jdbc.oracle.OracleJDBCConnectionPool.borrowConnection(OracleJDBCConnectionPool.java:613)
at oracle.ucp.jdbc.oracle.OracleConnectionConnectionPool.borrowConnection(OracleConnectionConnectionPool.java:97)
at oracle.ucp.jdbc.PoolDataSourceImpl.getConnection(PoolDataSourceImpl.java:1989)
at oracle.ucp.jdbc.PoolDataSourceImpl.access$500(PoolDataSourceImpl.java:198)
at oracle.ucp.jdbc.PoolDataSourceImpl$30.build(PoolDataSourceImpl.java:4353)
at oracle.ucp.jdbc.PoolDataSourceImpl.getConnection(PoolDataSourceImpl.java:1924)
at oracle.ucp.jdbc.PoolDataSourceImpl.getConnection(PoolDataSourceImpl.java:1887)
at oracle.ucp.jdbc.PoolDataSourceImpl.getConnection(PoolDataSourceImpl.java:1872)
at sun.reflect.GeneratedMethodAccessor165.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:343)
at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:198)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
at org.springframework.data.jdbc.aop.ConnectionInterceptor.invoke(ConnectionInterceptor.java:46)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)
at org.springframework.data.jdbc.aop.ConnectionInterceptor.invoke(ConnectionInterceptor.java:46)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)
at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:93)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:212)
at com.sun.proxy.$Proxy15.getConnection(Unknown Source)
at org.hibernate.engine.jdbc.connections.internal.DatasourceConnectionProviderImpl.getConnection(DatasourceConnectionProviderImpl.java:122)
at org.hibernate.internal.NonContextualJdbcConnectionAccess.obtainConnection(NonContextualJdbcConnectionAccess.java:35)
at org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.acquireConnectionIfNeeded(LogicalConnectionManagedImpl.java:106)
at org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.getPhysicalConnection(LogicalConnectionManagedImpl.java:136)
at org.hibernate.internal.SessionImpl.connection(SessionImpl.java:541)
at org.springframework.orm.hibernate5.HibernateTransactionManager.doBegin(HibernateTransactionManager.java:480)
at org.springframework.transaction.support.AbstractPlatformTransactionManager.getTransaction(AbstractPlatformTransactionManager.java:378)
at org.springframework.transaction.interceptor.TransactionAspectSupport.createTransactionIfNecessary(TransactionAspectSupport.java:474)
at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:289)
at org.springframework.transaction.aspectj.AbstractTransactionAspect.ajc$around$org_springframework_transaction_aspectj_AbstractTransactionAspect$1$2a73e96c(AbstractTransactionAspect.aj:70)
at markit.valuations.sql.TradeQueries.loadMultipleTradesByValuationDate(TradeQueries.java:184)
at com.markit.valuations.ironbridge.trade.TradeLoader.load_aroundBody2(TradeLoader.java:66)
at com.markit.valuations.ironbridge.trade.TradeLoader$AjcClosure3.run(TradeLoader.java:1)
at org.springframework.cache.aspectj.AbstractCacheAspect.ajc$around$org_springframework_cache_aspectj_AbstractCacheAspect$1$2bc714b5proceed(AbstractCacheAspect.aj:64)
at org.springframework.cache.aspectj.AbstractCacheAspect$AbstractCacheAspect$1.invoke(AbstractCacheAspect.aj:71)
at org.springframework.cache.interceptor.CacheAspectSupport.invokeOperation(CacheAspectSupport.java:365)
at org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:420)
at org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:345)
at org.springframework.cache.aspectj.AbstractCacheAspect.ajc$around$org_springframework_cache_aspectj_AbstractCacheAspect$1$2bc714b5(AbstractCacheAspect.aj:80)
at com.markit.valuations.ironbridge.trade.TradeLoader.load(TradeLoader.java:56)
