Jira(4.4.3) instance keeps losing connectiont to database (oracle 10g (10.2.4)), with errors as below
This is observed every monday at the same time (8-9am which is the busiest period of the week). DBAs say no active sessions during that outage period. Not sure if Thread dumps can reveal anything ...!!
2012-02-20 08:41:39,126 TP-Processor620 ERROR jsmith63 516x255496x21 1fjn1it 166.15.148.121 /secure/IssueNavigator.jspa [NoModule] There was an error getting a DBCP datasource.
52930 java.lang.RuntimeException: Unable to obtain a connection from the underlying connection pool
52931 at org.ofbiz.core.entity.jdbc.interceptors.connection.ConnectionTracker.trackConnection(ConnectionTracker.java:59)
52932 at org.ofbiz.core.entity.transaction.DBCPConnectionFactory.trackConnection(DBCPConnectionFactory.java:149)
52933 at org.ofbiz.core.entity.transaction.DBCPConnectionFactory.getConnection(DBCPConnectionFactory.java:69)
52934 at org.ofbiz.core.entity.ConnectionFactory.tryGenericConnectionSources(ConnectionFactory.java:69)
52935 at org.ofbiz.core.entity.transaction.JNDIFactory.getConnection(JNDIFactory.java:146)
52936 at org.ofbiz.core.entity.TransactionFactory.getConnection(TransactionFactory.java:101)
52937 at org.ofbiz.core.entity.ConnectionFactory.getConnection(ConnectionFactory.java:59)
52938 at org.ofbiz.core.entity.jdbc.SQLProcessor.getConnection(SQLProcessor.java:367)
52939 at org.ofbiz.core.entity.jdbc.SQLProcessor.prepareStatement(SQLProcessor.java:515)
52940 at org.ofbiz.core.entity.GenericDAO.selectListIteratorByCondition(GenericDAO.java:1034)
52941 at org.ofbiz.core.entity.GenericDAO.selectByAnd(GenericDAO.java:605)
52942 at org.ofbiz.core.entity.GenericHelperDAO.findByAnd(GenericHelperDAO.java:131)
52943 at org.ofbiz.core.entity.GenericDelegator.findByAnd(GenericDelegator.java:790)
52944 at org.ofbiz.core.entity.GenericDelegator.findByAnd(GenericDelegator.java:775)
52945 at org.ofbiz.core.entity.GenericDelegator.findByAnd(GenericDelegator.java:752)
52946 at sun.reflect.GeneratedMethodAccessor32.invoke(Unknown Source)
52947 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
52948 at java.lang.reflect.Method.invoke(Method.java:597)
52949 at com.atlassian.multitenant.impl.MultiTenantComponentFactoryImpl$AbstractMultiTenantAwareInvocationHandler.invokeInternal(MultiTenantCompone ntFactoryImpl.java:181)
52950 at com.atlassian.multitenant.impl.MultiTenantComponentFactoryImpl$MultiTenantAwareMethodInterceptor.intercept(MultiTenantComponentFactoryImpl .java:230)
52951 at org.ofbiz.core.entity.GenericDelegator$$EnhancerByCGLIB$$77976a1f.findByAnd(<generated>)
52952 at com.atlassian.jira.ofbiz.DefaultOfBizDelegator.findByAnd(DefaultOfBizDelegator.java:92)
52953 at com.atlassian.jira.user.OfBizUserHistoryStore.updateHistoryItemNoChecks(OfBizUserHistoryStore.java:123)
52954 at com.atlassian.jira.user.CachingUserHistoryStore$2.run(CachingUserHistoryStore.java:124)
52955 at com.atlassian.util.concurrent.ManagedLocks$ManagedLockImpl.withLock(ManagedLocks.java:333)
52956 at com.atlassian.jira.user.CachingUserHistoryStore.addHistoryItem(CachingUserHistoryStore.java:93)
52957 at com.atlassian.jira.user.SessionBasedAnonymousUserHistoryStore$2.run(SessionBasedAnonymousUserHistoryStore.java:101)
52958 at com.atlassian.util.concurrent.ManagedLocks$ManagedLockImpl.withLock(ManagedLocks.java:333)
52959 at com.atlassian.jira.user.SessionBasedAnonymousUserHistoryStore.addHistoryItem(SessionBasedAnonymousUserHistoryStore.java:83)
52960 at com.atlassian.jira.user.DefaultUserHistoryManager.addItemToHistory(DefaultUserHistoryManager.java:41)
52961 at com.atlassian.jira.user.DefaultUserHistoryManager.addItemToHistory(DefaultUserHistoryManager.java:29)
52962 at com.atlassian.jira.user.DefaultUserProjectHistoryManager.addProjectToHistory(DefaultUserProjectHistoryManager.java:36)
52963 at com.atlassian.jira.user.DefaultUserProjectHistoryManager.addProjectToHistory(DefaultUserProjectHistoryManager.java:42)
52964 at com.atlassian.jira.issue.search.searchers.transformer.ProjectSearchInputTransformer.setProjectIdInSession(ProjectSearchInputTransformer.ja va:173)
52965 at com.atlassian.jira.issue.search.searchers.transformer.ProjectSearchInputTransformer.populateFromQuery(ProjectSearchInputTransformer.java:8 0)
52966 at com.atlassian.jira.web.action.issue.SearchDescriptionEnabledAction.populateFieldValuesHolderFromQuery(SearchDescriptionEnabledAction.java: 68)
52967 at com.atlassian.jira.web.action.issue.IssueNavigator.validateAndGetRedirectForQuery(IssueNavigator.java:756)
52968 at com.atlassian.jira.web.action.issue.IssueNavigator._doExecutePopulateFromSearchRequest(IssueNavigator.java:317)
52969 at com.atlassian.jira.web.action.issue.IssueNavigator.doExecute(IssueNavigator.java:221)
52970 at webwork.action.ActionSupport.execute(ActionSupport.java:165)
52971 at com.atlassian.jira.action.JiraActionSupport.execute(JiraActionSupport.java:76)
52972 at webwork.interceptor.DefaultInterceptorChain.proceed(DefaultInterceptorChain.java:39)
52973 at webwork.interceptor.NestedInterceptorChain.proceed(NestedInterceptorChain.java:31)
52974 at webwork.interceptor.ChainedInterceptor.intercept(ChainedInterceptor.java:16)
52975 at webwork.interceptor.DefaultInterceptorChain.proceed(DefaultInterceptorChain.java:35)
52976 at webwork.dispatcher.GenericDispatcher.executeAction(GenericDispatcher.java:205)
53098 Caused by: java.sql.SQLException: IO Error: Connection reset
53099 at oracle.jdbc.driver.T4CConnection.logon(T4CConnection.java:421)
53100 at oracle.jdbc.driver.PhysicalConnection.<init>(PhysicalConnection.java:531)
53101 at oracle.jdbc.driver.T4CConnection.<init>(T4CConnection.java:221)
53102 at oracle.jdbc.driver.T4CDriverExtension.getConnection(T4CDriverExtension.java:32)
53103 at oracle.jdbc.driver.OracleDriver.connect(OracleDriver.java:503)
53104 at java.sql.DriverManager.getConnection(DriverManager.java:582)
53105 at java.sql.DriverManager.getConnection(DriverManager.java:154)
53106 at org.apache.commons.dbcp.DriverManagerConnectionFactory.createConnection(DriverManagerConnectionFactory.java:97)
53107 at org.apache.commons.dbcp.PoolableConnectionFactory.makeObject(PoolableConnectionFactory.java:300)
53108 at org.apache.commons.pool.impl.GenericObjectPool.borrowObject(GenericObjectPool.java:816)
53109 at org.apache.commons.dbcp.PoolingDataSource.getConnection(PoolingDataSource.java:140)
53110 at org.ofbiz.core.entity.transaction.DBCPConnectionFactory$1.call(DBCPConnectionFactory.java:153)
53111 at org.ofbiz.core.entity.transaction.DBCPConnectionFactory$1.call(DBCPConnectionFactory.java:150)
53112 at org.ofbiz.core.entity.jdbc.interceptors.connection.ConnectionTracker.trackConnection(ConnectionTracker.java:53)
53113 ... 166 more
53114 Caused by: java.net.SocketException: Connection reset
53115 at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:96)
53116 at java.net.SocketOutputStream.write(SocketOutputStream.java:136)
53117 at oracle.net.ns.Packet.send(Packet.java:385)
53118 at oracle.net.ns.ConnectPacket.send(ConnectPacket.java:173)
53119 at oracle.net.ns.NSProtocol.connect(NSProtocol.java:283)
53120 at oracle.jdbc.driver.T4CConnection.connect(T4CConnection.java:1042)
53121 at oracle.jdbc.driver.T4CConnection.logon(T4CConnection.java:301)
53122 ... 179 more