I need a solution
So we use SEMP and the service keeps stoping once a week. This has been happening the past few weeks. In event manager we get a java -1 error at the same time the SEMP service stops. This is the last parts of the scm_log and scm-server log. Any help would be apreciated, thanks.
SCM_LOG
2013-04-14 02:34:47 StandardWrapperValve[ConsoleServlet]: Servlet.service() for servlet ConsoleServlet threw exception
javax.servlet.ServletException: Servlet execution threw an exception
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:221)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:145)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:210)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:596)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:433)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:955)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:139)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:596)
at org.apache.catalina.valves.CertificatesValve.invoke(CertificatesValve.java:198)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:594)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:433)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:955)
at org.apache.catalina.core.StandardContext.invoke(StandardContext.java:2460)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:133)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:596)
at org.apache.catalina.valves.ErrorDispatcherValve.invoke(ErrorDispatcherValve.java:119)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:594)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:117)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:594)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:433)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:955)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:127)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:596)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:433)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:955)
at org.apache.coyote.tomcat4.CoyoteAdapter.service(CoyoteAdapter.java:157)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:875)
at org.apache.coyote.http11.Http11BaseProtocol$Http11ConnectionHandler.processConnection(Http11BaseProtocol.java:665)
at org.apache.tomcat.util.net.PoolTcpEndpoint.processSocket(PoolTcpEndpoint.java:528)
at org.apache.tomcat.util.net.LeaderFollowerWorkerThread.runIt(LeaderFollowerWorkerThread.java:81)
at org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:689)
at java.lang.Thread.run(Thread.java:619)
----- Root Cause -----
java.lang.OutOfMemoryError: GC overhead limit exceeded
2013-04-14 02:34:51 StandardWrapper[:ConsoleServlet]: Waiting for 1 instance(s) to be deallocated
SCM-Server log
2013-04-10 20:19:34.957 SEVERE: Unknown response. in: com.sygate.scm.server.task.SecurityDataTask
com.sygate.scm.server.util.ServerException: Unknown response.
at com.sygate.scm.server.task.SecurityDataTask.processThreatCon(SecurityDataTask.java:301)
at com.sygate.scm.server.task.SecurityDataTask.run(SecurityDataTask.java:90)
at java.util.TimerThread.mainLoop(Timer.java:512)
at java.util.TimerThread.run(Timer.java:462)
2013-04-11 00:19:34.713 SEVERE: Unknown response. in: com.sygate.scm.server.task.SecurityDataTask
com.sygate.scm.server.util.ServerException: Unknown response.
at com.sygate.scm.server.task.SecurityDataTask.processThreatCon(SecurityDataTask.java:301)
at com.sygate.scm.server.task.SecurityDataTask.run(SecurityDataTask.java:90)
at java.util.TimerThread.mainLoop(Timer.java:512)
at java.util.TimerThread.run(Timer.java:462)
2013-04-11 04:19:34.595 SEVERE: Unknown response. in: com.sygate.scm.server.task.SecurityDataTask
com.sygate.scm.server.util.ServerException: Unknown response.
at com.sygate.scm.server.task.SecurityDataTask.processThreatCon(SecurityDataTask.java:301)
at com.sygate.scm.server.task.SecurityDataTask.run(SecurityDataTask.java:90)
at java.util.TimerThread.mainLoop(Timer.java:512)
at java.util.TimerThread.run(Timer.java:462)
2013-04-11 08:19:34.554 SEVERE: Unknown response. in: com.sygate.scm.server.task.SecurityDataTask
com.sygate.scm.server.util.ServerException: Unknown response.
at com.sygate.scm.server.task.SecurityDataTask.processThreatCon(SecurityDataTask.java:301)
at com.sygate.scm.server.task.SecurityDataTask.run(SecurityDataTask.java:90)
at java.util.TimerThread.mainLoop(Timer.java:512)
at java.util.TimerThread.run(Timer.java:462)
2013-04-11 12:19:34.436 SEVERE: Unknown response. in: com.sygate.scm.server.task.SecurityDataTask
com.sygate.scm.server.util.ServerException: Unknown response.
at com.sygate.scm.server.task.SecurityDataTask.processThreatCon(SecurityDataTask.java:301)
at com.sygate.scm.server.task.SecurityDataTask.run(SecurityDataTask.java:90)
at java.util.TimerThread.mainLoop(Timer.java:512)
at java.util.TimerThread.run(Timer.java:462)
2013-04-11 16:19:34.685 SEVERE: Unknown response. in: com.sygate.scm.server.task.SecurityDataTask
com.sygate.scm.server.util.ServerException: Unknown response.
at com.sygate.scm.server.task.SecurityDataTask.processThreatCon(SecurityDataTask.java:301)
at com.sygate.scm.server.task.SecurityDataTask.run(SecurityDataTask.java:90)
at java.util.TimerThread.mainLoop(Timer.java:512)
at java.util.TimerThread.run(Timer.java:462)
2013-04-11 20:19:35.111 SEVERE: Unknown response. in: com.sygate.scm.server.task.SecurityDataTask
com.sygate.scm.server.util.ServerException: Unknown response.
at com.sygate.scm.server.task.SecurityDataTask.processThreatCon(SecurityDataTask.java:301)
at com.sygate.scm.server.task.SecurityDataTask.run(SecurityDataTask.java:90)
at java.util.TimerThread.mainLoop(Timer.java:512)
at java.util.TimerThread.run(Timer.java:462)
2013-04-14 00:00:46.861 SEVERE: Unknown Exception in: com.sygate.scm.server.task.PackageTask
java.lang.OutOfMemoryError: GC overhead limit exceeded
at com.sybase.jdbc2.tds.Tds.paramArray(Tds.java:2946)
at com.sybase.jdbc2.jdbc.ParamManager.<init>(ParamManager.java:100)
at com.sybase.jdbc2.jdbc.SybPreparedStatement.countParams(SybPreparedStatement.java:1166)
at com.sybase.jdbc2.jdbc.SybPreparedStatement.<init>(SybPreparedStatement.java:69)
at com.sybase.jdbc2.jdbc.SybCallableStatement.<init>(SybCallableStatement.java:44)
at com.sybase.jdbc2.jdbc.SybConnection.prepareCall(SybConnection.java:961)
at com.sybase.jdbc2.jdbc.SybConnection.isClosed(SybConnection.java:1157)
at org.apache.commons.dbcp.DelegatingConnection.isClosed(DelegatingConnection.java:346)
at org.apache.commons.dbcp.PoolingDataSource$PoolGuardConnectionWrapper.isClosed(PoolingDataSource.java:190)
at com.sygate.scm.server.db.util.DatabaseUtilities.closeConnection(DatabaseUtilities.java:502)
at com.sygate.scm.server.metadata.MetadataManager.getLastestUsnForCollection(MetadataManager.java:173)
at com.sygate.scm.server.configmanager.ConfigManager.getLatestUsnForCollection(ConfigManager.java:1315)
at com.sygate.scm.server.task.PackageTask.run(PackageTask.java:224)
at java.util.TimerThread.mainLoop(Timer.java:512)
at java.util.TimerThread.run(Timer.java:462)
2013-04-14 00:00:48.078 SEVERE: Unknown Exception in: com.sygate.scm.server.task.AgentOnlineStatusTask
java.lang.OutOfMemoryError: GC overhead limit exceeded
at java.lang.StringCoding$StringEncoder.encode(StringCoding.java:232)
at java.lang.StringCoding.encode(StringCoding.java:272)
at java.lang.String.getBytes(String.java:946)
at com.sybase.jdbc2.utils.PureConverter.fromUnicode(PureConverter.java:54)
at com.sybase.jdbc2.tds.TdsDataOutputStream.toBytes(TdsDataOutputStream.java:973)
at com.sybase.jdbc2.tds.TdsOutputStream.stringToByte(TdsOutputStream.java:285)
at com.sybase.jdbc2.tds.DbrpcToken.send(DbrpcToken.java:60)
at com.sybase.jdbc2.tds.Tds.rpc(Tds.java:850)
at com.sybase.jdbc2.jdbc.SybCallableStatement.sendRpc(SybCallableStatement.java:446)
at com.sybase.jdbc2.jdbc.SybCallableStatement.executeQuery(SybCallableStatement.java:100)
at com.sybase.jdbc2.jdbc.SybConnection.isClosed(SybConnection.java:1160)
at org.apache.commons.dbcp.DelegatingConnection.isClosed(DelegatingConnection.java:346)
at org.apache.commons.dbcp.PoolingDataSource$PoolGuardConnectionWrapper.isClosed(PoolingDataSource.java:190)
at com.sygate.scm.server.db.util.DatabaseUtilities.closeConnection(DatabaseUtilities.java:502)
at com.sygate.scm.server.task.AgentOnlineStatusTask.updateDatabase(AgentOnlineStatusTask.java:156)
at com.sygate.scm.server.task.AgentOnlineStatusTask.run(AgentOnlineStatusTask.java:86)
at java.util.TimerThread.mainLoop(Timer.java:512)
at java.util.TimerThread.run(Timer.java:462)
2013-04-14 00:00:55.846 SEVERE: Unknown Exception in: com.sygate.scm.server.task.SecurityAlertNotifyTask
java.lang.OutOfMemoryError: GC overhead limit exceeded
2013-04-14 00:00:56.548 SEVERE: Unknown Exception
java.lang.OutOfMemoryError: GC overhead limit exceeded
2013-04-14 00:01:02.695 SEVERE: Unknown Exception in: com.sygate.scm.server.task.AgentLogCollector
java.lang.OutOfMemoryError: GC overhead limit exceeded
2013-04-14 00:01:03.490 SEVERE: Unknown Exception in: com.sygate.scm.server.task.AgentLogCollector
java.lang.OutOfMemoryError: GC overhead limit exceeded
2013-04-14 00:01:03.880 SEVERE: Unknown Exception
java.lang.OutOfMemoryError: GC overhead limit exceeded
2013-04-14 00:01:04.692 SEVERE: Unknown Exception in: com.sygate.scm.server.task.AgentLogCollector
java.lang.OutOfMemoryError: GC overhead limit exceeded
2013-04-14 00:01:32.522 SEVERE: Unknown Exception in: com.sygate.scm.server.task.StateCheckpointTask
java.lang.OutOfMemoryError: GC overhead limit exceeded
2013-04-14 00:07:33.613 SEVERE: Unknown Exception in: com.sygate.scm.server.task.StateCheckpointTask
java.lang.OutOfMemoryError: GC overhead limit exceeded
2013-04-14 00:19:34.359 SEVERE: Unknown Exception in: com.sygate.scm.server.util.ProxyServerConfigManager
java.lang.OutOfMemoryError: GC overhead limit exceeded
2013-04-14 00:19:35.139 SEVERE: Unknown Exception in: com.sygate.scm.server.task.SecurityDataTask
java.lang.OutOfMemoryError: GC overhead limit exceeded
2013-04-14 00:19:35.498 SEVERE: Unknown Exception
java.lang.OutOfMemoryError: GC overhead limit exceeded
2013-04-14 00:19:36.294 SEVERE: Unknown Exception
org.apache.commons.dbcp.SQLNestedException: Cannot get a connection, pool error Could not create a validated object, cause: GC overhead limit exceeded
at org.apache.commons.dbcp.PoolingDataSource.getConnection(PoolingDataSource.java:104)
at org.apache.commons.dbcp.BasicDataSource.getConnection(BasicDataSource.java:880)
at com.sygate.scm.server.db.util.DatabaseUtilities.getDefaultDatabaseConnection(DatabaseUtilities.java:285)
at com.sygate.scm.server.db.util.DatabaseUtilities.getDefaultDatabaseConnection(DatabaseUtilities.java:276)
at com.sygate.scm.server.db.util.DatabaseUtilities.getNextUSN_SP(DatabaseUtilities.java:639)
at com.sygate.scm.server.db.util.DatabaseUtilities.getNextUSN(DatabaseUtilities.java:623)
at com.sygate.scm.server.util.logging.ServerSystemLogRecord.doLog(ServerSystemLogRecord.java:119)
at com.sygate.scm.server.util.logging.DbLogHandler.publish(DbLogHandler.java:119)
at java.util.logging.Logger.log(Logger.java:458)
at com.sygate.scm.server.util.ServerLogger.log(ServerLogger.java:394)
at com.sygate.scm.server.util.ServerLogger.log(ServerLogger.java:373)
at com.sygate.scm.server.util.ServerLogger.log(ServerLogger.java:368)
at com.sygate.scm.server.db.util.DatabaseUtilities.getDefaultDatabaseConnection(DatabaseUtilities.java:307)
at com.sygate.scm.server.db.util.DatabaseUtilities.getDefaultDatabaseConnection(DatabaseUtilities.java:276)
at com.sygate.scm.server.db.util.DatabaseUtilities.getNextUSN_SP(DatabaseUtilities.java:639)
at com.sygate.scm.server.db.util.DatabaseUtilities.getNextUSN(DatabaseUtilities.java:623)
at com.sygate.scm.server.util.logging.ServerSystemLogRecord.doLog(ServerSystemLogRecord.java:119)
at com.sygate.scm.server.util.logging.DbLogHandler.publish(DbLogHandler.java:119)
at java.util.logging.Logger.log(Logger.java:458)
at com.sygate.scm.server.util.ServerLogger.log(ServerLogger.java:394)
at com.sygate.scm.server.util.ServerLogger.log(ServerLogger.java:373)
at com.sygate.scm.server.task.SecurityDataTask.run(SecurityDataTask.java:103)
at java.util.TimerThread.mainLoop(Timer.java:512)
at java.util.TimerThread.run(Timer.java:462)
Caused by: java.util.NoSuchElementException: Could not create a validated object, cause: GC overhead limit exceeded
at org.apache.commons.pool.impl.GenericObjectPool.borrowObject(GenericObjectPool.java:1008)
at org.apache.commons.dbcp.PoolingDataSource.getConnection(PoolingDataSource.java:96)
... 23 more
2013-04-14 00:19:35.498 SEVERE: DomainId: null
SiteId: 27195BC8AC100023018BF72372CA550A
ServerId: 81C6C9E8AC10002301704BE4282F3AD2
SystemEventId: 1281
EventDesc: GC overhead limit exceeded
MessageId: -1
ErrorCode: -1
java.lang.OutOfMemoryError: GC overhead limit exceeded
2013-04-14 00:21:15.587 SEVERE: Unknown Exception in: com.sygate.scm.server.task.AgentLastCheckInTask
java.lang.OutOfMemoryError: GC overhead limit exceeded
2013-04-14 02:34:47.497 SEVERE: Unknown Exception in: com.sygate.scm.server.consolemanager.RequestHandler
java.lang.OutOfMemoryError: GC overhead limit exceeded
2013-04-14 02:34:50.258 SEVERE: Unknown Exception in: com.sygate.scm.server.consolemanager.RequestHandler
java.lang.OutOfMemoryError: GC overhead limit exceeded
2013-04-14 02:34:50.726 SEVERE: Shuting down server ...