У меня есть несколько узлов, на которых работает JBoss 6, и приложение, размещенное на нем. В последнее время я видел, что большинство/все из них жалуются на высокую загрузку и загрузку ЦП, что приводит к замедлению работы приложения.
Итак, прошлой ночью, когда проблема начала всплывать на поверхность, я сделал дамп потока JAVA и увидел следующее в файле nohup.out.
"http-/0.0.0.0:8091-Poller" daemon prio=10 tid=0x00007f4cb0171800 nid=0x2a1d in Object.wait() [0x00007f4d01dc0000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007426621b0> (a org.apache.tomcat.util.net.JIoEndpoint$Poller)
at org.apache.tomcat.util.net.JIoEndpoint$Poller.run(JIoEndpoint.java:732)
- locked <0x00000007426621b0> (a org.apache.tomcat.util.net.JIoEndpoint$Poller)
at java.lang.Thread.run(Thread.java:745)
"ContainerBackgroundProcessor[StandardEngine[jboss.web]]" daemon prio=10 tid=0x00007f4cc0056000 nid=0x2a18 waiting on condition [0x00007f4d022c5000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.run(ContainerBase.java:1563)
at java.lang.Thread.run(Thread.java:745)
"ConnectionValidator" daemon prio=10 tid=0x00007f4cb4051800 nid=0x2a12 waiting on condition [0x00007f4d023c6000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x0000000740c29b68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2176)
at org.jboss.jca.core.connectionmanager.pool.validator.ConnectionValidator$ConnectionValidatorRunner.run(ConnectionValidator.java:263)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
"http-/0.0.0.0:8091-8" daemon prio=10 tid=0x00007f4ce8046000 nid=0x2a2e in Object.wait() [0x00007f4d0055c000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007c1965448> (a com.hazelcast.spi.impl.BasicInvocationFuture)
at com.hazelcast.spi.impl.BasicInvocationFuture.pollResponse(BasicInvocationFuture.java:265)
- locked <0x00000007c1965448> (a com.hazelcast.spi.impl.BasicInvocationFuture)
at com.hazelcast.spi.impl.BasicInvocationFuture.waitForResponse(BasicInvocationFuture.java:216)
at com.hazelcast.spi.impl.BasicInvocationFuture.get(BasicInvocationFuture.java:193)
at com.hazelcast.spi.impl.BasicInvocationFuture.get(BasicInvocationFuture.java:173)
at com.hazelcast.map.impl.proxy.MapProxySupport.invokeOperation(MapProxySupport.java:404)
at com.hazelcast.map.impl.proxy.MapProxySupport.getInternal(MapProxySupport.java:223)
at com.hazelcast.map.impl.proxy.MapProxyImpl.get(MapProxyImpl.java:84)
at com.ericsson.cac.sprint.adapters.hazelcast.HazelHelper.getObjectFromMap(HazelHelper.java:97)
at com.ericsson.cac.sprint.adapters.hazelcast.CacheAOP.testAop(CacheAOP.java:415)
at sun.reflect.GeneratedMethodAccessor126.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:621)
at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:610)
at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:68)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:92)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:653)
at com.ericsson.cac.sprint.adapters.QueryUsageProxyService$$EnhancerBySpringCGLIB$$21767524.queryPrepaidBalanceAndThresholdInfo(<generated>)
at com.ericsson.cac.sprint.selfcare.workflow.throttling.ThrottlingWorkflowImpl.getThrottleInfo(ThrottlingWorkflowImpl.java:127)
at com.ericsson.sprint.msdp.selfcare.controllers.abstracts.AbstractMyAccountController.usage(AbstractMyAccountController.java:529)
at sun.reflect.GeneratedMethodAccessor173.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.springframework.web.method.support.InvocableHandlerMethod.invoke(InvocableHandlerMethod.java:215)
at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:132)
at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:104)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandleMethod(RequestMappingHandlerAdapter.java:781)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:721)
at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:83)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:943)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:877)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:966)
at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:857)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:734)
"hz._hzInstance_1_prod.event-5" prio=10 tid=0x00007f4cc8707000 nid=0x2abb waiting on condition [0x00007f4c85fde000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000074f0b9ef0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at com.hazelcast.util.executor.StripedExecutor$Worker.run(StripedExecutor.java:173)
"hz._hzInstance_1_prod.event-4" prio=10 tid=0x00007f4cc8704800 nid=0x2aba waiting on condition [0x00007f4c860df000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000074f0ba1e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at com.hazelcast.util.executor.StripedExecutor$Worker.run(StripedExecutor.java:173)
и еще много подобных ошибок.
Я также приложил полный дамп Jboss
Любая идея, почему это может происходить?
