Tomcat 死锁了，找不到啥原因，求指导一下。服务器内存使用正常，但是 CPU 异常基本四核 390%+

2018-12-09 14:55:42 +08:00
 wangmeixin
"http-nio-8080-exec-75" #141 daemon prio=5 os_prio=0 tid=0x00007f8d68048000 nid=0x649c runnable [0x00007f8d3452b000]
   java.lang.Thread.State: RUNNABLE
	at org.springframework.core.SerializableTypeWrapper.forTypeProvider(SerializableTypeWrapper.java:141)
	at org.springframework.core.SerializableTypeWrapper$TypeProxyInvocationHandler.invoke(SerializableTypeWrapper.java:211)
	at com.sun.proxy.$Proxy4.getRawType(Unknown Source)
	at sun.reflect.generics.reflectiveObjects.ParameterizedTypeImpl.equals(ParameterizedTypeImpl.java:168)
	at sun.reflect.GeneratedMethodAccessor32.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.springframework.core.SerializableTypeWrapper$TypeProxyInvocationHandler.invoke(SerializableTypeWrapper.java:220)
	at com.sun.proxy.$Proxy4.equals(Unknown Source)
	at org.springframework.util.ObjectUtils.nullSafeEquals(ObjectUtils.java:252)
	at org.springframework.core.ResolvableType.equals(ResolvableType.java:681)
	at org.springframework.util.ConcurrentReferenceHashMap$Segment.findInChain(ConcurrentReferenceHashMap.java:571)
	at org.springframework.util.ConcurrentReferenceHashMap$Segment.getReference(ConcurrentReferenceHashMap.java:439)
	at org.springframework.util.ConcurrentReferenceHashMap.getReference(ConcurrentReferenceHashMap.java:238)
	at org.springframework.util.ConcurrentReferenceHashMap.get(ConcurrentReferenceHashMap.java:217)
	at org.springframework.core.ResolvableType.forType(ResolvableType.java:1018)
	at org.springframework.core.ResolvableType.forType(ResolvableType.java:998)
	at org.springframework.core.ResolvableType.forTypes(ResolvableType.java:725)
	at org.springframework.core.ResolvableType.getInterfaces(ResolvableType.java:341)
	at org.springframework.core.ResolvableType.as(ResolvableType.java:308)
	at org.springframework.core.ResolvableType.asCollection(ResolvableType.java:274)
	at org.springframework.core.convert.TypeDescriptor.getElementTypeDescriptor(TypeDescriptor.java:305)
	at org.springframework.core.convert.TypeDescriptor.elementTypeDescriptor(TypeDescriptor.java:321)
	at org.springframework.core.convert.support.CollectionToArrayConverter.convert(CollectionToArrayConverter.java:66)
	at org.springframework.core.convert.support.ConversionUtils.invokeConverter(ConversionUtils.java:35)
	at org.springframework.core.convert.support.GenericConversionService.convert(GenericConversionService.java:178)
	at org.springframework.beans.TypeConverterDelegate.convertIfNecessary(TypeConverterDelegate.java:161)
	at org.springframework.beans.TypeConverterDelegate.convertIfNecessary(TypeConverterDelegate.java:93)
	at org.springframework.beans.TypeConverterSupport.doConvert(TypeConverterSupport.java:64)
	at org.springframework.beans.TypeConverterSupport.convertIfNecessary(TypeConverterSupport.java:47)
	at org.springframework.validation.DataBinder.convertIfNecessary(DataBinder.java:582)
	at org.springframework.web.method.annotation.AbstractNamedValueMethodArgumentResolver.resolveArgument(AbstractNamedValueMethodArgumentResolver.java:102)
	at org.springframework.web.method.support.HandlerMethodArgumentResolverComposite.resolveArgument(HandlerMethodArgumentResolverComposite.java:79)
	at org.springframework.web.method.support.InvocableHandlerMethod.getMethodArgumentValues(InvocableHandlerMethod.java:157)
	at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:124)
	at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:104)
	at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandleMethod(RequestMappingHandlerAdapter.java:748)
	at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:689)
	at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:83)
	at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:945)
	at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:876)
	at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:931)
	at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:833)
	at javax.servlet.http.HttpServlet.service(HttpServlet.java:661)
	at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:807)
	at javax.servlet.http.HttpServlet.service(HttpServlet.java:742)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:88)
	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:108)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:198)
	at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
	at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:478)
	at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:140)
	at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:80)
	at org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:624)
	at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:87)
	at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:342)
	at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:799)
	at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66)
	at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:868)
	at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1455)
	at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
	- locked <0x00000006d075cd78> (a org.apache.tomcat.util.net.NioEndpoint$NioSocketWrapper)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
	at java.lang.Thread.run(Thread.java:748)
2262 次点击
所在节点    程序员
11 条回复
sagaxu
2018-12-09 15:56:15 +08:00
非自旋锁，死锁时 CPU 占用应该接近零。是不是有什么地方死循环了。
lihongjie0209
2018-12-09 16:01:51 +08:00
同意楼上, CPU 占用高的锁应该是活锁
aboutyang
2018-12-09 16:51:14 +08:00
查看下是哪个线程占用了 CPU， 再把对应的线程栈信息贴出来。

现在贴的线程，正在对 request 请求数据进行转换。 如果是这个线程占用 CPU，可以编辑接收请求的对象， 把对象中的属性一个个删除，看看是哪个字段转换时候出的问题。
wangmeixin
2018-12-09 17:34:59 +08:00
@aboutyang 1
li24361
2018-12-09 17:46:04 +08:00
感觉像是序列化的问题
loveCoding
2018-12-09 17:54:33 +08:00
个人认为问题出在 tomcat 和 spring-mvc 的几率应该不大, 优先查自己的业务代码,结合阿里巴巴开源的 https://github.com/alibaba/arthas 诊断工具
alvinbone88
2018-12-09 19:20:42 +08:00
先说 Spring 的版本是多少
然后看看是不是这个 bug
https://jira.spring.io/browse/SPR-11335
wangmeixin
2018-12-09 19:30:31 +08:00
@alvinbone88 4.0.0.RELEASE,应该不是这个 bug 引起的
alvinbone88
2018-12-09 20:46:43 +08:00
4.0.0.RELEASE，那基本就是这个 bug 了
这个 bug 是在 4.0.1.RELEASE 里修复的
Github 上的提交说明里注明了是 equals()造成的性能问题
https://github.com/spring-projects/spring-framework/commit/d96b91a57bfab0ecb7543f687d297dc5db9414ee
然后调用栈里刚好有 equals()

当然如果觉得我说得不对，可以自己用 visualVM 或者上面老哥推荐的东西查看 CPU 时间；如果是远程服务器，自己搜 jmx 怎么远程
wangmeixin
2018-12-09 21:52:40 +08:00
@alvinbone88 谢啦，我研究一波
wangmeixin
2018-12-10 14:36:15 +08:00
```java
Attaching to process ID 4902, please wait...
Debugger attached successfully.
Server compiler detected.
JVM version is 25.161-b12
Deadlock Detection:

No deadlocks found.

Thread 23113: (state = IN_JAVA)
- java.util.concurrent.ConcurrentHashMap.get(java.lang.Object) @bci=127, line=946 (Compiled frame; information may be imprecise)
- java.lang.reflect.WeakCache.get(java.lang.Object, java.lang.Object) @bci=92, line=121 (Compiled frame)
- java.lang.reflect.Proxy.getProxyClass0(java.lang.ClassLoader, java.lang.Class[]) @bci=22, line=419 (Compiled frame)
- java.lang.reflect.Proxy.newProxyInstance(java.lang.ClassLoader, java.lang.Class[], java.lang.reflect.InvocationHandler) @bci=33, line=719 (Compiled frame)
- org.springframework.core.SerializableTypeWrapper.forTypeProvider(org.springframework.core.SerializableTypeWrapper$TypeProvider) @bci=111, line=148 (Compiled frame)
- org.springframework.core.SerializableTypeWrapper$TypeProxyInvocationHandler.invoke(java.lang.Object, java.lang.reflect.Method, java.lang.Object[]) @bci=104, line=216 (Compiled frame)
- com.sun.proxy.$Proxy4.getActualTypeArguments() @bci=9 (Compiled frame)
- sun.reflect.generics.reflectiveObjects.ParameterizedTypeImpl.equals(java.lang.Object) @bci=62, line=193 (Compiled frame)
- sun.reflect.GeneratedMethodAccessor32.invoke(java.lang.Object, java.lang.Object[]) @bci=44 (Compiled frame)
- sun.reflect.DelegatingMethodAccessorImpl.invoke(java.lang.Object, java.lang.Object[]) @bci=6, line=43 (Compiled frame)
- java.lang.reflect.Method.invoke(java.lang.Object, java.lang.Object[]) @bci=56, line=498 (Compiled frame)
- org.springframework.core.SerializableTypeWrapper$TypeProxyInvocationHandler.invoke(java.lang.Object, java.lang.reflect.Method, java.lang.Object[]) @bci=128, line=220 (Compiled frame)
- com.sun.proxy.$Proxy4.equals(java.lang.Object) @bci=16 (Compiled frame)
- org.springframework.util.ObjectUtils.nullSafeEquals(java.lang.Object, java.lang.Object) @bci=19, line=252 (Compiled frame)
- org.springframework.core.ResolvableType.equals(java.lang.Object) @bci=27, line=681 (Compiled frame)
- org.springframework.util.ConcurrentReferenceHashMap$Segment.findInChain(org.springframework.util.ConcurrentReferenceHashMap$Reference, java.lang.Object, int) @bci=43, line=571 (Compiled frame)
- org.springframework.util.ConcurrentReferenceHashMap$Segment.doTask(int, java.lang.Object, org.springframework.util.ConcurrentReferenceHashMap$Task) @bci=79, line=462 (Compiled frame)
- org.springframework.util.ConcurrentReferenceHashMap.doTask(java.lang.Object, org.springframework.util.ConcurrentReferenceHashMap$Task) @bci=14, line=351 (Compiled frame)
- org.springframework.util.ConcurrentReferenceHashMap.put(java.lang.Object, java.lang.Object, boolean) @bci=28, line=252 (Compiled frame)
- org.springframework.util.ConcurrentReferenceHashMap.put(java.lang.Object, java.lang.Object) @bci=4, line=243 (Compiled frame)
- org.springframework.core.ResolvableType.forType(java.lang.reflect.Type, org.springframework.core.SerializableTypeWrapper$TypeProvider, org.springframework.core.ResolvableType$VariableResolver) @bci=59, line=1021 (Compiled frame)
- org.springframework.core.ResolvableType.forType(java.lang.reflect.Type, org.springframework.core.ResolvableType$VariableResolver) @bci=3, line=998 (Compiled frame)
- org.springframework.core.ResolvableType.forTypes(java.lang.reflect.Type[], org.springframework.core.ResolvableType$VariableResolver) @bci=20, line=725 (Compiled frame)
- org.springframework.core.ResolvableType.getInterfaces() @bci=31, line=341 (Compiled frame)
- org.springframework.core.ResolvableType.as(java.lang.Class) @bci=25, line=308 (Compiled frame)
- org.springframework.core.ResolvableType.asCollection() @bci=3, line=274 (Compiled frame)
- org.springframework.core.convert.TypeDescriptor.getElementTypeDescriptor() @bci=31, line=305 (Compiled frame)
- org.springframework.core.convert.TypeDescriptor.elementTypeDescriptor(java.lang.Object) @bci=3, line=321 (Compiled frame)
- org.springframework.core.convert.support.CollectionToArrayConverter.convert(java.lang.Object, org.springframework.core.convert.TypeDescriptor, org.springframework.core.convert.TypeDescriptor) @bci=71, line=66 (Compiled frame)
- org.springframework.core.convert.support.ConversionUtils.invokeConverter(org.springframework.core.convert.converter.GenericConverter, java.lang.Object, org.springframework.core.convert.TypeDescriptor, org.springframework.core.convert.TypeDescriptor) @bci=4, line=35 (Compiled frame)
- org.springframework.core.convert.support.GenericConversionService.convert(java.lang.Object, org.springframework.core.convert.TypeDescriptor, org.springframework.core.convert.TypeDescriptor) @bci=112, line=178 (Compiled frame)
- org.springframework.beans.TypeConverterDelegate.convertIfNecessary(java.lang.String, java.lang.Object, java.lang.Object, java.lang.Class, org.springframework.core.convert.TypeDescriptor) @bci=79, line=161 (Compiled frame)
- org.springframework.beans.TypeConverterDelegate.convertIfNecessary(java.lang.Object, java.lang.Class, org.springframework.core.MethodParameter) @bci=24, line=93 (Compiled frame)
- org.springframework.beans.TypeConverterSupport.doConvert(java.lang.Object, java.lang.Class, org.springframework.core.MethodParameter, java.lang.reflect.Field) @bci=24, line=64 (Compiled frame)
- org.springframework.beans.TypeConverterSupport.convertIfNecessary(java.lang.Object, java.lang.Class, org.springframework.core.MethodParameter) @bci=5, line=47 (Compiled frame)
- org.springframework.validation.DataBinder.convertIfNecessary(java.lang.Object, java.lang.Class, org.springframework.core.MethodParameter) @bci=7, line=582 (Compiled frame)
- org.springframework.web.method.annotation.AbstractNamedValueMethodArgumentResolver.resolveArgument(org.springframework.core.MethodParameter, org.springframework.web.method.support.ModelAndViewContainer, org.springframework.web.context.request.NativeWebRequest, org.springframework.web.bind.support.WebDataBinderFactory) @bci=146, line=102 (Compiled frame)
- org.springframework.web.method.support.HandlerMethodArgumentResolverComposite.resolveArgument(org.springframework.core.MethodParameter, org.springframework.web.method.support.ModelAndViewContainer, org.springframework.web.context.request.NativeWebRequest, org.springframework.web.bind.support.WebDataBinderFactory) @bci=49, line=79 (Compiled frame)
- org.springframework.web.method.support.InvocableHandlerMethod.getMethodArgumentValues(org.springframework.web.context.request.NativeWebRequest, org.springframework.web.method.support.ModelAndViewContainer, java.lang.Object[]) @bci=105, line=157 (Compiled frame)
- org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(org.springframework.web.context.request.NativeWebRequest, org.springframework.web.method.support.ModelAndViewContainer, java.lang.Object[]) @bci=4, line=124 (Compiled frame)
- org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(org.springframework.web.context.request.ServletWebRequest, org.springframework.web.method.support.ModelAndViewContainer, java.lang.Object[]) @bci=4, line=104 (Compiled frame)
- org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandleMethod(javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse, org.springframework.web.method.HandlerMethod) @bci=230, line=748 (Compiled frame)
- org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse, org.springframework.web.method.HandlerMethod) @bci=89, line=689 (Compiled frame)
- org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse, java.lang.Object) @bci=7, line=83 (Compiled frame)
- org.springframework.web.servlet.DispatcherServlet.doDispatch(javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse) @bci=304, line=945 (Compiled frame)
- org.springframework.web.servlet.DispatcherServlet.doService(javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse) @bci=319, line=876 (Compiled frame)
- org.springframework.web.servlet.FrameworkServlet.resetContextHolders(javax.servlet.http.HttpServletRequest, org.springframework.context.i18n.LocaleContext, org.springframework.web.context.request.RequestAttributes) @bci=5, line=1020 (Compiled frame)
- org.springframework.web.servlet.FrameworkServlet.processRequest(javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse) @bci=80, line=947 (Compiled frame)
- org.springframework.web.servlet.FrameworkServlet.doPost(javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse) @bci=3, line=833 (Compiled frame)
- javax.servlet.http.HttpServlet.service(javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse) @bci=149, line=661 (Compiled frame)
- org.springframework.web.servlet.FrameworkServlet.service(javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse) @bci=32, line=807 (Compiled frame)
- javax.servlet.http.HttpServlet.service(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=30, line=742 (Compiled frame)
- org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=304, line=231 (Compiled frame)
- org.apache.catalina.core.ApplicationFilterChain.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=101, line=166 (Compiled frame)
- org.apache.tomcat.websocket.server.WsFilter.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse, javax.servlet.FilterChain) @bci=21, line=52 (Compiled frame)
```

这是一个专为移动设备优化的页面（即为了让你能够在 Google 搜索结果里秒开这个页面），如果你希望参与 V2EX 社区的讨论，你可以继续到 V2EX 上打开本讨论主题的完整版本。

https://www.v2ex.com/t/515847

V2EX 是创意工作者们的社区，是一个分享自己正在做的有趣事物、交流想法，可以遇见新朋友甚至新机会的地方。

V2EX is a community of developers, designers and creative people.

© 2021 V2EX