以前都是好好的,最近出现了 oom。
问题
开始是: java.lang.OutOfMemoryError: Java heap space
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
2019-06-14 11:02:41.678 ERROR 13789 --- [nio-8082-exec-3] c.e.p.s.c.c.core.ELDictionaryController : 系统异常 org.springframework.web.util.NestedServletException: Handler dispatch failed; nested exception is java.lang.OutOfMemoryError: Java heap space at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1053) ~[spring-webmvc-5.1.5.RELEASE.jar!/:5.1.5.RELEASE] at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:942) ~[spring-webmvc-5.1.5.RELEASE.jar!/:5.1.5.RELEASE] at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1005) [spring-webmvc-5.1.5.RELEASE.jar!/:5.1.5.RELEASE] at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:897) [spring-webmvc-5.1.5.RELEASE.jar!/:5.1.5.RELEASE] at javax.servlet.http.HttpServlet.service(HttpServlet.java:634) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:882) [spring-webmvc-5.1.5.RELEASE.jar!/:5.1.5.RELEASE] at javax.servlet.http.HttpServlet.service(HttpServlet.java:741) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:53) [tomcat-embed-websocket-9.0.16.jar!/:9.0.16] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.springframework.web.filter.FormContentFilter.doFilterInternal(FormContentFilter.java:92) [spring-web-5.1.5.RELEASE.jar!/:5.1.5.RELEASE] at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) [spring-web-5.1.5.RELEASE.jar!/:5.1.5.RELEASE] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.filters.RemoteIpFilter.doFilter(RemoteIpFilter.java:845) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.filters.RemoteIpFilter.doFilter(RemoteIpFilter.java:902) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at com.alibaba.druid.support.http.WebStatFilter.doFilter(WebStatFilter.java:123) [druid-1.1.5.jar!/:1.1.5] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at com.lkk.platform.system.controller.filter.CORSFilter.doFilter(CORSFilter.java:55) [erdp_system_controller-2.0.0-GA.jar!/:2.0.0-GA] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:200) [spring-web-5.1.5.RELEASE.jar!/:5.1.5.RELEASE] at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) [spring-web-5.1.5.RELEASE.jar!/:5.1.5.RELEASE] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:200) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:490) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:139) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:74) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.valves.RemoteIpValve.invoke(RemoteIpValve.java:679) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:343) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:408) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:834) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1415) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_212] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_212] at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) [tomcat-embed-core-9.0.16.jar!/:9.0.16] at java.lang.Thread.run(Thread.java:748) [na:1.8.0_212] Caused by: java.lang.OutOfMemoryError: Java heap space at java.util.jar.Manifest.read(Manifest.java:270) ~[na:1.8.0_212] at sun.security.util.SignatureFileVerifier.processImpl(SignatureFileVerifier.java:276) ~[na:1.8.0_212] at sun.security.util.SignatureFileVerifier.process(SignatureFileVerifier.java:263) ~[na:1.8.0_212] at java.util.jar.JarVerifier.processEntry(JarVerifier.java:318) ~[na:1.8.0_212] at java.util.jar.JarVerifier.update(JarVerifier.java:230) ~[na:1.8.0_212] at java.util.jar.JarInputStream.read(JarInputStream.java:212) ~[na:1.8.0_212] at java.util.zip.ZipInputStream.closeEntry(ZipInputStream.java:140) ~[na:1.8.0_212] at java.util.zip.ZipInputStream.getNextEntry(ZipInputStream.java:118) ~[na:1.8.0_212] at java.util.jar.JarInputStream.getNextEntry(JarInputStream.java:142) ~[na:1.8.0_212] at java.util.jar.JarInputStream.getNextJarEntry(JarInputStream.java:179) ~[na:1.8.0_212] at org.apache.catalina.webresources.JarWarResourceSet.getArchiveEntries(JarWarResourceSet.java:117) ~[tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.webresources.AbstractArchiveResourceSet.getResource(AbstractArchiveResourceSet.java:253) ~[tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.webresources.StandardRoot.getResourceInternal(StandardRoot.java:281) ~[tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.webresources.Cache.getResource(Cache.java:62) ~[tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.webresources.StandardRoot.getResource(StandardRoot.java:216) ~[tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.webresources.StandardRoot.getClassLoaderResource(StandardRoot.java:225) ~[tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.loader.WebappClassLoaderBase.findClassInternal(WebappClassLoaderBase.java:2299) ~[tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.apache.catalina.loader.WebappClassLoaderBase.findClass(WebappClassLoaderBase.java:863) ~[tomcat-embed-core-9.0.16.jar!/:9.0.16] at org.springframework.boot.web.embedded.tomcat.TomcatEmbeddedWebappClassLoader.findClassIgnoringNotFound(TomcatEmbeddedWebappClassLoader.java:121) ~[spring-boot-2.1.3.RELEASE.jar!/:2.1.3.RELEASE] at org.springframework.boot.web.embedded.tomcat.TomcatEmbeddedWebappClassLoader.doLoadClass(TomcatEmbeddedWebappClassLoader.java:86) ~[spring-boot-2.1.3.RELEASE.jar!/:2.1.3.RELEASE] at org.springframework.boot.web.embedded.tomcat.TomcatEmbeddedWebappClassLoader.loadClass(TomcatEmbeddedWebappClassLoader.java:68) ~[spring-boot-2.1.3.RELEASE.jar!/:2.1.3.RELEASE] at org.apache.catalina.loader.WebappClassLoaderBase.loadClass(WebappClassLoaderBase.java:1186) ~[tomcat-embed-core-9.0.16.jar!/:9.0.16] at ch.qos.logback.classic.spi.PackagingDataCalculator.loadClass(PackagingDataCalculator.java:204) ~[logback-classic-1.2.3.jar!/:na] at ch.qos.logback.classic.spi.PackagingDataCalculator.bestEffortLoadClass(PackagingDataCalculator.java:228) ~[logback-classic-1.2.3.jar!/:na] at ch.qos.logback.classic.spi.PackagingDataCalculator.computeBySTEP(PackagingDataCalculator.java:135) ~[logback-classic-1.2.3.jar!/:na] at ch.qos.logback.classic.spi.PackagingDataCalculator.populateFrames(PackagingDataCalculator.java:100) ~[logback-classic-1.2.3.jar!/:na] at ch.qos.logback.classic.spi.PackagingDataCalculator.calculate(PackagingDataCalculator.java:58) ~[logback-classic-1.2.3.jar!/:na] at ch.qos.logback.classic.spi.ThrowableProxy.calculatePackagingData(ThrowableProxy.java:142) ~[logback-classic-1.2.3.jar!/:na] at ch.qos.logback.classic.spi.LoggingEvent.<init>(LoggingEvent.java:122) ~[logback-classic-1.2.3.jar!/:na] at ch.qos.logback.classic.Logger.buildLoggingEventAndAppend(Logger.java:419) ~[logback-classic-1.2.3.jar!/:na] at ch.qos.logback.classic.Logger.filterAndLog_0_Or3Plus(Logger.java:383) ~[logback-classic-1.2.3.jar!/:na] at ch.qos.logback.classic.Logger.error(Logger.java:538) ~[logback-classic-1.2.3.jar!/:na]
512M 不够吗? 很有可能啊...
增加内存到1G 后仍然出现问题:Failed to mark a promise as failure because it has failed already: [DefaultChannelPromise@33a99639(failure: io.netty.handler.codec.EncoderException: java.lang.OutOfMemoryError: GC overhead limit exceeded), io.netty.handler.codec.EncoderException: java.lang.OutOfMemoryError: GC overhead limit exceeded
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
java.lang.OutOfMemoryError: GC overhead limit exceeded 2019-06-17 09:00:42.648 WARN 1993 --- [erverWorker-8-3] o.a.d.r.exchange.codec.ExchangeCodec : [DUBBO] Fail to encode response: Response [id=319633, version=2.0.2, status=20, event=false, error=null, result=RpcResult [result=null, exception=org.springframework.transaction.CannotCreateTransactionException: Could not open JDBC Connection for transaction; nested exception is java.lang.OutOfMemoryError: GC overhead limit exceeded]], send bad_response info instead, cause: GC overhead limit exceeded, dubbo version: 2.7.1, current host: 192.168.11.183 java.lang.OutOfMemoryError: GC overhead limit exceeded 2019-06-17 09:00:43.907 ERROR 1993 --- [20884-thread-52] o.a.dubbo.rpc.filter.ExceptionFilter : [DUBBO] Got unchecked and undeclared exception which called by 192.168.11.183. service: com.elead.platform.system.domain.service.ELCommonCodeRegulationService, method: GetCode, exception: java.lang.OutOfMemoryError: GC overhead limit exceeded, dubbo version: 2.7.1, current host: 192.168.11.183
这就奇怪了! 注意到 出现次数比较多是 com.lkk.platform.system.domain.service.ELCommonCodeRegulationService, method: GetCode,
@Transactional(readOnly = false) public String GetCode(String name){ RLock rlock = redissonManager.getRedisson().getLock(name); boolean getLock = false; try{ getLock = rlock.tryLock(3, 20, TimeUnit.SECONDS); if (getLock){ ELCodeDef elCodeDef = findCommonCode(name); super.updateById(elCodeDef); return elCodeDef.getCode(); } }catch (Exception ex){ ex.printStackTrace(); }finally { if (getLock) { rlock.unlock(); } } return ""; }
而
@Autowired
RedissonManager redissonManager;
分析
由此怀疑这个地方有些问题。 虽然出现了oom, 但是进程没有死, 似乎依然可以响应某些请求,于是把线程dump 下来, 观察一番,发现 redisson-netty 竟然有上千个
就是这个
"redisson-netty-25-32" #808 prio=5 os_prio=0 tid=0x00007f7ec0187800 nid=0x3625 runnable [0x00007f7e77d6c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000000e90b27a0> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000000e90b27f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000000e90b2708> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:786) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:434) at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748)
太不正常了! 但是 这里的redisson-netty- 仍然是 RUNNABLE 状态, 看起来也不是问题啊! 仔细检查了下, 也没发现死锁啊!!
那就不是线程问题吗?
redisson 的bug 吗? redisson 的官网 的issue 搜索一番,无果。 郁闷了! 而且我的 redisson 版本是 3.1.1, 已经很新的了吧!!
堆栈分析吧!!把java 的heap 拔下来,
jps -l, 然后 jmap -dump:format=b,file=dumpFileName pid
看到有些异常:
肯定不是 spring 的classloader 吧。
看到 netty 的PoolThreadCache 比较可疑啊, 还有 mybatis。
Biggest Top-Level Dominator Packages 跟之前一样的提示, 一个是netty的 PollThreadCache, 一个是netty 的epoll, 还有是redision, 还有是sun 的EPollArrayWrapper, 还有mybatis,其他 也看不出什么来啊!
分析只能到此为止了吗? io.netty.buffer.PoolThreadCache 是什么东东? 我不熟悉啊! 看过netty 源码, 已经全忘了!
是内存泄漏吗? 好像也看不出来。 不太确定。 网上搜索看看吧!!
还是从redision 入手吧。 咦, redision 的用法好像不太对哦!!! 改一下吧:
@Autowired
RedissonClient redissonClient;
==>
@Autowired
RedissonManager redissonManager;
RLock rlock = redissonClient.getLock(name); ==> RLock lock = redissonManager.getRedisson().getLock(name);
而RedissonManager如下:
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
import org.apache.commons.lang3.StringUtils; import org.redisson.Redisson; import org.redisson.api.RedissonClient; import org.redisson.config.Config; import org.redisson.config.ReadMode; import org.redisson.config.SentinelServersConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Component; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @Component public class RedissonManager { @Autowired RedisTemplate<String, Object> redisTemplate; @Value("${spring.redis.password}") private String redisPassword; @Value("${spring.redis.port}") private String redisPort; @Value("${spring.redis.host}") private String redisHost; @Value("${spring.redis.timeout}") private String redisTimeout; @Value("${spring.redis.sentinel.node}") private String redisSentinelNode; @Value("${spring.redis.sentinel.master}") private String redisSentinelMaster; @Bean public RedissonClient getRedisson() { Config config = new Config(); if (StringUtils.isNotEmpty(redisPort)) { config.useSingleServer().setAddress("redis://" + redisHost + ":" + redisPort).setPassword(redisPassword); } else if (StringUtils.isNotEmpty(redisSentinelNode)) { String[] nodes = redisSentinelNode.split(","); List<String> newNodes = new ArrayList(nodes.length); Arrays.stream(nodes).forEach((index) -> newNodes.add(index.startsWith("redis://") ? index : "redis://" + index)); SentinelServersConfig serverConfig = config.useSentinelServers() .addSentinelAddress(newNodes.toArray(new String[0])) .setMasterName(redisSentinelMaster) .setReadMode(ReadMode.SLAVE) .setTimeout(Integer.valueOf(redisTimeout)); if(StringUtils.isNotEmpty(redisPassword)){ serverConfig.setPassword(redisPassword); } } return Redisson.create(config); } }
改了就好了!!突然自己明白了, 原来就是这个redision 用法错误导致的!!
不信? 重新拔下来heap dump 分析一下:
最大的 com.mysql.cj.jdbc.AbandonedConnectionCleanupThread 才占用2m, 不是什么问题。 可见已经没有了什么
PoolThreadCache 已经下滑到了第七位, 总占用7M ,38个对象,看起来正常了许多!! :
总结
花了2天时间终于搞定!!
其实上面的 thread dump 和 heap dump 已经给出了比较明显的答案了!! 就是 PoolThreadCache 占用了 过多的内存, 其原因就是 PoolThreadCache 错误的创建了 太多!———— 本来应该是单例的 对象, 被搞成了 prototype, 你说是不是引起了大错!!! 一个 PoolThreadCache占用内存差不多196,000byte, 921个就 是 180516000 byte 也就是 差不多 下图的180M, 一类对象就 180M, 总共才1G, 当然会不够用!!
其实 从错误日志也可以 分析出来一些, 在创建需要比较大的内存的对象的时候, 就会出现 oom, 因为内存确实已经不够了啊!! (这也是为什么 ELCommonCodeRegulationService 的 GetCode 方法调用的时候,出现了很多oom。 但是又不是绝对的。 因为其他 地方也可以创建大内存对象)
其实只要再多问几个问题就知道了答案: 这个对象为什么出现了这么多次, 占用这么多内存呢?? 这个是正常的吗? 如果能够很早认识到这些问题,并回答之, 那么问题就不是大问题了,就不会浪费很多时间了!