东师理想云平台故障处理一例
pstree -p `ps -e | grep python | awk '{print $1}'`
kill -9 `ps -e|grep python | awk '{print $1}'`
vi /etc/crontab
# 解决思路
export MALLOC_ARENA_MAX=1
# Java程序在Linux上运行虚拟内存耗用很大
https://blog.csdn.net/u010686469/article/details/77319599
#Java 进程占用 VIRT 虚拟内存超高的问题研究
https://www.cnblogs.com/seasonsluo/p/java_virt.html
pmap -x 32520 | grep anon
# 重启两个占用虚拟内存大的进程
ps -ef | grep tomcat
kill -9 PID
service rabbitmq-server restart
119486 java8
33419 java7
119486
# 查看内存溢出日志
cat /var/log/messages | grep 'Out of memory' -C 5
#Out of memory: Kill process 解决
https://blog.51cto.com/qiangsh/2066747
[root@localhost ~]# cat /var/log/messages | grep 'Out of memory' -C 5 Aug 20 18:15:59 localhost kernel: [71037] 89 71037 20282 117 5 0 0 cleanup Aug 20 18:15:59 localhost kernel: [71039] 89 71039 20256 162 6 0 0 bounce Aug 20 18:15:59 localhost kernel: [72029] 0 72029 35007 155 0 0 0 crond Aug 20 18:15:59 localhost kernel: [72329] 0 72329 19143 82 1 0 0 sendmail Aug 20 18:15:59 localhost kernel: [72338] 0 72338 2275 16 17 0 0 sh Aug 20 18:15:59 localhost kernel: Out of memory: Kill process 56436 (find) score 267 or sacrifice child Aug 20 18:15:59 localhost kernel: Killed process 56436, UID 0, (find) total-vm:19784052kB, anon-rss:10437824kB, file-rss:8kB Aug 20 18:20:15 localhost NTP: 20 Aug 18:20:15 ntpdate[72927]: adjust time server 185.198.26.172 offset 0.017783 sec Aug 20 18:30:15 localhost NTP: 20 Aug 18:30:14 ntpdate[74577]: adjust time server 54.183.204.201 offset -0.012002 sec Aug 20 18:32:29 localhost kernel: sh invoked oom-killer: gfp_mask=0x200da, order=0, oom_adj=0, oom_score_adj=0 Aug 20 18:32:29 localhost kernel: sh cpuset=/ mems_allowed=0-1 -- Aug 20 18:32:29 localhost kernel: [74838] 0 74838 26489 37 18 0 0 awk Aug 20 18:32:29 localhost kernel: [74839] 0 74839 26308 28 21 0 0 sed Aug 20 18:32:29 localhost kernel: [74841] 0 74841 2275 14 0 0 0 sh Aug 20 18:32:29 localhost kernel: [74842] 0 74842 1541 29 17 0 0 sshpass Aug 20 18:32:29 localhost kernel: [74843] 497 74843 2275 13 19 0 0 sh Aug 20 18:32:29 localhost kernel: Out of memory: Kill process 51244 (find) score 408 or sacrifice child Aug 20 18:32:29 localhost kernel: Killed process 51244, UID 0, (find) total-vm:29107212kB, anon-rss:15434620kB, file-rss:4kB Aug 20 18:40:12 localhost NTP: 20 Aug 18:40:12 ntpdate[76243]: adjust time server 103.105.51.156 offset 0.012008 sec Aug 20 18:50:12 localhost NTP: 20 Aug 18:50:11 ntpdate[77522]: adjust time server 103.105.51.156 offset -0.002601 sec Aug 20 18:57:15 localhost kernel: java invoked oom-killer: gfp_mask=0x201da, order=0, oom_adj=0, oom_score_adj=0 Aug 20 18:57:15 localhost kernel: java cpuset=/ mems_allowed=0-1 -- Aug 20 18:57:15 localhost kernel: [78302] 89 78302 20282 227 16 0 0 cleanup Aug 20 18:57:15 localhost kernel: [78342] 0 78342 34942 90 0 0 0 crond Aug 20 18:57:15 localhost kernel: [78348] 0 78348 26519 45 1 0 0 freemem.sh Aug 20 18:57:15 localhost kernel: [78376] 0 78376 75 9 1 0 0 sync Aug 20 18:57:15 localhost kernel: [78377] 0 78377 297 11 0 0 0 sh Aug 20 18:57:15 localhost kernel: Out of memory: Kill process 30747 (find) score 848 or sacrifice child Aug 20 18:57:15 localhost kernel: Killed process 30747, UID 0, (find) total-vm:58109856kB, anon-rss:30773196kB, file-rss:4kB Aug 20 19:00:06 localhost NTP: 20 Aug 19:00:06 ntpdate[115182]: adjust time server 74.208.26.225 offset 0.030441 sec Aug 20 19:10:06 localhost NTP: 20 Aug 19:10:06 ntpdate[59888]: adjust time server 206.55.191.142 offset -0.023210 sec Aug 20 19:20:06 localhost NTP: 20 Aug 19:20:06 ntpdate[6326]: adjust time server 66.79.136.240 offset 0.017423 sec Aug 20 19:30:06 localhost NTP: 20 Aug 19:30:06 ntpdate[148943]: adjust time server 45.79.1.70 offset -0.016261 sec -- Oct 16 11:59:42 localhost kernel: [107168] 0 107168 14347 2 0 0 0 sftp-server Oct 16 11:59:42 localhost kernel: [108234] 89 108234 20246 224 1 0 0 pickup Oct 16 11:59:42 localhost kernel: [108512] 89 108512 20282 228 0 0 0 cleanup Oct 16 11:59:42 localhost kernel: [108514] 0 108514 20331 291 2 0 0 local Oct 16 11:59:42 localhost kernel: [108516] 89 108516 20256 224 0 0 0 bounce Oct 16 11:59:42 localhost kernel: Out of memory: Kill process 100492 (find) score 912 or sacrifice child Oct 16 11:59:42 localhost kernel: Killed process 100492, UID 0, (find) total-vm:62385608kB, anon-rss:31424580kB, file-rss:4kB Oct 16 12:00:07 localhost NTP: 16 Oct 12:00:07 ntpdate[108604]: adjust time server 72.30.35.89 offset -0.003301 sec Oct 16 12:10:06 localhost NTP: 16 Oct 12:10:06 ntpdate[109099]: adjust time server 198.255.68.106 offset -0.005266 sec Oct 16 12:20:06 localhost NTP: 16 Oct 12:20:06 ntpdate[110347]: adjust time server 44.190.6.254 offset 0.019827 sec
# 扩大JVM内存
vi /usr/local/tomcat7/bin/catalina.sh
# 原配置
#JAVA_OPTS="-Xms256m -Xmx512m -Xss1024K -XX:PermSize=128m -XX:MaxPermSize=256m"
# 32G内存参考配置
JAVA_OPTS="-server -Xms10g -Xmx10g -XX:PermSize=1g -XX:MaxPermSize=2g -Xshare:off -Xmn1024m"
cat /usr/local/tomcat7/logs/catalina.out | grep 'com.alibaba.druid.pool.GetConnectionTimeoutException' -C 5
Caused by: com.alibaba.druid.pool.GetConnectionTimeoutException: wait millis 30000, active 50, runningSqlCount 3 : INSERT INTO T_TK_QUESTION_BASE (QUESTION_ID_CHAR,QUESTION_TITLE,QUESTION_TIPS,QUESTION_TYPE_ID,QUESTION_TYPE_NAME,QUESTION_DIFFICULT_ID,QUESTION_DIFFICULT_NAME,QUESTION_DIFFICULT_STAR,QUESTION_ANSWER,CREATE_TIME,CREATE_PERSON,B_USE,SOURCE_ID,TS,USE_COUNT,USE_RANGE,KG_ZG,FILE_ID,HEIGHT,PRODUCT_ID,CHECK_STATUS,CHECK_MESSAGE,PARENT_ID_CHAR,JSON_QUESTION,JSON_ANSWER,APP_TYPE,OPTIONS_COUNT,SUBJECT_ID,HAVE_CHILD,CONTENT_MD5,CONTENT_MD5_NEW_UNIQUE) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) at com.alibaba.druid.pool.DruidDataSource.getConnectionInternal(DruidDataSource.java:1071) at com.alibaba.druid.pool.DruidDataSource.getConnectionDirect(DruidDataSource.java:898)
数据库连接池不释放。解决思路：
//数据源配置 druid.setInitialSize(20); druid.setMinIdle(10); druid.setMaxActive(50); druid.setMaxWait(600000); druid.setTimeBetweenEvictionRunsMillis(60000); druid.setMinEvictableIdleTimeMillis(300000); druid.setValidationQuery("SELECT 'x'"); druid.setTestWhileIdle(true); druid.setTestOnBorrow(false); druid.setTestOnReturn(false); druid.setMaxPoolPreparedStatementPerConnectionSize(20); //增加回收机制 druid.setRemoveAbandoned(true); druid.setRemoveAbandonedTimeoutMillis(300); druid.setLogAbandoned(false);
