Linux 排查思路
[root@master qqc]# pidstat -u 5 1
Linux 3.10.0-1160.76.1.el7.x86_64 (master) 10/12/2022 _x86_64_ (4 CPU)
10:43:37 AM UID PID %usr %system %guest %CPU CPU Command
10:43:42 AM 0 9 0.00 0.20 0.00 0.20 1 rcu_sched
10:43:42 AM 65534 370 0.40 0.20 0.00 0.60 0 node_exporter
10:43:42 AM 997 2800 0.20 0.20 0.00 0.40 0 zabbix_agent2
10:43:42 AM 0 4137 1.20 0.40 0.00 1.60 3 kubelet
10:43:42 AM 0 10967 0.40 0.00 0.00 0.40 2 dockerd
10:43:42 AM 0 11773 0.60 0.20 0.00 0.80 0 kube-controller
10:43:42 AM 0 11987 0.80 0.40 0.00 1.20 0 etcd
10:43:42 AM 0 11996 0.20 0.00 0.00 0.20 0 kube-scheduler
10:43:42 AM 0 11997 2.79 0.80 0.00 3.59 1 kube-apiserver
10:43:42 AM 0 12098 0.20 0.00 0.00 0.20 3 coredns
10:43:42 AM 0 12133 0.20 0.20 0.00 0.40 3 coredns
10:43:42 AM 0 30572 0.00 0.20 0.00 0.20 1 flanneld
Average: UID PID %usr %system %guest %CPU CPU Command
Average: 0 9 0.00 0.20 0.00 0.20 - rcu_sched
Average: 65534 370 0.40 0.20 0.00 0.60 - node_exporter
Average: 997 2800 0.20 0.20 0.00 0.40 - zabbix_agent2
Average: 0 4137 1.20 0.40 0.00 1.60 - kubelet
Average: 0 10967 0.40 0.00 0.00 0.40 - dockerd
Average: 0 11773 0.60 0.20 0.00 0.80 - kube-controller
Average: 0 11987 0.80 0.40 0.00 1.20 - etcd
Average: 0 11996 0.20 0.00 0.00 0.20 - kube-scheduler
Average: 0 11997 2.79 0.80 0.00 3.59 - kube-apiserver
Average: 0 12098 0.20 0.00 0.00 0.20 - coredns
Average: 0 12133 0.20 0.20 0.00 0.40 - coredns
Average: 0 30572 0.00 0.20 0.00 0.20 - flanneld
$ pidstat -p 24344
16:14:55 UID PID %usr %system %guest %wait %CPU CPU Command
1.top排查占有cpu过高的进程
2.perf看进程调研关系
# -g开启调用关系分析，-p指定php-fpm的进程号21515
$ perf top -g -p 21515
perf top -g -p 30365
3.查找对应的函数
pidstat -p 25133 查找进程
pstree |grep 进程名 看父进程
5730
# 记录性能事件，等待大约15秒后按 Ctrl+C 退出
$ perf record -g
# 查看报告
$ perf report
调小oom进程机制 就不容易因为内存问题而被omm kill
echo -16 > /proc/$(pidof sshd)/oom_adj
# 清理文件页、目录项、Inodes等各种缓存
$ echo 3 > /proc/sys/vm/drop_caches
# 每隔3秒输出一组数据
$ vmstat 3
procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----
r b swpd free buff cache si so bi bo in cs us sy id wa st
procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----
r b swpd free buff cache si so bi bo in cs us sy id wa st
0 0 0 6601824 97620 1098784 0 0 0 0 62 322 0 0 100 0 0
0 0 0 6601700 97620 1098788 0 0 0 0 57 251 0 0 100 0 0
0 0 0 6601320 97620 1098788 0 0 0 3 52 306 0 0 100 0 0
0 0 0 6601452 97628 1098788 0 0 0 27 63 326 0 0 100 0 0
2 0 0 6601328 97628 1098788 0 0 0 44 52 299 0 0 100 0 0
0 0 0 6601080 97628 1098792 0 0 0 0 56 285 0 0 100 0 0
# -a 表示显示每个内存分配请求的大小以及地址
# -p 指定案例应用的PID号
yum install bcc-tools
/usr/share/bcc/tools/memleak
pmap -x 3671
$ /usr/share/bcc/tools/memleak -a -p $(pidof app)
WARNING: Couldn't find .text section in /app
WARNING: BCC can't handle sym look ups for /app
addr = 7f8f704732b0 size = 8192
addr = 7f8f704772d0 size = 8192
addr = 7f8f704712a0 size = 8192
addr = 7f8f704752c0 size = 8192
32768 bytes in 4 allocations from stack
[unknown] [app]
[unknown] [app]
start_thread+0xdb [libpthread-2.27.so]
# grep表示只保留包含active的指标（忽略大小写）
# sort表示按照字母顺序排序
$ cat /proc/meminfo | grep -i active | sort
Active(anon): 167976 kB
Active(file): 971488 kB
Active: 1139464 kB
Inactive(anon): 720 kB
Inactive(file): 2109536 kB
Inactive: 2110256 kB
$ dmesg | grep -i "Out of memory"
Out of memory: Kill process 9329 (java) score 321 or sacrifice child
