记录-线上服务器系统盘爆满排查记录

 

开始告警

root@iZbp1aqs02dp5tbo0dt8rjZ:/mysql_data# df -hT
Filesystem     Type      Size  Used Avail Use% Mounted on
udev           devtmpfs   16G     0   16G   0% /dev
tmpfs          tmpfs     3.1G  708K  3.1G   1% /run
/dev/vda1      ext4       40G   34G  3.5G  91% /
tmpfs          tmpfs      16G     0   16G   0% /dev/shm
tmpfs          tmpfs     5.0M     0  5.0M   0% /run/lock
tmpfs          tmpfs      16G     0   16G   0% /sys/fs/cgroup
/dev/vdb1      ext4      2.0T  1.6T  286G  85% /data
tmpfs          tmpfs     3.1G     0  3.1G   0% /run/user/0
/dev/vdc1      ext4      2.9T  237G  2.6T   9% /mysql_data


root@iZbp1aqs02dp5tbo0dt8rjZ:/mysql_data# df -hT
Filesystem     Type      Size  Used Avail Use% Mounted on
udev           devtmpfs   16G     0   16G   0% /dev
tmpfs          tmpfs     3.1G  708K  3.1G   1% /run
/dev/vda1      ext4       40G   35G  2.8G  93% /
tmpfs          tmpfs      16G     0   16G   0% /dev/shm
tmpfs          tmpfs     5.0M     0  5.0M   0% /run/lock
tmpfs          tmpfs      16G     0   16G   0% /sys/fs/cgroup
/dev/vdb1      ext4      2.0T  1.6T  286G  85% /data
tmpfs          tmpfs     3.1G     0  3.1G   0% /run/user/0
/dev/vdc1      ext4      2.9T  237G  2.6T   9% /mysql_data

 

root@iZbp1aqs02dp5tbo0dt8rjZ:/mysql_data# df -hT
Filesystem     Type      Size  Used Avail Use% Mounted on
udev           devtmpfs   16G     0   16G   0% /dev
tmpfs          tmpfs     3.1G  708K  3.1G   1% /run
/dev/vda1      ext4       40G   37G  1.5G  97% /       # 即将爆满
tmpfs          tmpfs      16G     0   16G   0% /dev/shm
tmpfs          tmpfs     5.0M     0  5.0M   0% /run/lock
tmpfs          tmpfs      16G     0   16G   0% /sys/fs/cgroup
/dev/vdb1      ext4      2.0T  1.6T  286G  85% /data
tmpfs          tmpfs     3.1G     0  3.1G   0% /run/user/0
/dev/vdc1      ext4      2.9T  237G  2.6T   9% /mysql_data

写入速度非常快

root@iZbp1aqs02dp5tbo0dt8rjZ:/# df -hT
Filesystem     Type      Size  Used Avail Use% Mounted on
udev           devtmpfs   16G     0   16G   0% /dev
tmpfs          tmpfs     3.1G  708K  3.1G   1% /run
/dev/vda1      ext4       40G   38G     0 100% /              # 磁盘爆满
tmpfs          tmpfs      16G     0   16G   0% /dev/shm
tmpfs          tmpfs     5.0M     0  5.0M   0% /run/lock
tmpfs          tmpfs      16G     0   16G   0% /sys/fs/cgroup
/dev/vdb1      ext4      2.0T  1.6T  286G  85% /data
tmpfs          tmpfs     3.1G     0  3.1G   0% /run/user/0

非常感谢这位大哥：https://www.fujieace.com/linux/root-100.html

 杀死进程

kill 444393

root@iZbp1aqs02dp5tbo0dt8rjZ:/# df -hT
Filesystem     Type      Size  Used Avail Use% Mounted on
udev           devtmpfs   16G     0   16G   0% /dev
tmpfs          tmpfs     3.1G  700K  3.1G   1% /run
/dev/vda1      ext4       40G  5.3G   33G  14% /   //降下来了
tmpfs          tmpfs      16G     0   16G   0% /dev/shm
tmpfs          tmpfs     5.0M     0  5.0M   0% /run/lock
tmpfs          tmpfs      16G     0   16G   0% /sys/fs/cgroup
/dev/vdb1      ext4      2.0T  1.6T  286G  85% /data
tmpfs          tmpfs     3.1G     0  3.1G   0% /run/user/0