free,sar,iostat,vmstat,pidstat,ipcs,watch,ss
http://blog.chinaunix.net/uid-20787846-id-1842125.html 系统性能监控工具
内存监测每2s以M,K,B为单位分别显示
[root@test1 ~]# free -o
total used free shared buffers cached
Mem: 3943580 3890992 52588 0 184736 2178076
Swap: 6291448 0 6291448
[root@test1 ~]# free -ot
total used free shared buffers cached
Mem: 3943580 3891984 51596 0 184764 2178548
Swap: 6291448 0 6291448
Total: 10235028 3891984 6343044
[root@221-comecs mnt]# free -s2 -m
total used free shared buffers cached
Mem: 995 562 433 0 95 337
-/+ buffers/cache: 129 866
Swap: 1635 0 1635
total used free shared buffers cached
Mem: 995 562 433 0 95 337
-/+ buffers/cache: 129 866
Swap: 1635 0 1635
[root@221-comecs mnt]# free -s2
total used free shared buffers cached
Mem: 1019820 575496 444324 0 97448 345320
-/+ buffers/cache: 132728 887092
Swap: 1675256 0 1675256
total used free shared buffers cached
Mem: 1019820 575504 444316 0 97448 345320
-/+ buffers/cache: 132736 887084
Swap: 1675256 0 1675256
[root@221-comecs mnt]# free -s2 -b
total used free shared buffers cached
Mem: 1044295680 589180928 455114752 0 99786752 353607680
-/+ buffers/cache: 135786496 908509184
Swap: 1715462144 0 1715462144
total used free shared buffers cached
Mem: 1044295680 589189120 455106560 0 99786752 353607680
-/+ buffers/cache: 135794688 908500992
Swap: 1715462144 0 1715462144
[root@109-com1 ~]# yum install iotop
[root@109-com1 asterisk]# watch -n1 -d free
Every 1.0s: free Thu Oct 9 14:53:01 2014
total used free shared buffers cached
Mem: 1019820 933400 86420 0 112452 515660
-/+ buffers/cache: 305288 714532
Swap: 1675256 456 1674800
[root@109-com1 asterisk]# watch -d vmstat
Every 2.0s: vmstat Thu Oct 9 14:58:28 2014
procs -----------memory---------- ---swap-- -----io---- --system-- -----cpu-----
r b swpd free buff cache si so bi bo in cs us sy id wa st
0 0 456 74072 112636 527608 0 0 0 17 21 40 0 0 99 0 0
[root@250-shiyan bin]# ps aux|head -1;ps aux|sort -rnk6|head 以RSS即第六列为准由大到小排序
[root@oracle5A flt]# mpstat -P ALL 1
Linux 2.6.32-431.el6.x86_64 (oracle5A) 10/15/2014 _x86_64_ (4 CPU)
06:27:23 PM CPU %usr %nice %sys %iowait %irq %soft %steal %guest %idle
06:27:24 PM all 0.75 0.00 0.25 0.00 0.00 0.00 0.00 0.00 99.00
06:27:24 PM 0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 100.00
06:27:24 PM 1 0.99 0.00 0.99 0.00 0.00 0.00 0.00 0.00 98.02
06:27:24 PM 2 1.00 0.00 1.00 0.00 0.00 0.00 0.00 0.00 98.00
06:27:24 PM 3 1.00 0.00 1.00 0.00 0.00 0.00 0.00 0.00 98.00
[root@221-comecs ~]# vmstat -a 1
procs -----------memory---------- ---swap-- -----io---- --system-- -----cpu-----
r b swpd free inact active si so bi bo in cs us sy id wa st
0 0 11244 72396 570260 272184 0 0 2 8 0 0 1 1 98 0 0
0 0 11244 72264 570284 272252 0 0 0 472 3906 8222 2 1 97 0 0
[root@221-comecs ~]# vmstat -f
13839576 forks
[root@221-comecs ~]# vmstat -ta 1
procs -----------memory---------- ---swap-- -----io---- --system-- -----cpu------ ---timestamp---
r b swpd free buff cache si so bi bo in cs us sy id wa st
0 0 11244 69196 95452 548540 0 0 2 8 0 0 1 1 98 0 0 2015-11-16 14:02:31 CST
0 0 11244 69188 95452 548560 0 0 0 0 3927 8126 2 1 97 0 0 2015-11-16 14:02:32 CST
[root@localhost ~]# vmstat -s
502168 total memory
91576 used memory
23196 active memory
25396 inactive memory
[root@localhost ~]# vmstat -m
Cache Num Total Size Pages
nf_conntrack_expect 0 0 240 16
nf_conntrack_ffffffff81b18540 0 0 312 12
fib6_nodes 22 59 64 59
ip6_dst_cache 13 20 384 10
ndisc_cache 1 15 256 15
ip6_mrt_cache 0 0 128 30
RAWv6 4 4 1024 4
UDPLITEv6 0 0 1024 4
[root@localhost ~]# vmstat -p /dev/sda2 1
sda2 reads read sectors writes requested writes
1794 77490 154 3232
1794 77490 154 3232
1794 77490 154 3232
加w会变宽，不会看起来那么拥挤
[root@221-comecs ~]# vmstat 1 -w
procs -------------------memory------------------ ---swap-- -----io---- --system-- -----cpu-------
r b swpd free buff cache si so bi bo in cs us sy id wa st
2 0 11244 70540 94760 545484 0 0 2 8 0 0 1 1 98 0 0
0 0 11244 71332 94760 545492 0 0 0 0 6286 13875 2 2 96 0 0
0 0 11244 71340 94760 545516 0 0 0 0 9728 22204 2 2 96 0 0
[root@221-comecs ~]# iostat -k -c 1
Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU)
avg-cpu: %user %nice %system %iowait %steal %idle
0.26 0.00 0.31 0.07 0.00 99.35
avg-cpu: %user %nice %system %iowait %steal %idle
0.77 0.00 0.51 0.00 0.00 98.72
[root@221-comecs ~]# iostat -k -d 1
Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU)
Device: tps kB_read/s kB_wrtn/s kB_read kB_wrtn
sda 1.23 3.36 103.60 6461574 199095821
dm-0 25.99 3.36 103.60 6457057 199095812
dm-1 0.00 0.00 0.00 1188 0
Device: tps kB_read/s kB_wrtn/s kB_read kB_wrtn
sda 21.78 0.00 8273.27 0 8356
dm-0 2068.32 0.00 8273.27 0 8356
dm-1 0.00 0.00 0.00 0 0
[root@221-comecs ~]# iostat -k -N 1
Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU)
avg-cpu: %user %nice %system %iowait %steal %idle
0.26 0.00 0.31 0.07 0.00 99.35
Device: tps kB_read/s kB_wrtn/s kB_read kB_wrtn
sda 1.23 3.36 103.61 6461602 199139673
VolGroup-lv_root 25.99 3.36 103.61 6457085 199139664
VolGroup-lv_swap 0.00 0.00 0.00 1188 0
[root@221-comecs ~]# iostat -cx 1
Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU)
avg-cpu: %user %nice %system %iowait %steal %idle
0.26 0.00 0.31 0.07 0.00 99.35
Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util
sda 0.00 24.76 0.09 1.14 6.72 207.21 174.21 0.02 19.55 4.53 0.56
dm-0 0.00 0.00 0.09 25.90 6.72 207.21 8.23 1.30 50.10 0.21 0.56
dm-1 0.00 0.00 0.00 0.00 0.00 0.00 8.00 0.00 1.10 0.79 0.00
[root@221-comecs ~]# iostat -k -p dm-0 1 -t
Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU)
12/11/2014 05:10:02 PM
avg-cpu: %user %nice %system %iowait %steal %idle
0.26 0.00 0.31 0.07 0.00 99.35
Device: tps kB_read/s kB_wrtn/s kB_read kB_wrtn
dm-0 26.00 3.36 103.64 6457133 199205400
[root@221-comecs ~]# iostat -k -n 1 -t
Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU)
12/11/2014 05:11:40 PM
Filesystem: rkB_nor/s wkB_nor/s rkB_dir/s wkB_dir/s rkB_svr/s wkB_svr/s ops/s rops/s wops/s
192.168.2.2:/fileserv 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
[root@221-comecs ~]# sar --help
Usage: sar [ options ] [ <interval> [ <count> ] ]
Options are:
[ -A ] [ -b ] [ -B ] [ -C ] [ -d ] [ -h ] [ -m ] [ -p ] [ -q ] [ -r ] [ -R ]
[ -S ] [ -t ] [ -u [ ALL ] ] [ -v ] [ -V ] [ -w ] [ -W ] [ -y ]
[ -I { <int> [,...] | SUM | ALL | XALL } ] [ -P { <cpu> [,...] | ALL } ]
[ -j { ID | LABEL | PATH | UUID | ... } ] [ -n { <keyword> [,...] | ALL } ]
[ -o [ <filename> ] | -f [ <filename> ] ] [ --legacy ]
[ -i <interval> ] [ -s [ <hh:mm:ss> ] ] [ -e [ <hh:mm:ss> ] ]
[root@221-comecs ~]# sar -h
Usage: sar [ options ] [ <interval> [ <count> ] ]
Main options and reports:
-b I/O and transfer rate statistics
-B Paging statistics
-d Block device statistics
-I { <int> | SUM | ALL | XALL }
Interrupts statistics
-m Power management statistics
-n { <keyword> [,...] | ALL }
Network statistics
Keywords are:
DEV Network interfaces
EDEV Network interfaces (errors)
NFS NFS client
NFSD NFS server
SOCK Sockets (v4)
IP IP traffic (v4)
EIP IP traffic (v4) (errors)
ICMP ICMP traffic (v4)
EICMP ICMP traffic (v4) (errors)
TCP TCP traffic (v4)
ETCP TCP traffic (v4) (errors)
UDP UDP traffic (v4)
SOCK6 Sockets (v6)
IP6 IP traffic (v6)
EIP6 IP traffic (v6) (errors)
ICMP6 ICMP traffic (v6)
EICMP6 ICMP traffic (v6) (errors)
UDP6 UDP traffic (v6)
-q Queue length and load average statistics
-r Memory utilization statistics
-R Memory statistics
-S Swap space utilization statistics
-u [ ALL ]
CPU utilization statistics
-v Kernel table statistics
-w Task creation and system switching statistics
-W Swapping statistics
-y TTY device statistics
不带选项执行会以10分钟为间隔报告自午夜起当天的CPU利用率。
[root@221-comecs ~]# sar
Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU)
12:00:01 AM CPU %user %nice %system %iowait %steal %idle
12:10:01 AM all 0.12 0.00 0.14 0.07 0.00 99.66
12:20:01 AM all 0.10 0.00 0.15 0.08 0.00 99.68
......
04:50:01 PM all 0.99 0.00 0.64 0.05 0.00 98.32
05:00:01 PM all 1.10 0.00 0.70 0.08 0.00 98.12
05:10:01 PM all 0.78 0.00 0.80 0.52 0.00 97.91
05:20:01 PM all 0.74 0.00 0.59 0.07 0.00 98.61
Average: all 0.44 0.00 0.40 0.09 0.00 99.07
[root@ipt ~]# sar -m 1
Linux 2.6.32-431.el6.x86_64 (ipt) 10/17/2014 _x86_64_ (2 CPU)
09:46:58 PM CPU MHz
09:46:59 PM all 2493.76
09:47:00 PM all 2493.76
[root@ipt ~]# sar -C 1
Linux 2.6.32-431.el6.x86_64 (ipt) 10/17/2014 _x86_64_ (2 CPU)
09:46:33 PM CPU %user %nice %system %iowait %steal %idle
09:46:34 PM all 0.00 0.00 0.50 0.00 0.00 99.50
09:46:35 PM all 0.00 0.00 0.00 0.00 0.00 100.00
[root@ipt ~]# sar -B 1
Linux 2.6.32-431.el6.x86_64 (ipt) 10/17/2014 _x86_64_ (2 CPU)
09:46:12 PM pgpgin/s pgpgout/s fault/s majflt/s pgfree/s pgscank/s pgscand/s pgsteal/s %vmeff
09:46:13 PM 0.00 8.08 37.37 0.00 111.11 0.00 0.00 0.00 0.00
09:46:14 PM 0.00 0.00 40.00 0.00 110.00 0.00 0.00 0.00 0.00
| pgpgin/s |
每秒从磁盘分到内存中的页面数,paged in 置入分页；页入 the system paged in from disk |
| pgpgout/s |
每秒从内存分到磁盘的页面数,paged out 页出；移出分页 the system paged out to disk |
| fault/s |
每秒的页面故障数 |
| majflt/s |
每秒的主要页面故障数 |
With older kernels, a block is of indeterminate size.
indeterminate adj. 不确定的；模糊的；含混的
[root@ipt ~]# sar -b 1
Linux 2.6.32-431.el6.x86_64 (ipt) 10/17/2014 _x86_64_ (2 CPU)
09:45:42 PM tps rtps wtps bread/s bwrtn/s
09:45:43 PM 0.00 0.00 0.00 0.00 0.00
09:45:44 PM 0.00 0.00 0.00 0.00 0.00
查看系统内存使用情况
[root@221-comecs ~]# sar -r 1
Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU)
05:25:18 PM kbmemfree kbmemused %memused kbbuffers kbcached kbcommit %commit
05:25:19 PM 68656 951164 93.27 68072 682032 194064 7.20
05:25:20 PM 68284 951536 93.30 68076 682228 194064 7.20
05:25:21 PM 68036 951784 93.33 68076 682568 194064 7.20
[root@221-comecs ~]# sar -S 1
Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU)
05:36:40 PM kbswpfree kbswpused %swpused kbswpcad %swpcad
05:36:41 PM 1675256 0 0.00 0 0.00
05:36:42 PM 1675256 0 0.00 0 0.00
05:36:43 PM 1675256 0 0.00 0 0.00
查看系统磁盘的读写情况。
[root@221-comecs ~]# sar -d 1
Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU)
05:26:26 PM DEV tps rd_sec/s wr_sec/s avgrq-sz avgqu-sz await svctm %util
05:26:27 PM dev8-0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
05:26:27 PM dev253-0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
05:26:27 PM dev253-1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
[root@221-comecs ~]# sar -dp 1
Linux 2.6.32-431.el6.x86_64 (221-comecs) 11/16/2015 _x86_64_ (4 CPU)
11:07:54 AM DEV tps rd_sec/s wr_sec/s avgrq-sz avgqu-sz await svctm %util
11:07:55 AM sda 4.21 0.00 42.11 10.00 0.05 11.00 3.25 1.37
11:07:55 AM VolGroup-lv_root 5.26 0.00 42.11 8.00 0.05 10.40 2.60 1.37
11:07:55 AM VolGroup-lv_swap 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
查看2号cpu及所有cpu使用情况
[root@221-comecs ~]# sar -P 2 1
Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU)
05:27:50 PM CPU %user %nice %system %iowait %steal %idle
05:27:51 PM 2 0.00 0.00 0.99 0.00 0.00 99.01
05:27:52 PM 2 2.02 0.00 0.00 0.00 0.00 97.98
05:27:53 PM 2 0.00 0.00 0.00 0.00 0.00 100.00
^C
[root@221-comecs ~]# sar -P ALL 1
Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU)
05:27:59 PM CPU %user %nice %system %iowait %steal %idle
05:28:00 PM all 0.76 0.00 0.76 0.25 0.00 98.24
05:28:00 PM 0 3.03 0.00 1.01 0.00 0.00 95.96
05:28:00 PM 1 1.01 0.00 1.01 1.01 0.00 96.97
05:28:00 PM 2 0.00 0.00 0.00 0.00 0.00 100.00
05:28:00 PM 3 0.00 0.00 0.00 0.00 0.00 100.00
[root@221-comecs ~]# sar -u 1
Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU)
05:29:27 PM CPU %user %nice %system %iowait %steal %idle
05:29:28 PM all 0.50 0.00 0.50 0.00 0.00 99.00
05:29:29 PM all 0.75 0.00 0.50 0.00 0.00 98.75
[root@rac02 ~]# sar -n DEV 1 0|egrep "IFACE|lo"
02:57:18 PM IFACE rxpck/s txpck/s rxbyt/s txbyt/s rxcmp/s txcmp/s rxmcst/s
02:57:19 PM lo 2223.00 2223.00 282223.00 282223.00 0.00 0.00 0.00
02:57:19 PM IFACE rxpck/s txpck/s rxbyt/s txbyt/s rxcmp/s txcmp/s rxmcst/s
02:57:20 PM lo 59.41 59.41 13728.71 13728.71 0.00 0.00 0.00
[root@221-comecs ~]# sar -n DEV 1
Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU)
05:30:50 PM IFACE rxpck/s txpck/s rxkB/s txkB/s rxcmp/s txcmp/s rxmcst/s
05:30:51 PM lo 0.00 0.00 0.00 0.00 0.00 0.00 0.00
05:30:51 PM eth0 23.00 24.00 3.67 2.34 0.00 0.00 0.00
[root@221-comecs ~]# sar -n EDEV 1
Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU)
05:32:24 PM IFACE rxerr/s txerr/s coll/s rxdrop/s txdrop/s txcarr/s rxfram/s rxfifo/s txfifo/s
05:32:25 PM lo 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
05:32:25 PM eth0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
[root@221-comecs ~]# sar -n SOCK 1
Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU)
05:31:47 PM totsck tcpsck udpsck rawsck ip-frag tcp-tw
05:31:48 PM 147 15 11 0 0 0
05:31:49 PM 147 15 11 0 0 0
05:31:50 PM 147 15 11 0 0 0
[root@221-comecs ~]# sar -v 1
Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU)
05:35:13 PM dentunusd file-nr inode-nr pty-nr
05:35:14 PM 1806 544 6747 34
05:35:15 PM 1806 544 6747 34
05:35:16 PM 1806 544 6747 34
###收集数据
[root@250-shiyan sa]# /usr/lib64/sa/sadc 1 10 sadc00
###sadc 是把数据写在一个二进制的文件中，如果想查看数据内容，需要用sadf工具来显示；如果不指定文件名，默认是屏幕。
###查看数据 两条命令都可以查看sadc00文件的数据，但sadf不太直观
[root@250-shiyan sa]# sar -f sadc00
Linux 2.6.32-431.el6.x86_64 (250-shiyan) 12/12/2014 _x86_64_ (1 CPU)
11:28:13 AM CPU %user %nice %system %iowait %steal %idle
11:28:14 AM all 0.00 0.00 0.00 1.00 0.00 99.00
11:28:15 AM all 0.00 0.00 0.00 0.00 0.00 100.00
11:28:16 AM all 0.99 0.00 0.00 0.00 0.00 99.01
11:28:17 AM all 0.00 0.00 0.00 0.00 0.00 100.00
11:28:18 AM all 0.00 0.00 0.99 0.00 0.00 99.01
11:28:19 AM all 0.00 0.00 0.00 0.00 0.00 100.00
11:28:20 AM all 0.00 0.00 0.00 0.00 0.00 100.00
11:28:21 AM all 0.00 0.00 0.00 0.00 0.00 100.00
11:28:22 AM all 0.00 0.00 0.00 1.00 0.00 99.00
Average: all 0.11 0.00 0.11 0.22 0.00 99.56
[root@250-shiyan sa]# sadf sadc00
250-shiyan 1 1418354966 all %user 0.99
250-shiyan 1 1418354966 all %nice 0.00
250-shiyan 1 1418354966 all %system 0.00
250-shiyan 1 1418354966 all %iowait 0.99
250-shiyan 1 1418354966 all %steal 0.00
250-shiyan 1 1418354966 all %idle 98.0
[root@109-com1 ~]# pidstat 1
Linux 2.6.32-431.el6.x86_64 (109-com1) 12/12/2014 _x86_64_ (4 CPU)
05:02:25 PM PID %usr %system %guest %CPU CPU Command
05:02:26 PM 5545 8.91 8.91 0.00 17.82 0 asterisk
05:02:26 PM 10216 0.00 1.98 0.00 1.98 1 FLTService
05:02:26 PM 12611 0.00 0.99 0.00 0.99 1 pidstat
[root@109-com1 ~]# pidstat -I 1
Linux 2.6.32-431.el6.x86_64 (109-com1) 12/12/2014 _x86_64_ (4 CPU)
05:01:25 PM PID %usr %system %guest %CPU CPU Command
05:01:26 PM 5545 9.90 7.92 0.00 4.59 0 asterisk
05:01:26 PM 10216 0.99 1.98 0.00 0.77 1 FLTService
05:01:26 PM 12595 0.00 0.99 0.00 0.26 1 pidstat
[root@109-com1 ~]# pidstat -h 1
Linux 2.6.32-431.el6.x86_64 (109-com1) 12/12/2014 _x86_64_ (4 CPU)
# Time PID %usr %system %guest %CPU CPU Command
1418374844 20 0.00 0.99 0.00 0.99 1 events/1
1418374844 5545 9.90 9.90 0.00 19.80 0 asterisk
1418374844 10216 0.99 1.98 0.00 2.97 1 FLTService
1418374844 12568 0.00 1.98 0.00 1.98 2 pidstat
[root@109-com1 ~]# pidstat 1 -l
Linux 2.6.32-431.el6.x86_64 (109-com1) 12/12/2014 _x86_64_ (4 CPU)
05:03:08 PM PID %usr %system %guest %CPU CPU Command
05:03:09 PM 5545 8.91 9.90 0.00 18.81 0 /usr/sbin/asterisk -f -vvvg -c
05:03:09 PM 10216 0.99 2.97 0.00 3.96 1 ./FLTService
05:03:09 PM 12628 0.00 0.99 0.00 0.99 1 pidstat 1 -l
[root@109-com1 ~]# pidstat 1 -C FL
Linux 2.6.32-431.el6.x86_64 (109-com1) 12/12/2014 _x86_64_ (4 CPU)
04:58:20 PM PID %usr %system %guest %CPU CPU Command
04:58:21 PM 10216 0.00 2.97 0.00 2.97 1 FLTService
04:58:21 PM PID %usr %system %guest %CPU CPU Command
04:58:22 PM 10216 1.00 2.00 0.00 3.00 1 FLTService
[root@109-com1 ~]# pidstat -d 1
Linux 2.6.32-431.el6.x86_64 (109-com1) 12/12/2014 _x86_64_ (4 CPU)
05:00:06 PM PID kB_rd/s kB_wr/s kB_ccwr/s Command
05:00:07 PM 347 0.00 87.13 0.00 jbd2/dm-0-8
05:00:07 PM 5545 0.00 39.60 0.00 asterisk
05:00:07 PM 10216 0.00 11.88 0.00 FLTService
05:00:07 PM PID kB_rd/s kB_wr/s kB_ccwr/s Command
05:00:08 PM 5545 0.00 32.00 0.00 asterisk
[root@109-com1 ~]# pidstat 1 -r
Linux 2.6.32-431.el6.x86_64 (109-com1) 12/12/2014 _x86_64_ (4 CPU)
05:04:27 PM PID minflt/s majflt/s VSZ RSS %MEM Command
05:04:28 PM 5545 4.95 0.00 2387204 193976 19.02 asterisk
05:04:28 PM 12666 253.47 0.00 101096 932 0.09 pidstat
05:04:30 PM PID minflt/s majflt/s VSZ RSS %MEM Command
05:04:31 PM 4790 1.00 0.00 197676 2220 0.22 snmpd
05:04:31 PM 5545 8.00 0.00 2387204 193976 19.02 asterisk
05:04:31 PM 12666 254.00 0.00 101096 964 0.09 pidstat
[root@109-com1 ~]# pidstat 1 -p 10216
Linux 2.6.32-431.el6.x86_64 (109-com1) 12/12/2014 _x86_64_ (4 CPU)
05:05:30 PM PID %usr %system %guest %CPU CPU Command
05:05:31 PM 10216 0.00 3.00 0.00 3.00 0 FLTService
05:05:32 PM 10216 1.00 3.00 0.00 4.00 0 FLTService
05:05:33 PM 10216 1.00 3.00 0.00 4.00 0 FLTService
05:05:34 PM 10216 1.00 3.00 0.00 4.00 0 FLTService
[root@109-com1 ~]# pidstat 1 -p 5545 -T ALL
Linux 2.6.32-431.el6.x86_64 (109-com1) 12/12/2014 _x86_64_ (4 CPU)
05:22:26 PM PID %usr %system %guest %CPU CPU Command
05:22:27 PM 5545 10.00 10.00 0.00 20.00 1 asterisk
05:22:26 PM PID usr-ms system-ms guest-ms Command
05:22:27 PM 5545 100 100 0 asterisk
05:22:28 PM 5545 8.00 10.00 0.00 18.00 1 asterisk
05:22:28 PM 5545 80 100 0 asterisk
[root@109-com1 ~]# pidstat 1 -r -p 5545 -T ALL
Linux 2.6.32-431.el6.x86_64 (109-com1) 12/12/2014 _x86_64_ (4 CPU)
05:23:21 PM PID minflt/s majflt/s VSZ RSS %MEM Command
05:23:22 PM 5545 5.00 0.00 2387204 193980 19.02 asterisk
05:23:21 PM PID minflt-nr majflt-nr Command
05:23:22 PM 5545 5 0 asterisk
[root@aster3 asterisk]# ipcs -l
------ Shared Memory Limits --------
max number of segments = 4096
max seg size (kbytes) = 67108864
max total shared memory (kbytes) = 17179869184
min seg size (bytes) = 1
------ Semaphore Limits --------
max number of arrays = 128
max semaphores per array = 250
max semaphores system wide = 32000
max ops per semop call = 32
semaphore max value = 32767
------ Messages: Limits --------
max queues system wide = 973
max size of message (bytes) = 65536
default max size of queue (bytes) = 65536
[root@aster3 asterisk]# ipcs -a
------ Shared Memory Segments --------
key shmid owner perms bytes nattch status
0x74000417 983040 root 600 4 0
0x740003ec 1638401 root 600 4 0
0x00000000 1998850 root 644 52 2
0x740003eb 1605635 root 600 4 0
0x00000000 2031620 root 644 16384 2
0x00000000 2064389 root 644 268 2
------ Semaphore Arrays --------
key semid owner perms nsems
0x000000a7 0 root 600 1
0x00000000 32769 apache 600 1
0x00000000 262146 apache 600 1
0x00000000 294915 apache 600 1
0x00000000 327684 apache 600 1
0x00000000 360453 apache 600 1
0x00000000 393222 apache 600 1
0x00000000 425991 apache 600 1
------ Message Queues --------
key msqid owner perms used-bytes messages
[root@aster3 asterisk]# ipcs -s
------ Semaphore Arrays --------
key semid owner perms nsems
0x000000a7 0 root 600 1
0x00000000 32769 apache 600 1
0x00000000 262146 apache 600 1
0x00000000 294915 apache 600 1
0x00000000 327684 apache 600 1
0x00000000 360453 apache 600 1
0x00000000 393222 apache 600 1
0x00000000 425991 apache 600 1
[root@aster3 asterisk]# ipcs -m
------ Shared Memory Segments --------
key shmid owner perms bytes nattch status
0x74000417 983040 root 600 4 0
0x740003ec 1638401 root 600 4 0
0x00000000 1998850 root 644 52 2
0x740003eb 1605635 root 600 4 0
0x00000000 2031620 root 644 16384 2
0x00000000 2064389 root 644 268 2
[root@aster3 asterisk]# ipcs -mi 2064389
Shared memory Segment shmid=2064389
uid=0 gid=0 cuid=0 cgid=0
mode=0644 access_perms=0644
bytes=268 lpid=3357 cpid=3340 nattch=2
att_time=Mon Dec 8 10:35:54 2014
det_time=Mon Dec 8 10:35:54 2014
change_time=Mon Dec 8 10:35:54 2014
[root@aster3 asterisk]# ipcs -mu
------ Shared Memory Status --------
segments allocated 6
pages allocated 9
pages resident 9
pages swapped 0
Swap performance: 0 attempts 0 successes
[root@aster3 asterisk]# ipcs -ml
------ Shared Memory Limits --------
max number of segments = 4096
max seg size (kbytes) = 4194303
max total shared memory (kbytes) = 1073741824
min seg size (bytes) = 1
[root@221-comecs ~]# watch -n1 -d "ss -no"
Every 1.0s: ss -no Thu Dec 18 11:18:00 2014
State Recv-Q Send-Q Local Address:Port Peer Address:Port
ESTAB 0 0 192.168.2.221:10001 124.89.70.40:4776 timer:(keepalive,103min,0)
ESTAB 0 1506 192.168.2.221:10001 36.46.211.248:1872 timer:(on,235ms,0)
ESTAB 0 1506 192.168.2.221:10001 36.46.211.248:4482 timer:(on,165ms,0)
ESTAB 0 0 192.168.2.221:22 192.168.2.84:54625 timer:(keepalive,114min,0)
ESTAB 0 149 192.168.2.221:10001 36.46.211.248:1432 timer:(on,181ms,0)
ESTAB 0 149 192.168.2.221:10001 36.46.211.248:20403 timer:(on,344ms,0)
ESTAB 0 0 192.168.2.221:10001 36.46.211.248:4277 timer:(keepalive,78min,0)
ESTAB 0 1506 192.168.2.221:10001 36.46.211.248:4932 timer:(on,321ms,0)
ESTAB 0 0 192.168.2.221:53198 192.168.2.2:1521
ESTAB 0 0 192.168.2.221:10001 123.138.91.202:2702 timer:(keepalive,34min,0)
ESTAB 0 149 192.168.2.221:10001 117.36.195.67:12850 timer:(on,343ms,0)
ESTAB 0 306 192.168.2.221:10001 36.46.211.248:2644 timer:(on,231ms,0)
ESTAB 0 0 192.168.2.221:10001 113.138.207.39:2302 timer:(keepalive,67min,0)
ESTAB 0 0 192.168.2.221:10001 210.74.130.138:50607 timer:(keepalive,109min,0)
ESTAB 0 0 192.168.2.221:53201 192.168.2.2:1521
[root@109-com1 ~]# watch -n1 -d "ss -n -o state fin-wait-1"
Every 1.0s: ss -n -o state fin-wait-1 Thu Dec 18 11:23:17 2014
Recv-Q Send-Q Local Address:Port Peer Address:Port
0 1 192.168.2.109:10000 223.104.11.173:30328 timer:(on,4.409ms,3)
0 1 192.168.2.109:10000 117.136.25.78:50720 timer:(on,3.505ms,0)
0 1 192.168.2.109:10000 223.104.11.172:26123 timer:(on,34sec,6)
0 1 192.168.2.109:10000 117.136.25.185:6084 timer:(on,20sec,5)
0 1 192.168.2.109:10000 117.136.25.189:60125 timer:(on,4.732ms,2)
0 21 192.168.2.109:10000 117.136.64.85:63212 timer:(on,39sec,8)
0 1 192.168.2.109:10000 223.104.11.176:27962 timer:(on,45sec,5)
0 1 192.168.2.109:10000 223.104.11.180:14531 timer:(on,1min1sec,6)
0 1 192.168.2.109:10000 117.136.4.27:14568 timer:(on,1min5sec,6)
0 1 192.168.2.109:10000 117.136.4.16:26374 timer:(on,19sec,2)
0 1 192.168.2.109:10000 117.136.4.22:24919 timer:(on,4.621ms,6)
0 1 192.168.2.109:10000 117.136.38.175:38252 timer:(on,1min1sec,5)
0 1 192.168.2.109:10000 117.136.25.180:52928 timer:(on,5.700ms,6)
0 1 192.168.2.109:10000 117.136.25.220:43257 timer:(on,5.912ms,6)
[root@109-com1 ~]# watch -n1 -d "netstat -not"
Every 1.0s: netstat -not Thu Dec 18 11:48:52 2014
Active Internet connections (w/o servers)
Proto Recv-Q Send-Q Local Address Foreign Address State Timer
tcp 0 0 192.168.2.109:10000 117.136.3.10:4607 SYN_RECV on (16.97/5/0)
tcp 0 0 192.168.2.109:10000 223.104.11.85:16549 ESTABLISHED keepalive (1126.16/0/0)
tcp 0 0 192.168.2.109:10000 117.136.25.189:15734 ESTABLISHED keepalive (3639.54/0/0)
tcp 0 0 192.168.2.109:10000 117.136.25.76:53721 ESTABLISHED keepalive (990.24/0/0)
tcp 0 0 192.168.2.109:10000 117.136.25.179:14765 ESTABLISHED keepalive (2948.78/0/0)
tcp 0 0 192.168.2.109:10000 117.136.25.136:30413 ESTABLISHED keepalive (5343.41/0/0)
tcp 0 0 192.168.2.109:10000 223.104.11.182:35807 ESTABLISHED keepalive (1776.77/0/0)
tcp 0 0 192.168.2.109:10000 117.136.25.189:42258 ESTABLISHED keepalive (2645.11/0/0)
tcp 0 0 192.168.2.109:10000 223.104.11.172:26158 ESTABLISHED keepalive (7132.59/0/0)
tcp 0 0 192.168.2.109:10000 223.104.11.85:6855 ESTABLISHED keepalive (6403.83/0/0)
tcp 0 0 192.168.2.109:10000 223.104.11.84:41843 ESTABLISHED keepalive (3607.52/0/0)
tcp 0 0 192.168.2.109:10000 223.104.11.175:46953 ESTABLISHED keepalive (3189.19/0/0)
tcp 0 0 192.168.2.109:10000 117.136.50.35:57643 ESTABLISHED keepalive (6432.29/0/0)
tcp 0 20 192.168.2.109:10000 117.136.25.142:45223 ESTABLISHED on (5.28/1/0)
tcp 0 20 192.168.2.109:10000 117.136.25.184:49946 ESTABLISHED on (29.47/5/0)
tcp 0 0 192.168.2.109:10000 223.104.11.176:64584 ESTABLISHED keepalive (1179.13/0/0)
[root@222-comecs ~]# watch -n1 -d "ss -ni"
Every 1.0s: ss -ni Mon Dec 22 11:57:39 2014
State Recv-Q Send-Q Local Address:Port Peer Address:Port
ESTAB 0 0 192.168.2.222:46839 192.168.2.220:1521
cubic wscale:7,6 rto:208 rtt:8.25/11 ato:40 cwnd:10 send 14.0Mbps rcv_rtt:6 rcv_space:14600
ESTAB 0 0 192.168.2.222:46797 192.168.2.220:1521
cubic wscale:7,6 rto:201 rtt:1.875/0.75 ato:40 cwnd:10 send 61.8Mbps rcv_rtt:8 rcv_space:14600
ESTAB 0 0 192.168.2.222:46786 192.168.2.220:1521
cubic wscale:7,6 rto:201 rtt:1.875/0.75 ato:40 cwnd:10 send 61.8Mbps rcv_rtt:8 rcv_space:14600
ESTAB 0 0 192.168.2.222:46798 192.168.2.220:1521
cubic wscale:7,6 rto:201 rtt:1.875/0.75 ato:40 cwnd:10 send 61.8Mbps rcv_rtt:12 rcv_space:14600
ESTAB 0 0 192.168.2.222:46819 192.168.2.220:1521
cubic wscale:7,6 rto:201 rtt:1.875/0.75 ato:40 cwnd:10 send 61.8Mbps rcv_rtt:9 rcv_space:14600
ESTAB 0 0 ::ffff:192.168.2.222:46674 ::ffff:192.168.2.220:1521
cubic wscale:7,6 rto:206 rtt:6.75/8.25 ato:40 cwnd:10 send 17.2Mbps rcv_rtt:2.125 rcv_space:14600
[root@222-comecs ~]# watch -n1 -d "ss -nmp"
Every 1.0s: ss -nmp Mon Dec 22 12:00:03 2014
State Recv-Q Send-Q Local Address:Port Peer Address:Port
ESTAB 0 0 192.168.2.222:46839 192.168.2.220:1521 users:(("asterisk",30976,157))
mem:(r0,w0,f0,t0)
ESTAB 0 0 192.168.2.222:46797 192.168.2.220:1521 users:(("asterisk",30976,73))
mem:(r0,w0,f0,t0)
ESTAB 0 0 192.168.2.222:46786 192.168.2.220:1521 users:(("asterisk",30976,51))
mem:(r0,w0,f0,t0)
ESTAB 0 0 192.168.2.222:46798 192.168.2.220:1521 users:(("asterisk",30976,75))
mem:(r0,w0,f0,t0)
ESTAB 0 0 192.168.2.222:46819 192.168.2.220:1521 users:(("asterisk",30976,117))
mem:(r0,w0,f0,t0)
ESTAB 0 0 ::ffff:192.168.2.222:46674 ::ffff:192.168.2.220:1521 users:(("java",28435,36))
mem:(r0,w0,f0,t0)
[root@222-comecs ~]# watch -n1 -d "ss -np"
Every 1.0s: ss -np Mon Dec 22 11:59:16 2014
State Recv-Q Send-Q Local Address:Port Peer Address:Port
ESTAB 0 0 192.168.2.222:46839 192.168.2.220:1521 users:(("asterisk",30976,157))
ESTAB 0 0 192.168.2.222:46797 192.168.2.220:1521 users:(("asterisk",30976,73))
ESTAB 0 0 192.168.2.222:46786 192.168.2.220:1521 users:(("asterisk",30976,51))
ESTAB 0 0 192.168.2.222:46798 192.168.2.220:1521 users:(("asterisk",30976,75))
ESTAB 0 0 192.168.2.222:46819 192.168.2.220:1521 users:(("asterisk",30976,117))
ESTAB 0 0 ::ffff:192.168.2.222:46674 ::ffff:192.168.2.220:1521 users:(("java",28435,36))
ESTAB 0 0 192.168.2.222:46809 192.168.2.220:1521 users:(("asterisk",30976,97))
ESTAB 0 0 192.168.2.222:46789 192.168.2.220:1521 users:(("asterisk",30976,57))
ESTAB 0 0 192.168.2.222:41986 192.168.2.220:1521 users:(("java",28345,21))
ESTAB 0 0 192.168.2.222:46812 192.168.2.220:1521 users:(("asterisk",30976,103))
ESTAB 0 0 192.168.2.222:10000 223.104.11.85:47208 users:(("asterisk",30976,18))
ESTAB 0 0 192.168.2.222:46785 192.168.2.220:1521 users:(("asterisk",30976,49))
ESTAB 0 0 192.168.2.222:46784 192.168.2.220:1521 users:(("asterisk",30976,47))
ESTAB 0 0 192.168.2.222:49205 192.168.2.220:1521 users:(("httpd",15354,16))
[root@222-comecs ~]# watch -n1 -d "ss -ne"
Every 1.0s: ss -ne Mon Dec 22 12:02:04 2014
State Recv-Q Send-Q Local Address:Port Peer Address:Port
ESTAB 0 0 192.168.2.222:46839 192.168.2.220:1521 ino:5102290 sk:ffff8800375a58c0
ESTAB 0 0 192.168.2.222:46797 192.168.2.220:1521 ino:5102205 sk:ffff88003df53640
ESTAB 0 0 192.168.2.222:46786 192.168.2.220:1521 ino:5102183 sk:ffff88003ca95040
ESTAB 0 0 192.168.2.222:46798 192.168.2.220:1521 ino:5102207 sk:ffff88003d230ac0
ESTAB 0 0 192.168.2.222:46819 192.168.2.220:1521 ino:5102249 sk:ffff88003d697780
ESTAB 0 0 ::ffff:192.168.2.222:46674 ::ffff:192.168.2.220:1521 ino:4463116 sk:ffff88003b55f040
ESTAB 0 0 192.168.2.222:46809 192.168.2.220:1521 ino:5102229 sk:ffff88003e39a880
ESTAB 0 0 192.168.2.222:46789 192.168.2.220:1521 ino:5102189 sk:ffff88000ab48740
ESTAB 0 0 192.168.2.222:41986 192.168.2.220:1521 ino:4057728 sk:ffff88003ca942c0
ESTAB 0 0 192.168.2.222:46812 192.168.2.220:1521 ino:5102235 sk:ffff88003d579040
ESTAB 0 0 192.168.2.222:10000 223.104.11.85:47208 timer:(keepalive,2min50sec,0) ino:9738746 sk:ffff88003b6d1000
ESTAB 0 0 192.168.2.222:46785 192.168.2.220:1521 ino:5102181 sk:ffff88003ca94980
ESTAB 0 0 192.168.2.222:46784 192.168.2.220:1521 ino:5102179 sk:ffff88003ca95700
ESTAB 0 0 192.168.2.222:49205 192.168.2.220:1521 uid:48 ino:9911907 sk:ffff88003b4c11c0
