一个内存问题
一个内存问题
现象
最近发现有一个python程序，在top的%VSZ显示中占用率很高，但是使用gc.garbage查看并没有发现有不可达的对象。
for c in threading.enumerate():
syslog(LOG_DEBUG, 'FFF> %s' % str(c))
rt = gc.collect()
syslog(LOG_DEBUG, "%d unreachable" % rt)
garbages = gc.garbage
syslog(LOG_DEBUG, "\n%d garbages:" % len(garbages))
for garbage in garbages:
syslog(LOG_DEBUG, "%s" % str(garbage))
Mem: 101988K used, 24272K free, 0K shrd, 0K buff, 41376K cached
CPU: 8% usr 75% sys 0% nic 0% idle 0% io 0% irq 16% sirq
Load average: 1.54 1.46 1.56 2/88 19505
PID PPID USER STAT VSZ %VSZ %CPU COMMAND
1944 1 root S < 8940 7% 17% /usr/sbin/ok_capwapc
2015 1 root S 1556 1% 8% {supervisor.sh} /bin/sh /lib/okos/sup
19328 758 root R 1552 1% 8% top
1954 1 nobody S 1004 1% 8% /usr/sbin/dnsmasq -C /var/etc/dnsmasq
3 2 root RW 0 0% 8% [ksoftirqd/0]
5396 1 root S < 50264 40% 0% /usr/bin/wifidog -s -d 60 #### 40% too high
601 1 root S 37316 30% 0% {clientevent.py} /usr/bin/python /lib #### 30% too high
5087 1 root S < 5480 4% 0% /usr/sbin/arpwatch -f /tmp/arpwatch/a
5099 1 root S < 5480 4% 0% /usr/sbin/arpwatch -f /tmp/arpwatch/a
5107 1 root S < 5480 4% 0% /usr/sbin/arpwatch -f /tmp/arpwatch/a
5117 1 root S < 5480 4% 0% /usr/sbin/arpwatch -f /tmp/arpwatch/a
824 1 root S 2432 2% 0% hostapd -g /var/run/hostapd/global -B
6664 1 root S 1676 1% 0% /sbin/syslogd -L -R log.networkworld3
3442 1 root S < 1588 1% 0% hostapd_cli -p /var/run/hostapd-wifi1
3263 1 root S < 1588 1% 0% hostapd_cli -p /var/run/hostapd-wifi1
3366 1 root S < 1588 1% 0% hostapd_cli -p /var/run/hostapd-wifi1
4347 1 root S < 1588 1% 0% hostapd_cli -p /var/run/hostapd-wifi0
3632 1 root S < 1588 1% 0% hostapd_cli -p /var/run/hostapd-wifi1
3519 1 root S < 1588 1% 0% hostapd_cli -p /var/run/hostapd-wifi1
^C762 1 root S < 1588 1% 0% hostapd_cli -p /var/run/hostapd-wifi1
查看/proc/601下的具体maps情况
oot@DESK /proc/601 [#]# cat maps
00400000-0051e000 r-xp 00000000 00:01 86 /usr/bin/python2.7
0052d000-0052e000 r--p 0011d000 00:01 86 /usr/bin/python2.7
0052e000-00558000 rw-p 0011e000 00:01 86 /usr/bin/python2.7
00558000-00561000 rwxp 00000000 00:00 0
00661000-007a0000 rwxp 00000000 00:00 0 [heap]
1.
751fd000-751fe000 ---p 00000000 00:00 0 # 0x0000_1000 4K
751fe000-753fd000 rw-p 00000000 00:00 0 # 0x0020_0000 2M
2.
753fd000-753fe000 ---p 00000000 00:00 0 # 0x0000_1000 4k
753fe000-755fd000 rw-p 00000000 00:00 0 # 0x0020_0000 2M
3.
755fd000-755fe000 ---p 00000000 00:00 0
755fe000-757fd000 rw-p 00000000 00:00 0
4.
757fd000-757fe000 ---p 00000000 00:00 0
757fe000-759fd000 rw-p 00000000 00:00 0
5.
759fd000-759fe000 ---p 00000000 00:00 0
759fe000-75bfd000 rw-p 00000000 00:00 0
6.
75bfd000-75bfe000 ---p 00000000 00:00 0
75bfe000-75dfd000 rw-p 00000000 00:00 0
7.
75dfd000-75dfe000 ---p 00000000 00:00 0
75dfe000-75ffd000 rw-p 00000000 00:00 0
8.
75ffd000-75ffe000 ---p 00000000 00:00 0
75ffe000-761fd000 rw-p 00000000 00:00 0
9.
761fd000-761fe000 ---p 00000000 00:00 0
761fe000-763fd000 rw-p 00000000 00:00 0
10.
763fd000-763fe000 ---p 00000000 00:00 0
763fe000-765fd000 rw-p 00000000 00:00 0
11.
765fd000-765fe000 ---p 00000000 00:00 0
765fe000-767fd000 rw-p 00000000 00:00 0
12.
767fd000-767fe000 ---p 00000000 00:00 0
767fe000-769fd000 rw-p 00000000 00:00 0
13.
769fd000-769fe000 ---p 00000000 00:00 0
769fe000-76bfd000 rw-p 00000000 00:00 0
76bfd000-76c9f000 rw-p 00000000 00:00 0 # 0x000A_2000 648K
76cb7000-76cb9000 r-xp 00000000 00:01 581 /usr/lib/python2.7/lib-dynload/syslog.so
76cb9000-76cc8000 ---p 00000000 00:00 0
76cc8000-76cc9000 r--p 00001000 00:01 581 /usr/lib/python2.7/lib-dynload/syslog.so
76cc9000-76cca000 rw-p 00002000 00:01 581 /usr/lib/python2.7/lib-dynload/syslog.so
76cca000-76d0b000 rw-p 00000000 00:00 0
76d0b000-76d0f000 r-xp 00000000 00:01 580 /usr/lib/python2.7/lib-dynload/select.so
76d0f000-76d1e000 ---p 00000000 00:00 0
76d1e000-76d1f000 r--p 00003000 00:01 580 /usr/lib/python2.7/lib-dynload/select.so
76d1f000-76d21000 rw-p 00004000 00:01 580 /usr/lib/python2.7/lib-dynload/select.so
76d21000-76d25000 r-xp 00000000 00:01 570 /usr/lib/python2.7/lib-dynload/strop.so
76d25000-76d34000 ---p 00000000 00:00 0
76d34000-76d35000 r--p 00003000 00:01 570 /usr/lib/python2.7/lib-dynload/strop.so
76d35000-76d37000 rw-p 00004000 00:01 570 /usr/lib/python2.7/lib-dynload/strop.so
76d37000-76d78000 rw-p 00000000 00:00 0
76d78000-76d7a000 r-xp 00000000 00:01 584 /usr/lib/python2.7/lib-dynload/fcntl.so
76d7a000-76d89000 ---p 00000000 00:00 0
76d89000-76d8a000 r--p 00001000 00:01 584 /usr/lib/python2.7/lib-dynload/fcntl.so
76d8a000-76d8b000 rw-p 00002000 00:01 584 /usr/lib/python2.7/lib-dynload/fcntl.so
76d8b000-76d8e000 r-xp 00000000 00:01 542 /usr/lib/python2.7/lib-dynload/_random.so
76d8e000-76d9d000 ---p 00000000 00:00 0
76d9d000-76d9e000 r--p 00002000 00:01 542 /usr/lib/python2.7/lib-dynload/_random.so
76d9e000-76d9f000 rw-p 00003000 00:01 542 /usr/lib/python2.7/lib-dynload/_random.so
76d9f000-76da7000 r-xp 00000000 00:01 575 /usr/lib/python2.7/lib-dynload/math.so
76da7000-76db6000 ---p 00000000 00:00 0
76db6000-76db7000 r--p 00007000 00:01 575 /usr/lib/python2.7/lib-dynload/math.so
76db7000-76db8000 rw-p 00008000 00:01 575 /usr/lib/python2.7/lib-dynload/math.so
76db8000-76dbb000 r-xp 00000000 00:01 577 /usr/lib/python2.7/lib-dynload/_heapq.so
76dbb000-76dca000 ---p 00000000 00:00 0
76dca000-76dcb000 r--p 00002000 00:01 577 /usr/lib/python2.7/lib-dynload/_heapq.so
76dcb000-76dcd000 rw-p 00003000 00:01 577 /usr/lib/python2.7/lib-dynload/_heapq.so
76dcd000-76dcf000 r-xp 00000000 00:01 585 /usr/lib/python2.7/lib-dynload/_bisect.so
76dcf000-76dde000 ---p 00000000 00:00 0
76dde000-76ddf000 r--p 00001000 00:01 585 /usr/lib/python2.7/lib-dynload/_bisect.so
76ddf000-76de0000 rw-p 00002000 00:01 585 /usr/lib/python2.7/lib-dynload/_bisect.so
76de0000-76de9000 r-xp 00000000 00:01 563 /usr/lib/python2.7/lib-d ynload/itertools.so
76de9000-76df8000 ---p 00000000 00:00 0
76df8000-76df9000 r--p 00008000 00:01 563 /usr/lib/python2.7/lib-dynload/itertools.so
76df9000-76dfc000 rw-p 00009000 00:01 563 /usr/lib/python2.7/lib-dynload/itertools.so
76dfc000-76e02000 r-xp 00000000 00:01 571 /usr/lib/python2.7/lib-dynload/operator.so
76e02000-76e11000 ---p 00000000 00:00 0
76e11000-76e12000 r--p 00005000 00:01 571 /usr/lib/python2.7/lib-dynload/operator.so
76e12000-76e13000 rw-p 00006000 00:01 571 /usr/lib/python2.7/lib-dynload/operator.so
76e13000-76e18000 r-xp 00000000 00:01 576 /usr/lib/python2.7/lib-dynload/_collections.so
76e18000-76e27000 ---p 00000000 00:00 0
76e27000-76e28000 r--p 00004000 00:01 576 /usr/lib/python2.7/lib-dynload/_collections.so
76e28000-76e29000 rw-p 00005000 00:01 576 /usr/lib/python2.7/lib-dynload/_collections.so
76e29000-76e2c000 r-xp 00000000 00:01 540 /usr/lib/python2.7/lib-dynload/cStringIO.so
76e2c000-76e3b000 ---p 00000000 00:00 0
76e3b000-76e3c000 r--p 00002000 00:01 540 /usr/lib/python2.7/lib-dynload/cStringIO.so
76e3c000-76e3d000 rw-p 00003000 00:01 540 /usr/lib/python2.7/lib-dynload/cStringIO.so
76e3d000-76f5e000 r-xp 00000000 00:01 1192 /usr/lib/libcrypto.so.1.0.0
76f5e000-76f6e000 ---p 00000000 00:00 0
76f6e000-76f7a000 r--p 00121000 00:01 1192 /usr/lib/libcrypto.so.1.0.0
76f7a000-76f7f000 rw-p 0012d000 00:01 1192 /usr/lib/libcrypto.so.1.0.0
76f7f000-76f80000 rw-p 00000000 00:00 0
76f80000-76fc6000 r-xp 00000000 00:01 1227 /usr/lib/libssl.so.1.0.0
76fc6000-76fd5000 ---p 00000000 00:00 0
76fd5000-76fd7000 r--p 00045000 00:01 1227 /usr/lib/libssl.so.1.0.0
76fd7000-76fd9000 rw-p 00047000 00:01 1227 /usr/lib/libssl.so.1.0.0
76fd9000-76fe0000 r-xp 00000000 00:01 572 /usr/lib/python2.7/lib-dynload/_ssl.so
76fe0000-76fef000 ---p 00000000 00:00 0
76fef000-76ff0000 r--p 00006000 00:01 572 /usr/lib/python2.7/lib-dynload/_ssl.so
76ff0000-76ff1000 rw-p 00007000 00:01 572 /usr/lib/python2.7/lib-dynload/_ssl.so
76ff1000-76ff3000 r-xp 00000000 00:01 583 /usr/lib/python2.7/lib-dynload/_functools.so
76ff3000-77002000 ---p 00000000 00:00 0
77002000-77003000 r--p 00001000 00:01 583 /usr/lib/python2.7/lib-dynload/_functools.so
77003000-77004000 rw-p 00002000 00:01 583 /usr/lib/python2.7/lib-dynload/_functools.so
77004000-7700f000 r-xp 00000000 00:01 558 /usr/lib/python2.7/lib-dynload/_socket.so
7700f000-7701e000 ---p 00000000 00:00 0
7701e000-7701f000 r--p 0000a000 00:01 558 /usr/lib/python2.7/lib-dynload/_socket.so
7701f000-77022000 rw-p 0000b000 00:01 558 /usr/lib/python2.7/lib-dynload/_socket.so
77022000-770e5000 rw-p 00000000 00:00 0
770f6000-770fd000 r-xp 00000000 00:01 582 /usr/lib/python2.7/lib-dynload/array.so
770fd000-7710c000 ---p 00000000 00:00 0
7710c000-7710d000 r--p 00006000 00:01 582 /usr/lib/python2.7/lib-dynload/array.so
7710d000-7710f000 rw-p 00007000 00:01 582 /usr/lib/python2.7/lib-dynload/array.so
7710f000-77118000 r-xp 00000000 00:01 565 /usr/lib/python2.7/lib-dynload/_sha512.so
77118000-77127000 ---p 00000000 00:00 0
77127000-77128000 r--p 00008000 00:01 565 /usr/lib/python2.7/lib-dynload/_sha512.so
77128000-77129000 rw-p 00009000 00:01 565 /usr/lib/python2.7/lib-dynload/_sha512.so
77129000-7712d000 r-xp 00000000 00:01 533 /usr/lib/python2.7/lib-dynload/_sha256.so
7712d000-7713c000 ---p 00000000 00:00 0
7713c000-7713d000 r--p 00003000 00:01 533 /usr/lib/python2.7/lib-dynload/_sha256.so
7713d000-7713e000 rw-p 00004000 00:01 533 /usr/lib/python2.7/lib-dynload/_sha256.so
7713e000-77141000 r-xp 00000000 00:01 551 /usr/lib/python2.7/lib-dynload/_sha.so
77141000-77150000 ---p 00000000 00:00 0
77150000-77151000 r--p 00002000 00:01 551 /usr/lib/python2.7/lib-dynload/_sha.so
77151000-77152000 rw-p 00003000 00:01 551 /usr/lib/python2.7/lib-dynload/_sha.so
77152000-77156000 r-xp 00000000 00:01 586 /usr/lib/python2.7/lib-dynload/binascii.so
77156000-77165000 ---p 00000000 00:00 0
77165000-77166000 r--p 00003000 00:01 586 /usr/lib/python2.7/lib-dynload/binascii.so
77166000-77167000 rw-p 00004000 00:01 586 /usr/lib/python2.7/lib-dynload/binascii.so
77167000-771a8000 rw-p 00000000 00:00 0
771a8000-771aa000 r-xp 00000000 00:01 545 /usr/lib/python2.7/lib-dynload/_md5.so
771aa000-771b9000 ---p 00000000 00:00 0
771b9000-771ba000 r--p 00001000 00:01 545 /usr/lib/python2.7/lib-dynload/_md5.so
771ba000-771bb000 rw-p 00002000 00:01 545 /usr/lib/python2.7/lib-dynload/_md5.so
771bb000-771c1000 r-xp 00000000 00:01 567 /usr/lib/python2.7/lib-dynload/_struct.so
771c1000-771d0000 ---p 00000000 00:00 0
771d0000-771d1000 r--p 00005000 00:01 567 /usr/lib/python2.7/lib-dynload/_struct.so
771d1000-771d3000 rw-p 00006000 00:01 567 /usr/lib/python2.7/lib-dynload/_struct.so
771d3000-771d6000 r-xp 00000000 00:01 561 /usr/lib/python2.7/lib-dynload/time.so
771d6000-771e5000 ---p 00000000 00:00 0
771e5000-771e6000 r--p 00002000 00:01 561 /usr/lib/python2.7/lib-dynload/time.so
771e6000-771e8000 rw-p 00003000 00:01 561 /usr/lib/python2.7/lib-dynload/time.so
771e9000-772ac000 rw-p 00000000 00:00 0
772ac000-77305000 r-xp 00000000 00:01 1490 /lib/libuClibc-0.9.33.2.so
77305000-77314000 ---p 00000000 00:00 0
77314000-77315000 r--p 00058000 00:01 1490 /lib/libuClibc-0.9.33.2.so
77315000-77316000 rw-p 00059000 00:01 1490 /lib/libuClibc-0.9.33.2.so
77316000-7731b000 rw-p 00000000 00:00 0
7731b000-7732e000 r-xp 00000000 00:01 1544 /lib/libgcc_s.so.1
7732e000-7733d000 ---p 00000000 00:00 0
7733d000-7733e000 r--p 00012000 00:01 1544 /lib/libgcc_s.so.1
7733e000-7733f000 rw-p 00013000 00:01 1544 /lib/libgcc_s.so.1
7733f000-77356000 r-xp 00000000 00:01 1501 /lib/libm-0.9.33.2.so
77356000-77365000 ---p 00000000 00:00 0
77365000-77366000 rw-p 00016000 00:01 1501 /lib/libm-0.9.33.2.so
77366000-77378000 r-xp 00000000 00:01 1163 /usr/lib/libz.so.1.2.7
77378000-77387000 ---p 00000000 00:00 0
77387000-77388000 r--p 00011000 00:01 1163 /usr/lib/libz.so.1.2.7
77388000-77389000 rw-p 00012000 00:01 1163 /usr/lib/libz.so.1.2.7
77389000-7738b000 r-xp 00000000 00:01 1545 /lib/libutil-0.9.33.2.so
7738b000-7739a000 ---p 00000000 00:00 0
7739a000-7739b000 rw-p 00001000 00:01 1545 /lib/libutil-0.9.33.2.so
7739b000-7739e000 r-xp 00000000 00:01 1503 /lib/libdl-0.9.33.2.so
7739e000-773ad000 ---p 00000000 00:00 0
773ad000-773ae000 r--p 00002000 00:01 1503 /lib/libdl-0.9.33.2.so
773ae000-773af000 rw-p 00003000 00:01 1503 /lib/libdl-0.9.33.2.so
773af000-773c5000 r-xp 00000000 00:01 1494 /lib/libpthread-0.9.33.2.so
773c5000-773d4000 ---p 00000000 00:00 0
773d4000-773d5000 r--p 00015000 00:01 1494 /lib/libpthread-0.9.33.2.so
773d5000-773d6000 rw-p 00016000 00:01 1494 /lib/libpthread-0.9.33.2.so
773d6000-773d8000 rw-p 00000000 00:00 0
773d8000-773e0000 r-xp 00000000 00:01 1495 /lib/ld-uClibc-0.9.33.2.so
773ee000-773ef000 rw-p 00000000 00:00 0
773ef000-773f0000 r--p 00007000 00:01 1495 /lib/ld-uClibc-0.9.33.2.so
773f0000-773f1000 rw-p 00008000 00:01 1495 /lib/ld-uClibc-0.9.33.2.so
773f1000-773f2000 rw-p 00000000 00:00 0
7fbf5000-7fc16000 rwxp 00000000 00:00 0 [stack]
7fff7000-7fff8000 r-xp 00000000 00:00 0 [vdso]
root@DESK /proc/601 [#]#
上面的maps结果显示，增加的内存是创建线程时给线程的栈。当线程退出时，线程栈不是马上回收的，glibc允许缓存一部分内存块，只有当内存块的大小超过 stack_cache_maxsize 时才会释放掉一部分内存块
查看了uClibc的代码发现stack_cache_maxsize是40MB, 在嵌入式设备中，这个值太大了。
llwang@compiler~/repos/master_for_AA-12.09/osdk_repos/build_dir/toolchain-mips_r2_gcc-4.6-linaro_uClibc-0.9.33.2/uClibc-0.9.33.2 $ grep -rn 'stack_cache_maxsize' *
libpthread/nptl/allocatestack.c:103:static size_t stack_cache_maxsize = 40 * 1024 * 1024; /* 40MiBi by default. */
libpthread/nptl/allocatestack.c:301: if (__builtin_expect (stack_cache_actsize > stack_cache_maxsize, 0))
libpthread/nptl/allocatestack.c:302: __free_stacks (stack_cache_maxsize);
修改
llwang@compiler~/repos/master_for_AA-12.09/osdk_repos $ cat toolchain/uClibc/patches-0.9.33.2/961-pthread_stack_cache_size.patch
--- a/libpthread/nptl/allocatestack.c
+++ b/libpthread/nptl/allocatestack.c
@@ -100,7 +100,7 @@
/* Cache handling for not-yet free stacks. */
/* Maximum size in kB of cache. */
-static size_t stack_cache_maxsize = 40 * 1024 * 1024; /* 40MiBi by default. */
+static size_t stack_cache_maxsize = 2 * 1024 * 1024; /* 2MiBi by default. */
static size_t stack_cache_actsize;
/* Mutex protecting this variable. */
参考
nicephil@gmail.com – 2018-1-2
