追踪代码申请的内存大小
tracemalloc
为python代码{%code%}添加如下代码：
import tracemalloc
tracemalloc.start()
f=open('mem_size.txt','w',encoding='utf8')
while-clause:
{% code %}
snapshot=tracemalloc.take_snapshot()
top_stats=snapshot.statistics('lineno')
for stat in top_stats:
f.write(str(stat)+'\n')
f.close()
追踪这片代码所涉及的内存申请过程，包括所申请内存的大小
对记录结果进行分析：
from collections import defaultdict
import matplotlib.pyplot as plt
with open('test.txt','r',encoding='utf8') as f:
contents=f.read()
contents=contents.strip().split('\n')
print(len(contents))
malloc_size=defaultdict(list)
for content in contents:
units=content.split(' ')
if units[-4]=='KiB,':
malloc_size[' '.join(units[:-5])].append(float(units[-5][5:])*1024)
elif units[-4]=='B,':
malloc_size[' '.join(units[:-5])].append(float(units[-5][5:]))
elif units[-4]=='MiB,':
malloc_size[' '.join(units[:-5])].append(float(units[-5][5:])*1024*1024)
else:
print(units)
continue
def isIncrement(ls,guanxing):
if len(ls)<2:
return False
o=0
score=0
cha=[ls[i+1]-ls[i] for i in range(len(ls)-1)]
for i in range(len(cha)):
if o*guanxing+cha[i]>1:
score+=1
o=cha[i]
if score/len(cha)>0.5:
return True
else:
return False
plt.rcParams['figure.dpi'] = 300
plt.rcParams['font.sans-serif'] = ['SimHei']
for ms in malloc_size:
# if ms.startswith('/home/tellw'):
# print(ms)
# plt.title(ms)
# plt.plot(list(range(len(malloc_size[ms]))),malloc_size[ms])
# plt.show()
guanxing=0.1
if isIncrement(malloc_size[ms],guanxing):
print(ms)
plt.title(ms)
plt.plot(list(range(len(malloc_size[ms]))),malloc_size[ms])
plt.show()
可以查看自己所写的代码所申请内存空间大小的变化情况
memory_profiler
import numpy as np
from memory_profiler import profile
@profile
def func():
for i in range(3):
print(np.random.rand(100000,10000))
func()
结果：
Line # Mem usage Increment Occurrences Line Contents
=============================================================
4 52.8 MiB 52.8 MiB 1 @profile
5 def func():
6 53.0 MiB 0.0 MiB 4 for i in range(3):
7 53.0 MiB 0.1 MiB 3 print(np.random.rand(10000,1000))
objgraph+gc
# -*- coding: utf-8 -*-
import gc, time
class OBJ(object):
pass
def show_cycle_reference():
a, b = OBJ(), OBJ()
a.attr_b = b
b.attr_a = a
if __name__ == '__main__':
gc.disable() # 这里是否disable事实上无所谓
gc.set_debug(gc.DEBUG_COLLECTABLE|gc.DEBUG_SAVEALL)
for _ in range(1):
show_cycle_reference()
gc.collect()
for o in gc.garbage:
print(o)
import pdb;pdb.set_trace()
time.sleep(5)
结果
gc: collectable <tuple 0x00000160C3B46020>
gc: collectable <cell 0x00000160C3D195D0>
gc: collectable <tuple 0x00000160C3D19D20>
gc: collectable <function 0x00000160C3D288B0>
gc: collectable <function 0x00000160C3D28820>
gc: collectable <cell 0x00000160C3D196F0>
gc: collectable <function 0x00000160C3D28700>
gc: collectable <cell 0x00000160C3D19840>
gc: collectable <cell 0x00000160C3D198D0>
gc: collectable <set 0x00000160C3D38200>
gc: collectable <tuple 0x00000160C3D48580>
gc: collectable <function 0x00000160C3D3EA70>
gc: collectable <function 0x00000160C3D28790>
gc: collectable <function 0x00000160C3D3EB00>
gc: collectable <tuple 0x00000160C3D19D80>
gc: collectable <dict 0x00000160C3D22F80>
gc: collectable <type 0x00000160C3C25FE0>
gc: collectable <staticmethod 0x00000160C3D19870>
gc: collectable <tuple 0x00000160C3D22940>
gc: collectable <member_descriptor 0x00000160C3D48500>
gc: collectable <member_descriptor 0x00000160C3D48AC0>
gc: collectable <member_descriptor 0x00000160C3D48B00>
gc: collectable <getset_descriptor 0x00000160C3D48B40>
gc: collectable <getset_descriptor 0x00000160C3D48B80>
gc: collectable <OBJ 0x00000160C3B6C8B0>
gc: collectable <OBJ 0x00000160C3B6E560>
gc: collectable <dict 0x00000160C3AE0700>
gc: collectable <dict 0x00000160C3AE0880>
('func', 'args', 'keywords', '__dict__', '__weakref__')
<cell at 0x00000160C3D195D0: type object at 0x00000160C3C25FE0>
(<cell at 0x00000160C3D195D0: type object at 0x00000160C3C25FE0>,)
<function partial.__new__ at 0x00000160C3D288B0>
<function partial.__call__ at 0x00000160C3D28820>
<cell at 0x00000160C3D196F0: str object at 0x00000160C3D217B0>
<function partial.__repr__ at 0x00000160C3D28700>
<cell at 0x00000160C3D19840: set object at 0x00000160C3D38200>
set()
(<cell at 0x00000160C3D196F0: str object at 0x00000160C3D217B0>, <cell at 0x00000160C3D19840: set object at 0x00000160C3D38200>, <cell at 0x00000160C3D198D0: function object at 0x00000160C3D28700>)
<function partial.__repr__ at 0x00000160C3D3EA70>
<function partial.__setstate__ at 0x00000160C3D3EB00>
n ', '__slots__': ('func', 'args', 'keywords', '__dict__', '__weakref__'), '__new__': <staticmethod(<function partial.__new__ at 0x00000160C3D288B0>)>, '__call__': <function partial.__call__ at 0x00000160C3D28820>, '__repr__': <function partial.__repr__ at 0x00000160C3D3EA70>, '__reduce__': <function partial.__reduce__ at 0x00000160C3D28790>, '__setstate__': <function partial.__setstate__ at 0x00000160C3D3EB00>, 'args': <member 'args' of 'partial' objects>, 'func': <member 'func' of 'partial' objects>, 'keywords': <member 'keywords' of 'partial' objects>, '__dict__': <attribute '__dict__' of 'partial' objects>, '__weakref__': <attribute '__weakref__' of 'partial' objects>}
<class 'functools.partial'>
<staticmethod(<function partial.__new__ at 0x00000160C3D288B0>)>
(<class 'functools.partial'>, <class 'object'>)
<member 'args' of 'partial' objects>
<member 'func' of 'partial' objects>
<member 'keywords' of 'partial' objects>
<attribute '__dict__' of 'partial' objects>
<attribute '__weakref__' of 'partial' objects>
<__main__.OBJ object at 0x00000160C3B6C8B0>
<__main__.OBJ object at 0x00000160C3B6E560>
{'attr_b': <__main__.OBJ object at 0x00000160C3B6E560>}
{'attr_a': <__main__.OBJ object at 0x00000160C3B6C8B0>}
-> time.sleep(5)
在gdb输入处键入
(Pdb) import objgraph
(Pdb) objgraph.show_backrefs(objgraph.at(0x00000160C3B6C8B0),5,filename='obj.dot')
得到描述对象之间关系（引用链）的图像
objgraph.show_growth
通过两次objgraph.show_growth()的调用环境差异得到内存中增加的对象。
# -*- coding: utf-8 -*-
import objgraph
_cache = []
class OBJ(object):
pass
def func_to_leak():
o = OBJ()
_cache.append(o)
# do something with o, then remove it from _cache
if True: # this seem ugly, but it always exists
return
_cache.remove(o)
if __name__ == '__main__':
objgraph.show_growth()
try:
func_to_leak()
except:
pass
print('after call func_to_leak')
objgraph.show_growth()
结果
function 2884 +2884
dict 1623 +1623
tuple 1399 +1399
wrapper_descriptor 1168 +1168
ReferenceType 864 +864
method_descriptor 799 +799
builtin_function_or_method 786 +786
getset_descriptor 474 +474
type 448 +448
cell 383 +383
after call func_to_leak
OBJ 1 +1
在调用完func_to_leak()之后，内存中还存在着OBJ实例
objgraph.show_most_common_types给出程序上下文中占用内存的数量较多的变量类型，默认给出最多的前十种
# -*- coding: utf-8 -*-
import objgraph, gc
class OBJ(object):
pass
def show_cycle_reference():
a, b = OBJ(), OBJ()
a.attr_b = b
b.attr_a = a
if __name__ == '__main__':
gc.disable()
for _ in range(50):
show_cycle_reference()
objgraph.show_most_common_types(20)
输出结果
function 2884
dict 1795
tuple 1616
wrapper_descriptor 1168
ReferenceType 864
method_descriptor 799
builtin_function_or_method 786
getset_descriptor 474
type 448
cell 382
member_descriptor 361
list 243
module 150
ModuleSpec 149
classmethod 118
property 104
SourceFileLoader 102
OBJ 100
frozenset 100
set 95
注释掉gc.disable语句，输出结果（30种最多的变量类型）
function 2884
dict 1741
tuple 1497
wrapper_descriptor 1168
ReferenceType 864
method_descriptor 799
builtin_function_or_method 786
getset_descriptor 474
type 448
cell 382
member_descriptor 361
list 243
module 150
ModuleSpec 149
classmethod 118
property 104
SourceFileLoader 102
frozenset 100
set 95
staticmethod 88
_tuplegetter 79
_abc_data 75
_NamedIntConstant 74
ABCMeta 67
OBJ 46
_SpecialGenericAlias 38
classmethod_descriptor 30
method 27
Pattern 23
FileFinder 17
可以看到，有一半的OBJ实例没有被回收，存在内存泄漏
pympler
from pympler import tracker,muppy,summary
import gc
tr=tracker.SummaryTracker()
print('memory total')
all_objects=muppy.get_objects()
sum1=summary.summarize(all_objects)
summary.print_(sum1)
while_clause:
gc.collect()
print('memory difference')
tr.print_diff()
loop_func()
打印结果
memory total
types | # objects | total size
============================ | =========== | ============
str | 460242 | 58.50 MB
code | 82833 | 32.64 MB
dict | 79885 | 30.18 MB
list | 254209 | 19.04 MB
type | 10445 | 12.90 MB
tuple | 105130 | 7.14 MB
set | 4085 | 5.96 MB
collections.OrderedDict | 6755 | 3.29 MB
inspect.Parameter | 29776 | 1.82 MB
weakref.ReferenceType | 16794 | 1.28 MB
abc.ABCMeta | 825 | 1.24 MB
cell | 30656 | 1.17 MB
int | 41339 | 1.13 MB
builtin_function_or_method | 14747 | 1.01 MB
function (__init__) | 5452 | 809.28 KB
memory difference
types | # objects | total size
======================= | =========== | ============
list | 88009 | 18.97 MB
str | 92362 | 7.04 MB
int | 20073 | 549.00 KB
code | 479 | 159.77 KB
type | 61 | 87.15 KB
dict | 246 | 73.92 KB
tuple | 163 | 8.27 KB
weakref.ReferenceType | 71 | 5.55 KB
function (__init__) | 28 | 4.16 KB
property | 39 | 3.05 KB
set | 12 | 3.03 KB
getset_descriptor | 47 | 2.94 KB
function (__repr__) | 19 | 2.82 KB
collections.deque | 3 | 2.23 KB
frozenset | 7 | 1.98 KB
2024-11-09 17:42:29.005537: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:465] Loaded cuDNN version 8907
memory difference
types | # objects | total size
================================================================ | =========== | ============
numpy.ndarray | 8 | 4.99 MB
list | 44109 | 3.74 MB
str | 44215 | 3.32 MB
int | 10073 | 275.54 KB
set | 8 | 194.69 KB
weakref.ReferenceType | 2002 | 156.41 KB
builtin_function_or_method | 1967 | 138.30 KB
dict | 41 | 10.74 KB
tuple | 80 | 4.25 KB
tensorflow.python.framework.ops.EagerTensor | 25 | 4.10 KB
type | 2 | 2.06 KB
tensorflow.python.framework.cpp_shape_inference_pb2.HandleData | 24 | 1.88 KB
tensorflow.python.framework.tensor_shape.TensorShape | 24 | 1.50 KB
keras.src.backend.tensorflow.core.Variable | 24 | 1.31 KB
tensorflow.python.ops.resource_variable_ops.ResourceVariable | 24 | 1.31 KB
memory difference
types | # objects | total size
=============== | =========== | ============
numpy.ndarray | 0 | 493.87 KB
str | 3 | 288 B
list | 3 | 240 B
memory difference
types | # objects | total size
=============== | =========== | ============
numpy.ndarray | 0 | 278.12 KB
memory difference
types | # objects | total size
=============== | =========== | ===============
numpy.ndarray | 0 | -857712 B
监测内存中变量类型的变化
guppy
安装pip install guppy3
from guppy import hpy
h=hpy()
while_clause:
gc.collect()
print(h.heap())
loop_func()
打印结果
Partition of a set of 2022181 objects. Total size = 267902754 bytes.
Index Count % Size % Cumulative % Kind (class / dict of class)
0 584430 29 71918335 27 71918335 27 str
1 100645 5 38919944 15 110838279 41 types.CodeType
2 338727 17 26151200 10 136989479 51 tuple
3 196620 10 19018176 7 156007655 58 bytes
4 209933 10 15817776 6 171825431 64 list
5 37466 2 14702720 5 186528151 70 dict (no owner)
6 95971 5 14587592 5 201115743 75 function
7 10204 1 13534480 5 214650223 80 type
8 4080 0 6264448 2 220914671 82 set
9 4956 0 5125624 2 226040295 84 dict of module
<2535 more rows. Type e.g. '_.more' to view.>
2024-11-12 17:56:23.562654: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:465] Loaded cuDNN version 8907
Partition of a set of 2026043 objects. Total size = 273632823 bytes.
Index Count % Size % Cumulative % Kind (class / dict of class)
0 584551 29 71927320 26 71927320 26 str
1 100643 5 38919472 14 110846792 41 types.CodeType
2 338659 17 26144944 10 136991736 50 tuple
3 196619 10 19018000 7 156009736 57 bytes
4 209943 10 15815584 6 171825320 63 list
5 37469 2 14704376 5 186529696 68 dict (no owner)
6 95823 5 14565096 5 201094792 73 function
7 10205 1 13536168 5 214630960 78 type
8 4088 0 6462272 2 221093232 81 set
9 90 0 5249132 2 226342364 83 numpy.ndarray
<2538 more rows. Type e.g. '_.more' to view.>
Partition of a set of 2026043 objects. Total size = 274138191 bytes.
Index Count % Size % Cumulative % Kind (class / dict of class)
0 584551 29 71927320 26 71927320 26 str
1 100643 5 38919472 14 110846792 40 types.CodeType
2 338659 17 26144944 10 136991736 50 tuple
3 196619 10 19018000 7 156009736 57 bytes
4 209943 10 15815584 6 171825320 63 list
5 37469 2 14704376 5 186529696 68 dict (no owner)
6 95823 5 14565096 5 201094792 73 function
7 10205 1 13536168 5 214630960 78 type
8 4088 0 6462272 2 221093232 81 set
9 90 0 5754852 2 226848084 83 numpy.ndarray
<2538 more rows. Type e.g. '_.more' to view.>
Partition of a set of 2026043 objects. Total size = 274422759 bytes.
Index Count % Size % Cumulative % Kind (class / dict of class)
0 584551 29 71927320 26 71927320 26 str
1 100643 5 38919472 14 110846792 40 types.CodeType
2 338659 17 26144944 10 136991736 50 tuple
3 196619 10 19018000 7 156009736 57 bytes
4 209943 10 15815584 6 171825320 63 list
5 37469 2 14704376 5 186529696 68 dict (no owner)
6 95823 5 14565096 5 201094792 73 function
7 10205 1 13536168 5 214630960 78 type
8 4088 0 6462272 2 221093232 81 set
9 90 0 6039644 2 227132876 83 numpy.ndarray
<2538 more rows. Type e.g. '_.more' to view.>
结果比较粗糙，只能看到numpy.ndarray类型变量所占内存空间有增有减，其他类型变量所占内存空间没有变化，没什么参考意义，不好用
对打印结果进行统计，代码如下，提取关键打印内容和画图的代码跟下一节的结果分析代码相同
import json
with open('test1.txt','r',encoding='utf8') as f:
contents=f.read()
lines=contents.strip().split('\n')
count=0
features={}
keys=[]
key_posts=['Count','Co%','Size','S%','Cumulative','Cu%']
for line in lines:
line=line.strip()
if line[0].isdigit():
if line[0]=='0':
count+=1
parts=line.split()
for i in range(6):
key=f'{parts[7]}_{key_posts[i]}'
if key in features:
features[key][0].append(count)
features[key][1].append(int(parts[i+1]))
else:
features[key]=[[count],[int(parts[i+1])]]
with open('test.json','w',encoding='utf8') as f:
json.dump(features,f,ensure_ascii=False,indent=4)
print(f'共有{count}次统计')
mem_top
安装pip install mem_top
from mem_top import mem_top
while_clause:
gc.collect()
print(mem_top())
loop_func()
打印结果
refs:
84842 <class 'set'> {'内当', '传圭', '过早', '司马', '牛郎织女', '远垂不', '大智如愚', '背本', '积雪', '亲操', '龃龉不', '结帐', '颠倒衣裳', '马革盛', '闲言淡',
45492 <class 'dict'> {'一丁不识': [['yī'], ['dīng'], ['bù'], ['shí']], '一丁点儿': [['yī'], ['dīng'], ['diǎn'], ['er']], '一不小心':
45492 <class 'dict'> {'一丁不识': [['yī'], ['dīng'], ['bù'], ['shí']], '一丁点儿': [['yī'], ['dīng'], ['diǎn'], ['er']], '一不小心':
22713 <class 'list'> [b'import', b'sys', 'sys', b'', 'sys', b'import', b'requests', 'requests', b'', 'requests', b'import
10000 <class 'list'> ['./A11_0', './A11_1', './A11_10', './A11_100', './A11_102', './A11_103', './A11_104', './A11_105', # 一万个文件名称
10000 <class 'list'> [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 2 # 一万个数据编号
7914 <class 'dict'> {9856192: <weakref at 0x704b9bab8d10; to 'type' at 0x9664c0 (type)>, 9829408: <weakref at 0x704b9bab
6730 <class 'dict'> {'get_logger': <function get_logger at 0x704b9a7deca0>, 'compat.v1.get_logger': <function get_logger
6702 <class 'list'> ['# Copyright 2015 The TensorFlow Authors. All Rights Reserved.\n', '#\n', '# Licensed under the Apa
6697 <class 'list'> ['# Copyright 2015 The TensorFlow Authors. All Rights Reserved.\n', '#\n', '# Licensed under the Apa
bytes:
4194520 {'内当', '传圭', '过早', '司马', '牛郎织女', '远垂不', '大智如愚', '背本', '积雪', '亲操', '龃龉不', '结帐', '颠倒衣裳', '马革盛', '闲言淡',
1922480 {'一丁不识': [['yī'], ['dīng'], ['bù'], ['shí']], '一丁点儿': [['yī'], ['dīng'], ['diǎn'], ['er']], '一不小心':
1922480 {'一丁不识': [['yī'], ['dīng'], ['bù'], ['shí']], '一丁点儿': [['yī'], ['dīng'], ['diǎn'], ['er']], '一不小心':
207616 {'sys': <module 'sys' (built-in)>, 'builtins': <module 'builtins' (built-in)>, '_frozen_importlib':
207616 {'get_logger': <function get_logger at 0x704b9a7deca0>, 'compat.v1.get_logger': <function get_logger
194680 [b'import', b'sys', 'sys', b'', 'sys', b'import', b'requests', 'requests', b'', 'requests', b'import
147544 {9856192: <weakref at 0x704b9bab8d10; to 'type' at 0x9664c0 (type)>, 9829408: <weakref at 0x704b9bab
131288 {'aten/atanh_/Meta', 'prims/igammac/CompositeExplicitAutograd', 'aten/_foreach_add.List/Meta', 'prim
131288 {'aten/atanh_/Meta', 'aten/_foreach_add.List/Meta', 'aten/bitwise_right_shift.Tensor/Meta', 'aten/lc
80056 ['./A11_0', './A11_1', './A11_10', './A11_100', './A11_102', './A11_103', './A11_104', './A11_105',
types:
210288 <class 'list'>
94866 <class 'function'>
70083 <class 'tuple'>
44454 <class 'dict'>
30584 <class 'cell'>
29776 <class 'inspect.Parameter'>
18627 <class 'weakref.ReferenceType'>
16396 <class 'builtin_function_or_method'>
12452 <class 'getset_descriptor'>
8791 <class 'type'>
mem_top里参数limit控制显示最高占用量的对象，refs是指包含最多引用的对象，bytes是指占用内存空间最多的对象，types是指数量最多的对象类型。代码实现细节参见denis-ryzhkov/mem_top。间隔一段时间长时间观测，就能发现端倪。python进程内存占用持续增高排查经验分享，mem_top统计整个进程的内存占用情况，只要程序A的代码不变且mem_top被包含在程序A的代码里，无论mem_top插在哪里都是相同的结果。
附上mem_top结果的分析代码，以观察什么因素在增高导致内存泄露
# 1. 提取要分析的关键内容
# with open('test.txt','r',encoding='utf8') as f:
# contents=f.read()
# lines=contents.strip().split('\n')
# new_lines=[]
# for line in lines:
# line=line.strip()
# if len(line)==0:
# continue
# if not line.startswith('2024-'):
# new_lines.append(line)
# with open('test1.txt','w',encoding='utf8') as f:
# f.write('\n'.join(new_lines))
# 2. 得到统计结果并持久化
# import json
# with open('test1.txt','r',encoding='utf8') as f:
# contents=f.read()
# lines=contents.strip().split('\n')
# count=0
# flag=1
# features={}
# keys=[] # keys变量好像没用
# for line in lines:
# line=line.strip()
# if line=='refs:':
# flag=1
# count+=1
# keys=[]
# continue
# if line =='bytes:':
# flag=2
# keys=[]
# continue
# if line=='types:':
# flag=3
# keys=[]
# continue
# if flag==1:
# parts=line.split()
# num=int(parts[0])
# key=''.join(parts[1:])+'refs'
# if key in keys:
# continue
# else:
# keys.append(key)
# if key in features:
# features[key][0].append(count)
# features[key][1].append(num)
# else:
# features[key]=[[count],[num]]
# elif flag==2:
# parts=line.split()
# num=int(parts[0])
# key=''.join(parts[1:])+'bytes'
# if key in keys:
# continue
# else:
# keys.append(key)
# if key in features:
# features[key][0].append(count)
# features[key][1].append(num)
# else:
# features[key]=[[count],[num]]
# elif flag==3:
# parts=line.split()
# num=int(parts[0])
# key=''.join(parts[1:])+'types'
# if key in keys:
# continue
# else:
# keys.append(key)
# if key in features:
# features[key][0].append(count)
# features[key][1].append(num)
# else:
# features[key]=[[count],[num]]
# with open('test.json','w',encoding='utf8') as f:
# json.dump(features,f,ensure_ascii=False,indent=4)
# print(f'共有{count}次统计')
# 3. 画图
import json
import matplotlib.pyplot as plt
features=None
with open('test.json','r',encoding='utf8') as f:
features=json.load(f)
plt.rcParams['figure.dpi'] = 300
plt.rcParams['font.sans-serif'] = ['SimHei']
for key in features:
plt.title(key)
plt.plot(features[key][0],features[key][1])
plt.show()
memprof
from memprof import *
@memprof(plot=True)
def func():
do_sth()
func()
代码运行目录下会生成func.log和func.png（描述func中的占用内存空间超过1MB的变量的内存使用量随着时间的变化情况）
参考链接：
使用gc、objgraph干掉python内存泄露与循环引用！
创建于202411052157，修改于202412012056
