python 內存監控模塊之memory_profiler

0. memory_profiler是干嘛的

This is a python module for monitoring memory consumption of a process as well as line-by-line analysis of memory consumption for python programs. It is a pure python module and has the psutil module as optional (but highly recommended) dependencies.

memory_profiler是監控python進程的神器，它可以分析出每一行代碼所增減的內存狀況。

1. 入門例子

#del3.py

import time
@profile
def my_func():
    a = [1] * (10 ** 6)
    b = [2] * (2 * 10 ** 7)
    time.sleep(10)
    del b
    del a
    print "+++++++++"

if __name__ == '__main__':
    my_func()

結果

$python -m memory_profiler del3.py
+++++++++
Filename: del3.py

Line #    Mem usage    Increment   Line Contents
================================================
     2   10.293 MiB    0.000 MiB   @profile
     3                             def my_func():
     4   17.934 MiB    7.641 MiB       a = [1] * (10 ** 6)
     5  170.523 MiB  152.590 MiB       b = [2] * (2 * 10 ** 7)
     6  170.527 MiB    0.004 MiB       time.sleep(10)
     7   17.938 MiB -152.590 MiB       del b
     8   10.305 MiB   -7.633 MiB       del a
     9   10.309 MiB    0.004 MiB       print "+++++++++"

代碼執行一遍，然后給出具體代碼在某一步占用的內存，通過內存加減可以看出某個對象的大小。

2. 對象不刪除，直接賦值內存是否會繼續增長

#對比1

@profile
def my_func():
    a = 'a' * 1024 * 1024 * 1024;
    a = 'a' * 1024 * 1024
    a = 'a' * 1024
    del a
    print "+++++++++"

if __name__ == '__main__':
    my_func()

結果

Line #    Mem usage    Increment   Line Contents
================================================
     1   10.293 MiB    0.000 MiB   @profile
     2                             def my_func():
     3 1034.301 MiB 1024.008 MiB       a = 'a' * 1024 * 1024 * 1024;
     4   11.285 MiB -1023.016 MiB       a = 'a' * 1024 * 1024
     5   11.285 MiB    0.000 MiB       a = 'a' * 1024
     6   11.285 MiB    0.000 MiB       del a
     7   11.289 MiB    0.004 MiB       print "+++++++++"

#對比2

@profile
def my_func():
    a = 'a' * 1024 * 1024 * 1024;
    del a
    a = 'a' * 1024 * 1024
    del a
    a = 'a' * 1024
    del a
    print "+++++++++"

if __name__ == '__main__':
    my_func()

結果

Line #    Mem usage    Increment   Line Contents
================================================
     1   10.293 MiB    0.000 MiB   @profile
     2                             def my_func():
     3 1034.301 MiB 1024.008 MiB       a = 'a' * 1024 * 1024 * 1024;
     4   10.297 MiB -1024.004 MiB       del a
     5   11.285 MiB    0.988 MiB       a = 'a' * 1024 * 1024
     6   11.285 MiB    0.000 MiB       del a
     7   11.285 MiB    0.000 MiB       a = 'a' * 1024
     8   11.285 MiB    0.000 MiB       del a
     9   11.289 MiB    0.004 MiB       print "+++++++++"

結論：是否 del對象沒有影響，新賦的值會替代舊的值

3. 對象賦值是否會增加同樣的內存

#對比1

@profile
def my_func():
    a = 'a' * 1024 * 1024 * 1024;
    b = a
    del a
    print "+++++++++"

if __name__ == '__main__':
    my_func()

結果

Line #    Mem usage    Increment   Line Contents
================================================
     1   10.293 MiB    0.000 MiB   @profile
     2                             def my_func():
     3 1034.301 MiB 1024.008 MiB       a = 'a' * 1024 * 1024 * 1024;
     4 1034.301 MiB    0.000 MiB       b = a
     5 1034.301 MiB    0.000 MiB       del a
     6 1034.305 MiB    0.004 MiB       print "+++++++++"

#對比2

@profile
def my_func():
    a = 'a' * 1024 * 1024 * 1024;
    b = a
    del a
    del b
    print "+++++++++"

if __name__ == '__main__':
    my_func()

結果

Line #    Mem usage    Increment   Line Contents
================================================
     1   10.297 MiB    0.000 MiB   @profile
     2                             def my_func():
     3 1034.305 MiB 1024.008 MiB       a = 'a' * 1024 * 1024 * 1024;
     4 1034.305 MiB    0.000 MiB       b = a
     5 1034.305 MiB    0.000 MiB       del a
     6   10.301 MiB -1024.004 MiB       del b
     7   10.305 MiB    0.004 MiB       print "+++++++++"

結論，把a賦值給b，內存沒有增加。但是只刪除其中一個對象的時候，內存不會減。

4. 另一種等價的啟動方式

from memory_profiler import profile
@profile(precision=4)
def my_func():
    a = 'a' * 1024 * 1024 * 1024;
    del a
    a = 'a' * 1024 * 1024
    del a
    a = 'a' * 1024
    del a
    print "+++++++++"

if __name__ == '__main__':
    my_func()

結果

$python -m memory_profiler del3.py
+++++++++
Filename: del3.py

Line #    Mem usage    Increment   Line Contents
================================================
     2  10.3867 MiB   0.0000 MiB   @profile(precision=4)
     3                             def my_func():
     4 1034.3945 MiB 1024.0078 MiB       a = 'a' * 1024 * 1024 * 1024;
     5  10.3906 MiB -1024.0039 MiB       del a
     6  11.3789 MiB   0.9883 MiB       a = 'a' * 1024 * 1024
     7  11.3789 MiB   0.0000 MiB       del a
     8  11.3789 MiB   0.0000 MiB       a = 'a' * 1024
     9  11.3789 MiB   0.0000 MiB       del a
    10  11.3828 MiB   0.0039 MiB       print "+++++++++"

5. 非python內置對象例子

from memory_profiler import profile
import networkx as nx

@profile(precision=4)
def my_func():
    a = 'a' * 1024 * 1024 * 1024;
    del a
    G = nx.Graph()
    G.add_node(1)
    G.add_nodes_from([i for i in range(10000)])
    G.add_nodes_from([i for i in range(10000, 20000)])
    G.add_edges_from([(1,2), (1,4), (2, 9), (4, 1), (3, 8)])
    del G
    print "++++++"

if __name__ == '__main__':
    my_func()

結果

$python del3.py
++++++
Filename: del3.py

Line #    Mem usage    Increment   Line Contents
================================================
     4  23.4844 MiB   0.0000 MiB   @profile(precision=4)
     5                             def my_func():
     6 1047.4922 MiB 1024.0078 MiB       a = 'a' * 1024 * 1024 * 1024;
     7  23.4883 MiB -1024.0039 MiB       del a 
     8  23.4883 MiB   0.0000 MiB       G = nx.Graph()
     9  23.4883 MiB   0.0000 MiB       G.add_node(1)
    10  31.3359 MiB   7.8477 MiB       G.add_nodes_from([i for i in range(10000)]) 
    11  36.9219 MiB   5.5859 MiB       G.add_nodes_from([i for i in range(10000, 20000)]) 
    12  36.9219 MiB   0.0000 MiB       G.add_edges_from([(1,2), (1,4), (2, 9), (4, 1), (3, 8)])
    13  25.9219 MiB -11.0000 MiB       del G
    14  25.9258 MiB   0.0039 MiB       print "++++++"

6. 類怎么使用呢

#del4.py

from memory_profiler import profile

class people:
    name = ''
    age = 0
    __weight = 0

    def __init__(self,n,a,w):
        self.name = n
        self.age = a
        self.__weight = w

    @profile(precision=4)
    def speak(self):
        a = 'a' * 1024
        b = 'b' * 1024 * 1024
        print("%s is speaking: I am %d years old" % (self.name,self.age))



if __name__ == '__main__':
    p = people('tom', 10, 30)
    p.speak()

結果

$python del4.py
tom is speaking: I am 10 years old
Filename: del4.py

Line #    Mem usage    Increment   Line Contents
================================================
    13   9.4219 MiB   0.0000 MiB       @profile(precision=4)
    14                                 def speak(self):  
    15   9.4258 MiB   0.0039 MiB           a = 'a' * 1024
    16  10.4297 MiB   1.0039 MiB           b = 'b' * 1024 * 1024
    17  10.4336 MiB   0.0039 MiB           print("%s is speaking: I am %d years old" % (self.name,self.age)) 

7. 隨時間內存統計

#test.py

import time

@profile
def test1():
    n = 10000
    a = [1] * n
    time.sleep(1)
    return a

@profile
def test2():
    n = 100000
    b = [1] * n
    time.sleep(1)
    return b

if __name__ == "__main__":
    test1()
    test2()

test.py 里有兩個兩個待分析的函數（@profile標識），為了形象地看出內存隨時間的變化，每個函數內sleep 1s，執行

mprof run test.py

如果執行成功，結果這樣

$ mprof run test.py
mprof: Sampling memory every 0.1s
running as a Python program...

結果會生成一個.dat文件，如"mprofile_20160716170529.dat"，里面記錄了內存隨時間的變化，可用下面的命令以圖片的形式展示出來：

mprof plot

8. API

memory_profiler提供很多包給第三方代碼，如

>>> from memory_profiler import memory_usage
>>> mem_usage = memory_usage(-1, interval=.2, timeout=1)
>>> print(mem_usage)
    [7.296875, 7.296875, 7.296875, 7.296875, 7.296875]

memory_usage(proc=-1, interval=.2, timeout=None)返回一段時間的內存值，其中proc=-1表示此進程，這里可以指定特定的進程號；interval=.2表示監控的時間間隔是0.2秒；timeout=1表示總共的時間段為1秒。那結果就返回5個值。

如果要返回一個函數的內存消耗，示例

def f(a, n=100):
     import time
     time.sleep(2)
     b = [a] * n
     time.sleep(1)
     return b

from memory_profiler import memory_usage
print memory_usage((f, (2,), {'n' : int(1e6)}))

這里執行了 f(1, n=int(1e6)) ，並返回在執行此函數時的內存消耗。

9. 優化實例

對比str & int

from datetime import datetime
@profile
def my_func():
    beg = datetime.now()
    a = {}
    for i in range(1000000):
        a[i] = i
        #a[str(i)] = i
    print "+++++++++"
    del a
    print "+++++++++"
    end = datetime.now()
    print "time:", end - beg

if __name__ == '__main__':
    my_func()

用a[i] = i，結果

+++++++++
+++++++++
time: 0:06:14.790899
Filename: int.py

Line #    Mem usage    Increment   Line Contents
================================================
     2   14.727 MiB    0.000 MiB   @profile
     3                             def my_func():
     4   14.734 MiB    0.008 MiB       beg = datetime.now()
     5   14.734 MiB    0.000 MiB       a = {}
     6   94.031 MiB   79.297 MiB       for i in range(1000000):
     7   94.031 MiB    0.000 MiB           a[i] = i
     8                                     #a[str(i)] = i
     9   86.402 MiB   -7.629 MiB       print "+++++++++"
    10   38.398 MiB  -48.004 MiB       del a
    11   38.398 MiB    0.000 MiB       print "+++++++++"
    12   38.398 MiB    0.000 MiB       end = datetime.now()
    13   38.406 MiB    0.008 MiB       print "time:", end - beg

用a[str(i)] = i，結果

+++++++++
+++++++++
time: 0:06:00.288052
Filename: int.py

Line #    Mem usage    Increment   Line Contents
================================================
     2   14.723 MiB    0.000 MiB   @profile
     3                             def my_func():
     4   14.730 MiB    0.008 MiB       beg = datetime.now()
     5   14.730 MiB    0.000 MiB       a = {}
     6  140.500 MiB  125.770 MiB       for i in range(1000000):
     7                                     #a[i] = i
     8  140.500 MiB    0.000 MiB           a[str(i)] = i
     9  132.871 MiB   -7.629 MiB       print "+++++++++"
    10   38.539 MiB  -94.332 MiB       del a
    11   38.539 MiB    0.000 MiB       print "+++++++++"
    12   38.539 MiB    0.000 MiB       end = datetime.now()
    13   38.547 MiB    0.008 MiB       print "time:", end - beg