python ThreadPoolExecutor使用后內存不斷消耗內存解決

from functools import wraps
import concurrent.futures
import time


def test_func(func):
    @wraps(func)
    def inner(*args, **kwargs):
        print ("start...")
        res = func(*args, **kwargs)
        print ("end...")
        return res
    return inner

@test_func
def test1():
    print ('run test1 ....')
    time.sleep(10)

    return 1

@test_func
def test2():
    print ('run test2 ....')
    time.sleep(10)

    return 1

@test_func
def test3():
    print ('run test3 ....')
    time.sleep(10)

    return 1

@test_func
def test4():
    print ('run test4 ....')
    time.sleep(10)

    return 1



def main():

    funcs = [test1, test2, test3, test4]

    executor = concurrent.futures.ThreadPoolExecutor(4)
     while True:
             for func in funcs:
                    executor.submit(func)
             print executor._work_queue.qsize()

 

通過運行代碼能發現在調用該程序之后，內存直線上升；在循環調用線程池時，進程會不斷的往線程池中扔任務，而不會判斷，等待線程池中是否存在空閑線程程；

解決方法

既然線程池使用的為無界隊列，那么就可以將類重寫，並使用有界隊列，如：

import queue
from concurrent.futures import ThreadPoolExecutor

class BoundThreadPollExecutor(ThreadPoolExecutor):
    def __init__(self, *args, **kwargs):
        super(BoundThreadPollExecutor, self).__init__(*args, **kwargs)
        self._work_queue = queue.Queue(4)