python 線程池與進程池簡單實現與等待全部線程結束

import time
import random
from concurrent.futures import ThreadPoolExecutor
from multiprocessing import Process, Pool

def worker(n, index):
    print('開始第{}個進程，第{}個線程'.format(n, index))
    t = random.random()
    time.sleep(t)
    print('結束第{}個進程，第{}個線程'.format(n, index))

def main(n):
    max_workers = 20 # 最大線程數
    pool = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix='Thread')
    i = 0
    while True:
        pool.submit(worker, n, i)
        i = i + 1

if __name__ == "__main__":
    pool1 = Pool(2) # 最大進程數2
    for i in range(1,3):
       pool1.apply_async(main, args=(i, ))
    pool1.close()
    pool1.join()

 

等待所有線程結束再進行操作

……

max_workers = 20 # 最大線程數
pool = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix='Thread')

task_list = []

while True:
    for _i, _n in enumerate(task_list):
        if _n.done():
            task_list.pop(_i)
    if len(task_list) < int(max_workers * 0.9):
        break  # 判斷工作線程數來決定是否添加新的提交新的任務
task_list.append(pool.submit(worker, infos, collection, search_params))

……

 等待所有線程結束再操作並獲取線程返回結果

from concurrent.futures import ThreadPoolExecutor, as_completed
……
task_list = []
    if True:
        for n in range(0, max_workers):
            ……
            task_list.append(pool.submit(uptxt, page_start, pasge_end, simhash_index_path_rand))
        while True:
            for lala in as_completed(task_list):
                res = lala.result()  # 獲取線程return內容
……    

 其他線程池相關操作：https://blog.csdn.net/xiaoyu_wu/article/details/102820384