Python3用多線程替代for循環提升程序運行速度

[本文出自天外歸雲的博客園]

優化前后新老代碼如下：

from git_tools.git_tool import get_collect_projects, QQNews_Git
from threading import Thread, Lock
import datetime

base_url = "http://git.xx.com"
project_members_commits_lang_info = {}
lock = Lock()
threads = []

'''
Author:zenkilan
'''


def count_time(func):
    def took_up_time(*args, **kwargs):
        start_time = datetime.datetime.now()
        ret = func(*args, **kwargs)
        end_time = datetime.datetime.now()
        took_up_time = (end_time - start_time).total_seconds()
        print(f"{func.__name__} execution took up time:{took_up_time}")
        return ret

    return took_up_time


def get_project_member_lang_code_lines(git, member, begin_date, end_date):
    global project_members_commits_lang_info
    global lock
    member_name = member["username"]
    r = git.get_user_info(member_name)
    if not r["id"]:
        return
    user_commits_lang_info = git.get_commits_user_lang_diff_between(r["id"], begin_date, end_date)
    if len(user_commits_lang_info) == 0:
        return
    lock.acquire()
    project_members_commits_lang_info.setdefault(git.project, dict())
    project_members_commits_lang_info[git.project][member_name] = user_commits_lang_info
    lock.release()


def get_project_lang_code_lines(project, begin_date, end_date):
    global threads
    git = QQNews_Git(project[1], base_url, project[0])
    project_members = git.get_project_members()
    if len(project_members) == 0:
        return
    for member in project_members:
        thread = Thread(target=get_project_member_lang_code_lines, args=(git, member, begin_date, end_date))
        threads.append(thread)
        thread.start()


@count_time
def get_projects_lang_code_lines(begin_date, end_date):
    """
    獲取項目代碼行語言相關統計——新方法（提升效率）
    應用多線程替代for循環
    並發訪問共享外部資源
    :return:
    """
    global project_members_commits_lang_info
    global threads
    for project in get_collect_projects():
        thread = Thread(target=get_project_lang_code_lines, args=(project, begin_date, end_date))
        threads.append(thread)
        thread.start()


@count_time
def get_projects_lang_code_lines_old(begin_date, end_date):
    """
    獲取項目代碼行語言相關統計——老方法（耗時嚴重）
    使用最基本的思路進行編程
    雙層for循環嵌套並且每層都包含耗時操作
    :return:
    """
    project_members_commits_lang_info = {}
    for project in get_collect_projects():
        git = QQNews_Git(project[1], base_url, project[0])
        project_members = git.get_project_members()
        user_commits_lang_info_dict = {}
        if len(project_members) == 0:
            continue
        for member in project_members:
            member_name = member["username"]
            r = git.get_user_info(member_name, debug=False)
            if not r["id"]:
                continue
            try:
                user_commits_lang_info = git.get_commits_user_lang_diff_between(r["id"], begin_date, end_date)
                if len(user_commits_lang_info) == 0:
                    continue
                user_commits_lang_info_dict[member_name] = user_commits_lang_info
                project_members_commits_lang_info[git.project] = user_commits_lang_info_dict
            except:
                pass
    return project_members_commits_lang_info


def test_results_equal(resultA, resultB):
    """
    測試方法
    :param resultA:
    :param resultB:
    :return:
    """
    print(resultA)
    print(resultB)
    assert len(str(resultA)) == len(str(resultB))


if __name__ == '__main__':
    from git_tools.config import begin_date, end_date

    get_projects_lang_code_lines(begin_date, end_date)
    for t in threads:
        t.join()
    old_result = get_projects_lang_code_lines_old(begin_date, end_date)
    test_results_equal(old_result, project_members_commits_lang_info)

老方法里外層for循環和內層for循環里均存在耗時操作：

1）git.get_project_members()

2）git.get_user_info(member_name, debug=False)

分兩步來優化，先里后外或先外后里都行。用多線程替換for循環，並發共享外部資源，加鎖避免寫沖突。

測試結果通過，函數運行時間裝飾器顯示（單位秒）：

get_projects_lang_code_lines execution took up time:1.85294

get_projects_lang_code_lines_old execution took up time:108.604177

速度提升了約58倍