使用pandas導出數據到excel速度測試

需要將數據庫中的數據導出到excel下載，由於數據量較大，導出時間太長，測試了不同的python模塊的速度。

pandas默認使用的是openpyxl，另外使用xlsxwriter對比。直接上代碼，數據庫中大概有23萬條數據，轉成excel的大小約為35m。

需要說明一下硬件：

CPU：Intel(R) Core(TM) i5-9300H CPU @ 2.40GHz

內存：8.0 GB

openpyxl：

import pandas as pd
import MySQLdb
from MySQLdb import cursors
import time

db_settings = {
    'user': '',
    'password': '',
    'db': '',
    'host': '127.0.0.1',
    'port': 3306,
    'cursorclass': cursors.DictCursor,
    'charset': 'utf8'
}


def main():
    start = time.time()
    con = MySQLdb.connect(**db_settings)
    cursor = con.cursor()
    cursor.execute('select * from test')
    data = cursor.fetchall()
    cursor.close()
    con.close()
    df = pd.DataFrame(data)
    writer = pd.ExcelWriter('output.xlsx', engine='openpyxl')
    df.to_excel(writer, sheet_name='Sheet1', index=False)
    writer.save()
    end = time.time()
    print(end - start)


if __name__ == '__main__':
    main()

最終，用時為：197.815589427948

xlsxwriter：

import pandas as pd
import MySQLdb
from MySQLdb import cursors
import time

db_settings = {
    'user': '',
    'password': '',
    'db': '',
    'host': '127.0.0.1',
    'port': 3306,
    'cursorclass': cursors.DictCursor,
    'charset': 'utf8'
}


def main():
    start = time.time()
    con = MySQLdb.connect(**db_settings)
    cursor = con.cursor()
    cursor.execute('select * from test')
    data = cursor.fetchall()
    cursor.close()
    con.close()
    df = pd.DataFrame(data)
    writer = pd.ExcelWriter('output.xlsx', engine='xlsxwriter')
    df.to_excel(writer, sheet_name='Sheet1', index=False)
    writer.save()
    end = time.time()
    print(end - start)


if __name__ == '__main__':
    main()

最終，用時為：102.69404006004333

直接使用xlsxwriter：

import xlsxwriter
import MySQLdb
from MySQLdb import cursors
import time


db_settings = {
    'user': '',
    'password': '',
    'db': '',
    'host': '127.0.0.1',
    'port': 3306,
    'cursorclass': cursors.DictCursor,
    'charset': 'utf8'
}


def main():
    start = time.time()
    con = MySQLdb.connect(**db_settings)
    cursor = con.cursor()
    cursor.execute('select * from test')
    data = cursor.fetchall()
    cursor.close()
    con.close()
    file_name = "output.xlsx"
    workbook = xlsxwriter.Workbook(file_name)
    worksheet = workbook.add_worksheet('sheet1')
    fields = []
    for key, value in data[0].items():
        fields.append(key)
    worksheet.write_row('A1', fields)
    i = 2
    for d in data:
        row_data = d.values()
        row = 'A' + str(i)
        worksheet.write_row(row, row_data)
        i += 1
    workbook.close()
    end = time.time()
    print(end - start)


if __name__ == '__main__':
    main()

最終時間：64.21226000785828

可以看出效率：

xlsxwriter > pandas+xlsxwriter > pandas+openpyxl