Python中sqlite3使用 executemany 批量插入數據

轉自：https://blog.csdn.net/TomorrowAndTuture/article/details/113978649

       Python 里邊 MySQL 和 sqlite 都是可以使用 executemany 批量插入大量數據的，而且效率基本上是普通插入的數量級提升。

使用 executemany 的好處效率高

我自己測試過，同樣的一萬多條數據，普通插入用時 54.5 秒，使用 executemany 用時 0.22 秒。效率這玩意兒，我就不多贅述了。

不過 sql 稍微有點區別的是，sqlite 是使用的 ? 作為占位符，而不是 %s，%d 之類的喲！正確方法的例子如下：

sql = 'insert into filelist (pkgKey, dirname, filenames, filetypes) values (?, ?, ?, ?);'

import sqlite3
 
class DbOperate(object):
    def __new__(cls, *args, **kwargs):
        if not hasattr(cls, "_instance"):
            cls._instance = super(DbOperate, cls).__new__(cls)
        return cls._instance
 
    def __init__(self, db_name):
        self.db_name = db_name
        self.connect = sqlite3.connect(self.db_name)
        self.cursor = self.connect.cursor()
 
    def __enter__(self):
        return self
 
    def __exit__(self, exc_type, exc_val, exc_tb):
        self.connect.close()
 
    def execute_sql(self, sql):
        try:
            self.cursor.execute(sql)
            self.connect.commit()
        except Exception as e:
            self.connect.rollback()
 
    def executemany_sql(self, sql, data_list):
        # example:
        # sql = 'insert into filelist (pkgKey, dirname, filenames, filetypes) values (?, ?, ?, ?);'
        # data_list = [(1, '/etc/sysconfig', 'openshift_option', 'f'), (1, '/usr/share/doc', 'adb-utils-1.6', 'd')]
        try:
            self.cursor.executemany(sql, data_list)
            self.connect.commit()
        except Exception as e:
            self.connect.rollback()
            raise Exception("executemany failed")
 
sqlite_path = "***.sqlite"
with DbOperate(sqlite_path) as db:
    t1 = time.clock()
    sql = 'insert into filelist (pkgKey, dirname, filenames, filetypes) values (?, ?, ?, ?);'
    data_list = [(1, '/etc/sysconfig', 'openshift_option', 'f'), (1, '/usr/share/doc', 'adb-utils-1.6', 'd')]
    db.executemany_sql(sql, data_list)
    t2 = time.clock()
    print('insert data into filelist cost %s seconds' % (t2 - t1))
    print('success insert data into filelist with %s' % sqlite_path)