Python中sqlite3使用 executemany 批量插入数据

转自：https://blog.csdn.net/TomorrowAndTuture/article/details/113978649

       Python 里边 MySQL 和 sqlite 都是可以使用 executemany 批量插入大量数据的，而且效率基本上是普通插入的数量级提升。

使用 executemany 的好处效率高

我自己测试过，同样的一万多条数据，普通插入用时 54.5 秒，使用 executemany 用时 0.22 秒。效率这玩意儿，我就不多赘述了。

不过 sql 稍微有点区别的是，sqlite 是使用的 ? 作为占位符，而不是 %s，%d 之类的哟！正确方法的例子如下：

sql = 'insert into filelist (pkgKey, dirname, filenames, filetypes) values (?, ?, ?, ?);'

import sqlite3
 
class DbOperate(object):
    def __new__(cls, *args, **kwargs):
        if not hasattr(cls, "_instance"):
            cls._instance = super(DbOperate, cls).__new__(cls)
        return cls._instance
 
    def __init__(self, db_name):
        self.db_name = db_name
        self.connect = sqlite3.connect(self.db_name)
        self.cursor = self.connect.cursor()
 
    def __enter__(self):
        return self
 
    def __exit__(self, exc_type, exc_val, exc_tb):
        self.connect.close()
 
    def execute_sql(self, sql):
        try:
            self.cursor.execute(sql)
            self.connect.commit()
        except Exception as e:
            self.connect.rollback()
 
    def executemany_sql(self, sql, data_list):
        # example:
        # sql = 'insert into filelist (pkgKey, dirname, filenames, filetypes) values (?, ?, ?, ?);'
        # data_list = [(1, '/etc/sysconfig', 'openshift_option', 'f'), (1, '/usr/share/doc', 'adb-utils-1.6', 'd')]
        try:
            self.cursor.executemany(sql, data_list)
            self.connect.commit()
        except Exception as e:
            self.connect.rollback()
            raise Exception("executemany failed")
 
sqlite_path = "***.sqlite"
with DbOperate(sqlite_path) as db:
    t1 = time.clock()
    sql = 'insert into filelist (pkgKey, dirname, filenames, filetypes) values (?, ?, ?, ?);'
    data_list = [(1, '/etc/sysconfig', 'openshift_option', 'f'), (1, '/usr/share/doc', 'adb-utils-1.6', 'd')]
    db.executemany_sql(sql, data_list)
    t2 = time.clock()
    print('insert data into filelist cost %s seconds' % (t2 - t1))
    print('success insert data into filelist with %s' % sqlite_path)