多線程之批量插入
背景
昨天在測試mysql的兩種批量更新時，由於需要入庫大量測試數據，反復執行插入腳本，過程繁瑣，檔次很低，測試完后我就想着寫個批量插入的小demo，然后又想寫個多線程的批量插入的demo，然后就有了下面的東西了……
環境
spring-boot 1.5.6 集成 mysql druid mybits 還有一些無關緊要的東西
代碼
線程類：
/**
* @ClassName InsertDataThread
* @Description <插入數據類>
* @Author zhaiyt
* @Date 2018/8/29 17:04
* @Version 1.0
*/
public class InsertDataThread extends Thread {
//日志
private final Logger logger = LoggerFactory.getLogger(InsertDataThread.class);
//數據訪問層
private UserEntityMapper userEntityMapper;
//具體插入批次
private int batch;
//插入的數據
private List<UserEntity> list;
public InsertDataThread(UserEntityMapper userMpper, List<UserEntity> li, int batch) {
this.userEntityMapper = userMpper;
this.list = li;
this.batch = batch;
}
@Override
public void run() {
try {
this.userEntityMapper.insertBatch(this.list);
logger.info("第" + this.batch + "批次插入成功");
} catch (Exception e) {
logger.error("第" + this.batch + "批次插入失敗");
}
}
}
===============================================================================
service層的多線程批量插入方法：
/**
* @param list
* @return int
* @Description <批量插入>
* @Author zhaiyt
* @Date 9:51 2018/8/29
* @Param [list]
*/
@Override
public int insertBatch(List<UserEntity> list) throws Exception {
PageHelper.offsetPage(0, 500000);
long start = System.currentTimeMillis();
List<UserEntity> listUser = userEntityMapper.selectAllUser();
int betch = 0;
if (CollectionUtils.isEmpty(listUser)) {
logger.error("表中無數據，需要改造測試");
return 0;
}
//根據數據量判斷是否使用多線程 選擇開啟線程數
if (listUser.size() > 1000000) {
betch = 10;
} else if (listUser.size() > 100000) {
betch = 5;
} else if (listUser.size() > 10000) {
betch = 3;
} else {
//不走多線程
long end = System.currentTimeMillis();
logger.error("查詢耗時:" + (end - start));
start = System.currentTimeMillis();
int count = userEntityMapper.insertBatch(listUser);
end = System.currentTimeMillis();
logger.error("插入耗時:" + (end - start));
return count;
}
//計數器
int size = 0;
//創建線程池
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(betch);
for (int i = 0; i < (Math.ceil(listUser.size() / 5000)); i++) {
int startLen = i * 5000;
int endLen = ((i + 1) * 5000 > listUser.size() ? listUser.size() - 1 : (i + 1) * 5000);
// 該線程處理
List<UserEntity> threadList = listUser.subList(startLen, endLen);
size = size + threadList.size();
fixedThreadPool.execute(new InsertDataThread(userEntityMapper, threadList, i));
}
System.err.println("插入數據總條數：" + size);
long end = System.currentTimeMillis();
logger.error("查詢耗時:" + (end - start));
return size;
}