Springboot 批量插入優化

• 優化背景
• 在一次批量導入數據過程中，3000條數據，postman請求耗時：5.65s
  
• 方案說明
• 代碼

      public void importSupplyOrder(MultipartFile file, String recycleId) {
          long methodStart = System.currentTimeMillis();
           //從數據庫查詢相關信息
          .................
          ArrayList<SupplyOrder> supplyOrders = new ArrayList<>();
          long forStart = System.currentTimeMillis();
          for (int i = 0; i <importSupplyOrderVos.size(); i++) {
             //參數判斷，寫入數據List 封裝
             .................
          }
          long forEnd = System.currentTimeMillis();
          saveBatch(supplyOrders);
  //        supplyOrderMapper.importSupplyOrder(supplyOrders);
          long insertEnd = System.currentTimeMillis();
          log.info("循環前消耗時間[{}]",forStart-methodStart);
          log.info("循環消耗時間[{}]",forEnd-forStart);
          log.info("insert消耗時間[{}]",insertEnd-forEnd);
      }

• 優化前方案：使用mybatisplus 定義的 saveBatch(....) 方法
• 優化后方案：在 xml 文件中使用sql 語句處理

      <insert id="importSupplyOrder">
          insert into supply_order (
          	<!--表字段-->
             .......
          )values
          <foreach collection="supplyOrders"  open="(" separator="),(" close=")" item="orderVo">
             <!--字段值-->
             .......               
          </foreach>
      </insert>

• 優化前后耗時對比
• 3000條數據，日志打印

  數據量		優化前	優化后	差值
  3000	循環前耗時	846	776	正常波動，忽略
  	循環耗時	30	21	正常波動，忽略
  	寫入耗時	4528	1121	3407
  10000	循環前耗時	1405	1344	正常波動，忽略
  	循環耗時	51	54	正常波動，忽略
  	寫入耗時	12074	3080	8994

  
  
• post請求對比
  
•  原因分析
• 優化前sql 執行log

  INSERT INTO supply_order ( supply_order, supply_user_id, supply_user_name, goods_id, goods_name, tax_code, car_num, gross_weight, tare_weight, deduct_weight, net_weight, unit, price, total_price, status, create_time, supply_order_time, operation_time, info_id, recycle_id, recycle_name, supply_credential_num ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )

• 優化后sql 執行log

  insert into supply_order ( supply_order, supply_user_id, supply_user_name, goods_id, goods_name, tax_code, size, car_num, gross_weight, tare_weight, deduct_weight, net_weight, unit, price, total_price, status, create_time, supply_order_time, operation_time, info_id, description, recycle_id, recycle_name, supply_credential_num, unit_id )values 
  ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ),
  ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ),
  ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ),

  
  
• 源碼分析

•  //com.baomidou.mybatisplus.extension.service.impl    ServiceImpl
      /**
       * 批量插入
       *
       * @param entityList
       * @param batchSize
       * @return
       */
      @Transactional(rollbackFor = Exception.class)
      @Override
      public boolean saveBatch(Collection<T> entityList, int batchSize) {
          int i = 0;
          String sqlStatement = sqlStatement(SqlMethod.INSERT_ONE);
          try (SqlSession batchSqlSession = sqlSessionBatch()) {
              for (T anEntityList : entityList) {
                  batchSqlSession.insert(sqlStatement, anEntityList);      //每條數據都會執行，
                  if (i >= 1 && i % batchSize == 0) {
                      batchSqlSession.flushStatements();
                  }
                  i++;
              }
              batchSqlSession.flushStatements();       //預寫入
          }
          return true;
      }

  //org.apache.ibatis.session.defaults ;    DefaultSqlSession
    @Override
    public int insert(String statement, Object parameter) {
      return update(statement, parameter);
    }
    @Override
    public int update(String statement, Object parameter) {
      try {
        dirty = true;
        MappedStatement ms = configuration.getMappedStatement(statement);
        return executor.update(ms, wrapCollection(parameter));
      } catch (Exception e) {
        throw ExceptionFactory.wrapException("Error updating database.  Cause: " + e, e);
      } finally {
        ErrorContext.instance().reset();
      }
    }

  //優化前   mybatisplus saveBatch(...) 方法
  //org.apache.ibatis.executor;  BatchExecutor
    @Override
    public int doUpdate(MappedStatement ms, Object parameterObject) throws SQLException {
      final Configuration configuration = ms.getConfiguration();
      final StatementHandler handler = configuration.newStatementHandler(this, ms, parameterObject, RowBounds.DEFAULT, null, null);
      final BoundSql boundSql = handler.getBoundSql();
      final String sql = boundSql.getSql();
      final Statement stmt;
      if (sql.equals(currentSql) && ms.equals(currentStatement)) {
        int last = statementList.size() - 1;
        stmt = statementList.get(last);
        applyTransactionTimeout(stmt);
       handler.parameterize(stmt);//fix Issues 322
        BatchResult batchResult = batchResultList.get(last);
        batchResult.addParameterObject(parameterObject);
      } else {
        Connection connection = getConnection(ms.getStatementLog());
        stmt = handler.prepare(connection, transaction.getTimeout());
        handler.parameterize(stmt);    //fix Issues 322
        currentSql = sql;
        currentStatement = ms;
        statementList.add(stmt);
        batchResultList.add(new BatchResult(ms, sql, parameterObject));
      }
    // handler.parameterize(stmt);
      handler.batch(stmt);
      return BATCH_UPDATE_RETURN_VALUE;
    }
  //優化后   
  // org.apache.ibatis.executor;    SimpleExecutor
    @Override
    public int doUpdate(MappedStatement ms, Object parameter) throws SQLException {
      Statement stmt = null;
      try {
        Configuration configuration = ms.getConfiguration();
        StatementHandler handler = configuration.newStatementHandler(this, ms, parameter, RowBounds.DEFAULT, null, null);
        stmt = prepareStatement(handler, ms.getStatementLog());
        return handler.update(stmt);
      } finally {
        closeStatement(stmt);
      }
    }

  
  
• 優化后需要考慮的問題
• mysql  有包最大允許限制   4,194,304字節，因此在處理過程中要手動對數據進行分割

  SHOW VARIABLES LIKE  'max_allowed_packet'

  
  
• 數據分割示例：

      batchSize = 1000    
          
  for(int i = 0;true;i++){
  
              if(((i+1)*batchSize)<supplyOrders.size()){
                  supplyOrderMapper.importSupplyOrder(supplyOrders.subList(i*batchSize,(i+1)*batchSize ));
              }else{
                  supplyOrderMapper.importSupplyOrder(supplyOrders.subList(i*batchSize,supplyOrders.size() ));
                  break;
              }
          }