利用 hutool工具類ExcelWriter 導出 百萬級數據及導出空的數據問題解決

一、Java 通過hutool工具類ExcelWriter 導出

運用到多線程分頁查詢

這個采用的是Java的utool工具類ExcelWriter 導出
踩過一些坑，盡量用一條sql 將所有數據查詢出來，否則再循環時查詢會隨着表數據的增大查詢速度會成倍增加，所以
建議用一條sql把查詢出結果。實測21列1.1w 多條數據查詢 4067ms左右。
還可以進一步優化。
1.大量數據導出，先調整一下前端請求的response的超時時間 timeout: 1000 * 60 * 10 // 調整個10分鍾
2.直接上Java代碼
注：page 和Ipage 主要是用到分頁，當前頁數和大小，自定義分頁查詢;
每個sheet的導入數據list容易數據過大，請自行優化，后期有時間再做這個優化

 controller 代碼：

/**
* 導出用戶信息
*/
@GetMapping("/exportUser")
public void exportUser(UserManagerReqParam user, HttpServletResponse response){
// 開始時間
long start = System.currentTimeMillis();
PageParam<User> userPageParam = new PageParam<>();
userPageParam.setCurrent(1);
userPageParam.setSize(10L);
IPage<UserManagerParam> userPage = userService.getUserInfoPage(userPageParam,user);
// 總共有多少條數據
Long total = userPage.getTotal();
// 用戶有很多，考慮2000條以上數據的導出 一個之多104w 行數據
Long rowMaxCount = 500000L;
// 每一次查詢條數
Long eachCount = 1000L;
// 這里不用PageParam<User> ，為了方便自由調整查詢條數
Page<User> pages = new Page<>();
pages.setCurrent(1);

// 控制list 大小
Long listSize = 2000L;
// list 分片的數量
int listNums = getPageSize(total,listSize);
List<UserManagerParam> list = new ArrayList<>();

String filepath = "";
// 查詢記錄數
//通過工具類創建writer
ExcelWriter writer = ExcelUtil.getBigWriter();
String fileName = "用戶信息表.xls";
if (total<=rowMaxCount){
if (total<= eachCount) {
pages.setSize(total);
userPage = userService.getUserInfoPage(pages, user);
exportExcel(userPage.getRecords(), 1, 1, response,writer);
// 導出
PoiExcelUtil.writeExcel(fileName,writer,response);
} else {
// 開啟多線程查詢，每eachCount(100)行數據為一個線程開始
int pageSize = getPageSize(total, eachCount);
// ExecutorService execservice = new ThreadPoolExecutor(4,10,200L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue(10));
ExecutorService execservice =Executors.newFixedThreadPool(15);
try {
List<Callable<List<UserManagerParam>>> tasks = new ArrayList<Callable<List<UserManagerParam>>>();
for (int i = 1; i <= pageSize; i++) {
Page<User> pagesIndex = new Page<>();
pagesIndex.setCurrent(i);
pagesIndex.setSize(eachCount);
Callable<List<UserManagerParam>> task = new AnalysisSalseTask(userService, user, pagesIndex);
tasks.add(task);
}
List<Future<List<UserManagerParam>>> futures = execservice.invokeAll(tasks);
if (futures != null && futures.size() > 0) {
for (Future<List<UserManagerParam>> future : futures) {
list.addAll(future.get());
}
}
execservice.shutdown();
tasks.clear();
long end = System.currentTimeMillis();
System.out.println("線程查詢數據用時:"+(end-start)+"ms");
} catch (Exception e) {
System.out.println("多線程查詢異常");
}
exportExcel(list, 1, 0, response, writer);
list.clear();
// 導出
PoiExcelUtil.writeExcel(fileName,writer,response);
}
} else {
// 分多少個 sheet
int pageSize = getPageSize(total,rowMaxCount);
eachCount = 1000L;
for (int i = 1; i <= pageSize; i++) {
list.clear();
int size = getPageSize(rowMaxCount, eachCount);
try {
ExecutorService execservice = new ThreadPoolExecutor(4, 10, 200L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue(10));
List<Callable<List<UserManagerParam>>> tasks = new ArrayList<Callable<List<UserManagerParam>>>();
for (int j = 1; j <= size; j++) {
Page<User> pagesIndex = new Page<>();
pagesIndex.setSize(eachCount);
pagesIndex.setCurrent((i-1) * size + j);
Callable<List<UserManagerParam>> task = new AnalysisSalseTask(userService, user, pagesIndex);
tasks.add(task);
}
List<Future<List<UserManagerParam>>> futures = execservice.invokeAll(tasks);
if (Objects.nonNull(futures) && futures.size() > 0) {
for (Future<List<UserManagerParam>> future : futures) {
list.addAll(future.get());
}
}
tasks.clear();
execservice.shutdown();
long end = System.currentTimeMillis();
System.out.println("線程查詢數據用時:"+(end-start)+"ms");
} catch (Exception e) {
System.out.println("多線程查詢異常");
}
// 序號 (i-1) * rowMaxCount + 1
int rowStart = new BigDecimal(i - 1).multiply(new BigDecimal(rowMaxCount)).add(new BigDecimal(1)).intValue();
//方法1： 導出到一個臨時文件，
// 然后合並小於50W行的Excel到100W行為一個Excel文件,此處跳過，直接是100W行為一個文件
// 然后將100W的每一個Excel文件進行壓縮

// 方法2：分多個sheet 導出
// 此時達到 rowMaxCount 行數據 list 導出到excel
exportExcel(list, rowStart, i-1, response, writer);
list.clear();
}
// 導出
PoiExcelUtil.writeExcel(fileName,writer,response);
}
}
private int getPageSize(Long total, Long eachCount) {
int pageSize = new BigDecimal(total).divide(new BigDecimal(eachCount),1).intValue();
int mod = new BigDecimal(total).divideAndRemainder(new BigDecimal(eachCount))[1].intValue();
if (mod > 0) {
pageSize = pageSize + 1;
// pageSize = new BigDecimal(pageSize).add(new BigDecimal(1)).intValue();
}
return pageSize;
}

/**
* @param list 導出的數據
* @param rowStart 開始的行數
* @param pages 一共有多少頁
*/
private void exportExcel(List<UserManagerParam> list,int rowStart,int pages, HttpServletResponse response,ExcelWriter writer) {

// 商品導出or模板
List<String> headerList;
String[] header = {
"序號", "用戶昵稱", "用戶名稱", "聯系電話", "會員等級",
"會員類型", "用戶積分", "狀態","消費金額", "實付金額",
"消費次數", "平均折扣", "充值金額","充值次數", "退款金額",
"退款次數", "累計積分","當前余額", "累計余額",
"注冊時間", "最近消費時間"
};
headerList = Arrays.asList(header);
writer.setSheet(pages);
Sheet sheet = writer.getSheet();
writer.merge(headerList.size() - 1, "用戶信息表");
writer.writeRow(headerList);
for (int i = 0; i < headerList.size(); i++) {
if (i==19 || i==20) {
sheet.setColumnWidth(i, 30 * 256);
} else {
sheet.setColumnWidth(i, 20 * 256);
}
}
// 如果要導出的數據為空，導出一個模板(可以換成最下面的代碼)
if (CollectionUtils.isEmpty(list)) {
PoiExcelUtil.writeExcel(response, writer);
return;
}
int row = rowStart;
int size = list.size();
for (UserManagerParam param : list) {
int firstRow = row + 1;
int lastRow = row + 1;
int col = -1;
// 序號
PoiExcelUtil.mergeIfNeed(writer, firstRow, lastRow, ++col, col,rowStart++);
// 用戶昵稱
String nickName = Objects.isNull(param.getNickName())?"":param.getNickName();
PoiExcelUtil.mergeIfNeed(writer, firstRow, lastRow, ++col, col,nickName);
// 用戶名稱
String realName = Objects.isNull(param.getRealName())?"":param.getRealName();
PoiExcelUtil.mergeIfNeed(writer, firstRow, lastRow, ++col, col,realName);
// 聯系電話
String userMobile = Objects.isNull(param.getUserMobile())?"":param.getUserMobile();
PoiExcelUtil.mergeIfNeed(writer, firstRow, lastRow, ++col, col,userMobile);
// 會員等級
PoiExcelUtil.mergeIfNeed(writer, firstRow, lastRow, ++col, col,param.getLevelName());
// 會員類型
String levelType = param.getLevelType() == 0 ? "普通會員": "付費會員";
PoiExcelUtil.mergeIfNeed(writer, firstRow, lastRow, ++col, col,levelType);
// 用戶積分
PoiExcelUtil.mergeIfNeed(writer, firstRow, lastRow, ++col, col,param.getScore());
// 狀態
String status = param.getStatus() == 0 ? "禁用": "正常";
PoiExcelUtil.mergeIfNeed(writer, firstRow, lastRow, ++col, col,status);
// 消費金額
PoiExcelUtil.mergeIfNeed(writer, firstRow, lastRow, ++col, col,param.getConsAmount());
// 實付金額
PoiExcelUtil.mergeIfNeed(writer, firstRow, lastRow, ++col, col,param.getActualAmount());
// 消費次數
PoiExcelUtil.mergeIfNeed(writer, firstRow, lastRow, ++col, col,param.getConsTimes());
// 平均折扣
PoiExcelUtil.mergeIfNeed(writer, firstRow, lastRow, ++col, col,param.getAverDiscount());

// 充值金額
PoiExcelUtil.mergeIfNeed(writer, firstRow, lastRow, ++col, col,param.getRechargeAmount());
// 充值次數
PoiExcelUtil.mergeIfNeed(writer, firstRow, lastRow, ++col, col,param.getRechargeTimes());

// 退款金額
PoiExcelUtil.mergeIfNeed(writer, firstRow, lastRow, ++col, col,param.getAfterSaleAmount());
// 退款次數
PoiExcelUtil.mergeIfNeed(writer, firstRow, lastRow, ++col, col,param.getAfterSaleTimes());
// 當前積分
// PoiExcelUtil.mergeIfNeed(writer, firstRow, lastRow, ++col, col,param.getCurrentScore());
// 累計積分
PoiExcelUtil.mergeIfNeed(writer, firstRow, lastRow, ++col, col,param.getSumScore());
// 當前余額
PoiExcelUtil.mergeIfNeed(writer, firstRow, lastRow, ++col, col,param.getCurrentBalance());
// 累計余額
PoiExcelUtil.mergeIfNeed(writer, firstRow, lastRow, ++col, col,param.getSumBalance());
// 注冊時間
String regTime = "";
if (Objects.nonNull(param.getUserRegtime())){
regTime = DateUtil.format(param.getUserRegtime(),"yyyy-MM-dd HH:mm:ss");
}
PoiExcelUtil.mergeIfNeed(writer, firstRow, lastRow, ++col, col,regTime);
// 最近消費時間
String recTime = "";
if (Objects.nonNull(param.getUserRegtime())){
recTime = DateUtil.format(param.getReConsTime(),"yyyy-MM-dd HH:mm:ss");
}
PoiExcelUtil.mergeIfNeed(writer, firstRow, lastRow, ++col, col,recTime);
row++;
}
}

多線程代碼：

AnalysisSalseTask.java

@Slf4j
public class AnalysisSalseTask implements Callable<List<UserManagerParam>> {

private UserService userService;
private UserManagerReqParam user;
private Page<User> pages;

public AnalysisSalseTask(UserService userService, UserManagerReqParam user, Page<User> pages) {
this.userService = userService;
this.user = user;
this.pages = pages;
}

@Override
public List<UserManagerParam> call() throws Exception {
IPage<UserManagerParam> userPage = userService.getUserInfoPage(pages,user);
return userPage.getRecords();
}
}

針對 ExcelWriter 的excel操作工具類：

PoiExcelUtil.java

package com.XXX.util;

import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

import cn.hutool.core.io.IORuntimeException;
import cn.hutool.poi.excel.BigExcelWriter;
import cn.hutool.poi.excel.ExcelWriter;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.util.CellRangeAddress;
import org.apache.poi.xssf.streaming.SXSSFSheet;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;

import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletResponse;

/**
* 功能: poi導出excel工具類
*/
public class PoiExcelUtil {

/**
* 合並單元格處理,獲取合並行
*
* @param sheet
* @return List<CellRangeAddress>
*/
public static List<CellRangeAddress> getCombineCell(Sheet sheet) {
List<CellRangeAddress> list = new ArrayList<CellRangeAddress>();
// 獲得一個 sheet 中合並單元格的數量
int sheetmergerCount = sheet.getNumMergedRegions();
// 遍歷所有的合並單元格
for (int i = 0; i < sheetmergerCount; i++) {
// 獲得合並單元格保存進list中
CellRangeAddress ca = sheet.getMergedRegion(i);
list.add(ca);
}
return list;
}

public static int getRowNum(List<CellRangeAddress> listCombineCell, Cell cell, Sheet sheet) {
int xr = 0;
int firstC = 0;
int lastC = 0;
int firstR = 0;
int lastR = 0;
for (CellRangeAddress ca : listCombineCell) {
// 獲得合並單元格的起始行, 結束行, 起始列, 結束列
firstC = ca.getFirstColumn();
lastC = ca.getLastColumn();
firstR = ca.getFirstRow();
lastR = ca.getLastRow();
if (cell.getRowIndex() >= firstR && cell.getRowIndex() <= lastR) {
if (cell.getColumnIndex() >= firstC && cell.getColumnIndex() <= lastC) {
xr = lastR;
}
}

}
return xr;

}

/**
* 判斷指定的單元格是否是合並單元格
*
* @param sheet
* @param row
* 行下標
* @param column
* 列下標
* @return
*/
public static boolean isMergedRegion(Sheet sheet, int row, int column) {
int sheetMergeCount = sheet.getNumMergedRegions();
for (int i = 0; i < sheetMergeCount; i++) {
CellRangeAddress range = sheet.getMergedRegion(i);
int firstColumn = range.getFirstColumn();
int lastColumn = range.getLastColumn();
int firstRow = range.getFirstRow();
int lastRow = range.getLastRow();
if (row >= firstRow && row <= lastRow) {
if (column >= firstColumn && column <= lastColumn) {
return true;
}
}
}
return false;
}

/**
* 如果需要合並的話，就合並
*/
public static void mergeIfNeed(ExcelWriter writer, int firstRow, int lastRow, int firstColumn, int lastColumn, Object content) {
if (lastRow - firstRow > 0 || lastColumn - firstColumn > 0) {
writer.merge(firstRow, lastRow, firstColumn, lastColumn, content, false);
} else {
writer.writeCellValue(firstColumn, firstRow, content);
}

}
public static void writeExcel(HttpServletResponse response, ExcelWriter writer) {
//response為HttpServletResponse對象
response.setContentType("application/vnd.ms-excel;charset=utf-8");
//test.xls是彈出下載對話框的文件名，不能為中文，中文請自行編碼
response.setHeader("Content-Disposition", "attachment;filename=1.xls");

ServletOutputStream servletOutputStream = null;
try {
servletOutputStream = response.getOutputStream();
writer.flush(servletOutputStream);
servletOutputStream.flush();
} catch (IORuntimeException | IOException e) {
e.printStackTrace();
} finally {
writer.close();
try {
if (servletOutputStream != null) {
servletOutputStream.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
public static void writeExcel(String filename, ExcelWriter writer, HttpServletResponse response){
ServletOutputStream ouputStream = null;
try {
filename = new String(filename.getBytes("UTF-8"), "ISO-8859-1");
response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
response.setHeader("Content-disposition", "attachment;filename=" + filename);
ouputStream = response.getOutputStream();
writer.flush(ouputStream);
ouputStream.flush();
Runtime.getRuntime().gc();
} catch (Exception e) {
e.printStackTrace();
} finally {
writer.close();
if (null != ouputStream) {
try {
ouputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
}

原文鏈接：https://blog.csdn.net/qq_26586953/article/details/109059564

注意：假入查詢數據為null,導出數據時會出現空的excel

解決方案：

創建個空的對象 並添加到list 里  
if (CollectionUtils.isEmpty(itemList)){
    UserManagerParam qualitySuperviseItem = new UserManagerParam();
    List<UserManagerParam> list= new ArrayList<>();
    list.add(qualitySuperviseItem);
    excelWriter.write(list, true);
}else{
    excelWriter.write(lists, true);
}