輕松解決百萬級大數據導出Excel內存溢出問題

隨着運營系統系統的日益龐大和數據量的持續增長，導出excel成了系統占用資源的大頭，最近系統頻繁出現OOM和響應慢的問題，和導出數據脫不了干系。

 

故周六抽時間寫了個工具類，用於異步導出，依賴 EasyExcel ，如果項目中已經引入，那么可以無縫接入。

 

 

 

1.支持游標數據

2.支持回調進度、成功、失敗

3.不會出現OOM

4.易於使用

 

 

使用：

        ExportWriter.New(out, data, DemoData.class)
                .dataSize(2000)
                .progress((progress) -> {
                    // 進度設置
                    log.info("當前進度......" + progress);
                    return null;
                })
                .success(() -> {
                    // 成功
                    log.info("下載完成......");
                    return null;
                })
                .error(() -> {
                    // 失敗
                    log.info("下載失敗......");
                    return null;
                })
                .asyncWrite();

 

 

源代碼：

package com.wangjie.lee.nice;

import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.write.builder.ExcelWriterBuilder;
import com.alibaba.excel.write.metadata.WriteSheet;

import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import java.util.function.Supplier;

/**
 * @author shaozhengmao
 * @create 2021-11-13 9:36 下午
 * @desc
 */
public class ExportWriter<T> {

    private static final ExecutorService THREAD_POOL_EXECUTOR = Executors.newFixedThreadPool(8);

    private OutputStream out;
    private Supplier<Void> callback = () -> null;
    private Supplier<Void> errorCallBack = () -> null;
    private Function<Integer, Void> progressCallback = (Integer p) -> null;
    private Iterable<T> iterable;
    private Class<T> head;

    private Long dataSize = 0L;

    public ExportWriter<T> dataSize(Long dataSize) {
        this.dataSize = dataSize;
        return this;
    }

    /**
     * 非list類型需要指定數據量
     *
     * @param dataSize
     * @return
     */
    public ExportWriter<T> dataSize(Integer dataSize) {
        if (this.dataSize==0L){
            this.dataSize = dataSize.longValue();
        }
        return this;
    }

    private ExportWriter(OutputStream out, Iterable<T> iterable, Class<T> head, Long dataSize) {
        this.out = out;
        this.iterable = iterable;
        this.head = head;
        this.dataSize = dataSize;
    }

    public static <T> ExportWriter<T> New(OutputStream out, Iterable<T> iterable, Class<T> head) {
        if (iterable instanceof List) {
            List<T> collection = (List<T>) iterable;
            return new ExportWriter<>(out, iterable, head, (long) collection.size());
        }
        return new ExportWriter<>(out, iterable, head, 0L);
    }

    public void asyncWrite() {
        THREAD_POOL_EXECUTOR.submit(() -> {
            try {
                write();
                this.out.close();
                this.callback.get();
            } catch (Exception e) {
                e.printStackTrace();
                this.errorCallBack.get();
            }
        });
    }

    public ExportWriter<T> success(Supplier<Void> callback) {
        this.callback = callback;
        return this;
    }

    private void write() {
        final ExcelWriterBuilder builder = EasyExcel.write(this.out);
        builder.head(this.head);
        final ExcelWriter writer = builder.build();
        WriteSheet writeSheet = new WriteSheet();
        writeSheet.setSheetName("DATA");
        writeSheet.setSheetNo(1);

        this.progressCallback.apply(0);
        AtomicLong count = new AtomicLong(0);
        iterable.iterator().forEachRemaining(e -> {
            if (count.get() % 10000 == 0) {
                final double progress = ((double) count.incrementAndGet() / (double) this.dataSize) * 100.0;
                if (progress > 100) {
                    // error size
                    this.progressCallback.apply(60);
                } else {
                    this.progressCallback.apply((int) progress);
                }
            }
            final ArrayList<T> data = new ArrayList<>();
            data.add(e);
            writer.write(data, writeSheet);
            count.getAndIncrement();
        });

        writer.finish();
        this.progressCallback.apply(100);
    }

    public ExportWriter<T> error(Supplier<Void> errorCallBack) {
        this.errorCallBack = errorCallBack;
        return this;
    }

    public ExportWriter<T> progress(Function<Integer, Void> progressCallback) {
        this.progressCallback = progressCallback;
        return this;
    }
}