node導出百萬條數據成excel文件

用的是xlsx包，由於v8默認內存限制是1.4G，當導出的數據太大的時候，內存會炸，由於csv格式的excel文件，本質上就是文本文件，所以在不調大內存上限的情況下，實現思路是分批處理，用流的方式追加到文件，第一批處理有header，后面追加沒有header。微軟excel能打開最大的行數是1048576，所以生成文件超過了也沒用。

const xlsx = require('xlsx')
const path = require('path')
const fs = require('fs')
let result = []
for (let i = 0; i < 1100000; i++) {
  result.push({
    hello: '哈哈哈' + i,
    world: '黑乎乎' + i,
    hehe2: '黑乎乎' + i,
    hehe3: '黑乎乎' + i,
    hehe4: '黑乎乎' + i,
    hehe5: '黑乎乎' + i,
    hehe6: '黑乎乎' + i,
    hehe7: '黑乎乎' + i,
    hehe8: '黑乎乎' + i,
    hehe9: '黑乎乎' + i,
    hehe10: '黑乎乎' + i,
    hehe11: '黑乎乎' + i,
    hehe12: '黑乎乎' + i,
    hehe13: '黑乎乎' + i,
    hehe14: '黑乎乎' + i,
    hehe15: '黑乎乎' + i,
  })
}

const add = async (data, filePath, hasHeader) => {
  return new Promise(resolve => {
    const jsonWorkSheet = xlsx.utils.json_to_sheet(data, {skipHeader: hasHeader});
    const stream = xlsx.stream.to_csv(jsonWorkSheet);
    const writeS = fs.createWriteStream(filePath, {flags: 'a'})
    stream.pipe(writeS)
    stream.on('end', function() {
      resolve()
    });
  })
} 

let filepath = path.resolve('world6.csv');

async function hello(result,filepath) {
  let chunk = []
  let num = 10000;
  let flag = false;
  while(result.length > 0) {
    chunk = result.splice(0,num)
    await add(chunk,filepath, flag)
    flag = true
  }
}
hello(result,filepath)

上面的實現是生成csv文件，如果要生成xlsx格式，可以用xlsx-writestream

var XLSXWriter = require('xlsx-writestream');
var fs = require('fs');

var writer = new XLSXWriter('mySpreadsheet.xlsx', {} /* options */);

writer.getReadStream().pipe(fs.createWriteStream('mySpreadsheet.xlsx'));

let result = []
for (let i = 0; i < 1500000; i++) {
  result.push({
    hello: '哈哈哈' + i,
    world: '黑乎乎' + i,
    hehe2: '黑乎乎' + i,
    hehe3: '黑乎乎' + i,
    hehe4: '黑乎乎' + i,
    hehe5: '黑乎乎' + i,
    hehe6: '黑乎乎' + i,
    hehe7: '黑乎乎' + i,
    hehe8: '黑乎乎' + i,
    hehe9: '黑乎乎' + i,
    hehe10: '黑乎乎' + i,
    hehe11: '黑乎乎' + i,
    hehe12: '黑乎乎' + i,
    hehe13: '黑乎乎' + i,
    hehe14: '黑乎乎' + i,
    hehe15: '黑乎乎' + i,
  })
}
result.forEach(item => {
    writer.addRow(item)
})

// Finalize the spreadsheet. If you don't do this, the readstream will not end.
writer.finalize();