前言
之前公司有個需求,導入幾十萬的數據,需要從excel讀取出來,再把重復的項合並起來導入數據庫,當時用程序寫的非常慢,光讀取數據半小時都下不來,感覺自己寫的程序太渣了.
思路
1.將Excel文件轉換成.csv文件
2.讀取.csv文件到DataTable里 (這個讀取速度非常快)
3.補充數據表的列名,修改數據類型
4.使用SqlBulkCopy將DataTable中的數據批量插入數據庫(這里就是瞬間插入的秘籍)
實現
下邊直接上代碼了 需要nuget安裝 Microsoft.Office.Interop.Excel
using System; using System.Collections.Generic; using System.ComponentModel; using System.Data; using System.Drawing; using System.Linq; using System.Text; using System.Threading.Tasks; using System.Windows.Forms; using System.Diagnostics; using System.IO; using System.Reflection; using Excel=Microsoft.Office.Interop.Excel; using System.Data.SqlClient; namespace excel轉csv { public partial class Form1 : Form { public Form1() { InitializeComponent(); } /// <summary> /// 將Csv文件轉換為XLS文件 /// </summary> /// <param name="FilePath">文件全路路徑</param> /// <returns>返回轉換后的Xls文件名</returns> public static string CSVSaveasXLS(string FilePath) { QuertExcel(); string _NewFilePath = ""; Excel.Application excelApplication; Excel.Workbooks excelWorkBooks = null; Excel.Workbook excelWorkBook = null; Excel.Worksheet excelWorkSheet = null; try {
//此時報錯:無法嵌入互操作類型“……”，請改用適用的接口的解決方法
//解決方案:選中項目中引入的dll，鼠標右鍵，選擇屬性，把“嵌入互操作類型”設置為False。
excelApplication = new Excel.ApplicationClass(); excelWorkBooks = excelApplication.Workbooks; excelWorkBook = ((Excel.Workbook)excelWorkBooks.Open(FilePath, Missing.Value, Missing.Value, Missing.Value, Missing.Value, Missing.Value, Missing.Value, Missing.Value, Missing.Value, Missing.Value, Missing.Value, Missing.Value, Missing.Value, Missing.Value, Missing.Value)); excelWorkSheet = (Excel.Worksheet)excelWorkBook.Worksheets[1]; excelApplication.Visible = false; excelApplication.DisplayAlerts = false; _NewFilePath = FilePath.Replace(".csv", ".xls"); excelWorkBook.SaveAs(_NewFilePath, Excel.XlFileFormat.xlAddIn, Missing.Value, Missing.Value, Missing.Value, Missing.Value, Excel.XlSaveAsAccessMode.xlNoChange, Missing.Value, Missing.Value, Missing.Value, Missing.Value, Missing.Value); excelWorkBook.Close(); QuertExcel(); // ExcelFormatHelper.DeleteFile(FilePath); //可以不用殺掉進程QuertExcel(); GC.Collect(System.GC.GetGeneration(excelWorkSheet)); GC.Collect(System.GC.GetGeneration(excelWorkBook)); GC.Collect(System.GC.GetGeneration(excelApplication)); } catch (Exception exc) { throw new Exception(exc.Message); } finally { GC.Collect(); } return _NewFilePath; } /// <summary> /// 將xls文件轉換為csv文件 /// </summary> /// <param name="FilePath">文件全路路徑</param> /// <returns>返回轉換后的csv文件名</returns> public static string XLSSavesaCSV(string FilePath) { QuertExcel(); string _NewFilePath = ""; Excel.Application excelApplication; Excel.Workbooks excelWorkBooks = null; Excel.Workbook excelWorkBook = null; Excel.Worksheet excelWorkSheet = null; try { excelApplication = new Excel.ApplicationClass(); excelWorkBooks = excelApplication.Workbooks; excelWorkBook = ((Excel.Workbook)excelWorkBooks.Open(FilePath, Missing.Value, Missing.Value, Missing.Value, Missing.Value, Missing.Value, Missing.Value, Missing.Value, Missing.Value, Missing.Value, Missing.Value, Missing.Value, Missing.Value, Missing.Value, Missing.Value)); excelWorkSheet = (Excel.Worksheet)excelWorkBook.Worksheets[1]; excelApplication.Visible = false; excelApplication.DisplayAlerts = false; _NewFilePath = FilePath.Replace(".xlsx", ".csv"); //excelWorkSheet._SaveAs(FilePath, Excel.XlFileFormat.xlCSVWindows, Missing.Value, Missing.Value, Missing.Value,Missing.Value,Missing.Value, Missing.Value, Missing.Value); excelWorkBook.SaveAs(_NewFilePath, Excel.XlFileFormat.xlCSV, Missing.Value, Missing.Value, Missing.Value, Missing.Value, Microsoft.Office.Interop.Excel.XlSaveAsAccessMode.xlNoChange, Missing.Value, Missing.Value, Missing.Value, Missing.Value, Missing.Value); QuertExcel(); //ExcelFormatHelper.DeleteFile(FilePath); } catch (Exception exc) { throw new Exception(exc.Message); } return _NewFilePath; } /// <summary> /// 刪除一個指定的文件 /// </summary> /// <param name="FilePath">文件路徑</param> /// <returns></returns> public static bool DeleteFile(string FilePath) { try { bool IsFind = File.Exists(FilePath); if (IsFind) { File.Delete(FilePath); } else { throw new IOException("指定的文件不存在"); } return true; } catch (Exception exc) { throw new Exception(exc.Message); } } /// <summary> /// 執行過程中可能會打開多個EXCEL文件 所以殺掉 /// </summary> private static void QuertExcel() { Process[] excels = Process.GetProcessesByName("EXCEL"); foreach (var item in excels) { item.Kill(); } } private void Button1_Click(object sender, EventArgs e) { //F:\資料\資料\demo\大數據瞬間入庫\WebApplication1\Content\客戶信息.csv //CSVSaveasXLS(textBox1.Text); //F:\資料\資料\demo\大數據瞬間入庫\WebApplication1\Content\客戶信息.xlsx //E:\客戶信息模板.xlsx var aa = XLSSavesaCSV(textBox1.Text); label1.Text = aa; //讓主線程停頓兩秒 負責下邊讀取csv文件太快會顯示占用 System.Threading.Thread.Sleep(2000); //2秒 var dt= OpenCSV(aa); dt.Columns["序號"].ColumnName = "Id"; //有數據則不能改類型 //dt.Columns["Id"].DataType = Type.GetType("System.Int32"); dt.Columns["公司名稱"].ColumnName = "Company"; dt.Columns["公司注冊地址"].ColumnName = "Address"; dt.Columns["公司營業范圍"].ColumnName = "Business"; dt.Columns["注冊資金"].ColumnName = "RegCapital"; dt.Columns["投保人數"].ColumnName = "EmployeesNum"; dt.Columns["法人姓名"].ColumnName = "Boss"; dt.Columns["備注"].ColumnName = "Remark"; dt.Columns.Add("Status", Type.GetType("System.Int32")).SetOrdinal(7); dt.Columns.Add("StatusTime", Type.GetType("System.DateTime")).SetOrdinal(8); dt.Columns.Add("Type", Type.GetType("System.Int32")).SetOrdinal(9); dt.Columns.Add("TypeTime", Type.GetType("System.DateTime")).SetOrdinal(10); dt.Columns.Add("Level", Type.GetType("System.String")).SetOrdinal(11); dt.Columns.Add("PrePurchase", Type.GetType("System.Decimal")).SetOrdinal(12); dt.Columns.Add("Description", Type.GetType("System.String")).SetOrdinal(13); dt.Columns.Add("YewuId", Type.GetType("System.Int32")).SetOrdinal(14); dt.Columns.Add("YewuName", Type.GetType("System.String")).SetOrdinal(15); dt.Columns.Add("IsDel", Type.GetType("System.Int32")).SetOrdinal(16); dt.Columns.Add("Createtime",Type.GetType("System.DateTime")).SetOrdinal(18); //有數據的列要修改類型進入以下方法 dt = UpdateDataTable(dt); var newCon = "server=127.0.0.1;database=its;uid=sa;pwd=1;"; SqlBulkCopyInsert(newCon, "MallCustomer", dt); } /// <summary> /// 修改數據表DataTable某一列的類型和記錄值(正確步驟：1.克隆表結構，2.修改列類型，3.修改記錄值，4.返回希望的結果) /// </summary> /// <param name="argDataTable">數據表DataTable</param> /// <returns>數據表DataTable</returns> private DataTable UpdateDataTable(DataTable argDataTable) { DataTable dtResult = new DataTable(); //克隆表結構 dtResult = argDataTable.Clone(); foreach (DataColumn col in dtResult.Columns) { //修改列的數據格式 switch (col.ColumnName) { case "Id": col.DataType = typeof(Int32); break; case "RegCapital": col.DataType = typeof(Decimal); break; case "EmployeesNum": col.DataType = typeof(Int32); break; } } string company = ""; foreach (DataRow row in argDataTable.Rows) { DataRow rowNew = dtResult.NewRow(); rowNew["Id"] =Convert.ToInt32(row["Id"].ToString() == "" ? 0 : row["Id"]).ToString(); var aaa = row["EmployeesNum"]; //修改記錄值 rowNew["EmployeesNum"] = Convert.ToInt32(row["EmployeesNum"].ToString()==""?0: row["EmployeesNum"]).ToString(); var aaa1 = row["RegCapital"]; rowNew["RegCapital"] = Convert.ToDecimal(row["RegCapital"].ToString() == "" ? 0 : row["RegCapital"]).ToString(); rowNew["Company"] = row["Company"]; rowNew["Address"] = row["Address"]; rowNew["Business"] = row["Business"]; rowNew["Boss"] = row["Boss"]; rowNew["Remark"] = row["Remark"]; rowNew["Status"] = row["Status"]; rowNew["StatusTime"] = row["StatusTime"]; rowNew["Type"] = row["Type"]; rowNew["TypeTime"] = row["TypeTime"]; rowNew["Level"] = row["Level"]; rowNew["PrePurchase"] = row["PrePurchase"]; rowNew["Description"] = row["Description"]; rowNew["YewuId"] = row["YewuId"]; rowNew["YewuName"] = row["YewuName"]; rowNew["IsDel"] = 0.ToString(); rowNew["Createtime"] = DateTime.Now.ToString(); //合並重復項 if (company == row["Company"].ToString()) { dtResult.Rows[dtResult.Rows.Count - 1]["Remark"] +=","+ row["Remark"].ToString(); } else { dtResult.Rows.Add(rowNew); company = row["Company"].ToString(); } } return dtResult; } /// <summary> /// 將CSV文件的數據讀取到DataTable中 /// </summary> /// <param name="fileName">CSV文件路徑</param> /// <returns>返回讀取了CSV數據的DataTable</returns> public static DataTable OpenCSV(string filePath) { //中文亂碼 如果亂碼 可以改下編碼格式 Encoding encoding = Encoding.Default; //Encoding.ASCII;// DataTable dt = new DataTable(); FileStream fs = new FileStream(filePath, System.IO.FileMode.Open, System.IO.FileAccess.Read); //StreamReader sr = new StreamReader(fs, Encoding.UTF8); StreamReader sr = new StreamReader(fs, encoding); //string fileContent = sr.ReadToEnd(); //encoding = sr.CurrentEncoding; //記錄每次讀取的一行記錄 string strLine = ""; //記錄每行記錄中的各字段內容 string[] aryLine = null; string[] tableHead = null; //標示列數 int columnCount = 0; //標示是否是讀取的第一行 bool IsFirst = true; //逐行讀取CSV中的數據 while ((strLine = sr.ReadLine()) != null) { //strLine = Common.ConvertStringUTF8(strLine, encoding); //strLine = Common.ConvertStringUTF8(strLine); if (IsFirst == true) { tableHead = strLine.Split(','); IsFirst = false; columnCount = tableHead.Length; //創建列 for (int i = 0; i < columnCount; i++) { DataColumn dc = new DataColumn(tableHead[i]); dt.Columns.Add(dc); } } else { if (!String.IsNullOrEmpty(strLine)) { aryLine = strLine.Split(','); DataRow dr = dt.NewRow(); for (int j = 0; j < columnCount; j++) { dr[j] = aryLine[j]; } dt.Rows.Add(dr); } } } if (aryLine != null && aryLine.Length > 0) { dt.DefaultView.Sort = tableHead[0] + " " + "asc"; } sr.Close(); fs.Close(); return dt; } #region 使用SqlBulkCopy將DataTable中的數據批量插入數據庫中 /// <summary> /// 注意：DataTable中的列需要與數據庫表中的列完全一致。/// </summary> /// <param name="conStr">數據庫連接串</param> /// <param name="strTableName">數據庫中對應的表名</param> /// <param name="dtData">數據集</param> public static void SqlBulkCopyInsert(string conStr, string strTableName, DataTable dtData) { try { using (SqlBulkCopy sqlRevdBulkCopy = new SqlBulkCopy(conStr)) //引用SqlBulkCopy { sqlRevdBulkCopy.DestinationTableName = strTableName; //數據庫中對應的表名 sqlRevdBulkCopy.NotifyAfter = dtData.Rows.Count; //有幾行數據 sqlRevdBulkCopy.WriteToServer(dtData); //數據導入數據庫 sqlRevdBulkCopy.Close(); //關閉連接 } } catch (Exception ex) {
//這里可能會報一些數據類型的錯誤,有以下幾種解決辦法
//1.DataTable的數據類型要與數據庫一致,列的順序,列名大小寫都要一致
//2.數據太長,如數據庫類型是varchar(50),當前數據太長超過50就會報錯,可以修改數據庫類型
throw (ex); } } #endregion } }
這樣百萬級數據就能瞬間插入數據庫了