Npgsql使用入門(三)【批量導入數據】

Program.cs代碼:

 class Program
    {
        static void Main(string[] args)
        {
            var test = new PgBulkCopyHelper<SingleBuilding>("bld_amap_gzmain");

            foreach (string pName in test.PropNames)
            {
                Console.WriteLine("name: {0},\t\ttype: {1}", pName, test.PropInfo[pName]);
            }

            //-----------------------------------------------------------------------------------------------
            //定義每次插入的最大數量限制
            int maxNum = 1; //100000;
            //初始化對應的數據表
            DataTable dataTable = test.InitDataTable();
            string connectionString = "Host=localhost;Username=king;Password=wu12345;Database=dellstore";
            List<List<SingleBuilding>> bldsList = new List<List<SingleBuilding>>();

            NpgsqlPolygon plg1 = new NpgsqlPolygon(10);
            plg1.Add(new NpgsqlPoint(0.0, 0.0));
            plg1.Add(new NpgsqlPoint(6.0, -1.0));
            plg1.Add(new NpgsqlPoint(5.0, 3.0));
            plg1.Add(new NpgsqlPoint(1.0, 2.0));


         

            NpgsqlPolygon plg2 = new NpgsqlPolygon(10);
            plg2.Add(new NpgsqlPoint(100.0, 10.0));
            plg2.Add(new NpgsqlPoint(40.0, 180.0));
            plg2.Add(new NpgsqlPoint(190.0, 60.0));
            plg2.Add(new NpgsqlPoint(10.0, 60.0));
            plg2.Add(new NpgsqlPoint(160.0, 180.0));

  
            List<SingleBuilding> sblist1 = new List<SingleBuilding>(){
              new SingleBuilding(){id=System.Guid.NewGuid(),
                                   tile_x=1,         
                                   tile_y=2,         
                                   bps_gc=plg1,
                                   bps_llc=plg2,

                                   cp_gc=new NpgsqlPoint(0,0),   
                                   cp_llc=new NpgsqlPoint(100,10),  
                                   name="測試文本1",
                                   bld_floor=111,     
                                   height=22           
              
              },
              new SingleBuilding(){id=System.Guid.NewGuid(),
                                   tile_x=1,         
                                   tile_y=2,         
                                   bps_gc=plg1,
                                   bps_llc=plg2,

                                   cp_gc=new NpgsqlPoint(0,0),   
                                   cp_llc=new NpgsqlPoint(100,10),  
                                   name="測試文本2",
                                   bld_floor=222,     
                                   height=444     
              }
            };

            bldsList.Add(sblist1);

            using (var conn = new NpgsqlConnection(connectionString))
            {
                conn.Open();

                foreach (List<SingleBuilding> blds in bldsList)
                {
                    if (blds != null && blds.Count > 0)
                    {
                        //填充數據
                        test.FillDataTable(blds, dataTable);
                    }

                    //判斷 dataTable 里面的數據量是否已經超過規定最大行數 maxNum
                    if (dataTable.Rows.Count>maxNum)
                    {
                        //如果是，則將 dataTable 里面的數據插入到數據庫中
                        test.BulkInsert(conn, dataTable);
                        //清空 dataTable 中的現有數據
                        dataTable.Clear();
                    }
                }
            }
        }
    }

    public class SingleBuilding
    {
        //創建數據表的SQL語句如下：
        /*
        CREATE TABLE bld_amap_gzmain (
            id uuid PRIMARY KEY NOT NULL,
            tile_x integer,             --x index of the map tile where the building is located
            tile_y integer,             --y index of the map tile where the building is located
            bps_gc polygon NOT NULL,    --the points of the bottom outline of the building, geodetic coordinates
            bps_llc polygon NOT NULL,   --the points of the bottom outline of the building, Latitude and longitude coordinates
            cp_gc point NOT NULL,       --the center point of the building, geodetic coordinates
            cp_llc point NOT NULL,      --the center point of the building, Latitude and longitude coordinates
            name text,
            bld_floor smallint,         --the number of floors of the building
            height real                 --the height of building
        );
        */

        public Guid id { get; set; }
        public int? tile_x { get; set; }
        public int? tile_y { get; set; }
        public NpgsqlPolygon bps_gc { get; set; }
        public NpgsqlPolygon bps_llc { get; set; }
        public NpgsqlPoint cp_gc { get; set; }
        public NpgsqlPoint cp_llc { get; set; }
        public string name { get; set; }
        public short? bld_floor { get; set; }
        public float? height { get; set; }
    }

---

PgBulkCopyHelper.cs代碼:

using Npgsql;
using System;
using System.Collections.Generic;
using System.Data;
using System.Globalization;
using System.Linq;
using System.Reflection;

namespace PgBulkCopyHelper
{
    /// <summary>
    /// 用以快速將大批量數據插入到postgresql中
    /// </summary>
    /// <typeparam name="TEntity"></typeparam>
    public class PgBulkCopyHelper<TEntity>
    {
        /// <summary>
        /// TEntity的屬性信息
        /// Dictionary(string "property_name", Type property_type)
        /// </summary>
        public Dictionary<string, Type> PropInfo { get; set; }
        /// <summary>
        /// TEntity的屬性名稱列表
        /// </summary>
        public List<string> PropNames { get; set; }
        /// <summary>
        /// 數據表全名：schema.tableName or tableName
        /// </summary>
        public string FullTableName { get; set; }

        /// <summary>
        /// 構造函數
        /// </summary>
        /// <param name="schema">數據表的schema，一般為public</param>
        /// <param name="tableName">數據表的名稱</param>
        public PgBulkCopyHelper(string schema, string tableName)
        {
            PropNames = new List<string>();
            PropInfo = new Dictionary<string, Type>();
            PropertyInfo[] typeArgs = GetPropertyFromTEntity();
            foreach (PropertyInfo tParam in typeArgs)
            {
                PropNames.Add(tParam.Name);
                PropInfo[tParam.Name] = tParam.PropertyType;
            }

            if (!string.IsNullOrWhiteSpace(tableName))
            {
                if (string.IsNullOrWhiteSpace(schema))
                {
                    FullTableName = tableName;
                }
                else
                    FullTableName = string.Format("{0}.{1}", schema, tableName);
            }
        }

        /// <summary>
        /// 構造函數
        /// </summary>
        /// <param name="tableName">數據表的名稱</param>
        public PgBulkCopyHelper(string tableName)
            :this(null, tableName)
        { }

        /// <summary>
        /// 獲取TEntity的屬性信息
        /// </summary>
        /// <returns>TEntity的屬性信息的列表</returns>
        private PropertyInfo[] GetPropertyFromTEntity()
        {
            Type t = typeof(TEntity);
            PropertyInfo[] typeArgs = t.GetProperties();
            return typeArgs;
        }

        /// <summary>
        /// 根據TEntity的屬性信息構造對應數據表
        /// </summary>
        /// <returns>只有字段信息的數據表</returns>
        public DataTable InitDataTable()
        {
            DataTable dataTable = new DataTable();
            
            foreach(PropertyInfo tParam in GetPropertyFromTEntity())
            {
                Type propType = tParam.PropertyType;
                //由於 DataSet 不支持 System.Nullable<> 類型，因此要先做判斷
                if ((propType.IsGenericType) && (propType.GetGenericTypeDefinition() == typeof(Nullable<>)))
                    propType = propType.GetGenericArguments()[0];
                dataTable.Columns.Add(tParam.Name, propType);
            }

            return dataTable;
        }

        /// <summary>
        /// 根據TEntity可枚舉列表填充給定的數據表
        /// </summary>
        /// <param name="entities">TEntity類型的可枚舉列表</param>
        /// <param name="dataTable">數據表</param>
        public void FillDataTable(IEnumerable<TEntity> entities, DataTable dataTable)
        {
            if (entities != null && entities.Count() > 0)
            {
                foreach (TEntity entity in entities)
                {
                    FillDataTable(entity, dataTable);
                }
            }
        }

        /// <summary>
        /// 在DataTable中插入單條數據
        /// </summary>
        /// <param name="entity">具體數據</param>
        /// <param name="dataTable">數據表</param>
        public void FillDataTable(TEntity entity, DataTable dataTable)
        {
            var dataRow = dataTable.NewRow();
            int colNum = dataTable.Columns.Count;
            PropertyInfo[] typeArgs = GetPropertyFromTEntity();
            for (int i = 0; i < colNum; i++)
            {
                dataRow[i] = typeArgs[i].GetValue(entity);
            }
            dataTable.Rows.Add(dataRow);
        }

        /// <summary>
        /// 通過PostgreSQL連接把dataTable中的數據整塊填充到數據庫對應的數據表中
        /// 注意，該函數不負責NpgsqlConnection的創建、打開以及關閉
        /// </summary>
        /// <param name="conn">PostgreSQL連接</param>
        /// <param name="dataTable">數據表</param>
        public void BulkInsert(NpgsqlConnection conn, DataTable dataTable)
        {
            var commandFormat = string.Format(CultureInfo.InvariantCulture, "COPY {0} FROM STDIN BINARY", FullTableName);
            using (var writer = conn.BeginBinaryImport(commandFormat))
            {
                foreach (DataRow item in dataTable.Rows)
                    writer.WriteRow(item.ItemArray);
            }
        }
    }
}

運行結果如圖:

---