.NET Core 3.0控制檯實現數據同步

寫作背景：應工作環境中存在一個數據庫實例多站點部署模式，每次同步數據都需要手動從本地導入目標站點數據庫，空餘之際寫了個簡單Demo;

技術點或Nuget元包：

.NET Core 3.0 Console;
Microsoft.Data.SqlClient -v 1.0.19269.1;
開發工具VS 2019 Pro x64 v16.3.3;
MS-SQLServer 2014 Enterprise;

實現目標：本地開發環境的正式數據同步到線上目標數據庫（單表全字段同結構模式）；

這裏使用本地環境同數據庫的不同實例模擬實現，假如數據庫【TestDB】爲源數據庫，【TestDB2】爲目標數據庫，都有共同的數據表對象【TestTB】；

Demo項目結構：

1.DHHelper封裝：基於Microsoft.Data.SqlClient 簡單的實現了所需的幾個方法，如下所示：

1.1 MsSqlHelper.cs 類實例代碼：

using System;
using System.Collections.Generic;
using System.Text;
using System.Data;
using Microsoft.Data.SqlTypes;
using Microsoft.Data.SqlClient;
using System.Reflection;

namespace DBHelper
{
    public sealed class MsSqlHelper
    {
        #region 單利模式
        /// <summary>
        /// 1.構造函數私有化
        /// </summary>
        private MsSqlHelper() { }

        /// <summary>
        /// 2.創建私有化靜態字段鎖
        /// </summary>
        private static readonly object _ObjLock = new object();

        /// <summary>
        /// 3.創建私有化類對象，接收類的實例化對象
        /// volatile 關鍵字促進線程安全，保障線程有序執行
        /// </summary>
        private static volatile MsSqlHelper _MsSqlHelper = null;

        /// <summary>
        /// 4.創建類實例化對象
        /// </summary>
        /// <returns></returns>
        public static MsSqlHelper GetSingleObj()
        {
            if (_MsSqlHelper == null)
            {
                lock (_ObjLock) //保證只有一線程操作
                {
                    if (_MsSqlHelper == null)
                    {
                        _MsSqlHelper = new MsSqlHelper();
                    }
                }
            }
            return _MsSqlHelper;
        }
        #endregion

        // 數據庫鏈接字符串
        private string _ConnString { get; set; }

        /// <summary>
        /// 初始化指定數據庫橋接字符串
        /// </summary>
        /// <param name="sqlConnStrBuilder">數據庫連接對象</param>
        public void RegisterConn(Connection conn)
        {
            var builder = new SqlConnectionStringBuilder
            {
                DataSource = conn.DataSource, //your_server.database.windows.net
                UserID = conn.UserID, //your_user  
                Password = conn.Password, //your_password  
                InitialCatalog = conn.InitialCatalog //your_database  
            };
            _ConnString = builder.ConnectionString;
        }

        #region 單個數據（添加，更新，刪除）
        /// <summary>  
        /// 執行SQL語句，返回影響的記錄數  
        /// Insert插入,Delete刪除,Update更新  
        /// </summary>  
        /// <param name="cmdText">sql語句</param>  
        /// <param name="sqlParams">[可選]sql參數化</param>  
        /// <returns>int：受影響的行數</returns>  
        public int ExecNonQuery(string cmdText, params SqlParameter[] sqlParams)
        {
            int rowsCount = 0;
            using (SqlConnection conn = new SqlConnection(_ConnString))  // 建立數據庫連接對象  
            {
                OpenConnection(conn);
                using (SqlCommand cmd = conn.CreateCommand())
                {
                    cmd.CommandType = CommandType.Text; //指定cmd命令類型爲文本類型(默認，可不寫);  
                    cmd.CommandText = cmdText; //sql語句或存儲過程                         
                    if (sqlParams != null && sqlParams.Length > 0) //檢查參數組是否有數據  
                    {
                        foreach (SqlParameter sqlParam in sqlParams)
                        {
                            //判斷參數是否爲null，是則轉爲數據庫接受的DBnull
                            if (sqlParam.Value == null)
                            {
                                sqlParam.Value = DBNull.Value;
                            }
                            cmd.Parameters.Add(sqlParam); //參數格式化，防止sql注入  
                        }
                    }
                    rowsCount = cmd.ExecuteNonQuery(); //執行非查詢命令，接收受影響行數，大於0的話表示添加成功  
                    cmd.Parameters.Clear();
                }
                CloseConnection(conn);
            }
            return rowsCount;
        }
        #endregion

        #region 查詢操作
        /// <summary>  
        /// 返回數據庫表DataTable  
        /// </summary>  
        /// <param name="cmdText">sql語句</param>  
        /// <param name="sqlParams">sql參數化</param>  
        /// <returns>DataTable</returns>  
        public DataTable GetDataTable(string cmdText, params SqlParameter[] sqlParams)
        {
            using (DataTable dt = new DataTable())
            {
                using (SqlConnection conn = new SqlConnection(_ConnString))
                {
                    using (SqlCommand cmd = conn.CreateCommand())
                    {
                        cmd.CommandText = cmdText;
                        if (sqlParams != null && sqlParams.Length > 0)
                        {
                            foreach (SqlParameter parameter in sqlParams)
                            {
                                //判斷參數是否爲null，是則轉爲數據庫接受的DBnull
                                if (parameter.Value == null)
                                {
                                    parameter.Value = DBNull.Value;
                                }
                                //參數格式化，防止sql注入  
                                cmd.Parameters.Add(parameter);
                            }
                        }
                        //適配器自動打開數據庫連接  
                        using (SqlDataAdapter da = new SqlDataAdapter(cmd))
                        {
                            da.Fill(dt);
                        }
                        cmd.Parameters.Clear();
                    }
                }
                return dt;
            }
        }

        /// <summary>
        /// 執行多sql語句，返回數據集
        /// </summary>
        /// <param name="sqlTuples">list：tabNames[可選],sql,SqlParameter[]</param>
        /// <returns>DataSet</returns>
        public DataSet GetDataSet(List<Tuple<string, string, SqlParameter[]>> sqlTuples)
        {
            using (DataSet ds = new DataSet())
            {
                string tabName = string.Empty; //tab名稱
                string cmdText = string.Empty; //sql語句   
                SqlParameter[] sqlParams = null;//sql參數化  

                if (sqlTuples != null && sqlTuples.Count > 0)
                {
                    foreach (var tuple in sqlTuples)
                    {
                        tabName = tuple.Item1; //tab名稱
                        cmdText = tuple.Item2; //sql語句   
                        sqlParams = tuple.Item3;//sql格式化參數
                        using (SqlConnection conn = new SqlConnection(_ConnString))
                        {
                            using (SqlCommand cmd = conn.CreateCommand())
                            {
                                cmd.CommandText = cmdText;
                                //檢查參數組是否有數據  
                                if (sqlParams != null && sqlParams.Length > 0)
                                {
                                    //cmd.Parameters.AddRange(sqlParams);
                                    foreach (SqlParameter parameter in sqlParams)
                                    {
                                        //判斷參數是否爲null，是則轉爲數據庫接受的DBnull
                                        if (parameter.Value == null)
                                        {
                                            parameter.Value = DBNull.Value;
                                        }
                                        //參數格式化，防止sql注入  
                                        cmd.Parameters.Add(parameter);
                                    }
                                }

                                //適配器自動打開數據庫連接  
                                using (SqlDataAdapter da = new SqlDataAdapter(cmd))
                                {
                                    if (string.IsNullOrWhiteSpace(tabName))
                                        da.Fill(ds);
                                    else
                                        da.Fill(ds, tabName); // 將tabName查詢結果集合填入DataSet中，並且將DataTable命名爲tabName  
                                }
                                cmd.Parameters.Clear();
                            }
                        }
                    }
                }
                return ds;
            }
        }
        #endregion

        #region 批量添加數據
        /// <summary>
        /// 批量插入數據【INSERT INTO [TABLE] VALUES】，並返回受影響的行數  
        /// </summary>
        /// <typeparam name="T"></typeparam>
        /// <param name="keyValuePair"></param>
        /// <returns></returns>
        public int InsertValuesToDB<T>(KeyValuePair<string,List<T>> keyValuePair) 
        {
            string sql = $"INSERT INTO [{keyValuePair.Key}] @NAME VALUES @VALUES;";
            var dicList = new List<Dictionary<string, string>>();
            foreach (var item in keyValuePair.Value)
            {
                dicList.Add(GetProperties(item));
            }

            var name = new StringBuilder();
            var values = new StringBuilder();
            for (int i = 0; i < dicList.Count; i++)
            {
                var dic = dicList[i];
                int columns = 0;
                var tmpValue = new StringBuilder();
                foreach (var item in dic)
                {
                    if (i == 0)
                    {
                        if (columns < dic.Count -1)
                        {
                            name.Append($"[{item.Key}],");
                            tmpValue.Append($"'{item.Value}',");
                        }
                        else
                        {
                            name.Append($"[{item.Key}]");
                            tmpValue.Append($"'{item.Value}'");
                        }
                    }
                    else if(i >0 && i < dic.Count - 1)
                    {
                        if (columns < dic.Count - 1)
                        {
                            tmpValue.Append($"'{item.Value}',");
                        }
                        else
                        {
                            tmpValue.Append($"'{item.Value}'");
                        }
                    }
                    else
                    {
                        if (columns < dic.Count - 1)
                        {
                            tmpValue.Append($"'{item.Value}',");
                        }
                        else
                        {
                            tmpValue.Append($"'{item.Value}'");
                        }
                    }
                    columns++;
                }
                if (i == 0)
                {
                    sql = sql.Replace("@NAME", $"({name.ToString()})");
                }
                if (i < dicList.Count - 1)
                {
                    values.AppendLine($"({tmpValue.ToString()}),");
                }
                else
                {
                    values.Append($"({tmpValue.ToString()})");
                }
            }
            sql = sql.Replace("@VALUES", values.ToString());
            return ExecNonQuery(sql);
        }

        /// <summary>
        /// 反射得到實體類(泛型T)的字段名稱和值
        /// </summary>
        /// <typeparam name="T"></typeparam>
        /// <param name="t"></param>
        /// <returns></returns>
        private Dictionary<string, string> GetProperties<T>(T t)
        {
            var ret = new Dictionary<string, string>();
            if (t == null)
                return null;

            PropertyInfo[] properties = t.GetType().GetProperties(BindingFlags.Instance | BindingFlags.Public);
            if (properties.Length <= 0)
                return null;

            foreach (PropertyInfo item in properties)
            {
                string name = item.Name; //實體類字段名稱
                string value = Convert.ToString(item.GetValue(t, null)); //該字段的值
                //Type type = item.PropertyType; //獲取值value的類型
                if (item.PropertyType.IsValueType || item.PropertyType.Name.StartsWith("String"))
                {
                    ret.Add(name, value); //在此可轉換value的類型
                }
            }
            return ret;
        }


        /// <summary>  
        /// 1.批量插入數據【Bulk】，並返回受影響的行數  
        /// </summary>  
        /// <param name="dt">DataTable</param> 
        /// <param name="dtName">表名稱</param>  
        /// <returns>int：受影響的行數</returns>  
        public int InsertBulkToDB(DataTable dt, string dtName)
        {
            string tableName = dtName ?? dt.TableName;
            int rowsCount = 0;
            using (SqlConnection conn = new SqlConnection(_ConnString))
            {
                OpenConnection(conn);
                using (SqlBulkCopy bulkCopy = new SqlBulkCopy(conn))
                {
                    if (dt != null && dt.Rows.Count != 0)
                    {
                        bulkCopy.DestinationTableName = tableName; //數據表名稱  
                        bulkCopy.BatchSize = dt.Rows.Count;
                        bulkCopy.WriteToServer(dt);
                        rowsCount = bulkCopy.BatchSize;
                    }
                    CloseConnection(conn);
                }
            }
            return rowsCount;
        }
        /// <summary>  
        /// 2.批量插入數據【Bulk】(加入內部事務)，並返回受影響的行數  
        /// </summary>  
        /// <param name="dt">DataTable</param> 
        /// <param name="dtName">表名稱</param>  
        /// <returns>int：受影響的行數</returns>  
        public int InsertBulkToDBByTransaction(DataTable dt, string dtName)
        {
            SqlTransaction transaction = null; // 創建事務對象  
            try
            {
                string tableName = dtName ?? dt.TableName;
                int rowsCount = 0;
                using (SqlConnection conn = new SqlConnection(_ConnString))
                {
                    OpenConnection(conn);
                    using (SqlBulkCopy bulkCopy = new SqlBulkCopy(_ConnString, SqlBulkCopyOptions.UseInternalTransaction))
                    {
                        if (dt != null && dt.Rows.Count != 0)
                        {
                            bulkCopy.DestinationTableName = tableName; //數據表名稱  
                            bulkCopy.BatchSize = dt.Rows.Count;
                            bulkCopy.WriteToServer(dt);
                            rowsCount = bulkCopy.BatchSize;
                        }
                        CloseConnection(conn);
                    }
                }
                return rowsCount;
            }
            catch (Exception ex)
            {
                transaction.Rollback(); //事務回滾  
                throw new Exception(ex.Message, ex);
            }
        }
        #endregion

        #region 開啓連接SqlConnection.Open  
        /// <summary>  
        /// 打開OracleConnection  
        /// </summary>  
        /// <param name="conn">數據庫連接對象</param>  
        private static void OpenConnection(SqlConnection conn)
        {
            try
            {
                if (conn.State == ConnectionState.Closed) conn.Open();
            }
            catch (Exception ex)
            {
                throw new Exception(ex.Message, null);
            }
        }
        #endregion

        #region 關閉連接，釋放資源  
        /// <summary>  
        /// 關閉Connection  
        /// </summary>  
        /// <param name="conn">數據庫(Oracle)連接對象</param>  
        private static void CloseConnection(SqlConnection conn)
        {
            if (conn.State == ConnectionState.Open)
            {
                conn.Close();
                conn.Dispose();//釋放資源  
            }
        }

        /// <summary>  
        /// 關閉DataReader  
        /// </summary>  
        /// <param name="dataReader">數據讀取器對象</param>  
        private static void CloseDataReader(SqlDataReader dataReader)
        {
            if (dataReader.IsClosed == false) dataReader.Close();
        }
        #endregion
    }
}

1.2 Connection.cs 類實例代碼：

using System;
using System.Collections.Generic;
using System.Text;

namespace DBHelper
{
    /// <summary>
    /// 數據庫橋接對象模型
    /// </summary>
    public class Connection
    {
        /// <summary>
        /// 數據庫IP
        /// </summary>
        public string DataSource { get; set; }
        /// <summary>
        /// 數據庫授權賬戶
        /// </summary>
        public string UserID { get; set; }
        /// <summary>
        /// 數據庫訪問密碼
        /// </summary>
        public string Password { get; set; }
        /// <summary>
        /// 數據庫名稱
        /// </summary>
        public string InitialCatalog { get; set; } 
    }
}

2.DataSyncHelper 控制檯調用測試：

using DBHelper;
using System;
using System.Collections.Generic;
using Microsoft.Data.SqlClient;
using System.Data;

namespace DataSyncHelper
{
    class Program
    {
        static void Main(string[] args)
        {
            Console.WriteLine("Hello World!");

            //源數據庫(本地)連接
            var conn = new Connection
            {
                DataSource = "192.168.10.228",
                UserID = "sa",
                Password = "123456",
                InitialCatalog = "TestDB"
            };

            //目標數據庫連接（多個目標數據庫添加List對象即可）
            var connList = new List<Connection> 
            {
                new Connection
                {
                    DataSource = "192.168.10.228",
                    UserID = "sa",
                    Password = "123456",
                    InitialCatalog = "TestDB2"
                }
            };


            Test1(conn);

            Test2(conn, connList);

        }

        /// <summary>
        /// 模擬數據庫批量插入數據【INSERT INTO [TABLE] VALUES】
        /// </summary>
        /// <param name="conn">源數據庫橋接對象</param>
        static void Test1(Connection conn) 
        {
            var ml = new List<TestTb>(); //模擬10條數據
            for (int i = 0; i < 10; i++)
            {
                ml.Add(new TestTb
                {
                    Id = Guid.NewGuid().ToString("N"),
                    Name = $"張三{i}",
                    No = i,
                    Birthday = DateTime.Now,
                    Remark = $"張三{i}"
                });
            }
            var keyValuePair = new KeyValuePair<string, List<TestTb>>("TestTb", ml);
            MsSqlHelper.GetSingleObj().RegisterConn(conn); //註冊conn橋接對象
            MsSqlHelper.GetSingleObj().InsertValuesToDB(keyValuePair);
        }


        /// <summary>
        /// 同步數據(一主多從)
        /// </summary>
        /// <param name="conn">源數據庫橋接對象</param>
        /// <param name="connList">目標數據庫橋接對象</param>
        static void Test2(Connection conn, List<Connection> connList) 
        {
            #region 源數據庫(本地)連接
            MsSqlHelper.GetSingleObj().RegisterConn(conn); //註冊conn橋接對象
            string sql = "SELECT * FROM [TestTb];";
            var dt = MsSqlHelper.GetSingleObj().GetDataTable(sql);
            dt.TableName = "TestTb";
            #endregion

            #region 目標數據庫連接（Demo演示中暫時只有一個）
            var dtDic = new Dictionary<Connection, DataTable>(); //原始數據集（保留原始表數據）
            foreach (var item in connList)
            {
                MsSqlHelper.GetSingleObj().RegisterConn(item); //註冊目標conn橋接對象
                var oldDt = MsSqlHelper.GetSingleObj().GetDataTable(sql); //查詢歷史數據
                dtDic.Add(item, oldDt);
                string delSql = "DELETE FROM [TestTb];"; //全表刪除歷史數據
                int rcount = MsSqlHelper.GetSingleObj().ExecNonQuery(delSql);
                int newRCount = MsSqlHelper.GetSingleObj().InsertBulkToDB(dt, dt.TableName);
            }
            #endregion
        }

    }

    /// <summary>
    /// 原始表模型
    /// </summary>
    public class TestTb 
    {
        public string Id { get; set; }
        public string Name { get; set; }
        public int? No { get; set; } 
        public DateTime? Birthday { get; set; } 
        public string Remark { get; set; }
    }
}

3.測試結果：

3.1 首先給源數據庫【TestDB】模擬10條數據 =》 Test1方法，執行結果如下：

3.2 同步目標數據庫【TestDB2】中的【TestTB】對象 =》 Test2方法，執行結果如下：

以上Demo演示完畢，可根據自己的實際情況修改調整。

.NET Core 3.0控制檯實現數據同步

Spring Cloud 部署時如何使用 Kubernetes 作爲註冊中心和配置中心

asp.net core 3.1學習筆記——DI依賴注入的幾種方式

Ionic 基本元素（ion-list）

Angular 組件通訊（@Input，@ViewChild）

Angular表單數據模型之雙向綁定（MVVM）實現用戶註冊

asp.net core 3.1學習筆記——作用域與對象的釋放行爲

https://yachay.unat.edu.pe/blog/index.php?comment_area=format_blog&comment_component=blog&comment_co

linux以太網驅動總結