C#批量增量更新數據

方法一：建立臨時表進行數據更新和插入

Step1 創建臨時表

/// <summary>
/// 創建臨時表存儲當前需要提交的數據
/// </summary>
/// <param name="databaseConnectionString">數據庫連接字符串</param>
/// <param name="tableName">當前更新導入的表名</param>
/// <returns></returns>
public static string CreateTempTable(string databaseConnectionString, string tableName)
{
    string tempTableName = string.Empty;
    string sql = string.Empty;
    //創建臨時表存儲當前需要提交的數據
    for (int i = 0; i < 20; i++)  //只是設定一個臨時表的序號，如果是多臺主機導入同一個數據庫的時候，在數據庫中可能會創建同名的臨時表
    {
        sql = string.Format("SELECT * INTO {0} FROM {1} where 1=2", tableName + "_TEMP" + i, tableName);
        try
        {
            ExecuteQuery(sql, databaseConnectionString);
            tempTableName = tableName + "_TEMP" + i;
            break;
        }
        catch //如果在創建當前的臨時表的過程中發生了錯誤，如當前臨時表已經存在或者被佔用，則繼續通過 序號i去創建下一張臨時表，直到創建成功爲止
        {
            continue;
        }
    }
    return tempTableName;
}

Step2 臨時表和數據庫數據進行對比，提取數據庫表中不存在的數據,然後插入當前數據庫表中不存在的數據

/// <summary>
/// 將臨時表和當前需要更新的表的數據進行對比，根據表內的唯一標識提取出在當前表中不存在的數據，然後批量插入新增的數據
/// </summary>
/// <param name="databaseConnectionString">數據庫連接字符串</param>
/// /// <param name="dt">更新需要提交數據的datatable</param>
/// <param name="tableName">當前更新導入的表名</param>
/// <param name="tempTableName">臨時表表名</param>
/// <returns></returns>
public static string GetUpdateData(string databaseConnectionString, DataTable dt, string tableName, string tempTableName)
{
    string errMess = string.Empty;
    try
    {
        string sql = string.Empty;
        string found = string.Empty;
        List<string> keyList = new List<string>();
        //提取需要批量插入的數據的唯一標識列表
        sql = string.Format("select key from {0}  except select key from {1}", tempTableName, tableName);
        found = string.Format("key= '{0}'", "@key");  //生成在當前的datatable查找的語句，key爲查找的關鍵字，當前使用@key來代替所需的關鍵字
        #region 批量插入數據庫中不存在的數據

        keyList = new List<string>();  //用於存儲當前查詢的唯一標識中在臨時表中存在而在數據庫表中不存在的數據列表
        keyList = GetSqlAsString(sql, null, databaseConnectionString);
        DataTable appendData = dt.Clone();
        for (int i = 0; i < keyList.Count; i++)
        {
            DataRow[] dr = dt.Select(found.Replace("@key", keyList[i]));
            foreach (var row in dr)
            {
                appendData.ImportRow(row);
            }
        }
        SqlBulkCopyByDatatable(databaseConnectionString, tableName, appendData);
        #endregion
        return errMess;
    }
    catch (Exception ex)
    {
        errMess = ex.Message;
        return errMess;
    }
}

Step3 根據臨時表更新當前導入表的數據

/// <summary>
/// 根據臨時表更新當前導入表的數據
/// </summary>
/// <param name="databaseConnectionString">數據庫連接字符串</param>
/// <param name="tableName">當前更新導入的表名</param>
/// <param name="tempTableName">臨時表表名</param>
/// <returns></returns>
public static string UpdateTableData(string databaseConnectionString, string tableName, string tempTableName)
{
    string errMess = string.Empty;
    try
    {
        var filedList = GetFileds(databaseConnectionString, tableName);  //獲取當前操作表的字段信息
        StringBuilder sql = new StringBuilder();  //構造更新當前表的所有字段的數據信息的更新語句
        sql.Append(string.Format("update {0} set ", tableName));
        foreach (var filed in filedList)
        {
            sql.Append(string.Format("{0} = {1}.{2},", filed, tempTableName, filed));
        }
        sql.Append("@");
        sql.Replace(",@", " ");
        sql.Append(string.Format("from {0} where tableName.key= {1}.key", tempTableName, tempTableName));  //利用唯一標識更新數據
        ExecuteQuery(sql.ToString(), databaseConnectionString);
        return errMess;
    }
    catch (Exception ex)
    {
        errMess = ex.Message;
        return errMess;
    }
}

Step4 刪除臨時表

/// <summary>
/// 刪除臨時表
/// </summary>
/// <param name="databaseConnectionString">數據庫連接字符串</param>
/// <param name="tempTableName">需要刪除的臨時表表名</param>
/// <returns></returns>
public static string DropTempTable(string databaseConnectionString, string tempTableName)
{
    string errMess = string.Empty;
    try
    {
        string sql = string.Empty;
        sql = string.Format("drop table {0}", tempTableName);
        ExecuteQuery(sql, databaseConnectionString);
        return errMess;
    }
    catch (Exception ex)
    {
        errMess = ex.Message;
        return errMess;
    }
}

附：在增量更新中用到的方法

/// <summary>
/// 獲取當前操作表的所有字段名
/// </summary>
/// <param name="connectionString">數據庫連接字符串</param>
/// <param name="tableName">當前操作的表名</param>
/// <returns></returns>
private static List<string> GetFileds(string databaseConnectionString, string tableName)
{
    List<string> _Fields = new List<string>();
    SqlConnection _Connection = new SqlConnection(databaseConnectionString);
    try
    {
        _Connection.Open();
        string key = GetTableKey(databaseConnectionString, tableName);
        string[] restrictionValues = new string[4];
        restrictionValues[0] = null; // Catalog
        restrictionValues[1] = null; // Owner
        restrictionValues[2] = tableName; // Table
        restrictionValues[3] = null; // Column

        using (DataTable dt = _Connection.GetSchema(SqlClientMetaDataCollectionNames.Columns, restrictionValues))
        {
            foreach (DataRow dr in dt.Rows)
            {
                var filedName = dr["column_name"].ToString();
                if (!filedName.Equals(key))  //不將當前表的主鍵添加進去
                    _Fields.Add(filedName);
            }
        }
    }
    catch (Exception ex)
    {
        throw ex;
    }
    finally
    {
        _Connection.Dispose();
    }
    return _Fields;
}

/// <summary>
/// 根據表名獲取主鍵字段
/// </summary>
/// <param name="databaseConnectionString">數據庫連接字符串</param>
/// <param name="TableName">表名</param>
/// <returns>主鍵字段</returns>
public static string GetTableKey(string databaseConnectionString, string TableName)
{
    try
    {
        if (TableName == "")
            return null;
        StringBuilder sb = new StringBuilder();
        sb.Append("SELECT COLUMN_NAME ");
        sb.Append("FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE ");
        sb.Append("WHERE TABLE_NAME=");
        sb.Append("'" + TableName + "'");
        string key = string.Empty;
        var keyList = GetSqlAsString(sb.ToString(), null, databaseConnectionString);
        if (keyList.Count > 0)
            key = keyList[0];
        return key;
    }
    catch
    {
        return null;
    }
}


/// <summary>
/// 數據批量插入方法
/// </summary>
/// <param name="connectionString">目標連接字符</param>
/// <param name="TableName">目標表</param>
/// <param name="dt">源數據</param>
public static void SqlBulkCopyByDatatable(string connectionString, string TableName, DataTable dt)
{
    using (var conn = new SqlConnection(connectionString))
    using (var sqlbulkcopy = new SqlBulkCopy(connectionString, SqlBulkCopyOptions.UseInternalTransaction))
    {
        try
        {
            sqlbulkcopy.BulkCopyTimeout = 600;
            sqlbulkcopy.DestinationTableName = TableName;
            for (int i = 0; i < dt.Columns.Count; i++)
            {
                sqlbulkcopy.ColumnMappings.Add(dt.Columns[i].ColumnName, dt.Columns[i].ColumnName);
            }
            sqlbulkcopy.WriteToServer(dt);
        }
        catch (System.Exception ex)
        {
            throw ex;
        }
    }
}

/// <summary>
/// 以字符串列表的方式返回查詢的結果集
/// </summary>
/// <param name="sqlText">查詢語句</param>
/// <param name="sqlParameters">事物</param>
/// <param name="databaseConnectionString">數據庫連接字符串</param>
/// <returns></returns>
static public List<string> GetSqlAsString(string sqlText, SqlParameter[] sqlParameters, string databaseConnectionString)
{
    List<string> result = new List<string>();
    SqlDataReader reader;
    SqlConnection connection = new SqlConnection(databaseConnectionString);
    using (connection)
    {
        SqlCommand sqlcommand = connection.CreateCommand();
        sqlcommand.CommandText = sqlText;
        if (sqlParameters != null)
        {
            sqlcommand.Parameters.AddRange(sqlParameters);
        }
        connection.Open();
        reader = sqlcommand.ExecuteReader();
        if (reader != null)
        {
            while (reader.Read())
            {
                var re = reader.GetString(0);
                if (!string.IsNullOrEmpty(re))
                    result.Add(re);
            }
        }
    }
    return result;
}

/// <summary>
/// 執行查詢結果
/// </summary>
/// <param name="sql">執行語句</param>
/// <param name="databaseConnectionString">數據庫連接字符串</param>
public static void ExecuteQuery(string sql, string databaseConnectionString)
{
    SqlConnection conn = new SqlConnection(databaseConnectionString);
    SqlCommand cmd = new SqlCommand(sql, conn);
    conn.Open();
    cmd.ExecuteNonQuery();
    conn.Close();
}

主體方法調用：

/// <summary>
/// 增量更新
/// </summary>
/// <param name="dt">更新數據表datatable形式</param>
/// <param name="tableName">更新的數據庫表名</param>
/// <returns></returns>
protected string UpdateExistData(DataTable dt, string tableName)
{
    string databaseConnectionString = string.Empty;
    databaseConnectionString = "server=GLASSESONION;uid=sa;pwd=;database=TJPTTC"; //初始化數據庫連接字符串
    string err = string.Empty;
    string tempTableName = string.Empty;
    //創建臨時表
    tempTableName = IncrementalUpdating.CreateTempTable(databaseConnectionString, tableName);
    //將當前數據導入到臨時表中
    IncrementalUpdating.SqlBulkCopyByDatatable(databaseConnectionString, tempTableName, dt);
    //更新數據庫，導入數據庫中不存在的數據
    err += IncrementalUpdating.GetUpdateData(databaseConnectionString, dt, tableName, tempTableName);
    //更新數據庫，更新當前數據庫已經存在的數據
    err += IncrementalUpdating.UpdateTableData(databaseConnectionString, tableName, tempTableName);
    err += IncrementalUpdating.DropTempTable(databaseConnectionString, tempTableName);
    return err;
}

PS:批量導入7個多g的數據大約花費時間一分二十多秒，批量增量更新導入只比批量導入花費時間多了十幾秒，速度很可觀。

參考：http://www.mamicode.com/info-detail-2239128.html
https://www.cnblogs.com/two/p/5224656.html
https://blog.csdn.net/mxw_133/article/details/79321710

方法二：利用存儲過程進行數據更新和插入

Step1:在數據庫中建立存儲過程

--聲明數據庫引用
use [databaseName];
go

--判斷是否存在存儲過程，如果存在則刪除
if exists(select * from sys.procedures where name='updateData')
drop procedure updateData;
go

--創建存儲過程
create procedure updateData
@p_Xml xml
as 
set nocount on 

begin 

with b as 
( 
select 
QueryExpression.Criteria.value('filed1[1]', 'nvarchar(255)') as filed1, 
QueryExpression.Criteria.value('filed2[1]', 'nvarchar(38)') as filed2, 
QueryExpression.Criteria.value('filed3[1]', 'nvarchar(14)') as filed3
from @p_Xml.nodes('/DocumentElement/Key') QueryExpression(Criteria) 
) 
-- Update 
merge into databaseName.dbo.tableName a
using b on a.keyed=b.keyed
when matched then
update 
set 
a.filed1=b.filed1,
a.filed2=b.filed2,
a.filed3=b.filed3
--INSERT
when not matched then
insert(filed1,filed2,filed3)
values(b.filed1,b.filed2,b.filed3);

end 
return @@error
go

Step 2 在代碼中調用存儲過程

/// <summary>
/// 調用存儲過程進行增量更新
/// </summary>
/// <param name="dt">更新數據集 datatable形式</param>
/// <param name="databaseConnectionString">數據庫連接字符串</param>
public static void AddFlagDescription(DataTable dt, string databaseConnectionString)
{
    try
    {
        SqlParameter[] paramArr = { new SqlParameter() { ParameterName = "@p_Xml", Value = ConvertDataTableToXML(dt) } };
        using (SqlConnection conn = new SqlConnection(databaseConnectionString))
        {
            conn.Open();
            using (SqlCommand cmd = conn.CreateCommand())
            {
                cmd.CommandType = CommandType.StoredProcedure;
                cmd.CommandText = "updateData";
                cmd.Parameters.AddRange(paramArr);
                cmd.ExecuteNonQuery();
            }
        }
    }
    catch (Exception ex)
    {
        throw ex;
    }
}
/// <summary>
/// 傳入需要更新數據集進行序列化
/// </summary>
/// <param name="dtKeys">返回序列化結果</param>
/// <returns></returns>
public static string ConvertDataTableToXML(DataTable dtKeys)
{
    dtKeys.TableName = "Key";
    System.IO.StringWriter tw = new System.IO.StringWriter();
    dtKeys.WriteXml(tw);
    return tw.ToString();
}

PS:這種方法沒有驗證完全，因爲之前做這個工作的時候需要導入Geometry類型的數據，存儲過程中在進行這種數據類型插入數據的時候總是會出現不可預計的錯誤，所以才另尋它法，但是這種方式對於非空間數據的批量更新支持還是良好的。

參考：https://www.cnblogs.com/mibing/p/5853302.html