doris stream load 導入

官網地址：https://doris.apache.org/zh-CN/administrator-guide/load-data/stream-load-manual.html#基本原理

doris 可以通過insert into 語句插入單條，或者批量插入，但是正式環境不推薦。

通過http client 進行數據導入

1.curl 格式 ,這個格式也是嘗試了好多次 8030 是FE的http端口，8040是BE的http端口

注意文件的內容最後一樣不要有空行，光標不能在空行

 

密碼和用戶名，都是doris的用戶名和密碼，不是服務器的。

-T 後面是上傳文件的路基，加上後綴名。我看官網上沒有加，

-H label 保證每次不重複 多個參數的時候 一個個的寫

  //密碼爲fe的密碼
  curl --location-trusted -u root -T /home/label.csv -H "label:label12"  http://192.168.0.18:8030/api/example_db/table4/_stream_load  
  curl --location-trusted -u root -T /home/label.csv -H "label:label14" -H "column_separator:,"  http://192.168.0.18:8030/api/example_db/table3/_stream_load

 

2.http client  

   1.doc 文件

      FE的請求老是報 no valid Basic authorization  網上說的307 跳轉導致的 我這邊是直接調用的BE 的機器

 public static void StreamLoadBEDoc()
        {
            string result = "";
            string host = "192.168.0.74";//be
            int port = 8040;//be接口 fe8030
            string database = "example_db";
            string table = "table3";
            string user = "root";
            string passwd = "";
            string load_file_name = "E:/label.csv";
            string label = Guid.NewGuid().ToString();
            string loadUrl = string.Format("http://{0}:{1}/api/{2}/{3}/_stream_load", host, port, database, table);
            HttpWebRequest request = BuildRequestDoc(loadUrl, user, passwd, label);
            //文件數據
            FileStream rdr = new FileStream(load_file_name, FileMode.Open);
            request.ContentLength = rdr.Length;
            Stream reqStream = request.GetRequestStream();
            byte[] inData = new byte[rdr.Length];
            int bytesRead = rdr.Read(inData, 0, Convert.ToInt32(rdr.Length));
            reqStream.Write(inData, 0, Convert.ToInt32(rdr.Length));
            rdr.Close();
            WebResponse response = request.GetResponse();
            Stream stream = response.GetResponseStream();
            if (stream != null)
            {

                var reader = new StreamReader(stream, Encoding.UTF8);
                result = reader.ReadToEnd();
                stream.Dispose();
                stream = null;
                reader.Dispose();
                reader = null;
            }
            request.Abort();
            request = null;
            response.Dispose();
            response = null;
        }

 public static HttpWebRequest BuildRequestDoc(string url, string user, string passwd, string label)
        {
            HttpWebRequest client = (HttpWebRequest)WebRequest.Create(url);
            client.Headers.Set("Authorization", "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes($"{user}:{passwd}")));
            client.Headers.Set("Expect", "100-continue");//固定
            client.Headers.Set("label", label);//labid 保持唯一
            client.Headers.Set("column_separator", ",");//列分隔符
            client.Headers.Set("Content-Type", "text/plain; charset=UTF-8");
            client.Method = HttpMethod.Put.Method;//put請求

            client.ServicePoint.ConnectionLimit = int.MaxValue; //加上請求設置，提高效率
            client.ServicePoint.Expect100Continue = false;
            client.ServicePoint.UseNagleAlgorithm = false;
            client.AllowWriteStreamBuffering = false;
            client.Proxy = null; //不使用代理
            client.KeepAlive = false; //不建立持久性連接
            return client;
        }

  2.json 傳輸

     json 和doc 差不多

     

 public static void StreamLoadBEJson(string jsonData)
        {
            string result = "";
            string host = "192.168.0.74";//be
            int port = 8040;//be接口
            string database = "example_db";
            string table = "table3";
            string user = "root";
            string passwd = "";
            string label = Guid.NewGuid().ToString();
            string loadUrl = string.Format("http://{0}:{1}/api/{2}/{3}/_stream_load", host, port, database, table);
            HttpWebRequest request = BuildRequestJson(loadUrl, user, passwd, label);
            //文件數據
            Stream outstream;
            byte[] _buffer = Encoding.GetEncoding("utf-8").GetBytes(jsonData);
            outstream = request.GetRequestStream();
            outstream.Write(_buffer, 0, _buffer.Length);
            outstream.Close();

            WebResponse response = request.GetResponse();
            Stream stream = response.GetResponseStream();
            if (stream != null)
            {
                var reader = new StreamReader(stream, Encoding.UTF8);
                result = reader.ReadToEnd();
                stream.Dispose();
                stream = null;
                reader.Dispose();
                reader = null;
            }
            request.Abort();
            request = null;
            response.Dispose();
            response = null;
        }

 public static HttpWebRequest BuildRequestJson(string url, string user, string passwd, string label)
        {
            HttpWebRequest client = (HttpWebRequest)WebRequest.Create(url);
            client.Headers.Set("Authorization", "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes($"{user}:{passwd}")));
            client.Headers.Set("Expect", "100-continue");//固定
            client.Headers.Set("label", label);//labid 保持唯一
            client.Headers.Set("column_separator", ",");//列分隔符
            client.Headers.Set("Content-Type", "text/plain; charset=UTF-8");
            client.Method = HttpMethod.Put.Method;//put請求
            client.Headers.Set("format", "json"); //json數據
            client.Headers.Set("strip_outer_array", "true");//序列化是數組的json格式
            client.ServicePoint.ConnectionLimit = int.MaxValue; //加上請求設置，提高效率
            client.ServicePoint.Expect100Continue = false;
            client.ServicePoint.UseNagleAlgorithm = false;
            client.AllowWriteStreamBuffering = false;
            client.Proxy = null; //不使用代理
            client.KeepAlive = false; //不建立持久性連接
            return client;
        }

 

返回格式

{
    "TxnId": 1003,
    "Label": "b6f3bc78-0d2c-45d9-9e4c-faa0a0149bee",
    "Status": "Success",
    "ExistingJobStatus": "FINISHED", // optional
    "Message": "OK",
    "NumberTotalRows": 1000000,
    "NumberLoadedRows": 1000000,
    "NumberFilteredRows": 1,
    "NumberUnselectedRows": 0,
    "LoadBytes": 40888898,
    "LoadTimeMs": 2144,
    "BeginTxnTimeMs": 1,
    "StreamLoadPutTimeMs": 2,
    "ReadDataTimeMs": 325,
    "WriteDataTimeMs": 1933,
    "CommitAndPublishTimeMs": 106,
    "ErrorURL": "http://192.168.1.1:8042/api/_load_error_log?file=__shard_0/error_log_insert_stmt_db18266d4d9b4ee5-abb00ddd64bdf005_db18266d4d9b4ee5_abb00ddd64bdf005"
}

status :success 就成功了

 

唯一遺憾的是FE的鏈接沒有測試通過。不知道有沒有老鐵解決了。