Angel在TI-ONE機器學習平臺上的應用

{"type":"doc","content":[{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"本文將着重介紹開發者如何在騰訊雲智能TI平臺上快速的搭建Angel任務模型，快速的落地自己的業務場景。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"分爲以下三個主題："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"智能鈦機器學習平臺(TI-ONE)介紹，介紹一下解決的問題和搭建使用的流程"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Angel on TI-ONE，介紹TI平臺對Angel的支持"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"快速搭建一個圖算法任務流,具體實例演示圖算法搭建"}]}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"什麼是TI-ONE"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"智能鈦機器學習平臺 ( TI-ONE ) 是爲 AI 工程師打造的一站式機器學習服務平臺，爲用戶提供從數據預處理、模型構建、模型訓練到模型評估的全流程開發支持。智能鈦機器學習平臺內置豐富的算法組件，支持多種算法框架，滿足AI從業者各種應用場景的需求，包括今天分享的主題，對Angel的支持。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"對一個AI從業者來說，TI-ONE平臺提供給其的價值能夠匹配其所期望的搭建機器學習算法模型而面臨的困境。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"算法AI、算法工程師在搭建模型時需要考慮的問題非常多，例如對個人開發者而言："}]},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"足夠的GPU資源難以獲得；"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"由於框架日新月異，維護各種算法框架的繁瑣性高；"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"機器學習和深度學習的算法學習和搭建的門檻很高；"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"調參和對比算法效果費時費力；"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"實際業務複雜多變，版本更迭需要加快；"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"總之，由於上述各種原因，上線產品變成了費時且成本高昂的過程，TI-ONE就是爲了針對的解決問題，由此應運而生。智能鈦機器學習平臺 ( TI-ONE ) 是爲 AI 工程師打造的一站式機器學習服務平臺，爲用戶提供從數據預處理、模型構建、模型訓練到模型評估的全流程開發支持。智能鈦機器學習平臺內置豐富的算法組件，支持多種算法框架，滿足AI從業者各種應用場景的需求，包括今天分享的主題，對Angel的支持。"}]},{"type":"image","attrs":{"src":"https:\/\/static001.geekbang.org\/infoq\/e5\/e55c967ab42a813914dab549dc85bf67.jpeg","alt":"圖片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"針對上面的問題，TI-ONE平臺提供了下列解決方案："}]},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"算力即購即用，可針對單次任務購買算力資源，多餘算力隨時隨退；"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"拖拽式任務設計流程，自由根據任務繪製任務流；"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"集成了業界常見機器（深度）學習框架，例如pytorch，tensorflow，pyspark，Angel等等，並進行了優化，縮短了用戶訓練調試時間；"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"內置了業界常見的算法框架，例如CNN，RNN，LPA，聚類，可視化等等，大大降低了AI應用落地的門檻，能夠讓更多的從業者享受AI帶來的紅利；"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"運行模式靈活，支持手工、定時、批量參數和重跑等模式，確保參數調整的即快又好；"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"支持一鍵部署，便於對外提供服務；"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"集成Notebook交互式建模十分便捷，助力開發者進行建模探索。"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"此外，我們也提供一些開源數據集資源，利於快速建模。"}]},{"type":"image","attrs":{"src":"https:\/\/static001.geekbang.org\/infoq\/5a\/5a9abeedd976b8fd9d0c5b2d9c4bc720.jpeg","alt":"圖片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"可視化建模"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"這裏着重介紹可視化建模部分畫布："}]},{"type":"image","attrs":{"src":"https:\/\/static001.geekbang.org\/infoq\/93\/9302548c3800b4de1110469ab21e06a9.jpeg","alt":"圖片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如此圖所示，從左到右依次是算法框架組件，用戶可以選用自己所需要的算法模塊，包括輸入，框架，可視化等等；中間畫布是可視化畫布圖，用戶將左側選中的算法模塊拖入，平臺會幫助用戶自動連線生成流程圖，用戶如果不滿意，也可以自主修改流程圖步驟；每點擊一箇中間畫布的算法模塊，右側會生成菜單欄，包括此算法模塊所有可以自定義的參數；最後用戶只需要運行此流程圖，平臺就會根據流程圖產生相應的結果，我們也可以看到運行日誌和一些中間結果數據等等。"}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"Angel on TI-ONE"}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"1. TI-ONE內置的Angel算法"}]},{"type":"image","attrs":{"src":"https:\/\/static001.geekbang.org\/infoq\/2b\/2b8da1589eaedb88e6a596cab824f63c.jpeg","alt":"圖片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"TI-ONE平臺中含有兩種Angel算法模塊，一是Spark on Angel框架，二是Angel算法組件。前者是平臺內置框架，運行用戶自定義的代碼；後者是平臺提供的算法組件，分爲圖算法，PyTONA 算法，機器學習算法。我們可以在說明文檔中看到每一種算法的使用說明，參數說明等。我們提供上圖中的Angel算法，以供用戶自由選擇使用。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"2. 用戶自定義代碼訓練"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下面介紹用戶自定義代碼訓練Angel算法模塊的使用方式："}]},{"type":"image","attrs":{"src":"https:\/\/static001.geekbang.org\/infoq\/fd\/fda8127c69efb86f138339a8b174948b.jpeg","alt":"圖片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"首先用戶從左側拖拽Spark on Angel的組件至中間畫布，此時點擊組件右側就會出現需要填寫的參數，主要分爲兩類，一是組件參數，包括作業jar包，主類名，程序參數，這裏就是用戶的自定義代碼部分，平臺也支持從騰訊雲中拉取用戶代碼；二是資源組件，包括Executor\/Driver\/Master\/PS節點類型，這是騰訊雲CVM實例機型,目前僅支持CPU機型，所有支持的機型均可以在右側下拉框內看到。相比於傳統的代碼提交，本平臺的方式顯得十分簡明直觀。平臺也提供了統一的日誌查看入口，我們可以通過騰訊雲CVM實例查看訓練日誌。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"快速搭建一個圖算法任務流"}]},{"type":"image","attrs":{"src":"https:\/\/static001.geekbang.org\/infoq\/86\/860e79973e0cb736d41e6e7143eabd3e.jpeg","alt":"圖片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"這裏以GraphSage爲例，展示如何在TI-ONE上應用Angel圖計算。我們將該算法模塊拖入中間畫布，當然COS數據集需要提前拖入，平臺將自動連接兩者；右側是我們拖入的GraphSage可以配置的參數，以及算法說明文檔鏈接，我們可以根據此說明文件看到該算法在平臺上的詳細用法說明。算法IO參數就是我們所需要填寫的參數類型，每一種參數類型的樣例和維度等信息，我們都可以從說明文檔中得知。"}]},{"type":"image","attrs":{"src":"https:\/\/static001.geekbang.org\/infoq\/a7\/a75a82950e4b8077e85a6d67b418f75d.jpeg","alt":"圖片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"GraphSage算法IO參數，包括：邊路徑，特徵路徑，標籤路徑，預測值輸出路徑，embedding 輸出路徑，模型輸出路徑，驗證標籤路徑，都支持自定義。除了上述IO參數，我們也支持自定義算法本身參數，常見的包括：batchSize，學習率，數據分區數，Ps分區數，均衡分區，Epoch，驗證集比例，特徵格式，採樣鄰居個數等等。資源參數也是可以自定義的參數，包括num-executors，spark.ps.instances，driver 節點資源類型等等，從最小的2核4G到64核256G都支持用戶申請，使用結束後，這些資源也將自動釋放，不需要手動維護資源。"}]},{"type":"image","attrs":{"src":"https:\/\/static001.geekbang.org\/infoq\/5f\/5fd506c11eb0bf686ec1211f5e2889f0.jpeg","alt":"圖片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"跑完算法模型之後，我們可以查看日誌，也支持根據日誌的類型級別取查看各種日誌；關於模型結果，我們將提供模型鏈接，直達用戶COS。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"總結下，從數據的上傳到算法的選擇和自定義、參數的填寫、流程圖的建立、日誌的保存、模型的構建、結果的展示，都可以在TI-ONE平臺上使用。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"今天的分享就到這裏，謝謝大家。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"嘉賓介紹："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"姚冕，騰訊雲智能鈦高級工程師"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"本文轉載自：DataFunTalk（ID：datafuntalk）"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"原文鏈接："},{"type":"link","attrs":{"href":"https:\/\/mp.weixin.qq.com\/s\/aYAMUu9cbHdke2AuX0voQg","title":"xxx","type":null},"content":[{"type":"text","text":"Angel在TI-ONE機器學習平臺上的應用"}]}]}]}