Google大規模監控系統--Monarch

{"type":"doc","content":[{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Monarch是Google的大規模監控系統，服務於Google全球大規模實時業務監控，其實現爲超大規模時序數據庫集羣，被公認爲當今全球最大規模的時序數據庫。Monarch本身沒有開源，主要信息來源於Google在2020年8月份發表在PVLDB上的一篇論文：","attrs":{}},{"type":"link","attrs":{"href":"https://www.vldb.org/pvldb/vol13/p3181-adams.pdf","title":"","type":null},"content":[{"type":"text","text":"Monarch: Google’s Planet-Scale In-Memory Time Series Database","attrs":{}}]},{"type":"text","text":"。這篇文章是Medium上一篇介紹Google Monarch的文章的中文翻譯，原文鏈接：","attrs":{}},{"type":"link","attrs":{"href":"https://medium.com/geekculture/understanding-monarch-googles-planet-scale-monitoring-system-60e59b63ac0c","title":"","type":null},"content":[{"type":"text","text":"Understanding Monarch, Google’s Planet-Scale Monitoring System","attrs":{}}]},{"type":"text","text":"。","attrs":{}}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"以下爲正文，斜體字是我加的備註：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Monarch是Google開發的大規模內存時序數據庫，用於Google的大部分內部系統（如Spanner, BigTable, Colossus, BlobStore）的可靠監控系統。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"與任何其他的Google服務一樣，它必須被設計爲大規模、高可用並且支持區域局域性。但與其他服務不同的是，Monarch需要儘可能少的依賴其他Google服務，這一點非常重要，因爲其他服務都在使用Monarch對自己進行監控，因此如果有相互依賴關係，任何一方的服務中斷都會影響到另一方。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Monarch是一種必須支持高可用性和分區性的服務，因此在最終一致性有延遲的情況下，它通過向客戶服務提供一定的提示來解決低一致性的問題。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"這意味着Monarch是AP系統，支持最終一致性而不是強一致性，這也是大規模分佈式系統設計的普遍做法。","attrs":{}}]}],"attrs":{}},{"type":"heading","attrs":{"align":null,"level":5},"content":[{"type":"text","text":"Monarch的數據存儲","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"數據以兩種格式存儲：","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"Leaves","attrs":{}},{"type":"text","text":"（葉子）組件在內存中存儲實際的監控數據","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"Logs","attrs":{}},{"type":"text","text":"（日誌）是持久化存儲，可用於在組件失敗時重放事件","attrs":{}}]}]}],"attrs":{}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"Leaf是內存中的緩存存儲模塊，Log用來做持久化和高可用，在服務失效的時候可用於快速恢復數據。","attrs":{}}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":5},"content":[{"type":"text","text":"Monarch的數據獲取","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Data Ingestion Pipeline（數據接收管道）遵循如下原則：","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"將客戶端服務的數據存儲在距離服務運行的zone越近越好，從而減少網絡延遲","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"將客戶服務的數據存儲在同一個Leaf中，因爲數據查詢很有可能被集中在該Leaf上，以獲得更快的查詢響應","attrs":{}}]}]}],"attrs":{}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"就近原則和內聚原則，從而儘可能提高訪問性能。","attrs":{}}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/36/369eab95f35f9ac3b588355be7c64186.webp","alt":"圖片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":"center","origin":null},"content":[{"type":"text","marks":[{"type":"size","attrs":{"size":9}}],"text":"Monarch模塊架構","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"以下情況將觸發數據傳輸：","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"Ingestion Router（數據接收路由器）","attrs":{}},{"type":"text","text":"將數據發送給葉子路由器","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"Leaf Router（葉子路由器）","attrs":{}},{"type":"text","text":"將數據發送給葉子","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"Range Assigner（分配器）","attrs":{}},{"type":"text","text":"決定存儲數據的葉子","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"數據接收路由器根據位置字段將時間序列數據分區，葉子路由器根據分配器的決定將數據分佈在葉子上。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/c3/c30910e451a0d64d36208d64e43f376f.webp","alt":"圖片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":"center","origin":null},"content":[{"type":"text","marks":[{"type":"size","attrs":{"size":9}}],"text":"數據格式模型","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"接收到的數據包含以下幾類：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"Targes（目標）","attrs":{}},{"type":"text","text":"用於識別數據生成的節點/服務/組件。以上圖爲例，目標字符串","attrs":{}},{"type":"text","marks":[{"type":"italic","attrs":{}},{"type":"strong","attrs":{}}],"text":"ComputeTask::sql-dba::db.server::aa::0876","attrs":{}},{"type":"text","text":"表示數據庫服務器的Borg任務。目標字符串的格式對於決定數據在葉子之間的存儲位置很重要，因爲目標範圍用於決定葉子之間的字典分片和負載均衡。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"Metrics（指標）","attrs":{}},{"type":"text","text":"以key-value（鍵值對）的格式包含指標信息，其中鍵是目標的指標類型，值是基於時間序列的數據點。支持的度量類型有boolean、int64、double、string、distribution或tuple。度量值可以是cumulative（累計值），也可以是gauge（測量值）。使用累計值的優點是偶爾的數據丟失對統計分佈的影響不大。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}},{"type":"strong","attrs":{}}],"text":"distribution","attrs":{}},{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"實際上是一個double list，由許多個 bucket組成，每個bucket裏有一個double值，用於表示數據分佈範圍的統計。舉個栗子：有一個訪問延遲的distribution定義了4個bucket: 0-10ms, 10-20ms, 20-30ms, 30ms以上，每個bucket裏的double數值定義了在這個延遲範圍內的訪問次數。","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/0b/0bb702264f995e2b9055438819787e8c.webp","alt":"圖片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"size","attrs":{"size":9}}],"text":" distribution類型示例","attrs":{}}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"數據可以以","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"Delta Time Series（增量時間序列）","attrs":{}},{"type":"text","text":"的格式發送，這樣只需要發送時間序列數據中的增量，而不是整個指標數據。這減少了數據的連續輸入，並且只需要在數據有變化的時候進行處理。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"Bucketing","attrs":{}},{"type":"text","text":"在將數據發送到數據接收管道之前的一段時間內負責聚合數據，從而減少網絡處理，並且可以實現批量插入。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"Admission windows","attrs":{}},{"type":"text","text":"用於丟棄在一定時間後接收到的數據，從而避免處理過期數據的壓力。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"因爲Bucketing會在一定時間內聚合數據，然後存到葉子並且持久化，這樣如果有數據來晚了，對應的Bucket已經完成了處理，就需要將遲到的數據丟棄。","attrs":{}}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":5},"content":[{"type":"text","text":"數據查詢","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Monarch提供了一個全局聯合查詢引擎，所有查詢都可以在全局級別啓動，Monarch負責將查詢路由到存儲相關數據的葉子，並整合來自葉子的響應。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"上面的模塊圖顯示了查詢相關的模塊：","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"Mixers","attrs":{}},{"type":"text","text":"將查詢分解爲子查詢，併合並來自子查詢的響應。根Mixer接收查詢並將它們發送到zone mixer，zone mixer進一步將其發送到葉子，從而形成查詢樹。Mixers還查詢索引服務器，將查詢限制在數據所在的zone或葉子。","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"Index servers","attrs":{}},{"type":"text","text":"（索引服務器）索引每個zone和葉子的數據，這些數據可用於確定查詢針對哪些葉子。","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"Evaluators","attrs":{}},{"type":"text","text":"從standing query生成響應，並將數據寫回葉子。","attrs":{}}]}]}],"attrs":{}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"Standing query類似SQL中的View（視圖）的概念，用戶可以自定義standing query，evaluator會定期查詢standing query的結果，並寫回對應的葉子，從而可以加快查詢速度。","attrs":{}}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Monarch支持如下查詢語義：","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Fetch","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Filter","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Join","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Align","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"GroupBy","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"Ad-hoc queries","attrs":{}},{"type":"text","text":"是來自系統外部用戶的查詢。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"Standing queries","attrs":{}},{"type":"text","text":"是類似於其他數據庫系統中的視圖的查詢，定期計算並存儲到Monarch中，以獲得更快的查詢響應。根據查詢的廣度，standing query可以在zone或root級別進行評估，將查詢空間最小化到特定zone的葉子，從而獲得更好的性能。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"Level analysis","attrs":{}},{"type":"text","text":"用於對查詢進行級別分析，基於不同的級別將查詢打破重組，用以進行認證和獲得更好的查詢局部性。查詢級別可以根據上面提到的查詢樹進行定義。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"Replica Resolution","attrs":{}},{"type":"text","text":"用於找出響應查詢的最佳副本。因爲查詢負載、系統配置、數據完整性等方面可能存在差異，因此某個副本可能更適合響應查詢請求。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"User Isolation","attrs":{}},{"type":"text","text":"用來限制任一用戶在系統中可以使用的內存量，以便其他遵守規則的用戶不受影響。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":5},"content":[{"type":"text","text":"性能","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Monarch分佈在橫跨五大洲的38個zone，執行大約400,000個任務。","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"截至2019年7月，Monarch存儲了近9500億時間序列，其高度優化的數據結構消耗了約750TB內存。","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"2019年7月，Monarch的內部部署每秒接收約4.4TB的數據。","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Monarch持續指數級增長，截至2019年7月，每秒服務超過600萬次查詢。","attrs":{}}]}]}],"attrs":{}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/84/846307d96094e00dab302466266af345.webp","alt":"圖片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":"center","origin":null},"content":[{"type":"text","marks":[{"type":"size","attrs":{"size":9}}],"text":"性能","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"size","attrs":{"size":9}}],"text":"封面圖片來自：https://unsplash.com/@seanstratton","attrs":{}}]}]}