圖解Janusgraph系列-分佈式id生成策略分析

{"type":"doc","content":[{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"JanusGraph - 分佈式id的生成策略"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"大家好,我是洋仔,JanusGraph圖解系列文章,"},{"type":"codeinline","content":[{"type":"text","text":"實時更新"}]},{"type":"text","text":"~"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"本次更新時間:2020-9-1"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"文章爲作者跟蹤源碼和查看官方文檔整理,如有任何問題,請聯繫我或在評論區指出,感激不盡!"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"**"},{"type":"codeinline","content":[{"type":"text","text":"圖數據庫網上資源太少,評論區評論 or 私信我,邀你加入“圖庫交流微信羣”,一起交流學習!"}]},{"type":"text","text":"**"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"源碼分析相關:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://liyangyang.blog.csdn.net/article/details/106674499","title":""},"content":[{"type":"text","text":"源碼圖庫-一文搞定janusgraph圖數據庫的本地源碼編譯(janusgraph source code compile)"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"圖解圖庫JanusGraph系列-一文知曉導入數據流程(待發布)"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"圖解圖庫JanusGraph系列-簡要分析查詢讀數據流程(待發布)"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"圖解圖庫JanusGraph系列-一文知曉鎖機制(本地鎖+分佈式鎖)(待發布)"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://mp.weixin.qq.com/s?__biz=MzAwODkwMDk4OQ==&mid=2247483936&idx=1&sn=1b0ebe5699ee267d4b0360c90c937492&chksm=9b669a32ac111324e66af490a24478b81ad5221a88af61897353a6f4d80ff6005b60fbbb8490&scene=126&sessionid=1598925790&key=4598b5ee8f6c495005e73b5167cd30412733226c9d6ce36","title":""},"content":[{"type":"text","text":"圖解圖庫JanusGraph系列-一文知曉分佈式id生成策略"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"圖解圖庫JanusGraph系列-一文知曉圖庫存儲分區策略(待發布)"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"存儲結構相關:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://mp.weixin.qq.com/s?_biz=MzAwODkwMDk4OQ==&mid=2247483894&idx=1&sn=0d7b98d8d7abf86bfacf8c86b694651d&chksm=9b6699e4ac1110f2626789d78aaf617dc02b7a9cdad320c5273172a6fa3a21d8f40d63958461&token=2053731774&lang=zhCN#rd","title":""},"content":[{"type":"text","text":"圖解圖庫JanusGraph系列-一文知曉圖數據底層存儲結構"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"其他:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://mp.weixin.qq.com/s?_biz=MzAwODkwMDk4OQ==&mid=2247483830&idx=1&sn=71ad0d9e0d5868ae15011b7744c0fe8f&chksm=9b6699a4ac1110b294487a6987be5392a5093405a7a40f58d4bca697a18d64000db1aeda0a6f&token=1631136587&lang=zhCN#rd","title":""},"content":[{"type":"text","text":"解惑圖數據庫!你知道什麼是圖數據庫嗎?"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"圖解圖庫JanusGraph系列-官方測試圖:諸神之圖分析(待發布)"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"**"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"源碼分析相關可查看github(求star~~)"}]},{"type":"text","text":"**: "},{"type":"link","attrs":{"href":"https://github.com/YYDreamer/janusgraph","title":""},"content":[{"type":"text","text":"https://github.com/YYDreamer/janusgraph"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下述流程高清大圖地址:"},{"type":"link","attrs":{"href":"https://www.processon.com/view/link/5f471b2e7d9c086b9903b629","title":""},"content":[{"type":"text","text":"https://www.processon.com/view/link/5f471b2e7d9c086b9903b629"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"版本:JanusGraph-0.5.2"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "},{"type":"text","marks":[{"type":"strong"}],"text":"轉載文章請保留以下聲明:"}]},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"作者:洋仔聊編程"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"微信公衆號:匠心Java"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"原文地址:"},{"type":"link","attrs":{"href":"https://xie.infoq.cn/article/20cfc44ff85cf6bf326f3f6f7","title":""},"content":[{"type":"text","text":"https://xie.infoq.cn/article/20cfc44ff85cf6bf326f3f6f7"}]}]}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"正文"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在介紹JanusGraph的分佈式ID生成策略之前,我們來簡單分析一下"},{"type":"codeinline","content":[{"type":"text","text":"分佈式ID"}]},{"type":"text","text":"應該滿足哪些特徵?"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"全局唯一"},{"type":"text","text":":必須保證ID是分佈式環境中全局性唯一的,這是基本要求"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"高性能"},{"type":"text","text":":高可用低延時,ID生成響應快;否則可能會成爲業務瓶頸"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"高可用"},{"type":"text","text":":提供分佈式id的生成的服務要保證高可用,不能隨隨便便就掛掉了,會對業務產生影響"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"趨勢遞增"},{"type":"text","text":":主要看業務場景,類似於圖存儲中節點的唯一id就儘量保持趨勢遞增;但是如果類似於電商訂單就儘量不要趨勢遞增,因爲趨勢遞增會被惡意估算出當天的訂單量和成交量,泄漏公司信息"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"接入方便"},{"type":"text","text":":如果是中間件,要秉着拿來即用的設計原則,在系統設計和實現上要儘可能的簡單"}]}]}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"一:常用分佈式id生成策略"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"當前"},{"type":"codeinline","content":[{"type":"text","text":"常用的"}]},{"type":"text","text":"分佈式id的生成策略主要分爲以下四種:"}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"UUID"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"數據庫+號段模式(優化:數據庫+號段+雙buffer)"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"基於Redis實現"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"雪花算法(SnowFlake)"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"還有一些其他的比如:基於數據庫自增id、數據庫多主模式等,這些在小併發的情況下可以使用,大併發的情況下就不太ok了"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"市面上有一些生成分佈式id的開源組件,包括滴滴基於"},{"type":"codeinline","content":[{"type":"text","text":"數據庫+號段"}]},{"type":"text","text":"實現的"},{"type":"codeinline","content":[{"type":"text","text":"TinyID"}]},{"type":"text","text":" 、百度基於"},{"type":"codeinline","content":[{"type":"text","text":"SnowFlake"}]},{"type":"text","text":"的"},{"type":"codeinline","content":[{"type":"text","text":"Uidgenerator"}]},{"type":"text","text":"、美團支持"},{"type":"codeinline","content":[{"type":"text","text":"號段"}]},{"type":"text","text":"和"},{"type":"codeinline","content":[{"type":"text","text":"SnowFlake"}]},{"type":"text","text":"的"},{"type":"codeinline","content":[{"type":"text","text":"Leaf"}]},{"type":"text","text":"等"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"那麼,在JanusGraph中分佈式id的生成是採用的什麼方式呢? "}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"二:JanusGraph的分佈式id策略"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在JanusGraph中,分佈式id的生成採用的是"},{"type":"codeinline","content":[{"type":"text","text":"數據庫+號段+雙buffer優化"}]},{"type":"text","text":"的模式; 下面我們來具體分析一下:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"分佈式id生成使用的數據庫就是JanusGraph當前使用的第三方存儲後端,這裏我們以使用的存儲後端"},{"type":"codeinline","content":[{"type":"text","text":"Hbase"}]},{"type":"text","text":"爲例;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"JanusGraph分佈式id生成所需元數據存儲位置:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在Hbase中有"},{"type":"codeinline","content":[{"type":"text","text":"column family 列族"}]},{"type":"text","text":"的概念; JanusGraph在初始化Hbase表時默認創建了9大列族,用於存儲不同的數據, 具體看《圖解圖庫JanusGraph系列-一文知曉圖數據底層存儲結構》;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"其中有一個列族"},{"type":"codeinline","content":[{"type":"text","text":"janusgraph_ids"}]},{"type":"text","text":"簡寫爲"},{"type":"codeinline","content":[{"type":"text","text":"i"}]},{"type":"text","text":"這個列族,主要存儲的就是JanusGraph分佈式id生成所需要的元數據!"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"JanusGraph的分佈式id的組成結構:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":"\t // 源碼中有一句話體現\n /*\t\t--- JanusGraphElement id bit format ---\n * [ 0 | count | partition | ID padding (if any) ]\n */"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"主要分爲4部分:"},{"type":"codeinline","content":[{"type":"text","text":"0、count、partition、ID padding(每個類型是固定值)"}]},{"type":"text","text":";"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"其實這4部分的順序在序列化爲二進制數據時,順序會有所改變;這裏只是標明瞭id的組成部分!"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"上述部分的"},{"type":"codeinline","content":[{"type":"text","text":"partition"}]},{"type":"text","text":" + "},{"type":"codeinline","content":[{"type":"text","text":"count"}]},{"type":"text","text":"來保證分佈式節點的唯一性;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"partition id:分區id值,JanusGraph默認分了32個邏輯分區;節點分到哪個分區採用的是"},{"type":"codeinline","content":[{"type":"text","text":"隨機分配"}]},{"type":"text","text":";"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"count:每個partition都有對應的一個count範圍:0-2的55次冪;JanusGraph每次拉取一部分的範圍作爲節點的count取值;JanusGraph保證了針對相同的partition,不會重複獲取同一個count值!"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"保證count在partition維度保持全局唯一性,就保證了生成的最終id的全局唯一性!!"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"*則分佈式id的唯一性保證,就在於"},{"type":"codeinline","content":[{"type":"text","text":"count"}]},{"type":"text","text":"基於"},{"type":"codeinline","content":[{"type":"text","text":"partition"}]},{"type":"text","text":"維度的唯一性!下面我們的分析也是着重在"},{"type":"codeinline","content":[{"type":"text","text":"count"}]},{"type":"text","text":"的獲取!* "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"JanusGraph分佈式id生成的主要邏輯流程如下圖所示:(推薦結合源碼分析觀看!)"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"分析過程中有一個概念爲"},{"type":"codeinline","content":[{"type":"text","text":"id block"}]},{"type":"text","text":":指當前獲取的號段範圍"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/29/29288622004088ff0034a30a0be8d1cd.png","alt":"在這裏插入圖片描述","title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"JanusGraph主要使用`"},{"type":"codeinline","content":[{"type":"text","text":"PartitionIDPool "}]},{"type":"text","text":"類來存儲不同類型的"},{"type":"codeinline","content":[{"type":"text","text":"StandardIDPool"}]},{"type":"text","text":"; 在"},{"type":"codeinline","content":[{"type":"text","text":"StandardIDPool"}]},{"type":"text","text":"中主要包含兩個id Block:"}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"current block:當前生成id使用的block"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"next block:double buffer中的另一個已經準備好的block"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"爲什麼要有兩個block呢?"},{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"主要是如果只有一個block的話,當我們在使用完當前的block時,需要阻塞等待區獲取下一個block,這樣便會導致分佈式id生成較長時間的阻塞等待block的獲取;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"怎麼優化上述問題呢? "},{"type":"codeinline","content":[{"type":"text","text":"double buffer"}]},{"type":"text","text":"; "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"除了當前使用的block,我們再存儲一個"},{"type":"codeinline","content":[{"type":"text","text":"next block"}]},{"type":"text","text":";當正在使用的block假設已經使用了50%,觸發"},{"type":"codeinline","content":[{"type":"text","text":"next block"}]},{"type":"text","text":"的異步獲取,如上圖的藍色部分所示;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"這樣當"},{"type":"codeinline","content":[{"type":"text","text":"current block"}]},{"type":"text","text":"使用完成後可以直接無延遲的切換到"},{"type":"codeinline","content":[{"type":"text","text":"next block"}]},{"type":"text","text":"如上圖中綠色部分所示; "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"在執行過程中可能會因爲一些異常導致節點id獲取失敗,則會進行重試;重試次數默認爲1000次;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":"private static final int MAX_PARTITION_RENEW_ATTEMPTS = 1000;\nfor (int attempt = 0; attempt < MAX_PARTITION_RENEW_ATTEMPTS; attempt++) {\n // 獲取id的過程\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"ps:上述所說的IDPool和block是基於當前"},{"type":"codeinline","content":[{"type":"text","text":"圖實例"}]},{"type":"text","text":"維度共用的!"}]}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"三:源碼分析"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在JanusGraph的源碼中,主要包含兩大部分和其他的一些組件:"}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Graph相關類:用於對節點、屬性、邊的操作"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Transaction相關類:用於在對數據或者Schema進行CURD時,進行事務處理"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"其他一些:分佈式節點id生成類;序列化類;第三方索引操作類等等"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Graph和Transaction相關類的類圖如下所示:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/c8/c8656968eb85ef0a325b3eb49314c62e.png","alt":"在這裏插入圖片描述","title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"分佈式id涉及到id生成的類圖如下所示:"}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/0f/0fcb50f0c85fac9e310ab8c78fe8f438.png","alt":"在這裏插入圖片描述","title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"初始數據:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" @Test\n public void addVertexTest(){\n List godProperties = new ArrayList<>();\n godProperties.add(T.label);\n godProperties.add(\"god\");\n\n godProperties.add(\"name\");\n godProperties.add(\"lyy\");\n\n godProperties.add(\"age\");\n godProperties.add(18);\n\n JanusGraphVertex godVertex = graph.addVertex(godProperties.toArray());\n\n assertNotNull(godVertex);\n }"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在"},{"type":"codeinline","content":[{"type":"text","text":"諸神之圖"}]},{"type":"text","text":"中添加一個name爲"},{"type":"codeinline","content":[{"type":"text","text":"lyy"}]},{"type":"text","text":"節點;看下執行流程,注意,此處主要分析的節點的分佈式id生成代碼!"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"**1、調用"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"JanusGraphBlueprintsGraph"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"類的"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"AddVertex方法"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"**"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" @Override\n public JanusGraphVertex addVertex(Object... keyValues) {\n // 添加節點\n return getAutoStartTx().addVertex(keyValues);\n }"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"**2、調用"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"JanusGraphBlueprintsTransaction"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"的"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"addVertex"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"方法**"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" public JanusGraphVertex addVertex(Object... keyValues) {\n // 。。。省略了其他的處理\n // 該處生成節點對象,包含節點的唯一id生成邏輯\n final JanusGraphVertex vertex = addVertex(id, label); \n // 。。。省略了其他的處理\n return vertex;\n }"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"**3、調用"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"StandardJanusGraphTx"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"的"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"addVertex"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"方法**"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" @Override\n public JanusGraphVertex addVertex(Long vertexId, VertexLabel label) {\n // 。。。省略了其他的處理\n if (vertexId != null) {\n vertex.setId(vertexId);\n } else if (config.hasAssignIDsImmediately() || label.isPartitioned()) {\n graph.assignID(vertex,label); // 爲節點分配正式的節點id!\n }\n // 。。。省略了其他的處理\n return vertex;\n }"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"**4、調用"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"VertexIDAssigner"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"的"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"assignID(InternalElement element, IDManager.VertexIDType vertexIDType)"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"方法**"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" private void assignID(InternalElement element, IDManager.VertexIDType vertexIDType) {\n // 開始獲取節點分佈式唯一id\n // 因爲一些異常導致獲取節點id失敗,進行重試,重試此爲默認爲1000次\n for (int attempt = 0; attempt < MAX_PARTITION_RENEW_ATTEMPTS; attempt++) {\n // 初始化一個partiiton id\n long partitionID = -1;\n // 獲取一個partition id\n // 不同類型的數據,partition id的獲取方式也有所不同\n if (element instanceof JanusGraphSchemaVertex) {\n // 爲partition id賦值\n }\n try {\n // 正式分配節點id, 依據partition id 和 節點類型\n assignID(element, partitionID, vertexIDType);\n } catch (IDPoolExhaustedException e) {\n continue; //try again on a different partition\n }\n assert element.hasId();\n // 。。。省略了其他代碼\n }\n }"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"**5、調用了"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"VertexIDAssigner"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"的"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"assignID(final InternalElement element, final long partitionIDl, final IDManager.VertexIDType userVertexIDType)"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"方法**"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" private void assignID(final InternalElement element, final long partitionIDl, final IDManager.VertexIDType userVertexIDType) {\n \n final int partitionID = (int) partitionIDl;\n\n // count爲分佈式id組成中的一部分,佔55個字節\n // 分佈式id的唯一性保證,就在於`count`基於`partition`維度的唯一性\n long count;\n if (element instanceof JanusGraphSchemaVertex) { // schema節點處理\n Preconditions.checkArgument(partitionID==IDManager.SCHEMA_PARTITION);\n count = schemaIdPool.nextID();\n } else if (userVertexIDType==IDManager.VertexIDType.PartitionedVertex) { // 配置的熱點節點,類似於`makeVertexLabel('product').partition()`的處理\n count = partitionVertexIdPool.nextID();\n } else { // 普通節點和邊類型的處理\n // 首先獲取當前partition敵營的idPool\n PartitionIDPool partitionPool = idPools.get(partitionID);\n // 如果當前分區對應的IDPool爲空,則創建一個默認的IDPool,默認size = 0\n if (partitionPool == null) {\n // 在PartitionIDPool中包含多種類型對應的StandardIDPool類型\n // StandardIDPool中包含對應的block信息和count信息\n partitionPool = new PartitionIDPool(partitionID, idAuthority, idManager, renewTimeoutMS, renewBufferPercentage);\n // 緩存下來\n idPools.putIfAbsent(partitionID,partitionPool);\n // 從緩存中再重新拿出\n partitionPool = idPools.get(partitionID);\n }\n // 確保partitionPool不爲空\n Preconditions.checkNotNull(partitionPool);\n // 判斷當前分區的IDPool是否枯竭;已經被用完\n if (partitionPool.isExhausted()) {\n // 如果被用完,則將該分區id放到對應的緩存中,避免之後獲取分區id再獲取到該分區id\n placementStrategy.exhaustedPartition(partitionID);\n // 拋出IDPool異常, 最外層捕獲,然後進行重試獲取節點id\n throw new IDPoolExhaustedException(\"Exhausted id pool for partition: \" + partitionID);\n }\n // 存儲當前類型對應的IDPool,因爲partitionPool中保存好幾個類型的IDPool\n IDPool idPool;\n if (element instanceof JanusGraphRelation) {\n idPool = partitionPool.getPool(PoolType.RELATION);\n } else {\n Preconditions.checkArgument(userVertexIDType!=null);\n idPool = partitionPool.getPool(PoolType.getPoolTypeFor(userVertexIDType));\n }\n try {\n // 重要!!!! 依據給定的IDPool獲取count值!!!!\n // 在此語句中設計 block的初始化 和 double buffer block的處理!\n count = idPool.nextID();\n partitionPool.accessed();\n } catch (IDPoolExhaustedException e) { // 如果該IDPool被用完,拋出IDPool異常, 最外層捕獲,然後進行重試獲取節點id\n log.debug(\"Pool exhausted for partition id {}\", partitionID);\n placementStrategy.exhaustedPartition(partitionID);\n partitionPool.exhaustedIdPool();\n throw e;\n }\n }\n\n // 組裝最終的分佈式id:[count + partition id + ID padding]\n long elementId;\n if (element instanceof InternalRelation) {\n elementId = idManager.getRelationID(count, partitionID);\n } else if (element instanceof PropertyKey) {\n elementId = IDManager.getSchemaId(IDManager.VertexIDType.UserPropertyKey,count);\n } else if (element instanceof EdgeLabel) {\n elementId = IDManager.getSchemaId(IDManager.VertexIDType.UserEdgeLabel, count);\n } else if (element instanceof VertexLabel) {\n elementId = IDManager.getSchemaId(IDManager.VertexIDType.VertexLabel, count);\n } else if (element instanceof JanusGraphSchemaVertex) {\n elementId = IDManager.getSchemaId(IDManager.VertexIDType.GenericSchemaType,count);\n } else {\n elementId = idManager.getVertexID(count, partitionID, userVertexIDType);\n }\n\n Preconditions.checkArgument(elementId >= 0);\n // 對節點對象賦值其分佈式唯一id\n element.setId(elementId);\n }"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"上述代碼,我們拿到了對應的IdPool,有兩種情況:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"第一次獲取分佈式id時,分區對應的IDPool初始化爲默認的size = 0的IDPool"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"分區對應的IDPool不是初次獲取"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"這兩種情況的處理,都在代碼"},{"type":"codeinline","content":[{"type":"text","text":"count = idPool.nextID()"}]},{"type":"text","text":"的"},{"type":"codeinline","content":[{"type":"text","text":"StandardIDPool"}]},{"type":"text","text":"類中的"},{"type":"codeinline","content":[{"type":"text","text":"nextID()"}]},{"type":"text","text":"方法中被處理!"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"**在分析該代碼之前,我們需要知道 "},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"PartitionIDPool "}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"和"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"StandardIDPool"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"的關係:**"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"每個partition都有一個對應的"},{"type":"codeinline","content":[{"type":"text","text":"PartitionIDPool extends EnumMap "}]},{"type":"text","text":" 是一個枚舉map類型;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"每一個"},{"type":"codeinline","content":[{"type":"text","text":"PartitionIDPool "}]},{"type":"text","text":"都有對應的不同類型的"},{"type":"codeinline","content":[{"type":"text","text":"StandardIDPool"}]},{"type":"text","text":":"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"NORMAL_VERTEX:用於vertex id的分配"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"UNMODIFIABLE_VERTEX:用於schema label id的分配"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"RELATION:用於edge id的分配"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在"},{"type":"codeinline","content":[{"type":"text","text":"StandardIDPool"}]},{"type":"text","text":"中包含多個字段,分別代表不同的含義,抽取幾個重要的字段進行介紹:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" private static final int RENEW_ID_COUNT = 100; \n private final long idUpperBound; // Block的最大值,默認爲2的55次冪\n private final int partition; // 當前pool對應的分區\n private final int idNamespace; // 標識pool爲那種類型的pool,上述的三種類型NORMAL_VERTEX、UNMODIFIABLE_VERTEX、RELATION;值爲當前枚舉值在枚舉中的位置\n\n private final Duration renewTimeout;// 重新獲取block的超時時間\n private final double renewBufferPercentage;// 雙buffer中,當第一個buffer block使用的百分比,到達配置的百分比則觸發other buffer block的獲取\n\n private IDBlock currentBlock; // 當前的block\n private long currentIndex; // 標識當前block使用到那一個位置\n private long renewBlockIndex; // 依據currentBlock.numIds()*renewBufferPercentage來獲取這個值,主要用於在當前的block在消費到某個index的時候觸發獲取下一個buffer block\n\n private volatile IDBlock nextBlock;// 雙buffer中的另外一個block\n\n private final ThreadPoolExecutor exec;// 異步獲取雙buffer的線程池\n"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"**6、調用了"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"StandardIDPool"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"類中的"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"nextID"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"方法**"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"經過上述分析,我們知道,分佈式唯一id的唯一性是由在partition維度下的count的值的唯一性來保證的;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"上述代碼通過調用IDPool的nextId來獲取count值; "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下述代碼就是獲取count的邏輯;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" @Override\n public synchronized long nextID() {\n // currentIndex標識當前的index小於current block的最大值\n assert currentIndex <= currentBlock.numIds();\n\n // 此處涉及兩種情況:\n // 1、分區對應的IDPool是第一次被初始化;則currentIndex = 0; currentBlock.numIds() = 0;\n // 2、分區對應的該IDPool不是第一次,但是此次的index正好使用到了current block的最後一個count\n if (currentIndex == currentBlock.numIds()) {\n try {\n // 將current block賦值爲next block\n // next block置空 並計算renewBlockIndex\n nextBlock();\n } catch (InterruptedException e) {\n throw new JanusGraphException(\"Could not renew id block due to interruption\", e);\n }\n }\n \n // 在使用current block的過程中,當current index == renewBlockIndex時,觸發double buffer next block的異步獲取!!!!\n if (currentIndex == renewBlockIndex) {\n // 異步獲取next block\n startIDBlockGetter();\n }\n \n // 生成最終的count\n long returnId = currentBlock.getId(currentIndex);\n // current index + 1\n currentIndex++;\n if (returnId >= idUpperBound) throw new IDPoolExhaustedException(\"Reached id upper bound of \" + idUpperBound);\n log.trace(\"partition({})-namespace({}) Returned id: {}\", partition, idNamespace, returnId);\n // 返回最終獲取的分區維度的全局唯一count\n return returnId;\n }"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"上述代碼中進行了兩次判斷:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"currentIndex == currentBlock.numIds():"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" * 第一次生成分佈式id:此處判斷即爲 0==0;然後生成新的block"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" * 非第一次生成分佈式id:等於情況下標識當前的block已經使用完了,需要切換爲next block"}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"currentIndex == renewBlockIndex"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" * renew index:標識index使用多少後開始獲取下一個double buffer 的next block;有一個默認值100,主要爲了兼容第一次分佈式id的生成;相等則會觸發異步獲取下一個next block"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下面我們分別對"},{"type":"codeinline","content":[{"type":"text","text":"nextBlock();"}]},{"type":"text","text":"邏輯和"},{"type":"codeinline","content":[{"type":"text","text":"startIDBlockGetter();"}]},{"type":"text","text":"進行分析;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"**7、調用了"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"StandardIDPool"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"類中的"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"nextBlock"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"方法**"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" private synchronized void nextBlock() throws InterruptedException {\n // 在分區對應的IDPool第一次使用時,double buffer的nextBlock爲空\n if (null == nextBlock && null == idBlockFuture) {\n // 異步啓動 獲取id block\n startIDBlockGetter();\n }\n\n // 也是在分區對應的IDPool第一次使用時,因爲上述爲異步獲取,所以在執行到這一步時nextBlock可能還沒拿到\n // 所以需要阻塞等待block的獲取\n if (null == nextBlock) {\n waitForIDBlockGetter();\n }\n\n // 將當前使用block指向next block\n currentBlock = nextBlock;\n // index清零\n currentIndex = 0;\n // nextBlock置空\n nextBlock = null;\n\n // renewBlockIndex用於雙buffer中,當第一個buffer block使用的百分比,到達配置的百分比則觸發other buffer block的獲取\n // 值current block 對應的count數量 - (值current block 對應的count數量 * 爲renewBufferPercentage配置的剩餘空間百分比)\n // 在使用current block的時候,當current index == renewBlockIndex時,觸發double buffer next block的異步獲取!!!!\n renewBlockIndex = Math.max(0,currentBlock.numIds()-Math.max(RENEW_ID_COUNT, Math.round(currentBlock.numIds()*renewBufferPercentage)));\n }"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"主要是做了三件事:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"1、block是否爲空,爲空的話則異步獲取一個block"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"2、nextBlock不爲空的情況下:next賦值到current、next置空、index置零"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"3、計算獲取下一個nextBlock的觸發index renewBlockIndex值"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"**8、調用了"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"StandardIDPool"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"類中的"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"startIDBlockGetter"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"方法**"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" private synchronized void startIDBlockGetter() {\n Preconditions.checkArgument(idBlockFuture == null, idBlockFuture);\n if (closed) return; //Don't renew anymore if closed\n //Renew buffer\n log.debug(\"Starting id block renewal thread upon {}\", currentIndex);\n // 創建一個線程對象,包含給定的權限控制類、分區、命名空間、超時時間\n idBlockGetter = new IDBlockGetter(idAuthority, partition, idNamespace, renewTimeout);\n // 提交獲取double buffer的線程任務,異步執行\n idBlockFuture = exec.submit(idBlockGetter);\n }"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"其中創建一個線程任務,提交到線程池"},{"type":"codeinline","content":[{"type":"text","text":"exec"}]},{"type":"text","text":"進行異步執行;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下面看下,線程類的"},{"type":"codeinline","content":[{"type":"text","text":"call"}]},{"type":"text","text":"方法主要是調用了"},{"type":"codeinline","content":[{"type":"text","text":" idAuthority.getIDBlock"}]},{"type":"text","text":"方法,這個方法主要是基於Hbase來獲取還未使用的block;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" /**\n * 獲取double buffer block的線程類\n */\n private static class IDBlockGetter implements Callable {\n\n // 省略部分代碼\n @Override\n public IDBlock call() {\n Stopwatch running = Stopwatch.createStarted();\n try {\n // 此處調用idAuthority 調用HBase進行佔用獲取Block\n IDBlock idBlock = idAuthority.getIDBlock(partition, idNamespace, renewTimeout);\n return idBlock;\n } catch (BackendException e) {}\n }\n }"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"**9、調用"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"ConsistentKeyIDAuthority"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"類的"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"getIDBlock"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"方法**"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" @Override\n public synchronized IDBlock getIDBlock(final int partition, final int idNamespace, Duration timeout) throws BackendException {\n \n // 開始時間\n final Timer methodTime = times.getTimer().start();\n\n // 獲取當前命名空間配置的blockSize,默認值10000;可自定義配置\n final long blockSize = getBlockSize(idNamespace);\n // 獲取當前命名空間配置的最大id值idUpperBound;值爲:2的55次冪大小\n final long idUpperBound = getIdUpperBound(idNamespace);\n // uniqueIdBitWidth標識uniqueId佔用的位數;uniqueId爲了兼容“關閉分佈式id唯一性保障”的開關情況,uniqueIdBitWidth默認值=4\n // 值:64-1(默認0)-5(分區佔用位數)-3(ID Padding佔用位數)-4(uniqueIdBitWidth) = 51;標識block中的上限爲2的51次冪大小\n final int maxAvailableBits = (VariableLong.unsignedBitLength(idUpperBound)-1)-uniqueIdBitWidth;\n\n // 標識block中的上限爲2的51次冪大小\n final long idBlockUpperBound = (1L < exhaustedUniquePIDs = new ArrayList<>(randomUniqueIDLimit);\n\n // 默認0.3秒 用於處理TemporaryBackendException異常情況(後端存儲出現問題)下:阻塞一斷時間,然後進行重試\n Duration backoffMS = idApplicationWaitMS;\n\n // 從開始獲取IDBlock開始,持續超時時間(默認2分鐘)內重試獲取IDBlock\n while (methodTime.elapsed().compareTo(timeout) < 0) {\n final int uniquePID = getUniquePartitionID(); // 獲取uniquePID,默認情況下“開啓分佈式id唯一性控制”,值 = 0; 當“關閉分佈式id唯一性控制”時爲一個隨機值\n final StaticBuffer partitionKey = getPartitionKey(partition,idNamespace,uniquePID); // 依據partition + idNamespace + uniquePID組裝一個RowKey\n try {\n long nextStart = getCurrentID(partitionKey); // 從Hbase中獲取當前partition對應的IDPool中被分配的最大值,用來作爲當前申請新的block的開始值\n if (idBlockUpperBound - blockSize <= nextStart) { // 確保還未被分配的id池中的id個數,大於等於blockSize\n // 相應處理\n }\n\n long nextEnd = nextStart + blockSize; // 獲取當前想要獲取block的最大值\n StaticBuffer target = null;\n\n // attempt to write our claim on the next id block\n boolean success = false;\n try {\n Timer writeTimer = times.getTimer().start(); // ===開始:開始進行插入自身的block需求到Hbase\n target = getBlockApplication(nextEnd, writeTimer.getStartTime()); // 組裝對應的Column: -nextEnd + 當前時間戳 + uid(唯一標識當前圖實例)\n final StaticBuffer finalTarget = target; // copy for the inner class\n BackendOperation.execute(txh -> { // 異步插入當前生成的RowKey 和 Column\n idStore.mutate(partitionKey, Collections.singletonList(StaticArrayEntry.of(finalTarget)), KeyColumnValueStore.NO_DELETIONS, txh);\n return true;\n },this,times);\n writeTimer.stop(); // ===結束:插入完成\n\n final boolean distributed = manager.getFeatures().isDistributed();\n Duration writeElapsed = writeTimer.elapsed(); // ===獲取方纔插入的時間耗時\n if (idApplicationWaitMS.compareTo(writeElapsed) < 0 && distributed) { // 判斷是否超過配置的超時時間,超過則報錯TemporaryBackendException,然後等待一斷時間進行重試\n throw new TemporaryBackendException(\"Wrote claim for id block [\" + nextStart + \", \" + nextEnd + \") in \" + (writeElapsed) + \" => too slow, threshold is: \" + idApplicationWaitMS);\n } else {\n\n assert 0 != target.length();\n final StaticBuffer[] slice = getBlockSlice(nextEnd); // 組裝下述基於上述Rowkey的Column的查找範圍:(-nextEnd + 0 : 0nextEnd + 最大值) \n\n final List blocks = BackendOperation.execute( // 異步獲取指定Rowkey和指定Column區間的值\n (BackendOperation.Transactional>) txh -> idStore.getSlice(new KeySliceQuery(partitionKey, slice[0], slice[1]), txh),this,times);\n if (blocks == null) throw new TemporaryBackendException(\"Could not read from storage\");\n if (blocks.isEmpty())\n throw new PermanentBackendException(\"It seems there is a race-condition in the block application. \" +\n \"If you have multiple JanusGraph instances running on one physical machine, ensure that they have unique machine idAuthorities\");\n\n if (target.equals(blocks.get(0).getColumnAs(StaticBuffer.STATIC_FACTORY))) { // 如果獲取的集合中,當前的圖實例插入的數據是第一條,則表示獲取block; 如果不是第一條,則獲取Block失敗\n // 組裝IDBlock對象\n ConsistentKeyIDBlock idBlock = new ConsistentKeyIDBlock(nextStart,blockSize,uniqueIdBitWidth,uniquePID);\n\n if (log.isDebugEnabled()) {\n idBlock, partition, idNamespace, uid);\n }\n\n success = true;\n return idBlock; // 返回\n } else { }\n }\n } finally {\n if (!success && null != target) { // 在獲取Block失敗後,刪除當前的插入; 如果沒有失敗,則保留當前的插入,在hbase中標識該Block已經被佔用\n //Delete claim to not pollute id space\n for (int attempt = 0; attempt < ROLLBACK_ATTEMPTS; attempt++) { // 回滾:刪除當前插入,嘗試次數5次\n }\n }\n }\n } catch (UniqueIDExhaustedException e) {\n // No need to increment the backoff wait time or to sleep\n log.warn(e.getMessage());\n } catch (TemporaryBackendException e) {\n backoffMS = Durations.min(backoffMS.multipliedBy(2), idApplicationWaitMS.multipliedBy(32));\n sleepAndConvertInterrupts(backoffMS); \\\n }\n }\n\n throw new TemporaryLockingException();\n }\n"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"主要的邏輯就是:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"組裝Rowkey"}]},{"type":"text","text":":partition + idNameSpace+unquePId"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"組裝Column"}]},{"type":"text","text":":-nextEnd+now time+uid"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"將"},{"type":"codeinline","content":[{"type":"text","text":"RowKey+Column"}]},{"type":"text","text":"插入Hbase"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"獲取的上述組裝的RowKey 基於(-nextEnd + 0 : -nextEnd + max)範圍的所有Column集合"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"判斷集合的第一個Column是不是當前插入的Column,是的話則佔用block成功,不是的話則佔用失敗,刪除剛纔佔用並進行重試"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"最終:異步獲取到了唯一佔用的Block,然後生成對應的唯一count,組裝最後的唯一id"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"整體的調用流程如下:"}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/2e/2e5b1cd3e7351ae4684096de9281122b.png","alt":"在這裏插入圖片描述","title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"四:其他類型的id生成"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"上述我們主要依據生成節點id(vertex id)的過程來進行分析"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在"},{"type":"codeinline","content":[{"type":"text","text":"JanusGraph"}]},{"type":"text","text":"中還包含"},{"type":"codeinline","content":[{"type":"text","text":"edge id"}]},{"type":"text","text":"、"},{"type":"codeinline","content":[{"type":"text","text":"property id"}]},{"type":"text","text":"、"},{"type":"codeinline","content":[{"type":"text","text":"schema label id"}]},{"type":"text","text":"等幾種的分佈式id生成"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"所有類型的分佈式id的生成主要思想和邏輯都幾乎相同,只是一些具體的邏輯可能有所不同,我們理解了"},{"type":"codeinline","content":[{"type":"text","text":"vertex id"}]},{"type":"text","text":"的分佈式id生成流程,其他的也可以理解了。"}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"1、property id的生成"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在JanusGraph中的"},{"type":"codeinline","content":[{"type":"text","text":"property"}]},{"type":"text","text":"的分佈式唯一id的生成,整體邏輯和"},{"type":"codeinline","content":[{"type":"text","text":"vertex id"}]},{"type":"text","text":"的生成邏輯大體相同;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"property id"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"的 生成和 "},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"vertex id"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"有兩點不同:"}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"ID的組成部分: 在"},{"type":"codeinline","content":[{"type":"text","text":"vertex id"}]},{"type":"text","text":"中組成部分包含"},{"type":"codeinline","content":[{"type":"text","text":"count"}]},{"type":"text","text":"+"},{"type":"codeinline","content":[{"type":"text","text":"partition"}]},{"type":"text","text":"+"},{"type":"codeinline","content":[{"type":"text","text":"ID Padding"}]},{"type":"text","text":"; 而在"},{"type":"codeinline","content":[{"type":"text","text":"property id"}]},{"type":"text","text":"中沒有"},{"type":"codeinline","content":[{"type":"text","text":"ID Padding"}]},{"type":"text","text":"部分,其組成爲"},{"type":"codeinline","content":[{"type":"text","text":"count + partition"}]}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" long id = (count<
發表評論
所有評論
還沒有人評論,想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.
相關文章