图解Janusgraph系列-分布式id生成策略分析

{"type":"doc","content":[{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"JanusGraph - 分布式id的生成策略"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"大家好,我是洋仔,JanusGraph图解系列文章,"},{"type":"codeinline","content":[{"type":"text","text":"实时更新"}]},{"type":"text","text":"~"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"本次更新时间:2020-9-1"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"文章为作者跟踪源码和查看官方文档整理,如有任何问题,请联系我或在评论区指出,感激不尽!"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"**"},{"type":"codeinline","content":[{"type":"text","text":"图数据库网上资源太少,评论区评论 or 私信我,邀你加入“图库交流微信群”,一起交流学习!"}]},{"type":"text","text":"**"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"源码分析相关:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://liyangyang.blog.csdn.net/article/details/106674499","title":""},"content":[{"type":"text","text":"源码图库-一文搞定janusgraph图数据库的本地源码编译(janusgraph source code compile)"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"图解图库JanusGraph系列-一文知晓导入数据流程(待发布)"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"图解图库JanusGraph系列-简要分析查询读数据流程(待发布)"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"图解图库JanusGraph系列-一文知晓锁机制(本地锁+分布式锁)(待发布)"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://mp.weixin.qq.com/s?__biz=MzAwODkwMDk4OQ==&mid=2247483936&idx=1&sn=1b0ebe5699ee267d4b0360c90c937492&chksm=9b669a32ac111324e66af490a24478b81ad5221a88af61897353a6f4d80ff6005b60fbbb8490&scene=126&sessionid=1598925790&key=4598b5ee8f6c495005e73b5167cd30412733226c9d6ce36","title":""},"content":[{"type":"text","text":"图解图库JanusGraph系列-一文知晓分布式id生成策略"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"图解图库JanusGraph系列-一文知晓图库存储分区策略(待发布)"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"存储结构相关:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://mp.weixin.qq.com/s?_biz=MzAwODkwMDk4OQ==&mid=2247483894&idx=1&sn=0d7b98d8d7abf86bfacf8c86b694651d&chksm=9b6699e4ac1110f2626789d78aaf617dc02b7a9cdad320c5273172a6fa3a21d8f40d63958461&token=2053731774&lang=zhCN#rd","title":""},"content":[{"type":"text","text":"图解图库JanusGraph系列-一文知晓图数据底层存储结构"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"其他:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://mp.weixin.qq.com/s?_biz=MzAwODkwMDk4OQ==&mid=2247483830&idx=1&sn=71ad0d9e0d5868ae15011b7744c0fe8f&chksm=9b6699a4ac1110b294487a6987be5392a5093405a7a40f58d4bca697a18d64000db1aeda0a6f&token=1631136587&lang=zhCN#rd","title":""},"content":[{"type":"text","text":"解惑图数据库!你知道什么是图数据库吗?"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"图解图库JanusGraph系列-官方测试图:诸神之图分析(待发布)"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"**"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"源码分析相关可查看github(求star~~)"}]},{"type":"text","text":"**: "},{"type":"link","attrs":{"href":"https://github.com/YYDreamer/janusgraph","title":""},"content":[{"type":"text","text":"https://github.com/YYDreamer/janusgraph"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下述流程高清大图地址:"},{"type":"link","attrs":{"href":"https://www.processon.com/view/link/5f471b2e7d9c086b9903b629","title":""},"content":[{"type":"text","text":"https://www.processon.com/view/link/5f471b2e7d9c086b9903b629"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"版本:JanusGraph-0.5.2"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "},{"type":"text","marks":[{"type":"strong"}],"text":"转载文章请保留以下声明:"}]},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"作者:洋仔聊编程"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"微信公众号:匠心Java"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"原文地址:"},{"type":"link","attrs":{"href":"https://xie.infoq.cn/article/20cfc44ff85cf6bf326f3f6f7","title":""},"content":[{"type":"text","text":"https://xie.infoq.cn/article/20cfc44ff85cf6bf326f3f6f7"}]}]}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"正文"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在介绍JanusGraph的分布式ID生成策略之前,我们来简单分析一下"},{"type":"codeinline","content":[{"type":"text","text":"分布式ID"}]},{"type":"text","text":"应该满足哪些特征?"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"全局唯一"},{"type":"text","text":":必须保证ID是分布式环境中全局性唯一的,这是基本要求"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"高性能"},{"type":"text","text":":高可用低延时,ID生成响应快;否则可能会成为业务瓶颈"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"高可用"},{"type":"text","text":":提供分布式id的生成的服务要保证高可用,不能随随便便就挂掉了,会对业务产生影响"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"趋势递增"},{"type":"text","text":":主要看业务场景,类似于图存储中节点的唯一id就尽量保持趋势递增;但是如果类似于电商订单就尽量不要趋势递增,因为趋势递增会被恶意估算出当天的订单量和成交量,泄漏公司信息"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"接入方便"},{"type":"text","text":":如果是中间件,要秉着拿来即用的设计原则,在系统设计和实现上要尽可能的简单"}]}]}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"一:常用分布式id生成策略"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"当前"},{"type":"codeinline","content":[{"type":"text","text":"常用的"}]},{"type":"text","text":"分布式id的生成策略主要分为以下四种:"}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"UUID"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"数据库+号段模式(优化:数据库+号段+双buffer)"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"基于Redis实现"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"雪花算法(SnowFlake)"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"还有一些其他的比如:基于数据库自增id、数据库多主模式等,这些在小并发的情况下可以使用,大并发的情况下就不太ok了"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"市面上有一些生成分布式id的开源组件,包括滴滴基于"},{"type":"codeinline","content":[{"type":"text","text":"数据库+号段"}]},{"type":"text","text":"实现的"},{"type":"codeinline","content":[{"type":"text","text":"TinyID"}]},{"type":"text","text":" 、百度基于"},{"type":"codeinline","content":[{"type":"text","text":"SnowFlake"}]},{"type":"text","text":"的"},{"type":"codeinline","content":[{"type":"text","text":"Uidgenerator"}]},{"type":"text","text":"、美团支持"},{"type":"codeinline","content":[{"type":"text","text":"号段"}]},{"type":"text","text":"和"},{"type":"codeinline","content":[{"type":"text","text":"SnowFlake"}]},{"type":"text","text":"的"},{"type":"codeinline","content":[{"type":"text","text":"Leaf"}]},{"type":"text","text":"等"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"那么,在JanusGraph中分布式id的生成是采用的什么方式呢? "}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"二:JanusGraph的分布式id策略"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在JanusGraph中,分布式id的生成采用的是"},{"type":"codeinline","content":[{"type":"text","text":"数据库+号段+双buffer优化"}]},{"type":"text","text":"的模式; 下面我们来具体分析一下:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"分布式id生成使用的数据库就是JanusGraph当前使用的第三方存储后端,这里我们以使用的存储后端"},{"type":"codeinline","content":[{"type":"text","text":"Hbase"}]},{"type":"text","text":"为例;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"JanusGraph分布式id生成所需元数据存储位置:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在Hbase中有"},{"type":"codeinline","content":[{"type":"text","text":"column family 列族"}]},{"type":"text","text":"的概念; JanusGraph在初始化Hbase表时默认创建了9大列族,用于存储不同的数据, 具体看《图解图库JanusGraph系列-一文知晓图数据底层存储结构》;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"其中有一个列族"},{"type":"codeinline","content":[{"type":"text","text":"janusgraph_ids"}]},{"type":"text","text":"简写为"},{"type":"codeinline","content":[{"type":"text","text":"i"}]},{"type":"text","text":"这个列族,主要存储的就是JanusGraph分布式id生成所需要的元数据!"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"JanusGraph的分布式id的组成结构:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":"\t // 源码中有一句话体现\n /*\t\t--- JanusGraphElement id bit format ---\n * [ 0 | count | partition | ID padding (if any) ]\n */"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"主要分为4部分:"},{"type":"codeinline","content":[{"type":"text","text":"0、count、partition、ID padding(每个类型是固定值)"}]},{"type":"text","text":";"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"其实这4部分的顺序在序列化为二进制数据时,顺序会有所改变;这里只是标明了id的组成部分!"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"上述部分的"},{"type":"codeinline","content":[{"type":"text","text":"partition"}]},{"type":"text","text":" + "},{"type":"codeinline","content":[{"type":"text","text":"count"}]},{"type":"text","text":"来保证分布式节点的唯一性;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"partition id:分区id值,JanusGraph默认分了32个逻辑分区;节点分到哪个分区采用的是"},{"type":"codeinline","content":[{"type":"text","text":"随机分配"}]},{"type":"text","text":";"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"count:每个partition都有对应的一个count范围:0-2的55次幂;JanusGraph每次拉取一部分的范围作为节点的count取值;JanusGraph保证了针对相同的partition,不会重复获取同一个count值!"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"保证count在partition维度保持全局唯一性,就保证了生成的最终id的全局唯一性!!"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"*则分布式id的唯一性保证,就在于"},{"type":"codeinline","content":[{"type":"text","text":"count"}]},{"type":"text","text":"基于"},{"type":"codeinline","content":[{"type":"text","text":"partition"}]},{"type":"text","text":"维度的唯一性!下面我们的分析也是着重在"},{"type":"codeinline","content":[{"type":"text","text":"count"}]},{"type":"text","text":"的获取!* "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"JanusGraph分布式id生成的主要逻辑流程如下图所示:(推荐结合源码分析观看!)"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"分析过程中有一个概念为"},{"type":"codeinline","content":[{"type":"text","text":"id block"}]},{"type":"text","text":":指当前获取的号段范围"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/29/29288622004088ff0034a30a0be8d1cd.png","alt":"在这里插入图片描述","title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"JanusGraph主要使用`"},{"type":"codeinline","content":[{"type":"text","text":"PartitionIDPool "}]},{"type":"text","text":"类来存储不同类型的"},{"type":"codeinline","content":[{"type":"text","text":"StandardIDPool"}]},{"type":"text","text":"; 在"},{"type":"codeinline","content":[{"type":"text","text":"StandardIDPool"}]},{"type":"text","text":"中主要包含两个id Block:"}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"current block:当前生成id使用的block"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"next block:double buffer中的另一个已经准备好的block"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"为什么要有两个block呢?"},{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"主要是如果只有一个block的话,当我们在使用完当前的block时,需要阻塞等待区获取下一个block,这样便会导致分布式id生成较长时间的阻塞等待block的获取;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"怎么优化上述问题呢? "},{"type":"codeinline","content":[{"type":"text","text":"double buffer"}]},{"type":"text","text":"; "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"除了当前使用的block,我们再存储一个"},{"type":"codeinline","content":[{"type":"text","text":"next block"}]},{"type":"text","text":";当正在使用的block假设已经使用了50%,触发"},{"type":"codeinline","content":[{"type":"text","text":"next block"}]},{"type":"text","text":"的异步获取,如上图的蓝色部分所示;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"这样当"},{"type":"codeinline","content":[{"type":"text","text":"current block"}]},{"type":"text","text":"使用完成后可以直接无延迟的切换到"},{"type":"codeinline","content":[{"type":"text","text":"next block"}]},{"type":"text","text":"如上图中绿色部分所示; "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"在执行过程中可能会因为一些异常导致节点id获取失败,则会进行重试;重试次数默认为1000次;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":"private static final int MAX_PARTITION_RENEW_ATTEMPTS = 1000;\nfor (int attempt = 0; attempt < MAX_PARTITION_RENEW_ATTEMPTS; attempt++) {\n // 获取id的过程\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"ps:上述所说的IDPool和block是基于当前"},{"type":"codeinline","content":[{"type":"text","text":"图实例"}]},{"type":"text","text":"维度共用的!"}]}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"三:源码分析"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在JanusGraph的源码中,主要包含两大部分和其他的一些组件:"}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Graph相关类:用于对节点、属性、边的操作"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Transaction相关类:用于在对数据或者Schema进行CURD时,进行事务处理"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"其他一些:分布式节点id生成类;序列化类;第三方索引操作类等等"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Graph和Transaction相关类的类图如下所示:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/c8/c8656968eb85ef0a325b3eb49314c62e.png","alt":"在这里插入图片描述","title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"分布式id涉及到id生成的类图如下所示:"}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/0f/0fcb50f0c85fac9e310ab8c78fe8f438.png","alt":"在这里插入图片描述","title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"初始数据:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" @Test\n public void addVertexTest(){\n List godProperties = new ArrayList<>();\n godProperties.add(T.label);\n godProperties.add(\"god\");\n\n godProperties.add(\"name\");\n godProperties.add(\"lyy\");\n\n godProperties.add(\"age\");\n godProperties.add(18);\n\n JanusGraphVertex godVertex = graph.addVertex(godProperties.toArray());\n\n assertNotNull(godVertex);\n }"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在"},{"type":"codeinline","content":[{"type":"text","text":"诸神之图"}]},{"type":"text","text":"中添加一个name为"},{"type":"codeinline","content":[{"type":"text","text":"lyy"}]},{"type":"text","text":"节点;看下执行流程,注意,此处主要分析的节点的分布式id生成代码!"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"**1、调用"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"JanusGraphBlueprintsGraph"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"类的"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"AddVertex方法"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"**"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" @Override\n public JanusGraphVertex addVertex(Object... keyValues) {\n // 添加节点\n return getAutoStartTx().addVertex(keyValues);\n }"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"**2、调用"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"JanusGraphBlueprintsTransaction"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"的"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"addVertex"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"方法**"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" public JanusGraphVertex addVertex(Object... keyValues) {\n // 。。。省略了其他的处理\n // 该处生成节点对象,包含节点的唯一id生成逻辑\n final JanusGraphVertex vertex = addVertex(id, label); \n // 。。。省略了其他的处理\n return vertex;\n }"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"**3、调用"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"StandardJanusGraphTx"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"的"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"addVertex"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"方法**"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" @Override\n public JanusGraphVertex addVertex(Long vertexId, VertexLabel label) {\n // 。。。省略了其他的处理\n if (vertexId != null) {\n vertex.setId(vertexId);\n } else if (config.hasAssignIDsImmediately() || label.isPartitioned()) {\n graph.assignID(vertex,label); // 为节点分配正式的节点id!\n }\n // 。。。省略了其他的处理\n return vertex;\n }"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"**4、调用"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"VertexIDAssigner"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"的"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"assignID(InternalElement element, IDManager.VertexIDType vertexIDType)"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"方法**"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" private void assignID(InternalElement element, IDManager.VertexIDType vertexIDType) {\n // 开始获取节点分布式唯一id\n // 因为一些异常导致获取节点id失败,进行重试,重试此为默认为1000次\n for (int attempt = 0; attempt < MAX_PARTITION_RENEW_ATTEMPTS; attempt++) {\n // 初始化一个partiiton id\n long partitionID = -1;\n // 获取一个partition id\n // 不同类型的数据,partition id的获取方式也有所不同\n if (element instanceof JanusGraphSchemaVertex) {\n // 为partition id赋值\n }\n try {\n // 正式分配节点id, 依据partition id 和 节点类型\n assignID(element, partitionID, vertexIDType);\n } catch (IDPoolExhaustedException e) {\n continue; //try again on a different partition\n }\n assert element.hasId();\n // 。。。省略了其他代码\n }\n }"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"**5、调用了"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"VertexIDAssigner"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"的"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"assignID(final InternalElement element, final long partitionIDl, final IDManager.VertexIDType userVertexIDType)"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"方法**"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" private void assignID(final InternalElement element, final long partitionIDl, final IDManager.VertexIDType userVertexIDType) {\n \n final int partitionID = (int) partitionIDl;\n\n // count为分布式id组成中的一部分,占55个字节\n // 分布式id的唯一性保证,就在于`count`基于`partition`维度的唯一性\n long count;\n if (element instanceof JanusGraphSchemaVertex) { // schema节点处理\n Preconditions.checkArgument(partitionID==IDManager.SCHEMA_PARTITION);\n count = schemaIdPool.nextID();\n } else if (userVertexIDType==IDManager.VertexIDType.PartitionedVertex) { // 配置的热点节点,类似于`makeVertexLabel('product').partition()`的处理\n count = partitionVertexIdPool.nextID();\n } else { // 普通节点和边类型的处理\n // 首先获取当前partition敌营的idPool\n PartitionIDPool partitionPool = idPools.get(partitionID);\n // 如果当前分区对应的IDPool为空,则创建一个默认的IDPool,默认size = 0\n if (partitionPool == null) {\n // 在PartitionIDPool中包含多种类型对应的StandardIDPool类型\n // StandardIDPool中包含对应的block信息和count信息\n partitionPool = new PartitionIDPool(partitionID, idAuthority, idManager, renewTimeoutMS, renewBufferPercentage);\n // 缓存下来\n idPools.putIfAbsent(partitionID,partitionPool);\n // 从缓存中再重新拿出\n partitionPool = idPools.get(partitionID);\n }\n // 确保partitionPool不为空\n Preconditions.checkNotNull(partitionPool);\n // 判断当前分区的IDPool是否枯竭;已经被用完\n if (partitionPool.isExhausted()) {\n // 如果被用完,则将该分区id放到对应的缓存中,避免之后获取分区id再获取到该分区id\n placementStrategy.exhaustedPartition(partitionID);\n // 抛出IDPool异常, 最外层捕获,然后进行重试获取节点id\n throw new IDPoolExhaustedException(\"Exhausted id pool for partition: \" + partitionID);\n }\n // 存储当前类型对应的IDPool,因为partitionPool中保存好几个类型的IDPool\n IDPool idPool;\n if (element instanceof JanusGraphRelation) {\n idPool = partitionPool.getPool(PoolType.RELATION);\n } else {\n Preconditions.checkArgument(userVertexIDType!=null);\n idPool = partitionPool.getPool(PoolType.getPoolTypeFor(userVertexIDType));\n }\n try {\n // 重要!!!! 依据给定的IDPool获取count值!!!!\n // 在此语句中设计 block的初始化 和 double buffer block的处理!\n count = idPool.nextID();\n partitionPool.accessed();\n } catch (IDPoolExhaustedException e) { // 如果该IDPool被用完,抛出IDPool异常, 最外层捕获,然后进行重试获取节点id\n log.debug(\"Pool exhausted for partition id {}\", partitionID);\n placementStrategy.exhaustedPartition(partitionID);\n partitionPool.exhaustedIdPool();\n throw e;\n }\n }\n\n // 组装最终的分布式id:[count + partition id + ID padding]\n long elementId;\n if (element instanceof InternalRelation) {\n elementId = idManager.getRelationID(count, partitionID);\n } else if (element instanceof PropertyKey) {\n elementId = IDManager.getSchemaId(IDManager.VertexIDType.UserPropertyKey,count);\n } else if (element instanceof EdgeLabel) {\n elementId = IDManager.getSchemaId(IDManager.VertexIDType.UserEdgeLabel, count);\n } else if (element instanceof VertexLabel) {\n elementId = IDManager.getSchemaId(IDManager.VertexIDType.VertexLabel, count);\n } else if (element instanceof JanusGraphSchemaVertex) {\n elementId = IDManager.getSchemaId(IDManager.VertexIDType.GenericSchemaType,count);\n } else {\n elementId = idManager.getVertexID(count, partitionID, userVertexIDType);\n }\n\n Preconditions.checkArgument(elementId >= 0);\n // 对节点对象赋值其分布式唯一id\n element.setId(elementId);\n }"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"上述代码,我们拿到了对应的IdPool,有两种情况:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"第一次获取分布式id时,分区对应的IDPool初始化为默认的size = 0的IDPool"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"分区对应的IDPool不是初次获取"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"这两种情况的处理,都在代码"},{"type":"codeinline","content":[{"type":"text","text":"count = idPool.nextID()"}]},{"type":"text","text":"的"},{"type":"codeinline","content":[{"type":"text","text":"StandardIDPool"}]},{"type":"text","text":"类中的"},{"type":"codeinline","content":[{"type":"text","text":"nextID()"}]},{"type":"text","text":"方法中被处理!"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"**在分析该代码之前,我们需要知道 "},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"PartitionIDPool "}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"和"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"StandardIDPool"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"的关系:**"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"每个partition都有一个对应的"},{"type":"codeinline","content":[{"type":"text","text":"PartitionIDPool extends EnumMap "}]},{"type":"text","text":" 是一个枚举map类型;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"每一个"},{"type":"codeinline","content":[{"type":"text","text":"PartitionIDPool "}]},{"type":"text","text":"都有对应的不同类型的"},{"type":"codeinline","content":[{"type":"text","text":"StandardIDPool"}]},{"type":"text","text":":"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"NORMAL_VERTEX:用于vertex id的分配"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"UNMODIFIABLE_VERTEX:用于schema label id的分配"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"RELATION:用于edge id的分配"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在"},{"type":"codeinline","content":[{"type":"text","text":"StandardIDPool"}]},{"type":"text","text":"中包含多个字段,分别代表不同的含义,抽取几个重要的字段进行介绍:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" private static final int RENEW_ID_COUNT = 100; \n private final long idUpperBound; // Block的最大值,默认为2的55次幂\n private final int partition; // 当前pool对应的分区\n private final int idNamespace; // 标识pool为那种类型的pool,上述的三种类型NORMAL_VERTEX、UNMODIFIABLE_VERTEX、RELATION;值为当前枚举值在枚举中的位置\n\n private final Duration renewTimeout;// 重新获取block的超时时间\n private final double renewBufferPercentage;// 双buffer中,当第一个buffer block使用的百分比,到达配置的百分比则触发other buffer block的获取\n\n private IDBlock currentBlock; // 当前的block\n private long currentIndex; // 标识当前block使用到那一个位置\n private long renewBlockIndex; // 依据currentBlock.numIds()*renewBufferPercentage来获取这个值,主要用于在当前的block在消费到某个index的时候触发获取下一个buffer block\n\n private volatile IDBlock nextBlock;// 双buffer中的另外一个block\n\n private final ThreadPoolExecutor exec;// 异步获取双buffer的线程池\n"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"**6、调用了"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"StandardIDPool"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"类中的"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"nextID"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"方法**"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"经过上述分析,我们知道,分布式唯一id的唯一性是由在partition维度下的count的值的唯一性来保证的;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"上述代码通过调用IDPool的nextId来获取count值; "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下述代码就是获取count的逻辑;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" @Override\n public synchronized long nextID() {\n // currentIndex标识当前的index小于current block的最大值\n assert currentIndex <= currentBlock.numIds();\n\n // 此处涉及两种情况:\n // 1、分区对应的IDPool是第一次被初始化;则currentIndex = 0; currentBlock.numIds() = 0;\n // 2、分区对应的该IDPool不是第一次,但是此次的index正好使用到了current block的最后一个count\n if (currentIndex == currentBlock.numIds()) {\n try {\n // 将current block赋值为next block\n // next block置空 并计算renewBlockIndex\n nextBlock();\n } catch (InterruptedException e) {\n throw new JanusGraphException(\"Could not renew id block due to interruption\", e);\n }\n }\n \n // 在使用current block的过程中,当current index == renewBlockIndex时,触发double buffer next block的异步获取!!!!\n if (currentIndex == renewBlockIndex) {\n // 异步获取next block\n startIDBlockGetter();\n }\n \n // 生成最终的count\n long returnId = currentBlock.getId(currentIndex);\n // current index + 1\n currentIndex++;\n if (returnId >= idUpperBound) throw new IDPoolExhaustedException(\"Reached id upper bound of \" + idUpperBound);\n log.trace(\"partition({})-namespace({}) Returned id: {}\", partition, idNamespace, returnId);\n // 返回最终获取的分区维度的全局唯一count\n return returnId;\n }"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"上述代码中进行了两次判断:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"currentIndex == currentBlock.numIds():"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" * 第一次生成分布式id:此处判断即为 0==0;然后生成新的block"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" * 非第一次生成分布式id:等于情况下标识当前的block已经使用完了,需要切换为next block"}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"currentIndex == renewBlockIndex"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" * renew index:标识index使用多少后开始获取下一个double buffer 的next block;有一个默认值100,主要为了兼容第一次分布式id的生成;相等则会触发异步获取下一个next block"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下面我们分别对"},{"type":"codeinline","content":[{"type":"text","text":"nextBlock();"}]},{"type":"text","text":"逻辑和"},{"type":"codeinline","content":[{"type":"text","text":"startIDBlockGetter();"}]},{"type":"text","text":"进行分析;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"**7、调用了"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"StandardIDPool"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"类中的"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"nextBlock"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"方法**"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" private synchronized void nextBlock() throws InterruptedException {\n // 在分区对应的IDPool第一次使用时,double buffer的nextBlock为空\n if (null == nextBlock && null == idBlockFuture) {\n // 异步启动 获取id block\n startIDBlockGetter();\n }\n\n // 也是在分区对应的IDPool第一次使用时,因为上述为异步获取,所以在执行到这一步时nextBlock可能还没拿到\n // 所以需要阻塞等待block的获取\n if (null == nextBlock) {\n waitForIDBlockGetter();\n }\n\n // 将当前使用block指向next block\n currentBlock = nextBlock;\n // index清零\n currentIndex = 0;\n // nextBlock置空\n nextBlock = null;\n\n // renewBlockIndex用于双buffer中,当第一个buffer block使用的百分比,到达配置的百分比则触发other buffer block的获取\n // 值current block 对应的count数量 - (值current block 对应的count数量 * 为renewBufferPercentage配置的剩余空间百分比)\n // 在使用current block的时候,当current index == renewBlockIndex时,触发double buffer next block的异步获取!!!!\n renewBlockIndex = Math.max(0,currentBlock.numIds()-Math.max(RENEW_ID_COUNT, Math.round(currentBlock.numIds()*renewBufferPercentage)));\n }"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"主要是做了三件事:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"1、block是否为空,为空的话则异步获取一个block"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"2、nextBlock不为空的情况下:next赋值到current、next置空、index置零"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"3、计算获取下一个nextBlock的触发index renewBlockIndex值"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"**8、调用了"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"StandardIDPool"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"类中的"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"startIDBlockGetter"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"方法**"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" private synchronized void startIDBlockGetter() {\n Preconditions.checkArgument(idBlockFuture == null, idBlockFuture);\n if (closed) return; //Don't renew anymore if closed\n //Renew buffer\n log.debug(\"Starting id block renewal thread upon {}\", currentIndex);\n // 创建一个线程对象,包含给定的权限控制类、分区、命名空间、超时时间\n idBlockGetter = new IDBlockGetter(idAuthority, partition, idNamespace, renewTimeout);\n // 提交获取double buffer的线程任务,异步执行\n idBlockFuture = exec.submit(idBlockGetter);\n }"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"其中创建一个线程任务,提交到线程池"},{"type":"codeinline","content":[{"type":"text","text":"exec"}]},{"type":"text","text":"进行异步执行;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下面看下,线程类的"},{"type":"codeinline","content":[{"type":"text","text":"call"}]},{"type":"text","text":"方法主要是调用了"},{"type":"codeinline","content":[{"type":"text","text":" idAuthority.getIDBlock"}]},{"type":"text","text":"方法,这个方法主要是基于Hbase来获取还未使用的block;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" /**\n * 获取double buffer block的线程类\n */\n private static class IDBlockGetter implements Callable {\n\n // 省略部分代码\n @Override\n public IDBlock call() {\n Stopwatch running = Stopwatch.createStarted();\n try {\n // 此处调用idAuthority 调用HBase进行占用获取Block\n IDBlock idBlock = idAuthority.getIDBlock(partition, idNamespace, renewTimeout);\n return idBlock;\n } catch (BackendException e) {}\n }\n }"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"**9、调用"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"ConsistentKeyIDAuthority"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"类的"},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"getIDBlock"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"方法**"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" @Override\n public synchronized IDBlock getIDBlock(final int partition, final int idNamespace, Duration timeout) throws BackendException {\n \n // 开始时间\n final Timer methodTime = times.getTimer().start();\n\n // 获取当前命名空间配置的blockSize,默认值10000;可自定义配置\n final long blockSize = getBlockSize(idNamespace);\n // 获取当前命名空间配置的最大id值idUpperBound;值为:2的55次幂大小\n final long idUpperBound = getIdUpperBound(idNamespace);\n // uniqueIdBitWidth标识uniqueId占用的位数;uniqueId为了兼容“关闭分布式id唯一性保障”的开关情况,uniqueIdBitWidth默认值=4\n // 值:64-1(默认0)-5(分区占用位数)-3(ID Padding占用位数)-4(uniqueIdBitWidth) = 51;标识block中的上限为2的51次幂大小\n final int maxAvailableBits = (VariableLong.unsignedBitLength(idUpperBound)-1)-uniqueIdBitWidth;\n\n // 标识block中的上限为2的51次幂大小\n final long idBlockUpperBound = (1L < exhaustedUniquePIDs = new ArrayList<>(randomUniqueIDLimit);\n\n // 默认0.3秒 用于处理TemporaryBackendException异常情况(后端存储出现问题)下:阻塞一断时间,然后进行重试\n Duration backoffMS = idApplicationWaitMS;\n\n // 从开始获取IDBlock开始,持续超时时间(默认2分钟)内重试获取IDBlock\n while (methodTime.elapsed().compareTo(timeout) < 0) {\n final int uniquePID = getUniquePartitionID(); // 获取uniquePID,默认情况下“开启分布式id唯一性控制”,值 = 0; 当“关闭分布式id唯一性控制”时为一个随机值\n final StaticBuffer partitionKey = getPartitionKey(partition,idNamespace,uniquePID); // 依据partition + idNamespace + uniquePID组装一个RowKey\n try {\n long nextStart = getCurrentID(partitionKey); // 从Hbase中获取当前partition对应的IDPool中被分配的最大值,用来作为当前申请新的block的开始值\n if (idBlockUpperBound - blockSize <= nextStart) { // 确保还未被分配的id池中的id个数,大于等于blockSize\n // 相应处理\n }\n\n long nextEnd = nextStart + blockSize; // 获取当前想要获取block的最大值\n StaticBuffer target = null;\n\n // attempt to write our claim on the next id block\n boolean success = false;\n try {\n Timer writeTimer = times.getTimer().start(); // ===开始:开始进行插入自身的block需求到Hbase\n target = getBlockApplication(nextEnd, writeTimer.getStartTime()); // 组装对应的Column: -nextEnd + 当前时间戳 + uid(唯一标识当前图实例)\n final StaticBuffer finalTarget = target; // copy for the inner class\n BackendOperation.execute(txh -> { // 异步插入当前生成的RowKey 和 Column\n idStore.mutate(partitionKey, Collections.singletonList(StaticArrayEntry.of(finalTarget)), KeyColumnValueStore.NO_DELETIONS, txh);\n return true;\n },this,times);\n writeTimer.stop(); // ===结束:插入完成\n\n final boolean distributed = manager.getFeatures().isDistributed();\n Duration writeElapsed = writeTimer.elapsed(); // ===获取方才插入的时间耗时\n if (idApplicationWaitMS.compareTo(writeElapsed) < 0 && distributed) { // 判断是否超过配置的超时时间,超过则报错TemporaryBackendException,然后等待一断时间进行重试\n throw new TemporaryBackendException(\"Wrote claim for id block [\" + nextStart + \", \" + nextEnd + \") in \" + (writeElapsed) + \" => too slow, threshold is: \" + idApplicationWaitMS);\n } else {\n\n assert 0 != target.length();\n final StaticBuffer[] slice = getBlockSlice(nextEnd); // 组装下述基于上述Rowkey的Column的查找范围:(-nextEnd + 0 : 0nextEnd + 最大值) \n\n final List blocks = BackendOperation.execute( // 异步获取指定Rowkey和指定Column区间的值\n (BackendOperation.Transactional>) txh -> idStore.getSlice(new KeySliceQuery(partitionKey, slice[0], slice[1]), txh),this,times);\n if (blocks == null) throw new TemporaryBackendException(\"Could not read from storage\");\n if (blocks.isEmpty())\n throw new PermanentBackendException(\"It seems there is a race-condition in the block application. \" +\n \"If you have multiple JanusGraph instances running on one physical machine, ensure that they have unique machine idAuthorities\");\n\n if (target.equals(blocks.get(0).getColumnAs(StaticBuffer.STATIC_FACTORY))) { // 如果获取的集合中,当前的图实例插入的数据是第一条,则表示获取block; 如果不是第一条,则获取Block失败\n // 组装IDBlock对象\n ConsistentKeyIDBlock idBlock = new ConsistentKeyIDBlock(nextStart,blockSize,uniqueIdBitWidth,uniquePID);\n\n if (log.isDebugEnabled()) {\n idBlock, partition, idNamespace, uid);\n }\n\n success = true;\n return idBlock; // 返回\n } else { }\n }\n } finally {\n if (!success && null != target) { // 在获取Block失败后,删除当前的插入; 如果没有失败,则保留当前的插入,在hbase中标识该Block已经被占用\n //Delete claim to not pollute id space\n for (int attempt = 0; attempt < ROLLBACK_ATTEMPTS; attempt++) { // 回滚:删除当前插入,尝试次数5次\n }\n }\n }\n } catch (UniqueIDExhaustedException e) {\n // No need to increment the backoff wait time or to sleep\n log.warn(e.getMessage());\n } catch (TemporaryBackendException e) {\n backoffMS = Durations.min(backoffMS.multipliedBy(2), idApplicationWaitMS.multipliedBy(32));\n sleepAndConvertInterrupts(backoffMS); \\\n }\n }\n\n throw new TemporaryLockingException();\n }\n"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"主要的逻辑就是:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"组装Rowkey"}]},{"type":"text","text":":partition + idNameSpace+unquePId"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"组装Column"}]},{"type":"text","text":":-nextEnd+now time+uid"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"将"},{"type":"codeinline","content":[{"type":"text","text":"RowKey+Column"}]},{"type":"text","text":"插入Hbase"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"获取的上述组装的RowKey 基于(-nextEnd + 0 : -nextEnd + max)范围的所有Column集合"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"判断集合的第一个Column是不是当前插入的Column,是的话则占用block成功,不是的话则占用失败,删除刚才占用并进行重试"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"最终:异步获取到了唯一占用的Block,然后生成对应的唯一count,组装最后的唯一id"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"整体的调用流程如下:"}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/2e/2e5b1cd3e7351ae4684096de9281122b.png","alt":"在这里插入图片描述","title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"四:其他类型的id生成"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"上述我们主要依据生成节点id(vertex id)的过程来进行分析"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在"},{"type":"codeinline","content":[{"type":"text","text":"JanusGraph"}]},{"type":"text","text":"中还包含"},{"type":"codeinline","content":[{"type":"text","text":"edge id"}]},{"type":"text","text":"、"},{"type":"codeinline","content":[{"type":"text","text":"property id"}]},{"type":"text","text":"、"},{"type":"codeinline","content":[{"type":"text","text":"schema label id"}]},{"type":"text","text":"等几种的分布式id生成"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"所有类型的分布式id的生成主要思想和逻辑都几乎相同,只是一些具体的逻辑可能有所不同,我们理解了"},{"type":"codeinline","content":[{"type":"text","text":"vertex id"}]},{"type":"text","text":"的分布式id生成流程,其他的也可以理解了。"}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"1、property id的生成"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在JanusGraph中的"},{"type":"codeinline","content":[{"type":"text","text":"property"}]},{"type":"text","text":"的分布式唯一id的生成,整体逻辑和"},{"type":"codeinline","content":[{"type":"text","text":"vertex id"}]},{"type":"text","text":"的生成逻辑大体相同;"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"property id"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"的 生成和 "},{"type":"codeinline","content":[{"type":"text","marks":[{"type":"strong"}],"text":"vertex id"}],"marks":[{"type":"strong"}]},{"type":"text","marks":[{"type":"strong"}],"text":"有两点不同:"}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"ID的组成部分: 在"},{"type":"codeinline","content":[{"type":"text","text":"vertex id"}]},{"type":"text","text":"中组成部分包含"},{"type":"codeinline","content":[{"type":"text","text":"count"}]},{"type":"text","text":"+"},{"type":"codeinline","content":[{"type":"text","text":"partition"}]},{"type":"text","text":"+"},{"type":"codeinline","content":[{"type":"text","text":"ID Padding"}]},{"type":"text","text":"; 而在"},{"type":"codeinline","content":[{"type":"text","text":"property id"}]},{"type":"text","text":"中没有"},{"type":"codeinline","content":[{"type":"text","text":"ID Padding"}]},{"type":"text","text":"部分,其组成为"},{"type":"codeinline","content":[{"type":"text","text":"count + partition"}]}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"java"},"content":[{"type":"text","text":" long id = (count<
發表評論
所有評論
還沒有人評論,想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.
相關文章