无需扩容成本，百亿集群数据过期性能优化看这就行

原創

2021-07-15 13:03

节点类型

单个节点容器规格

说明

shardServer存储节点

16CPU、64G内存、1T磁盘

1个分片存储约30亿数据

3个分片预计存储100亿数据

mongos及config server节点

8CPU\/8G内存\/50G磁盘

一个代理和一个config server节点复用同一个容器"}}},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"2.2 集群部署架构"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"该业务数据不是很重要，为了节省成本，因此我们采用2+1模式部署，也就是：2mongod+1arbiter模式，同城机房部署，部署架构图如下图所示："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https:\/\/static001.geekbang.org\/infoq\/a9\/a931a25db0c37054ae2a301386bc56c1.png","alt":"图片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"考虑到数据重要性不高，通过2mongod+1arbiter模式即可满足用户要求，同时可以最大化节省成本。"}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"三、性能优化过程"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"该集群优化过程按照如下两个步骤优化：业务使用集群前的性能优化、业务使用过程中的性能优化。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"业务提前建好查询对应的最优索引，同时创建过期索引："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"text"},"content":[{"type":"text","text":"\ndb.dailyCloudOperateInfo.createIndex( { \"createTime\": 1 }, { expireAfterSeconds: 604800} )"}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"3.1 业务使用集群前的性能优化"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"和业务沟通确定，业务每条数据都携带有一个设备标识userId，同时业务查询更新等都是根据userId维度查询该设备下面的单条或者一批数据，因此片建选择userId。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"分片方式"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"为了充分散列数据到3个分片，因此选择hash分片方式，这样数据可以最大化散列，同时可以满足同一个userId数据落到同一个分片，保证查询效率。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"预分片"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"mongodb如果分片片建为hashed分片，则可以提前做预分片，这样就可以保证数据写进来的时候比较均衡的写入多个分片。预分片的好处可以规避非预分片情况下的chunk迁移问题，最大化提升写入性能。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"text"},"content":[{"type":"text","text":"\nsh.shardCollection(\"cloud_track.dailyCloudOperateInfo\", {userId:\"hashed\"}, false, { numInitialChunks: 8192} )\n"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"就近读"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"客户端增加secondaryPreferred配置，优先读从节点。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"禁用enable Majority Read Concern"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"禁用该功能后Read Concern majority将会报错，Read Concern majority功能注意是避免脏读，和业务沟通业务没该需求，因此可以直接关闭。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"mongodb默认使能了enable Majority Read Concern，该功能开启对性能有一定影响，参考："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"MongoDB readConcern 原理解析："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"https:\/\/developer.aliyun.com\/article\/60553"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"OPPO百万级高并发MongoDB集群性能数十倍提升优化实践："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"https:\/\/mongoing.com\/archives\/29934"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"存储引擎cacheSize规格选择"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"单个容器规格：16CPU、64G内存、7T磁盘，考虑到全量迁移过程中对内存压力，内存碎片等压力会比较大，为了避免OOM，设置cacheSize=42G。"}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"3.2 业务使用过程中遇到的问题及性能优化"}]},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"1）第一轮优化：存储引擎优化"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"业务高峰期主要是数据写入和更新，内存脏数据较多，当脏数据比例达到一定比例后用户读写请求对应线程将会阻塞，用户线程也会去淘汰内存中的脏数据page，最终写性能下降明显。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"wiredtiger存储引擎cache淘汰策略相关的几个配置如下:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https:\/\/static001.geekbang.org\/infoq\/ab\/ab7893eb65833ce784dbe1db0d919227.png","alt":"图片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"由于业务全量迁移数据是持续性的大流量写，而不是突发性的大流量写，因此eviction_target、eviction_trigger、eviction_dirty_target、eviction_dirty_trigger几个配置用处不大，这几个参数阀值只是在短时间突发流量情况下调整才有用。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"但是，在持续性长时间大流量写的情况下，我们可以通过提高wiredtiger存储引擎后台线程数来解决脏数据比例过高引起的用户请求阻塞问题，淘汰脏数据的任务最终交由evict模块后台线程来完成。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"全量大流量持续性写存储引擎优化如下："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"text"},"content":[{"type":"text","text":"\ndb.adminCommand( { setParameter : 1, \"wiredTigerEngineRuntimeConfig\" : \"eviction=(threads_min=4, threads_max=20)\"})"}]},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"2）第一轮优化后存在的问题"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"经过存储引擎后台线程数的调优后，数据写入和更新瓶颈解决，写入和更新过程时延都很平滑。但是随着一周后开始数据过期，业务写开始大量抖动，如下所示："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https:\/\/static001.geekbang.org\/infoq\/52\/52d804815fb9be94435aa0eb71b5e5ed.png","alt":"图片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"从上图可以看出平均时延极端情况下甚至达到了几百ms，这是业务完全接受不了的。通过mongostat监控发现如下现象："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"主节点mongostat监控统计"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https:\/\/static001.geekbang.org\/infoq\/af\/af883e2eb2480823f259160b27821ff2.png","alt":"图片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"从上面的监控可以看出，三个分片的每个主节点只有4000左右的写更新操作(注意：实际上还有4万左右的delete操作，由于主节点过期delete不会统计，因此只能通过从节点查看，详见后面分析，实际单个分片主节点写和删除操作4.4W\/s)，写流量很低。但是，监控中的脏数据比例持续性的超过20%，超过20%后业务的请求就需要进行脏数据淘汰，最终造成业务请求阻塞抖动。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"通过前面的分析可以看出，业务正常峰值写入3个分片差不多10W\/s。一星期后，七天前的数据需要过期，这时候过期删除的ops也需要delete删除10w\/S，由于这时候新数据同样按照10w\/s写入，因此集群就需要支持20w\/s的ops操作才能支持。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"显然，3个分片支持不了持续性的20w\/s左右的ops操作，因此如何不扩容情况下支撑业务需求将是一大难点。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"为何ttl过期主节点没用delete统计"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https:\/\/static001.geekbang.org\/infoq\/4b\/4bce188119306f8892b61be29d0a4e65.png","alt":"图片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"上图为mongodb单机模块架构图，主节点默认启用一个TTLMonitor线程，借助查询引起模块实时扫描过期索引，然后把满足条件的数据删除。整个删除过程没有走command命令处理模块，而命令计数操作只会在command模块计数，因此主节点的过期删除不会有delete操作。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"命令处理模块处理流程及其计数统计详见："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"mongodb内核源码模块化设计与实现专栏"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"更多mongodb模块化源码设计实现详见："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"https:\/\/github.com\/y123456yz\/reading-and-annotate-mongodb-3.6"}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"3.3 第二轮优化(过期删除放凌晨低峰期)-不可行"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"从前面的分析可以看出，我们三个分片支撑不了持续性20W\/S的更新和删除操作，因此我们考虑业务改造使用方式，把过期删除确定到凌晨低峰期。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"但是业务上线后出现其他的问题，由于业务凌晨会持续性的批量拉取分析过去几天的数据，如果过期和批量读数据叠加到一起，严重影响业务查询效率。最终，该方案不可行，如果不扩容，则需要其他方案。"}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"3.4 第三轮方案优化(天维度建表，过期删表)"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"为了不增加成本，同时3个分片又支撑不了20W\/s的读写删除等导致，为了尽量通过3个分片不扩容条件下来满足用户需求，因此转变方式，通过删表的方式来避免过期，具体实现如下："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"业务改造代码，以天为单位建表，每天的数据存到对应的表中。"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"建表后业务启用预分片功能，确保数据均衡分布到三个分片。"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"业务保存8天数据，第九天凌晨删除第一天的表"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 该方案业务时延统计对应收益如下："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https:\/\/static001.geekbang.org\/infoq\/46\/46f61dbfa09b645087d64a7809a3e86d.png","alt":"图片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https:\/\/static001.geekbang.org\/infoq\/c8\/c83d31b7066fbce52eb6d71f2bccfe90.png","alt":"图片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如上图所示，通过过期方式的优化，最终问题彻底解决，并且读写时延控制在0.5ms-2ms。"}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"四、优化收益总结"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"通过前面的一系列优化，最终没有扩容，并且解决了业务过期和大量数据写入更新引起的时延抖动问题，总体收益如下："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"改造优化前时延：经常几百ms抖动"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"改造优化后时延：0.5-2ms"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"作者介绍"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"杨亚洲，"},{"type":"text","text":"前滴滴出行专家工程师，现任OPPO文档数据库mongodb负责人，负责数万亿级数据量文档数据库mongodb内核研发、性能优化及运维工作，一直专注于分布式缓存、高性能服务端、数据库、中间件等相关研发。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"本文转载自：dbaplus社群（ID：dbaplus）"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"原文链接："},{"type":"link","attrs":{"href":"https:\/\/mp.weixin.qq.com\/s\/hlbmoeGbXi550Xhsa18WOA","title":"xxx","type":null},"content":[{"type":"text","text":"无需扩容成本，百亿集群数据过期性能优化看这就行！"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}}]}