探秘RocketMQ事务机制，如何保证消息零丢失

{"type":"doc","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"size","attrs":{"size":14}},{"type":"color","attrs":{"color":"#40A9FF","name":"blue"}},{"type":"strong","attrs":{}}],"text":"真正的大师永远怀着一颗学徒的心","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"引言","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"事务的概念就不用多说了，我相信阅读文章的童鞋都是有着非常深刻的认识。我们都知道","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"MQ","attrs":{}}],"attrs":{}},{"type":"text","text":"可以实现微服务之间的异步以及解耦，那么引入","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"MQ","attrs":{}}],"attrs":{}},{"type":"text","text":"之后，如何实现微服务之间的数据一致性是一个值得思考的问题。RocketMQ事务消息正是解决这个问题的解决方案。另外事务消息也是为了解决消息丢失问题。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"哪些场景会出现消息丢失","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在分析","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"RocketMQ","attrs":{}}],"attrs":{}},{"type":"text","text":"事务消息之前，我们先来分析下引入消息中间件之后，整个消息链路在哪些场景会出现消息丢失的异常情况。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"当我们支付订单之后，我们账户的购物积分也会进行相应的积分调整。我们结合下面的订单服务、","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"RocketMQ","attrs":{}}],"attrs":{}},{"type":"text","text":"、积分服务的简化交互图来看，我们来分析下整个链路中可能会出现的消息丢失问题。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/ac/aca80c6217c1b84c597f500dad584ac2.png","alt":null,"title":"","style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"color","attrs":{"color":"#40A9FF","name":"blue"}},{"type":"strong","attrs":{}}],"text":"场景1：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在订单服务向RocketMQ发送订单成功生成的消息的时候，可能由于网络抖动的问题导致订单消息没能正常投递到RocketMQ，导致消息丢失。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"color","attrs":{"color":"#40A9FF","name":"blue"}},{"type":"strong","attrs":{}}],"text":"场景2：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"那么假如订单服务以及RocketMQ之间的网络没问题，消息正常被RocketMQ接收到了，那么会存在消息丢失的情况吗？答案是肯定的，这和RocketMQ的持久化机制有关系，当消息到达RocketMQ之后，并不是立马落盘存储，而是存储在page cache中的。如果此时出现服务器断电或者宕机情况，那么还没来得及落盘的消息数据就有可能丢失。另外即使是落到磁盘当中，如果出现磁盘坏道的话，依然会出现消息数据的可能。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"color","attrs":{"color":"#40A9FF","name":"blue"}},{"type":"strong","attrs":{}}],"text":"场景3：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如果前面两种场景都没问题，积分服务拿到订单消息了。还会出现消息丢失的问题吗？答案依然是肯定的。即便是积分服务拿到了订单消息，当积分服务自动提交消息offset到RocketMQ中，但是此时如果出现宕机或者积分服务挂了，没有将本该增加的积分进行处理，此时也就出现了消息丢失的情况。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"事务消息机制原理","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"half消息","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"所谓的","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"RocketMQ","attrs":{}}],"attrs":{}},{"type":"text","text":"事务机制，其实是","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"RocketMQ","attrs":{}}],"attrs":{}},{"type":"text","text":"提供了一种","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"half","attrs":{}}],"attrs":{}},{"type":"text","text":"消息的机制。当订单服务接受到订单支付信息后，订单服务会发送","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"half","attrs":{}}],"attrs":{}},{"type":"text","text":"消息到","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"RocketMQ","attrs":{}}],"attrs":{}},{"type":"text","text":"中，这个","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"half","attrs":{}}],"attrs":{}},{"type":"text","text":"消息是不被消费者所见的。怎么理解这个","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"half","attrs":{}}],"attrs":{}},{"type":"text","text":"信息呢，按照我自己的理解，就是它实现了一半的消息功能，只在生产端可见，在消费端不可见。另外这个","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"half","attrs":{}}],"attrs":{}},{"type":"text","text":"信息相当于一种","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"RocketMQ","attrs":{}}],"attrs":{}},{"type":"text","text":"的可用性探测，如果","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"half","attrs":{}}],"attrs":{}},{"type":"text","text":"消息都发送失败的话，就不必再进行下游业务的一系列操作了。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/3a/3ac5f18cde2c6cc16447b7b54853afaa.png","alt":null,"title":"","style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如果此时用于探测RocketMQ的可用性的half消息发送失败了，那么说明此时订单服务与RocketMQ存在异常，则会对之前订单进行一系列的回滚操作。如果half消息被成功投递，则需要进行本地事务操作，更新订单状态。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如果本地事务执行失败了怎么办，订单服务可以发送rollback请求，将之前的half消息从RocketMQ中进行删除，不再进行后续的消息投递。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"half消息原理分析","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"上文提到","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"half","attrs":{}}],"attrs":{}},{"type":"text","text":"消息不被消费端可见，那么这个","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"half","attrs":{}}],"attrs":{}},{"type":"text","text":"消息是怎么实现在","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"RocketMQ","attrs":{}}],"attrs":{}},{"type":"text","text":"中不被积分服务所见的呢？","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"订单服务发送","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"half","attrs":{}}],"attrs":{}},{"type":"text","text":"消息，实际并不是将消息投递到积分服务订阅的","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"topic","attrs":{}}],"attrs":{}},{"type":"text","text":"，而是将消息投递到","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"RocketMQ","attrs":{}}],"attrs":{}},{"type":"text","text":"中的","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"RMQ_SYS_TRANS_HALF_TOPIC","attrs":{}}],"attrs":{}},{"type":"text","text":"对应的","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"messeageQueue","attrs":{}}],"attrs":{}},{"type":"text","text":"。由于积分服务并没有订阅这个","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"Topic","attrs":{}}],"attrs":{}},{"type":"text","text":"，所以这个消息对于积分服务是不可见的。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"另外有个","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"OP_TOPIC","attrs":{}}],"attrs":{}},{"type":"text","text":"用于记录对应","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"half","attrs":{}}],"attrs":{}},{"type":"text","text":"消息的","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"commit/rollback","attrs":{}}],"attrs":{}},{"type":"text","text":"状态。大致的交互如下如所示：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/12/123c165576508f20e7f29a9e66bbfeb3.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如果订单服务","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"half","attrs":{}}],"attrs":{}},{"type":"text","text":"消息发送失败了，由于网络原因或者","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"RocketMQ","attrs":{}}],"attrs":{}},{"type":"text","text":"挂了，那么此时需要执行一些回滚操作，让订单进行关闭。因为订单信息无法通知到下游服务了。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/fb/fb21322de4081f672343f3a0d4b66072.png","alt":null,"title":"","style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"那么如果","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"half","attrs":{}}],"attrs":{}},{"type":"text","text":"消息已经写入","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"RocketMQ","attrs":{}}],"attrs":{}},{"type":"text","text":"中，但是本地事务执行失败又该怎么办呢？也就是说当订单服务接收到","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"half","attrs":{}}],"attrs":{}},{"type":"text","text":"消息写入成功的响应后，更新订单信息时发生了异常，无法完成状态更新。那么此时订单服务需要发送","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"rollback","attrs":{}}],"attrs":{}},{"type":"text","text":"的请求给","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"RocketMQ","attrs":{}}],"attrs":{}},{"type":"text","text":"，通知其将原来的","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"half","attrs":{}}],"attrs":{}},{"type":"text","text":"信息进行删除。如果本地事务执行成功，则需要发送commit请求给","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"RocketMQ","attrs":{}}],"attrs":{}},{"type":"text","text":"，","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"RocketMQ","attrs":{}}],"attrs":{}},{"type":"text","text":"会将原先存在","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"RMQ_SYS_TRANS_HALF_TOPIC","attrs":{}}],"attrs":{}},{"type":"text","text":"中的消息重新投递到积分服务订阅的","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"TOPIC","attrs":{}}],"attrs":{}},{"type":"text","text":"中去，这样积分服务就可以正常消费信息进行下一步的积分操作了。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"再考虑一种情况，如果订单服务发送","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"commit","attrs":{}}],"attrs":{}},{"type":"text","text":"或者","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"rollback","attrs":{}}],"attrs":{}},{"type":"text","text":"请求未正常投递到","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"RocketMQ","attrs":{}}],"attrs":{}},{"type":"text","text":"中，","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"RocketMQ","attrs":{}}],"attrs":{}},{"type":"text","text":"不知道","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"half","attrs":{}}],"attrs":{}},{"type":"text","text":"消息到底是对应的本地事务到底是执行成功了还是执行失败了。针对这种情况，订单服务需要提供状态回查接口，","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"RocketMQ","attrs":{}}],"attrs":{}},{"type":"text","text":"定时检测是否还有没有处理的","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"half","attrs":{}}],"attrs":{}},{"type":"text","text":"消息，当存在这样的消息时，","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"RocketMQ","attrs":{}}],"attrs":{}},{"type":"text","text":"调用回查接口确认本地事务执行情况。执行失败的则删除","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"half","attrs":{}}],"attrs":{}},{"type":"text","text":"消息，执行成功则重新投递消息。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/a8/a8fb7fc3ac5a14b34731f1cd6c15b160.png","alt":null,"title":"","style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"总结","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"通过上文的分析，订单服务和","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"RocketMQ","attrs":{}}],"attrs":{}},{"type":"text","text":"之间的交互，通过事务消息机制可以保证消息可以被可靠投递。至少在订单服务和","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"RocketMQ","attrs":{}}],"attrs":{}},{"type":"text","text":"之间不会出现消息丢失的问题。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"我是慕枫，感谢各位小伙伴点赞、收藏和评论，文章持续跟新，我们下期再见！","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"微信搜索：","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"慕枫技术笔记","attrs":{}},{"type":"text","text":"，优质文章持续更新，我们有学习打卡的群可以拉你进，一起努力冲击大厂，另外有很多学习以及面试的材料提供给大家。","attrs":{}}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/7f/7ff6f4cf7dc92ed189149727ace4dc43.gif","alt":null,"title":"","style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}}]}