如何讓消息隊列達到最大吞吐量？

{"type":"doc","content":[{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"你在使用消息隊列的時候關注過吞吐量嗎？","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"思考過吞吐量的影響因素嗎？","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"考慮過怎麼提高嗎？","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"總結過最佳實踐嗎？","attrs":{}}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"本文帶你一起探討下消息隊列消費端高吞吐的 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"Go","attrs":{}}],"attrs":{}},{"type":"text","text":" 框架實現。Let’s go!","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"關於吞吐量的一些思考","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"寫入消息隊列吞吐量取決於以下兩個方面","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":1,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"網絡帶寬","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":1,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"消息隊列（比如Kafka）寫入速度","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":"none"},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"最佳吞吐量是讓其中之一打滿，而一般情況下內網帶寬都會非常高，不太可能被打滿，所以自然就是講消息隊列的寫入速度打滿，這就就有兩個點需要平衡","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":1,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"批量寫入的消息量大小或者字節數多少","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":1,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"延遲多久寫入","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":"none"},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"go-zero 的 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"PeriodicalExecutor","attrs":{}}],"attrs":{}},{"type":"text","text":" 和 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"ChunkExecutor","attrs":{}}],"attrs":{}},{"type":"text","text":" 就是爲了這種情況設計的","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"從消息隊列裏消費消息的吞吐量取決於以下兩個方面","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":1,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"消息隊列的讀取速度，一般情況下消息隊列本身的讀取速度相比於處理消息的速度都是足夠快的","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":1,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"處理速度，這個依賴於業務","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":"none"},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"這裏有個核心問題是不能不考慮業務處理速度，而讀取過多的消息到內存裏，否則可能會引起兩個問題：","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":1,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"內存佔用過高，甚至出現OOM，","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"pod","attrs":{}}],"attrs":{}},{"type":"text","text":" 也是有 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"memory limit","attrs":{}}],"attrs":{}},{"type":"text","text":" 的","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":1,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"停止 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"pod","attrs":{}}],"attrs":{}},{"type":"text","text":" 時堆積的消息來不及處理而導致消息丟失","attrs":{}}]}]}],"attrs":{}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"解決方案和實現","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/ee/eedff4bbd5fde6638ebccccf9410cb22.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"借用一下 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"Rob Pike","attrs":{}}],"attrs":{}},{"type":"text","text":" 的一張圖，這個跟隊列消費異曲同工。左邊4個 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"gopher","attrs":{}}],"attrs":{}},{"type":"text","text":" 從隊列裏取，右邊4個 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"gopher","attrs":{}}],"attrs":{}},{"type":"text","text":" 接過去處理。比較理想的結果是左邊和右邊速率基本一致，沒有誰浪費，沒有誰等待，中間交換處也沒有堆積。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"我們來看看 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"go-zero","attrs":{}}],"attrs":{}},{"type":"text","text":" 是怎麼實現的：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"Producer","attrs":{}}],"attrs":{}},{"type":"text","text":" 端","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"go"},"content":[{"type":"text","text":" for {\n select {\n case