高性能-Nginx多進程高併發、低時延、高可靠機制在百萬級緩存(redis、memcache)代理中間件中的應用

{"type":"doc","content":[{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"關於作者","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 前滴滴出行技術專家，現任OPPO文檔數據庫mongodb負責人，負責oppo千萬級峯值TPS/十萬億級數據量文檔數據庫mongodb內核研發及運維工作，一直專注於分佈式緩存、高性能服務端、數據庫、中間件等相關研發。後續持續分享《MongoDB內核源碼設計、性能優化、最佳運維實踐》，Github賬號地址:","attrs":{}},{"type":"link","attrs":{"href":"https://github.com/y123456yz","title":null},"content":[{"type":"text","marks":[{"type":"underline","attrs":{}}],"text":"https://github.com/y123456yz","attrs":{}}]}]},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"1. 開發背景","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"     現有開源緩存代理中間件有twemproxy、codis等，其中twemproxy爲單進程單線程模型，只支持memcache單機版和redis單機版，都不支持集羣版功能。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"     由於twemproxy無法利用多核特性，因此性能低下，短連接QPS大約爲3W，長連接QPS大約爲13W，同時某些場景時延抖動厲害。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"      爲了適應公有云平臺上業務方的高併發需求，因此決定藉助於twemproxy來做二次開發，把nginx的高性能、高可靠、高併發機制引入到twemproxy中，通過master+多worker進程來實現七層轉發功能。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"2    Twemproxy","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"2.1  Twemproxy簡介","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" Twemproxy 是一個快速的單線程代理程序，支持 ","attrs":{}},{"type":"link","attrs":{"href":"http://www.oschina.net/p/memcached","title":null},"content":[{"type":"text","text":"Memcached","attrs":{}}]},{"type":"text","text":" ASCII協議和更新的","attrs":{}},{"type":"link","attrs":{"href":"http://www.oschina.net/p/redis","title":null},"content":[{"type":"text","text":"Redis","attrs":{}}]},{"type":"text","text":"協議。它全部用C寫成，使用Apache 2.0 License授權。支持以下特性：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"i)速度快","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"ii)輕量級","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"iii)維護持久的服務器連接","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"iiii)啓用請求和響應的管道","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"iiiii)支持代理到多個後端緩存服務器","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"iiiii)同時支持多個服務器池","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"iiiiii)多個服務器自動分享數據","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"iiiiiii)可同時連接後端多個緩存集羣","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"iiiiiiii)實現了完整的 ","attrs":{}},{"type":"link","attrs":{"href":"https://github.com/twitter/twemproxy/blob/master/notes/memcache.txt","title":null},"content":[{"type":"text","text":"memcached ascii","attrs":{}}]},{"type":"text","text":" 和 ","attrs":{}},{"type":"link","attrs":{"href":"https://github.com/twitter/twemproxy/blob/master/notes/redis.md","title":null},"content":[{"type":"text","text":"redis","attrs":{}}]},{"type":"text","text":" 協議.","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"iiiiiiiii)服務器池配置簡單，通過一個 YAML 文件即可","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"iiiiiiiiii)一致性hash","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"iiiiiiiiii)詳細的監控統計信息","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"iiiiiiiiiii)支持 Linux, *BSD, OS X and Solaris (SmartOS)","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"iiiiiiiiiiii)支持設置HashTag","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"iiiiiiiiiiiiiii)連接複用，內存複用，提高效率","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"2.2 memcache緩存集羣拓撲結構","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/b5/b5c2ba907ca3360b18c9fdd010816cb0.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":"center","origin":null},"content":[{"type":"text","text":"圖1 twemproxy緩存集羣拓撲圖","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 如上圖所示，實際應用中業務程序通過輪詢不同的twemproxy來提高qps，同時實現負載均衡。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}},{"type":"strong","attrs":{}}],"text":"說明:官方memcache沒有集羣版和持久化功能，集羣版和持久化功能由我們自己內部開發完成。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"2.3 推特原生twemproxy瓶頸","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 如今twemproxy憑藉其高性能的優勢, 在很多互聯網公司得到了廣泛的應用，已經佔據了其不可動搖的地位, 然而在實際的生產環境中, 存在以下缺陷，如下：","attrs":{}}]},{"type":"numberedlist","attrs":{"start":1,"normalizeStart":1},"content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":1,"align":null,"origin":null},"content":[{"type":"text","text":"單進程單線程, 無法充分發揮服務器多核cpu的性能","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":2,"align":null,"origin":null},"content":[{"type":"text","text":"當twemproxy qps短連接達到8000後，消耗cpu超過70%，時延陡增。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":3,"align":null,"origin":null},"content":[{"type":"text","text":"大流量下造成IO阻塞，無法處理更多請求，qps上不去，業務時延飆升","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":4,"align":null,"origin":null},"content":[{"type":"text","text":"維護成本高，如果想要充分發揮服務器的所有資源包括cpu、 網絡io等，就必須建立多個twemproxy實例，維護成本高","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":5,"align":null,"origin":null},"content":[{"type":"text","text":"擴容、升級不便","attrs":{}}]}],"attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 原生twemproxy進程呈現了下圖現象：一個人幹活，多個人圍觀。多核服務器只有一個cpu在工作，資源沒有得到充分利用。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/be/be9b636cd1b1b0864c3aadbd21f37d81.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"3. Nginx","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" nginx是俄羅斯軟件工程師Igor Sysoev開發的免費開源web服務器軟件，聚焦於高性能，高併發和低內存消耗問題，因此成爲業界公認的高性能服務器，並逐漸成爲業內主流的web服務器。主要特點有：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"完全藉助epoll機制實現異步操作，避免阻塞。","attrs":{}}]},{"type":"numberedlist","attrs":{"start":1,"normalizeStart":1},"content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":1,"align":null,"origin":null},"content":[{"type":"text","text":"重複利用現有服務器的多核資源。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":2,"align":null,"origin":null},"content":[{"type":"text","text":"充分利用CPU 親和性（affinity），把每個進程與固定CPU綁定在一起，給定的 CPU 上儘量長時間地運行而不被遷移到其他處理器的傾向性，減少進程調度開銷。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":3,"align":null,"origin":null},"content":[{"type":"text","text":"請求響應快","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":4,"align":null,"origin":null},"content":[{"type":"text","text":"支持模塊化開發，擴展性好","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":5,"align":null,"origin":null},"content":[{"type":"text","text":"Master+多worker進程方式，確保worker進程可靠工作。當worker進程出錯時，可以快速拉起新的worker子進程來提供服務。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":6,"align":null,"origin":null},"content":[{"type":"text","text":"內存池、連接池等細節設計保障低內存消耗。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":7,"align":null,"origin":null},"content":[{"type":"text","text":"熱部署支持，master與worker進程分離設計模式，使其具有熱部署功能。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":8,"align":null,"origin":null},"content":[{"type":"text","text":"升級方便，升級過程不會對業務造成任何傷害。","attrs":{}}]}],"attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Nginx多進程提供服務過程如下圖所示：","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"4    Nginx master+worker多進程機制在twemproxy中的應用","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"4.1  爲什麼選擇nginx多進程機制做爲參考？","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" Twemproxy和nginx都屬於網絡io密集型應用，都屬於七層轉發應用，時延要求較高，應用場景基本相同。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" Nginx充分利用了多核cpu資源，性能好，時延低。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"4.2  Master-worker多進程機制原理","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" Master-worker進程機制採用一個master進程來管理多個worker進程。每一個worker進程都是繁忙的，它們在真正地提供服務，master進程則很“清閒”，只負責監控管理worker進程, 包含：接收來自外界的信號，向各worker進程發送信號，監控worker進程的運行狀態，當worker進程退出後(異常情況下)，會自動重新啓動新的worker進程。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" worker進程負責處理客戶端的網絡請求，多個worker進程同時處理來自客戶端的不同請求，worker進程數可配置。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"4.3 多進程關鍵性能問題點","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" master-worker多進程模式需要解決的問題主要有：","attrs":{}}]},{"type":"numberedlist","attrs":{"start":1,"normalizeStart":1},"content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":1,"align":null,"origin":null},"content":[{"type":"text","text":"linux內核低版本(2.6以下版本), “驚羣”問題","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":2,"align":null,"origin":null},"content":[{"type":"text","text":"linux內核低版本(2.6以下版本),負載均衡問題","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":3,"align":null,"origin":null},"content":[{"type":"text","text":"linux內核高版本(3.9以上版本)新特性如何利用","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":4,"align":null,"origin":null},"content":[{"type":"text","text":"如何確保進程見高可靠通信","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":5,"align":null,"origin":null},"content":[{"type":"text","text":"如何減少worker進程在不同cpu切換的開銷","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":6,"align":null,"origin":null},"content":[{"type":"text","text":"master進程如何彙總各個工作進程的監控數據","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":7,"align":null,"origin":null},"content":[{"type":"text","text":"worker進程異常，如何快速恢復","attrs":{}}]}],"attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":" 4.3.1  linux內核低版本關鍵技術問題","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 由於linux低內核版本缺陷，因此存在”驚羣”、負載不均問題，解決辦法完全依賴應用層代碼保障。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"  4.3.1.1 如何解決“驚羣”問題","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 當客戶端發起連接後，由於所有的worker子進程都監聽着同一個端口，內核協議棧在檢測到客戶端連接後，會激活所有休眠的worker子進程，最終只會有一個子進程成功建立新連接，其他子進程都會accept失敗。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" Accept失敗的子進程是不應該被內核喚醒的，因爲它們被喚醒的操作是多餘的，佔用本不應該被佔用的系統資源，引起不必要的進程上下文切換，增加了系統開銷，同時也影響了客戶端連接的時延。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" “驚羣”問題是多個子進程同時監聽同一個端口引起的，因此解決的方法是同一時刻只讓一個子進程監聽服務器端口，這樣新連接事件只會喚醒唯一正在監聽端口的子進程。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 因此“驚羣”問題通過非阻塞的accept鎖來實現進程互斥accept()，其原理是：在worker進程主循環中非阻塞trylock獲取accept鎖，如果trylock成功，則此進程把監聽端口對應的fd通過epoll_ctl()加入到本進程自由的epoll事件集；如果trylock失敗，則把監聽fd從本進程對應的epoll事件集中清除。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" Nginx實現了兩套互斥鎖：基於原子操作和信號量實現的互斥鎖、基於文件鎖封裝的互斥鎖。考慮到鎖的平臺可移植性和通用性，改造twemproxy選擇時，選擇文件鎖實現。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 如果獲取accept鎖成功的進程佔用鎖時間過長，那麼其他空閒進程在這段時間內無法獲取到鎖，從而無法接受新的連接。最終造成客戶端連接相應時間變長，qps低，同時引起負載嚴重不均衡。爲了解決該問題，選擇通過post事件隊列方式來提高性能，trylock獲取到accept鎖成功的進程，其工作流程如下：","attrs":{}}]},{"type":"numberedlist","attrs":{"start":1,"normalizeStart":1},"content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":1,"align":null,"origin":null},"content":[{"type":"text","text":"trylock獲取accept鎖成功","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":2,"align":null,"origin":null},"content":[{"type":"text","text":"通過epoll_wait獲取所有的事件信息，把監聽到的所有accept事件信息加入accept_post列表，把已有連接觸發的讀寫事件信息加入read_write_post列表。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":3,"align":null,"origin":null},"content":[{"type":"text","text":"執行accept_post列表中的所有事件","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":4,"align":null,"origin":null},"content":[{"type":"text","text":"Unlock鎖","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":5,"align":null,"origin":null},"content":[{"type":"text","text":"執行read_write_post列表中的事件。","attrs":{}}]}],"attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" Worker進程主循環工作流程圖如下：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/9b/9b676e734e45d2fce6f82236c38fde9c.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 從上圖可以看出，worker進程藉助epoll來實現網絡異步收發，客戶端連接twemproxy的時候，worker進程循環檢測客戶端的各種網絡事件和後端memcached的網絡事件，並進行相應的處理。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" twemproxy各個進程整體網絡i/o處理過程圖如下：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/10/10afd31239524fa569cca334af2ee186.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"4.3.1.2     如何解決“負載均衡“問題","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 在多個子進程爭搶處理同一個新連接事件時，一定只有一個worker子進程最終會成功建立連接，隨後，它會一直處理這個連接直到連接關閉。這樣，如果有的子進程“運氣”很好，它們搶着建立並處理了大部分連接，其他子進程就只能處理少量連接，這對多核cpu架構下的應用很不利。理想情況下，每個子進程應該是平等的，每個worker子進程應該大致平均的處理客戶端連接請求。如果worker子進程負載不均衡，必然影響整體服務的性能。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" nginx通過連接閾值機制來實現負載均衡，其原理如下：每個進程都有各自的最大連接數閾值max_threshold和當前連接閾值數local_threshold，和當前連接數閾值，進程每接收一個新的連接，local_threshold增一，連接斷開後，local_threashold減一。如果local_threshold超過max_threshold，則不去獲取accept鎖，把accept機會留給其他進程，同時把local_threshold減1，這樣下次就有機會獲取accept鎖，接收客戶端連接了。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 在實際業務應用中，有的業務採用長連接和twemproxy建立連接，連接數最大可能就幾百連接，如果設置max_threshold閾值過大，多個連接如果同時壓到twemproxy，則很容易引起所有連接被同一個進程獲取從而造成不均衡。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 爲了儘量減少負載不均衡，在實際應用中，新增了epoll_wait超時時間配置選項，把該超時時間設短，這樣減少空閒進程在epoll_wait上的等待事件，從而可以更快相應客戶端連接，並有效避免負載不均衡。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"4.3.2 Linux內核高版本TCP REUSEPORT特性如何利用","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"4.3.2.1 什麼是reuseport？","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" reuseport是一種套接字複用機制，它允許你將多個套接字bind在同一個IP地址/端口對上，這樣一來，就可以建立多個服務來接受到同一個端口的連接。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"4.3.2.2 支持reuseport和不支持reuseport的區別","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 如果linux內核版本小於3.9，則不支持reuseport(注:部分centos發行版在低版本中已經打了reuseport patch,所以部分linux低版本發行版本也支持該特性)。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 不支持該特性的內核，一個ip+port組合，只能被監聽bind一次。這樣在多核環境下，往往只能有一個線程（或者進程）是listener，也就是同一時刻只能由一個進程或者線程做accept處理，在高併發情況下，往往這就是性能瓶頸。其網絡模型如下:","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/f9/f997c7018c455f1b8dc7485ac81977ff.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 在Linux kernel 3.9帶來了reuseport特性，它可以解決上面的問題，其網絡模型如下:","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/ce/ce98e92a5cfad18a6675fd074ffdf163.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" reuseport是支持多個進程或者線程綁定到同一端口，提高服務器程序的吞吐性能，其優點體現在如下幾個方面:","attrs":{}}]},{"type":"numberedlist","attrs":{"start":1,"normalizeStart":1},"content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":1,"align":null,"origin":null},"content":[{"type":"text","text":"允許多個套接字 bind()/listen() 同一個TCP/UDP端口","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":2,"align":null,"origin":null},"content":[{"type":"text","text":"每一個線程擁有自己的服務器套接字","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":3,"align":null,"origin":null},"content":[{"type":"text","text":"在服務器套接字上沒有了鎖的競爭，因爲每個進程一個服務器套接字","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":4,"align":null,"origin":null},"content":[{"type":"text","text":"內核層面實現負載均衡","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":5,"align":null,"origin":null},"content":[{"type":"text","text":"安全層面，監聽同一個端口的套接字只能位於同一個用戶下面","attrs":{}}]}],"attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"4.3.3 Master進程和worker進程如何通信？","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 由於master進程需要實時獲取worker進程的工作狀態，並實時彙總worker進程的各種統計信息，所以選擇一種可靠的進程間通信方式必不可少。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 在twemproxy改造過程中，直接參考nginx的信號量機制和channel機制(依靠socketpair)來實現父子進程見通信。Master進程通過信號量機制來檢測子進程是否異常，從而快速直接的反應出來；此外，藉助socketpair，封裝出channel接口來完成父子進程見異步通信，master進程依靠該機制來統計子進程的各種統計信息並彙總，通過獲取來自master的彙總信息來判斷整個twemproxy中間件的穩定性、可靠性。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 配置下發過程：主進程接收實時配置信息，然後通過channel機制發送給所有的worker進程，各個worker進程收到配置信息後應答給工作進程。流程如下:","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/7e/7ed043247fa75d2ea26aed963dcc40ef.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 獲取監控信息流程和配置下發流程基本相同，master進程收到各個工作進程的應答後，由master進程做統一彙總，然後發送給客戶端。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"4.3.4 如何減少worker進程在不同cpu切換的開銷","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" CPU 親和性（affinity） 就是進程要在某個給定的 CPU 上儘量長時間地運行而不被遷移到其他處理器的傾向性。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" Linux 內核進程調度器天生就具有被稱爲 軟 CPU 親和性（affinity） 的特性，這意味着進程通常不會在處理器之間頻繁遷移。這種狀態正是我們希望的，因爲進程遷移的頻率小就意味着產生的負載小。具體參考sched_setaffinity函數。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"4.3.5 worker進程異常如何減少對業務的影響？","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 在實際線上環境中，經常出現這樣的情況:某個多線程服務跑幾個月後，因爲未知原因進程掛了，最終造成整個服務都會不可用。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 這時候，master+多worker的多進程模型就體現了它的優勢，如果代碼有隱藏的並且不容易觸發的bug，某個時候如果某個請求觸發了這個bug，則處理這個請求的worker進程會段錯誤退出。但是其他worker進程不會收到任何的影響，也就是說如果一個改造後的twemproxy起了20個worker進程，某個時候一個隱藏bug被某個請求觸發，則只有處理該請求的進程段錯誤異常，其他19個進程不會受到任何影響，該隱藏bug觸發後影響面僅爲5%。如果是多線程模型，則影響面會是100%。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 如果某個worker進程掛了，master父進程會感知到這個信號，然後重新拉起一個worker進程，實現瞬間無感知”拉起”恢復。以下爲模擬觸發異常段錯誤流程：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/47/472a3f8ef5ba587585007a0861a35a1e.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 如上圖所示，殺掉31420 worker進程後，master進程會立馬在拉起一個31451工作進程，實現了快速恢復。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 多進程異常，自動”拉起”功能源碼，可以參考如下demo：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://github.com/y123456yz/reading-code-of-nginx-1.9.2/blob/master/nginx-1.9.2/src/demo.c","title":""},"content":[{"type":"text","text":"https://github.com/y123456yz/reading-code-of-nginx-1.9.2/blob/master/nginx-1.9.2/src/demo.c","attrs":{}}]}]},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"5    網絡優化","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"5.1 網卡多隊列","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 在實際上線後，發現軟中斷過高，幾乎大部分都集中在一個或者幾個CPU上，嚴重影響客戶端連接和數據轉發，qps上不去，時延抖動厲害。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" RSS（Receive Side Scaling）是網卡的硬件特性，實現了多隊列，可以將不同的流分發到不同的CPU上。支持RSS的網卡，通過多隊列技術，每個隊列對應一箇中斷號，通過對每個中斷的綁定，可以實現網卡中斷在cpu多核上的分配，最終達到負載均衡的作用。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"5.2 可怕的40ms","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 原生twemproxy在線上跑得過程中，發現時延波動很大，抓包發現其中部分數據包應答出現了40ms左右的時延，拉高了整體時延抓包如下(藉助tcprstat工具)：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/73/735ba3e7d9643b4b6ccaae4f8a926b3a.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 解決辦法如下：在recv系統調用後，調用一次setsockopt函數，設置TCP_QUICKACK。代碼修改如下：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/c8/c8c57da792fce6feb11949745955c845.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"6   Twemproxy改造前後性能對比   (時延、qps對比)","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"6.1  線上真實流量時延對比","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"6.1.1  改造前線上twemproxy集羣時延","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 線上集羣完全採用開源twemproxy做代理，架構如下：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/bf/bf766fe1c3158f02e24c194f2463e13b.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 未改造前線上twemproxy+memcache集羣，qps=5000~6000，長連接，客戶端時延分佈如下圖所示：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/48/48832be4c5d3994487da62374488f054.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/ab/ab89cfe5fb355762ed33eb05217dfc29.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 在twemproxy機器上使用tcprstat監控到的網卡時延如下:","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/cf/cfac237be6d37f1f9ea2e895320cf8cf.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 從上面兩個圖可以看出，採用原生twemproxy,時延高，同時抖動厲害。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"6.1.2 參照nginx改造後的twemproxy時延","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 線上集羣一個twemproxy採用官方原生twemproxy，另一個爲改造後的twemproxy，其中改造後的twemproxy配置worker進程數爲1，保持和原生開源twemproxy進程數一致，架構如下：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/28/28d4683aa59daf6dfc926dd1e8af3b6c.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 替換線上集羣兩個代理中的一個後(影響50%流量)，長連接，qps=5000~6000，客戶端埋點監控時延分佈如下:","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/33/333564180ea71637ecab34a3cb338ebe.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/0f/0f04ad10259670cb2c300028edf72ade.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/b9/b913b61b464572a2e630b127df45cebb.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 替換兩個proxy中的一個後，使用tcprstat在代理集羣上面查看兩個代理的時延分佈如下:","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 原生twemproxy節點機器上的時延分佈:","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/49/498622e68731c76299e1500ec713516d.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 另一個改造後的twemproxy節點機器上的時延分佈：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/9f/9fe673fec7eef1043fea4e58f2167f75.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":" 總結：","attrs":{}},{"type":"text","text":"替換線上兩個proxy中的一個後，客戶端時間降低了一倍，如果線上集羣兩個代理都替換爲改造後的twemproxy，客戶端監控時延預計會再降低一倍，總體時延降低3倍左右。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 此外，從監控可以看出，改造後的twemproxy時延更低，更加穩定，無任何波動。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"6.2 參考nginx多進程改造後的twemproxy線下壓測結果(開啓reuseport功能)","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 監聽同一個端口，數據長度100字節，壓測結果如下：","attrs":{}}]},{"type":"numberedlist","attrs":{"start":1,"normalizeStart":1},"content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":1,"align":null,"origin":null},"content":[{"type":"text","text":"   linux內核版本:linux-3.10","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":2,"align":null,"origin":null},"content":[{"type":"text","text":"   物理機機型： M10(48 cpu)","attrs":{}}]}],"attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/74/74a84620b61a0e49d963c0a7d8c7dd42.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 多進程監聽同一個端口，數據長度150字節，壓測結果如下：","attrs":{}}]},{"type":"numberedlist","attrs":{"start":1,"normalizeStart":1},"content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":1,"align":null,"origin":null},"content":[{"type":"text","text":"   linux內核版本:linux-3.10","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":2,"align":null,"origin":null},"content":[{"type":"text","text":"   物理機機型： TS60 (24 cpu)","attrs":{}}]}],"attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/69/69d5ff1fe9cf2061cdf119d71677d52e.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"7   總結","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"7.1 多進程、多線程機制選擇","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 選擇參照nginx多進程機制，而不選擇多線程實現原因主要有:","attrs":{}}]},{"type":"numberedlist","attrs":{"start":1,"normalizeStart":1},"content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":1,"align":null,"origin":null},"content":[{"type":"text","text":"多進程機制無鎖操作，實現更容易","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":2,"align":null,"origin":null},"content":[{"type":"text","text":" 多進程的代理，整個worker進程無任何鎖操作，性能更好","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":3,"align":null,"origin":null},"content":[{"type":"text","text":" 如果是多線程方式，如果代碼出現bug段錯誤，則整個進程掛掉，整個服務不可用。而如果是多進程方式，因爲bug觸發某個worker進程段錯誤異常，其他工作進程不會受到如何影響，20個worker進程，如果觸發異常，同一時刻只有有1/20的流量受到影響。而如果是多線程模式，則100%的流量會受到影響。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":4,"align":null,"origin":null},"content":[{"type":"text","text":" worker進程異常退出後，master進程立馬感知拉起一個新進程提供服務，可靠性更高。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":5,"align":null,"origin":null},"content":[{"type":"text","text":" 配置熱加載、程序熱升級功能實現更加容易","attrs":{}}]}],"attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"7.2 參照nginx改造後的twemproxy特性","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 支持nginx幾乎所有的優秀特性，同時也根據自己實際情況新增加了自有特性:","attrs":{}}]},{"type":"numberedlist","attrs":{"start":1,"normalizeStart":1},"content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":1,"align":null,"origin":null},"content":[{"type":"text","text":"master+多worker進程機制","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":2,"align":null,"origin":null},"content":[{"type":"text","text":"適配所有linux內核版本，內核低版本驚羣問題避免支持","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":3,"align":null,"origin":null},"content":[{"type":"text","text":"quic_ack支持","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":4,"align":null,"origin":null},"content":[{"type":"text","text":"reuser_port適配支持","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":5,"align":null,"origin":null},"content":[{"type":"text","text":"worker進程異常，master進程自動拉起功能支持","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":6,"align":null,"origin":null},"content":[{"type":"text","text":"90%、95%、98%、100%平均時延統計功能支持","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":7,"align":null,"origin":null},"content":[{"type":"text","text":"memcache單機版、集羣版支持","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":8,"align":null,"origin":null},"content":[{"type":"text","text":"redis單機版、集羣版支持","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":9,"align":null,"origin":null},"content":[{"type":"text","text":"二進制協議、文本協議同時支持","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":10,"align":null,"origin":null},"content":[{"type":"text","text":"redis、memcache集羣在線擴容、縮容、數據遷移支持，擴縮容、數據遷移過程對業務無任何影響。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":11,"align":null,"origin":null},"content":[{"type":"text","text":"多租戶支持，一個代理可以接多個memcache、redis集羣，並支持混部。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":12,"align":null,"origin":null},"content":[{"type":"text","text":"mget、gets、sets等批量處理命令優化處理","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":13,"align":null,"origin":null},"content":[{"type":"text","text":"慢響應日誌記錄功能支持","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":14,"align":null,"origin":null},"content":[{"type":"text","text":"內存參數實時修改支持","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":15,"align":null,"origin":null},"content":[{"type":"text","text":"詳細的集羣監控統計功能","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":16,"align":null,"origin":null},"content":[{"type":"text","text":"CPU親緣性自添加","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":17,"align":null,"origin":null},"content":[{"type":"text","text":"內存配置動態實時修改","attrs":{}}]}],"attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"7.3 後期計劃","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"   添加如下功能:","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"     i) 配置文件熱加載支持。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"     ii) 代碼熱升級功能支持。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"7.4  長遠規劃展望","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 抽象出一款類似nginx的高性能代理軟件，nginx支持http協議，我們的支持tcp協議代理，覆蓋nginx所有功能，包括前面提到的所有功能，同時支持模塊化開發。這樣，很多的tcp協議代理就無需關心網絡架構底層實現，只需要根據需要開發對應的協議解析模塊，和自己關心的統計、審計等功能功能，降低開發成本。現有開源的中間件，很大一部分都是tcp的，有自己的私有tcp協議，把這個抽象出來，開發成本會更低","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"對nginx有興趣的可以源碼分析參考：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://github.com/y123456yz/reading-code-of-nginx-1.9.2","title":null},"content":[{"type":"text","text":"https://github.com/y123456yz/reading-code-of-nginx-1.9.2","attrs":{}}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"內核網卡時延分析工具:","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://github.com/y123456yz/tcprstat","title":null},"content":[{"type":"text","text":"https://github.com/y123456yz/tcprstat","attrs":{}}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"twemproxy源碼分析:","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://github.com/y123456yz/Reading-and-comprehense-twemproxy0.4.1","title":null},"content":[{"type":"text","text":"https://github.com/y123456yz/Reading-and-comprehense-twemproxy0.4.1","attrs":{}}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"內核協議棧延遲確認機制：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"https://github.com/y123456yz/Reading-and-comprehense-linux-Kernel-network-protocol-stack","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"中間件、分佈式存儲、高性能服務端開發等開發:","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://github.com/y123456yz/middleware_development_learning","title":null},"content":[{"type":"text","text":"https://github.com/y123456yz/middleware_development_learning","attrs":{}}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}}]}