生產環境的 Service Mesh 流量劫持怎麼搞？

{"type":"doc","content":[{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"背景"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"ServiceMesh 社區使用 iptables 實現流量劫持，這個機制在百度生產環境使用會遇到一些問題，因此，我們探索了其他的流量劫持方式，如基於服務發現的流量劫持機制、基於 SDK 的流量劫持機制、基於固定 Virutal IP 的流量劫持機制等。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"本文主要介紹"},{"type":"text","marks":[{"type":"strong"}],"text":"基於服務發現的流量劫持機制，這個機制是在服務發現步驟 \"僞造\" 地址來完成流量劫持。"}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"基於 iptables 流量劫持機制"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"我們先簡單的看看社區的流量劫持方案，首先看下 Inbound 流量劫持，如圖1 所示："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"numberedlist","attrs":{"start":null,"normalizeStart":1},"content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":1,"align":null,"origin":null},"content":[{"type":"text","text":"所有入站流量都會經過 iptables；"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":2,"align":null,"origin":null},"content":[{"type":"text","text":"iptables 忽略非 TCP 以及訪問 istio 管理端口的流量，將其他入站流量轉發給 Envoy；"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":3,"align":null,"origin":null},"content":[{"type":"text","text":"Envoy 處理完後再將流量轉發給 App；"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":4,"align":null,"origin":null},"content":[{"type":"text","text":"上述流量又一次經過 iptables 規則匹配，iptables 將具備以下特點的流量當做 Envoy 發出的流量直接轉發給目的地："}]},{"type":"paragraph","attrs":{"indent":0,"number":5,"align":null,"origin":null},"content":[{"type":"text","text":"Envoy 發出的；"}]},{"type":"paragraph","attrs":{"indent":0,"number":6,"align":null,"origin":null},"content":[{"type":"text","text":"輸出設備時 lo；"}]},{"type":"paragraph","attrs":{"indent":0,"number":7,"align":null,"origin":null},"content":[{"type":"text","text":"目的地時127.0.0.1。"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"至此 iptables 完成了 Envoy 對 Inbound 流量的劫持和轉發。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https:\/\/static001.geekbang.org\/infoq\/9d\/9d761a2edd38061763bfe404b6775501.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":"center","origin":null},"content":[{"type":"text","text":"圖1 iptables 流量劫持"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"接下來咱們再來看看 Outbound 流量劫持，如圖2所示："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"numberedlist","attrs":{"start":null,"normalizeStart":1},"content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":1,"align":null,"origin":null},"content":[{"type":"text","text":"App 給 Server 發送流量；"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":2,"align":null,"origin":null},"content":[{"type":"text","text":"iptables 將滿足以下條件的流量轉發給 Envoy："}]},{"type":"paragraph","attrs":{"indent":0,"number":3,"align":null,"origin":null},"content":[{"type":"text","text":"不是 Envoy 發出的；"}]},{"type":"paragraph","attrs":{"indent":0,"number":4,"align":null,"origin":null},"content":[{"type":"text","text":"輸出設備不是 lo；"}]},{"type":"paragraph","attrs":{"indent":0,"number":5,"align":null,"origin":null},"content":[{"type":"text","text":"目的地址不是 localhost。"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":6,"align":null,"origin":null},"content":[{"type":"text","text":"Envoy 處理完後，選定 Server 的一個 endpoint 轉發流量；"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":7,"align":null,"origin":null},"content":[{"type":"text","text":"iptables 將滿足以下條件的流量直接發給目的地，也就是 Server："}]},{"type":"paragraph","attrs":{"indent":0,"number":8,"align":null,"origin":null},"content":[{"type":"text","text":"Envoy 發出的；"}]},{"type":"paragraph","attrs":{"indent":0,"number":9,"align":null,"origin":null},"content":[{"type":"text","text":"輸出設備不是 lo。"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https:\/\/static001.geekbang.org\/infoq\/e0\/e05be8c41d16ba528cd5d3283185e1dc.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":"center","origin":null},"content":[{"type":"text","text":"圖2 iptables 劫持 outbound 流量"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"至此，iptables 完成了 inbound 和 outbound 的流量劫持，該機制的好處在於可以透明的劫持業務流量，但是在百度生產環境使用時存在一些問題："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"可管控性差，內網各容器網絡沒有隔離，iptables 是全局工具，可被其它用戶修改，導致流量劫持異常；它作爲單機流量管理工具，沒有成熟的平臺\/產品進行統一的管理。"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"大併發場景下存在一定的轉發性能問題；規則數過多時變更時延大。"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"因此我們探索了其他的劫持機制，接下來我來介紹下百度生產環境正在使用的流量劫持機制——基於服務發現的流量劫持機制。"}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"基於服務發現的流量劫持機制"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"先來看下該機制的設計思路，服務流量根據方向的不同，可以分爲 Outbound 和 Inbound。如圖3 所示，有兩個服務：Client 和 Server，Client 的 Envoy 記爲 EnvoyC，Server的 Envoy 記爲 EnvoyS（本質是一樣的，只不過爲了表述方便取了不同的名字）。EnvoyC 要劫持的流量是來自在相同機器上的 Client 發出的 Outbound 流量，而 EnvoyS 要劫持的流量大部分是來自不同機器上的服務發給 Server 的流量。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"這兩種流量的劫持機制可以分開設計，考慮到 ServiceMesh 常用的策略都在 EnvoyC 上生效，因此我們先設計了 EnvoyC 劫持 Outbound 流量的方案。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https:\/\/static001.geekbang.org\/infoq\/e6\/e680e0889bf52801eab7f632d0fa10ff.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":"center","origin":null},"content":[{"type":"text","text":"圖3 ServiceMesh 流量劫持"}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"Outbound 流量劫持"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"一個完整的請求大概要經歷域名解析（或者是服務發現）、建立連接、發送請求這幾個步驟，現在 iptables 用不了，其他依賴 Kernel 的劫持方案暫時也用不了，我們將目光轉向第一步——服務發現。百度生產環境的服務基本都依賴 Naming 系統來解析服務真實的 ip 列表，我們只需要讓 Naming 系統返回 Envoy 的 ip 地址，就能將服務的 Outbound 流量劫持到 Envoy。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如圖4 所示，Naming Agent 是單機上負責服務發現的 Agent。Client 在發送請求前，會先去 Naming Agent 問：我想給 Server 發個請求，請給我他的地址。這時候 Naming Agent 就會把 Envoy 的地址當成 Server 的地址告訴 Client。接下來 Client 就會乖乖的把請求發給 Envoy，Envoy 再根據一系列的策略把請求轉發給 Server。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https:\/\/static001.geekbang.org\/infoq\/20\/208dcc29f7229a39faa301593de80d0a.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":"center","origin":null},"content":[{"type":"text","text":"圖4 Outbound 流量劫持"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"這種劫持機制的好處在於改造事項集中在 Naming 系統，使用該 Naming 系統的服務都能通過該方案透明的完成 Outbound 流量劫持。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"另外，基於 Naming 系統的流量劫持機制可以動態回傳流量治理參數給業務服務，如超時、重試等。這種能力的其中一個用途是可以避免 Mesh 劫持後的多級重試導致服務雪崩，具體做法如圖5 所示，當業務流量被 Envoy 劫持後，Envoy 會通過 Naming Agent 將業務服務的重試次數置爲0。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"image","attrs":{"src":"https:\/\/static001.geekbang.org\/infoq\/54\/54c5800a4d36c135fb6c64939cda1ed9.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":"center","origin":null},"content":[{"type":"text","text":"圖5 動態回傳流量治理配置"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"此外，爲了降低數據面（Envoy）故障時對業務服務的影響，我們還增加了數據面自動容災、主動關閉 Mesh 等能力："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"數據面故障自動容災能力"},{"type":"text","text":"：當 Envoy 異常時，Naming Agent 會自動返回 Server 實際的實例列表，此時，Client 會自動回退爲非 Mesh 劫持模式。"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"主動關閉 Mesh 劫持"},{"type":"text","text":"：用戶也可以主動關閉 Mesh 劫持，此時，Client 也會自動回退爲非 Mesh 劫持模式。"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"至此，Envoy 能夠劫持 Outbound 流量，但是，只有 Outbound 流量劫持能力的 Envoy 是不完整的，對於入口限流等功能，還需要具備 Inbound 流量劫持的能力。"}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"Inbound 流量劫持"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Inbound 流量主要來自其他機器，我們無法再依賴單機的 Naming Agent 僞造地址，得另尋出路。還是基於 Naming 系統的思路，EnvoyS 和 Server 是同機部署的，他們對外提供的地址，唯一的區別在於端口，因此，只要我們能更換 EnvoyC 訪問 Server 時的端口，就能將 Inbound 流量劫持到 EnvoyS。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如圖4 所示，EgressPort 接收 Outbound 流量，IngressPort 接收 Inbound 流量。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"numberedlist","attrs":{"start":null,"normalizeStart":1},"content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":1,"align":null,"origin":null},"content":[{"type":"text","text":"控制面（Istio）將 EnvoyS 的 IngressPort 作爲 Server 的端口下發給 EnvoyC；"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":2,"align":null,"origin":null},"content":[{"type":"text","text":"EnvoyC 將訪問 Server 的流量轉發到 IngressPort，被 EnvoyS 收到。"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":3,"align":null,"origin":null},"content":[{"type":"text","text":"EnvoyS 再將流量轉發到 Server 服務端口 NamedPort。"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https:\/\/static001.geekbang.org\/infoq\/20\/20ea3c57ed7cafbff7b34e0038711603.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"至此，Envoy 具備了部分 Inbound 流量劫持能力，爲什麼說是部分呢，因爲這種機制無法劫持入口服務的流量。入口服務的上游（Client）是外部服務，它的配置不受 Istio 控制，也就無法採用該機制完成流量劫持，後續需進一步完善該能力。"}]},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"Inbound 流量劫持中的坑"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"除了上面提到的問題，Inbound 流量劫持還存在一些坑。我們發現當 EnvoyS 劫持了 Inbound 流量後，"},{"type":"text","marks":[{"type":"strong"}],"text":"L3\/L4 層通信協議的部分健康檢查機制失效。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"italic"}],"text":"L3\/L4 層通信協議的"},{"type":"text","marks":[{"type":"italic"},{"type":"strong"}],"text":"主動健康檢查"},{"type":"text","marks":[{"type":"italic"}],"text":"部分功能失效"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"原因：L3\/L4 層通信協議的主動健康檢查默認是檢查端口存活，當流量被劫持到 EnvoyS 後，該功能實際檢查的是 EnvoyS 的 IngressPort 端口存活，也就無法反饋 Server NamedPort 端口存活情況。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"我們目前採用的解決方案是採用兩段式主動健康檢查機制，兩段分別是："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"numberedlist","attrs":{"start":null,"normalizeStart":1},"content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":1,"align":null,"origin":null},"content":[{"type":"text","text":"Envoy 間健康檢查：EnvoyC 對 EnvoyS 的健康檢查，該健康檢查能夠反饋 EnvoyS 和 Server 的狀態。"}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":2,"align":null,"origin":null},"content":[{"type":"text","text":"Envoy 和本地 Service 間健康檢查：EnvoyS 檢查 Server 端口存活情況，檢查結果由 EnvoyS 反饋給 EnvoyC。 "}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"italic"}],"text":"L3\/L4 層通信 協議的"},{"type":"text","marks":[{"type":"italic"},{"type":"strong"}],"text":"異常點驅逐"},{"type":"text","marks":[{"type":"italic"}],"text":"（被動健康檢查）功能失效"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"原因：L3\/L4 層通信協議的"},{"type":"text","marks":[{"type":"strong"}],"text":"異常點驅逐條件是連接異常"},{"type":"text","text":"，當流量被劫持到 EnvoyS 後，該功能實際上檢查的是 EnvoyC 能否正常的跟 EnvoyS 建立連接，而不是 Server。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"我們目前採用的解決方案是完善L3\/L4 層通信協議的驅逐條件，增加訪問超時作爲驅逐條件。因此，當 Server 異常時，EnvoyC 會因爲一直無法得到應答，而將該下游標記爲異常。"}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"總結"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"最後簡單的對比下上述兩種方案："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https:\/\/static001.geekbang.org\/infoq\/33\/33782f0b8782256c750dcd920dafb690.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"基於服務發現的流量劫持機制目前已應用在百度App、信息流、百度地圖等業務線的數百個服務、數萬個實例上。這種流量劫持機制能夠減少轉發性能的損耗，具備數據面故障自動容災能力，能夠動態回傳流量治理參數。但是該機制也缺失一些能力：無法劫持入口服務的流量，後續我們將進一步補齊該能力。"}]}]}