Kubernetes實戰：高可用集羣的搭建和部署

{"type":"doc","content":[{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"​​​​​​​​摘要：官方只提到了一句“使用負載均衡器將 apiserver 暴露給工作節點”，而這恰恰是部署過程中需要解決的重點問題。","attrs":{}}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"本文分享自華爲雲社區","attrs":{}},{"type":"link","attrs":{"href":"https://bbs.huaweicloud.com/blogs/285683?utm_source=infoq&utm_medium=bbs-ex&utm_campaign=other&utm_content=content","title":"","type":null},"content":[{"type":"text","text":"《Kubernetes 高可用集羣落地二三事》","attrs":{}}]},{"type":"text","text":"，作者：zuozewei。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"一、高可用拓撲","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"可以設置 HA 集羣：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"使用堆疊（stacked）控制平面節點，其中etcd 節點與控制平面節點共存；","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"使用外部 etcd 節點，其中 etcd 在與控制平面不同的節點上運行；","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在設置 HA 集羣之前，應該仔細考慮每種拓撲的優缺點。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"1、堆疊（Stacked） etcd 拓撲","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/e2/e2152f1c305663e7fa4cb1c76b473cb3.jpeg","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"主要特點：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"etcd 分佈式數據存儲集羣堆疊在 kubeadm 管理的控制平面節點上，作爲控制平面的一個組件運行。","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"每個控制平面節點運行 kube-apiserver，kube-scheduler 和 kube-controller-manager實例。","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"kube-apiserver 使用 LB 暴露給工作節點。","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"每個控制平面節點創建一個本地 etcd 成員（member），這個 etcd 成員只與該節點的 kube-apiserver 通信。這同樣適用於本地 kube-controller-manager 和 kube-scheduler 實例。","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"簡單概況：","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"每個 master 節點上運行一個 apiserver 和 etcd, etcd 只與本節點 apiserver 通信","attrs":{}},{"type":"text","text":"。","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"這種拓撲將控制平面和 etcd 成員耦合在同一節點上。相對使用外部 etcd 集羣，設置起來更簡單，而且更易於副本管理。","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"然而堆疊集羣存在耦合失敗的風險。如果一個節點發生故障，則 etcd 成員和控制平面實例都將丟失，並且冗餘會受到影響。可以通過添加更多控制平面節點來降低此風險。應該爲 HA 集羣運行至少三個堆疊的控制平面節點（防止腦裂）。","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"這是 kubeadm 中的默認拓撲。當使用 kubeadm init和 kubeadm join --control-plane時，在控制平面節點上會自動創建本地 etcd 成員。","attrs":{}}]}]}],"attrs":{}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"2、外部 etcd 拓撲","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/c9/c91cf7fcc20cb6015b236d5ad2f19bfc.jpeg","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"主要特點：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"具有外部 etcd 的 HA 集羣是一種這樣的拓撲，其中 etcd 分佈式數據存儲集羣在獨立於控制平面節點的其他節點上運行。","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"就像堆疊的 etcd 拓撲一樣，外部 etcd拓撲中的每個控制平面節點都運行 kube-apiserver，kube-scheduler和 kube-controller-manager 實例。","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"同樣 kube-apiserver 使用負載均衡器暴露給工作節點。但是，etcd 成員在不同的主機上運行，​​每個 etcd 主機與每個控制平面節點的 kube-apiserver 通信。","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"簡單概況： ","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"etcd 集羣運行在單獨的主機上，每個 etcd 都與 apiserver 節點通信。","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"這種拓撲結構解耦了控制平面和 etcd 成員。因此，它提供了一種 HA 設置，其中失去控制平面實例或者 etcd 成員的影響較小，並且不會像堆疊的 HA 拓撲那樣影響集羣冗餘。","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"但是，此拓撲需要兩倍於堆疊 HA 拓撲的主機數量。具有此拓撲的 HA 集羣至少需要三個用於控制平面節點的主機和三個用於 etcd 節點的主機。","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"需要單獨設置外部 etcd 集羣。","attrs":{}}]}]}],"attrs":{}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"3、小結","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"官方這裏主要是解決了高可用場景下 apiserver 與 etcd 集羣的關係，以及控制平面節點防止單點故障。但是集羣對外訪問接口不可能將三個 apiserver 都暴露出去，一個節點掛掉時還是不能自動切換到其他節點。官方只提到了一句“使用負載均衡器將 apiserver 暴露給工作節點”，而這恰恰是部署過程中需要解決的重點問題。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Notes: 此處的負載均衡器並不是 kube-proxy，此處的 Load Balancer 是針對 apiserver 的。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"最後，我們總結一下兩種拓撲：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"堆疊（Stacked） etcd 拓撲：設置簡單，易於副本管理，不過存在耦合失敗風險。如果節點發生故障，則 etcd 成員和控制平面實例有丟失的可能，推薦測試開發環境；","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"外部 etcd 拓撲：解耦了控制平面和etcd 成員，不會像堆疊的 HA 拓撲那樣有影響集羣冗餘的風險，不過需要兩倍於堆疊 HA 拓撲的主機數量，設置相對複雜，推薦生產環境。","attrs":{}}]}]}],"attrs":{}},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"二、部署架構","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"以下是我們在測試環境所用的部署架構：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/a6/a64e36d00709d04da6b2aacada8a349b.jpeg","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"這裏採用kubeadm 方式搭建高可用 k8s 集羣，k8s 集羣的高可用實際是 k8s 各核心組件的高可用，這裏使用","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"主備","attrs":{}},{"type":"text","text":"模式：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/82/82f964d74db65183c296eb53c0cc90ad.jpeg","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"apiserver","attrs":{}},{"type":"text","text":" 通過keepalived+haproxy 實現高可用，當某個節點故障時觸發 keepalived vip 轉移，haproxy 負責將流量負載到 apiserver 節點；","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"controller-manager","attrs":{}},{"type":"text","text":" k8s 內部通過選舉方式產生領導者(由 --leader-elect選型控制，默認爲true)，同一時刻集羣內只有一個 controller-manager 組件運行，其餘處於 backup 狀態；","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"scheduler","attrs":{}},{"type":"text","text":" k8s 內部通過選舉方式產生領導者(由 --leader-elect選型控制，默認爲true)，同一時刻集羣內只有一個scheduler組件運行，其餘處於 backup 狀態；","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"etcd","attrs":{}},{"type":"text","text":" 通過運行kubeadm 方式自動創建集羣來實現高可用，部署的節點數爲奇數，3節點方式最多容忍一臺機器宕機。","attrs":{}}]}]}],"attrs":{}},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"三、環境示例","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"主機列表：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/9d/9d59921d835554d292c424696e0e53d6.jpeg","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"這裏共有 12 臺主機，3 臺 controlplane，9 臺 worker。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"四、核心組件","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"1、haproxy","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://github.com/haproxy/haproxy","title":"","type":null},"content":[{"type":"text","text":"haproxy","attrs":{}}]},{"type":"text","text":" 提供高可用性，負載均衡，基於 TCP 和 HTTP 的代理，支持數以萬記的併發連接。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"haproxy 可安裝在主機上，也可使用 docker 容器實現。文本採用第一種。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"創建配置文件 /etc/haproxy/haproxy.cfg，重要配置以中文註釋標出：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"text"},"content":[{"type":"text","text":"#---------------------------------------------------------------------\n# Example configuration for a possible web application. See the\n# full configuration options online.\n#\n# https://www.haproxy.org/download/2.1/doc/configuration.txt\n# https://cbonte.github.io/haproxy-dconv/2.1/configuration.html\n#\n#---------------------------------------------------------------------\n\n#---------------------------------------------------------------------\n# Global settings\n#---------------------------------------------------------------------\nglobal\n # to have these messages end up in /var/log/haproxy.log you will\n # need to:\n #\n # 1) configure syslog to accept network log events. This is done\n # by adding the '-r' option to the SYSLOGD_OPTIONS in\n # /etc/sysconfig/syslog\n #\n # 2) configure local2 events to go to the /var/log/haproxy.log\n # file. A line like the following can be added to\n # /etc/sysconfig/syslog\n #\n # local2.* /var/log/haproxy.log\n #\n log 127.0.0.1 local2\n\n# chroot /var/lib/haproxy\n pidfile /var/run/haproxy.pid\n maxconn 4000\n# user haproxy\n# group haproxy\n # daemon\n\n # turn on stats unix socket\n stats socket /var/lib/haproxy/stats\n\n#---------------------------------------------------------------------\n# common defaults that all the 'listen' and 'backend' sections will\n# use if not designated in their block\n#---------------------------------------------------------------------\ndefaults\n mode http\n log global\n option httplog\n option dontlognull\n option http-server-close\n option forwardfor except 127.0.0.0/8\n option redispatch\n retries 3\n timeout http-request 10s\n timeout queue 1m\n timeout connect 10s\n timeout client 1m\n timeout server 1m\n timeout http-keep-alive 10s\n timeout check 10s\n maxconn 3000\n\n#---------------------------------------------------------------------\n# main frontend which proxys to the backends\n#---------------------------------------------------------------------\nfrontend kubernetes-apiserver\n mode tcp\n bind *:9443 ## 監聽9443端口\n # bind *:443 ssl # To be completed ....\n\n acl url_static path_beg -i /static /images /javascript /stylesheets\n acl url_static path_end -i .jpg .gif .png .css .js\n\n default_backend kubernetes-apiserver\n\n#---------------------------------------------------------------------\n# round robin balancing between the various backends\n#---------------------------------------------------------------------\nbackend kubernetes-apiserver\n mode tcp # 模式tcp\n balance roundrobin # 採用輪詢的負載算法\n# k8s-apiservers backend # 配置apiserver，端口6443\n server k8s-master-1 xxx.16.106.208:6443 check\n server k8s-master-2 xxx.16.106.80:6443 check\n server k8s-master-3 xxx.16.106.14:6443 check","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"分別在三個 master 節點啓動 haproxy。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"2、keepalived","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"keepalived 是以 VRRP(虛擬路由冗餘協議)協議爲基礎, 包括一個 master 和多個backup。 master 劫持 vip 對外提供服務。master 發送組播，backup 節點收不到 vrrp 包時認爲 master 宕機，此時選出剩餘優先級最高的節點作爲新的 master, 劫持 vip。keepalived是保證高可用的重要組件。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://github.com/osixia/docker-keepalived","title":"","type":null},"content":[{"type":"text","text":"keepalived","attrs":{}}]},{"type":"text","text":" 可安裝在主機上，也可使用 docker 容器實現。文本採用第一種。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"配置 keepalived.conf, 重要部分以中文註釋標出：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"text"},"content":[{"type":"text","text":"! Configuration File for keepalived\nglobal_defs {\n router_id k8s-master-1\n}\nvrrp_script chk_haproxy {\n script \"/bin/bash -c 'if [[ $(netstat -nlp | grep 9443) ]]; then exit 0; else exit 1; fi'\" # haproxy 檢測\n interval 2 # 每2秒執行一次檢測\n weight 11 # 權重變化\n}\nvrrp_instance VI_1 {\n state MASTER # backup節點設爲BACKUP\n interface eth0\n virtual_router_id 50 # id設爲相同，表示是同一個虛擬路由組\n priority 100 # 初始權重\n authentication {\n auth_type PASS\n auth_pass 1111\n }\n virtual_ipaddress {\n 172.16.106.187 # vip\n }\n track_script {\n chk_haproxy\n }\n}","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"vrrp_script 用於檢測 haproxy 是否正常。如果本機的haproxy 掛掉，即使 keepalived 劫持vip，也無法將流量負載到 apiserver。","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"我所查閱的網絡教程全部爲檢測進程, 類似 killall -0 haproxy。這種方式用在主機部署上可以，但容器部署時，在keepalived 容器中無法知道另一個容器 haproxy 的活躍情況，因此我在此處通過檢測端口號來判斷haproxy 的健康狀況。","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"weight 可正可負。爲正時檢測成功 +weight，相當與節點檢測失敗時本身 priority 不變，但其他檢測成功節點 priority 增加。爲負時檢測失敗本身 priority 減少。","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"另外很多文章中沒有強調 nopreempt 參數，意爲不可搶佔，此時 master 節點失敗後，backup 節點也不能接管 vip，因此我將此配置刪去。","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"分別在三臺節點啓動keepalived，查看 keepalivedmaster日誌：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"text"},"content":[{"type":"text","text":"Dec 25 15:52:45 k8s-master-1 Keepalived_vrrp[12562]: VRRP_Script(chk_haproxy) succeeded # haproxy檢測成功\nDec 25 15:52:46 k8s-master-1 Keepalived_vrrp[12562]: VRRP_Instance(VI_1) Changing effective priority from 100 to 111 # priority增加\nDec 25 15:54:06 k8s-master-1 Keepalived_vrrp[12562]: VRRP_Instance(VI_1) Transition to MASTER STATE\nDec 25 15:54:06 k8s-master-1 Keepalived_vrrp[12562]: VRRP_Instance(VI_1) Received advert with lower priority 111, ours 111, forcing new election\nDec 25 15:54:07 k8s-master-1 Keepalived_vrrp[12562]: VRRP_Instance(VI_1) Entering MASTER STATE\nDec 25 15:54:07 k8s-master-1 Keepalived_vrrp[12562]: VRRP_Instance(VI_1) setting protocol VIPs. # 設置vip \nDec 25 15:54:07 k8s-master-1 Keepalived_vrrp[12562]: Sending gratuitous ARP on eth0 for 172.16.106.187\nDec 25 15:54:07 k8s-master-1 Keepalived_vrrp[12562]: VRRP_Instance(VI_1) Sending/queueing gratuitous ARPs on eth0 for 172.16.106.187\nDec 25 15:54:07 k8s-master-1 Keepalived_vrrp[12562]: Sending gratuitous ARP on eth0 for 172.16.106.187\nDec 25 15:54:07 k8s-master-1 Keepalived_vrrp[12562]: Sending gratuitous ARP on eth0 for 172.16.106.187\nDec 25 15:54:07 k8s-master-1 Keepalived_vrrp[12562]: Sending gratuitous ARP on eth0 for 172.16.106.187\nDec 25 15:54:07 k8s-master-1 Keepalived_vrrp[12562]: Sending gratuitous ARP on eth0 for 172.16.106.187\nDec 25 15:54:07 k8s-master-1 avahi-daemon[756]: Registering new address record for 172.16.106.187 on eth0.IPv4.\nDec 25 15:54:10 k8s-master-1 kubelet: E1225 15:54:09.999466 1047 kubelet_node_status.go:442] Error updating node status, will retry: failed to patch status \"{\\\"status\\\":{\\\"$setElementOrder/conditions\\\":[{\\\"type\\\":\\\"NetworkUnavailable\\\"},{\\\"type\\\":\\\"MemoryPressure\\\"},{\\\"type\\\":\\\"DiskPressure\\\"},{\\\"type\\\":\\\"PIDPressure\\\"},{\\\"type\\\":\\\"Ready\\\"}],\\\"addresses\\\":[{\\\"address\\\":\\\"172.16.106.187\\\",\\\"type\\\":\\\"InternalIP\\\"},{\\\"address\\\":\\\"k8s-master-1\\\",\\\"type\\\":\\\"Hostname\\\"},{\\\"$patch\\\":\\\"replace\\\"}],\\\"conditions\\\":[{\\\"lastHeartbeatTime\\\":\\\"2020-12-25T07:54:09Z\\\",\\\"type\\\":\\\"MemoryPressure\\\"},{\\\"lastHeartbeatTime\\\":\\\"2020-12-25T07:54:09Z\\\",\\\"type\\\":\\\"DiskPressure\\\"},{\\\"lastHeartbeatTime\\\":\\\"2020-12-25T07:54:09Z\\\",\\\"type\\\":\\\"PIDPressure\\\"},{\\\"lastHeartbeatTime\\\":\\\"2020-12-25T07:54:09Z\\\",\\\"type\\\":\\\"Ready\\\"}]}}\" for node \"k8s-master-1\": Patch \"https://apiserver.demo:6443/api/v1/nodes/k8s-master-1/status?timeout=10s\": write tcp 172.16.106.208:46566->172.16.106.187:6443: write: connection reset by peer\nDec 25 15:54:11 k8s-master-1 Keepalived_vrrp[12562]: Sending gratuitous ARP on eth0 for 172.16.106.187\nDec 25 15:54:11 k8s-master-1 Keepalived_vrrp[12562]: VRRP_Instance(VI_1) Sending/queueing gratuitous ARPs on eth0 for 172.16.106.187\nDec 25 15:54:11 k8s-master-1 Keepalived_vrrp[12562]: Sending gratuitous ARP on eth0 for 172.16.106.187\nDec 25 15:54:11 k8s-master-1 Keepalived_vrrp[12562]: Sending gratuitous ARP on eth0 for 172.16.106.187\nDec 25 15:54:11 k8s-master-1 Keepalived_vrrp[12562]: Sending gratuitous ARP on eth0 for 172.16.106.187\nDec 25 15:54:11 k8s-master-1 Keepalived_vrrp[12562]: Sending gratuitous ARP on eth0 for 172.16.106.187\nDec 25 15:54:12 k8s-master-1 Keepalived_vrrp[12562]: Sending gratuitous ARP on eth0 for 172.16.106.187","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"查看 master vip:","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"text"},"content":[{"type":"text","text":"[root@k8s-master-1 ~]# ip a|grep eth0\n2: eth0: mtu 1500 qdisc mq state UP group default qlen 1000\n inet 172.16.106.208/24 brd 172.16.106.255 scope global noprefixroute dynamic eth0\n inet 172.16.106.187/32 scope global eth0","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"可以看到 vip 已綁定到 keepalived master","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下面進行破壞性測試：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"暫停 keepalived master 節點 haproxy:","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"text"},"content":[{"type":"text","text":"[root@k8s-master-1 ~]# service haproxy stop\nRedirecting to /bin/systemctl stop haproxy.service","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"查看 keepalived k8s-master-1 節點日誌:","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"text"},"content":[{"type":"text","text":"Dec 25 15:58:31 k8s-master-1 Keepalived_vrrp[12562]: /bin/bash -c 'if [[ $(netstat -nlp | grep 9443) ]]; then exit 0; else exit 1; fi' exited with status 1\nDec 25 15:58:31 k8s-master-1 Keepalived_vrrp[12562]: VRRP_Script(chk_haproxy) failed\nDec 25 15:58:31 k8s-master-1 Keepalived_vrrp[12562]: VRRP_Instance(VI_1) Changing effective priority from 111 to 100\nDec 25 15:58:32 k8s-master-1 Keepalived_vrrp[12562]: VRRP_Instance(VI_1) Received advert with higher priority 111, ours 100\nDec 25 15:58:32 k8s-master-1 Keepalived_vrrp[12562]: VRRP_Instance(VI_1) Entering BACKUP STATE\nDec 25 15:58:32 k8s-master-1 Keepalived_vrrp[12562]: VRRP_Instance(VI_1) removing protocol VIPs.","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"可以看到 haproxy 檢測失敗，priority 降低，同時另一節點 priority 比 k8s-master-1 節點高， k8s-master-1 置爲 backup","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"查看 k8s-master-2 節點 keepalived日誌：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"text"},"content":[{"type":"text","text":"Dec 25 15:58:35 k8s-master-2 Keepalived_vrrp[3661]: VRRP_Instance(VI_1) Transition to MASTER STATE\nDec 25 15:58:35 k8s-master-2 Keepalived_vrrp[3661]: VRRP_Instance(VI_1) Received advert with lower priority 111, ours 111, forcing new election\nDec 25 15:58:36 k8s-master-2 Keepalived_vrrp[3661]: VRRP_Instance(VI_1) Entering MASTER STATE\nDec 25 15:58:36 k8s-master-2 Keepalived_vrrp[3661]: VRRP_Instance(VI_1) setting protocol VIPs.\nDec 25 15:58:36 k8s-master-2 Keepalived_vrrp[3661]: Sending gratuitous ARP on eth0 for 172.16.106.187\nDec 25 15:58:36 k8s-master-2 avahi-daemon[740]: Registering new address record for 172.16.106.187 on eth0.IPv4.","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"可以看到 k8s-master-2 被選舉爲新的 master。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"五、安裝部署","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"1、安裝 docker / kubelet","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"參考上文 ","attrs":{}},{"type":"link","attrs":{"href":"https://zuozewei.blog.csdn.net/article/details/107419381","title":"","type":null},"content":[{"type":"text","text":"使用 kubeadm 安裝單master kubernetes 集羣（腳本版）","attrs":{}}]}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"2、初始化第一個 master","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"kubeadm.conf 爲初始化的配置文件：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"text"},"content":[{"type":"text","text":"[root@master01 ~]# more kubeadm-config.yaml \napiVersion: kubeadm.k8s.io/v1beta2\nkind: ClusterConfiguration\nkubernetesVersion: v1.16.4\napiServer:\n certSANs: #填寫所有kube-apiserver節點的hostname、IP、VIP\n - k8s-master-1\n - k8s-master-2\n - k8s-master-3\n - k8s-worker-1\n - apiserver.demo\n.....\ncontrolPlaneEndpoint: \"172.27.34.130:6443\"\nnetworking:\n podSubnet: \"10.244.0.0/16\"","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"初始化 k8s-master-1：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"text"},"content":[{"type":"text","text":"# kubeadm init\n# 根據您服務器網速的情況，您需要等候 3 - 10 分鐘\nkubeadm init --config=kubeadm-config.yaml --upload-certs\n\n# 配置 kubectl\nrm -rf /root/.kube/\nmkdir /root/.kube/\ncp -i /etc/kubernetes/admin.conf /root/.kube/config\n\n# 安裝 calico 網絡插件\n# 參考文檔 https://docs.projectcalico.org/v3.13/getting-started/kubernetes/self-managed-onprem/onpremises\necho \"安裝calico-3.13.1\"\nkubectl apply -f calico-3.13.1.yaml","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"3、初始化第二、三個 master 節點","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"可以和第一個Master節點一起初始化第二、三個Master節點，也可以從單Master節點調整過來，只需要：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"增加 Master 的LoadBalancer","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"將所有節點的 /etc/hosts文件中 apiserver.demo 解析爲LoadBalancer 的地址","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"添加第二、三個Master節點","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"初始化 master 節點的 token 有效時間爲 2 小時","attrs":{}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"這裏我們演示第一個Master節點初始化2個小時後再初始化：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"text"},"content":[{"type":"text","text":"# 只在 第一個 master 上執行\n[root@k8s-master-1 ~]# kubeadm init phase upload-certs --upload-certs\nI1225 16:25:00.247925 19101 version.go:252] remote version is much newer: v1.20.1; falling back to: stable-1.19\nW1225 16:25:01.120802 19101 configset.go:348] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io]\n[upload-certs] Storing the certificates in Secret \"kubeadm-certs\" in the \"kube-system\" Namespace\n[upload-certs] Using certificate key:\n5c120930eae91fc19819f1cbe71a6986a78782446437778cc0777062142ef1e6","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"獲得 join 命令：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"text"},"content":[{"type":"text","text":"# 只在 第一個 master 節點上執行\n[root@k8s-master-1 ~]# kubeadm token create --print-join-command\nW1225 16:26:27.642047 20949 configset.go:348] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io]\nkubeadm join apiserver.demo:6443 --token kab883.kyw62ylnclbf3mi6 --discovery-token-ca-cert-hash sha256:566a7142ed059ab5dee403dd4ef6d52cdc6692fae9c05432e240bbc08420b7f0 ","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"則，第二、三個 master 節點的 join 命令如下：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"text"},"content":[{"type":"text","text":"# 命令行中，前面爲獲得的 join 命令，control-plane 指定的爲獲得的 certificate key\nkubeadm join apiserver.demo:6443 --token kab883.kyw62ylnclbf3mi6 \\\n--discovery-token-ca-cert-hash sha256:566a7142ed059ab5dee403dd4ef6d52cdc6692fae9c05432e240bbc08420b7f0 \\\n--control-plane --certificate-key 5c120930eae91fc19819f1cbe71a6986a78782446437778cc0777062142ef1e6","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"檢查 master 初始化結果:","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"text"},"content":[{"type":"text","text":"[root@k8s-master-1 ~]# kubectl get nodes\nNAME STATUS ROLES AGE VERSION\nk8s-master-1 Ready master 2d v1.19.2\nk8s-master-2 Ready master 2d v1.19.2\nk8s-master-3 Ready master 2d v1.19.2","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"4、初始化 worker節點","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"針對所有的 worker 節點執行:","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"text"},"content":[{"type":"text","text":"# 只在 worker 節點執行\n# 替換 x.x.x.x 爲 ApiServer LoadBalancer 的 IP 地址\nexport MASTER_IP=x.x.x.x\n# 替換 apiserver.demo 爲初始化 master 節點時所使用的 APISERVER_NAME\nexport APISERVER_NAME=apiserver.demo\necho \"${MASTER_IP} ${APISERVER_NAME}\" >> /etc/hosts\n\n# 替換爲前面 kubeadm token create --print-join-command 的輸出結果\nkubeadm join apiserver.demo:6443 --token kab883.kyw62ylnclbf3mi6 --discovery-token-ca-cert-hash sha256:566a7142ed059ab5dee403dd4ef6d52cdc6692fae9c05432e240bbc08420b7f0 ","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"檢查 worker 初始化結果:","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"text"},"content":[{"type":"text","text":"[root@k8s-master-1 ~]# kubectl get nodes\nNAME STATUS ROLES AGE VERSION\nk8s-master-1 Ready master 2d v1.19.2\nk8s-master-2 Ready master 2d v1.19.2\nk8s-master-3 Ready master 2d v1.19.2\nk8s-worker-1 Ready 2d v1.19.2\nk8s-worker-2 Ready 2d v1.19.2\nk8s-worker-3 Ready 2d v1.19.2\nk8s-worker-4 Ready 2d v1.19.2\nk8s-worker-5 Ready 2d v1.19.2\nk8s-worker-6 Ready 2d v1.19.2\nk8s-worker-7 Ready 2d v1.19.2\nk8s-worker-8 Ready 2d v1.19.2\nk8s-worker-9 Ready 2d v1.19.2","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"本文資料：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://github.com/zuozewei/blog-example/tree/master/Kubernetes/k8s-install-HA-cluster","title":"","type":null},"content":[{"type":"text","text":"https://github.com/zuozewei/blog-example/tree/master/Kubernetes/k8s-install-HA-cluster","attrs":{}}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"參考資料：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"[1]：","attrs":{}},{"type":"link","attrs":{"href":"https://www.kuboard.cn/install/install-kubernetes.html","title":"","type":null},"content":[{"type":"text","text":"https://www.kuboard.cn/install/install-kubernetes.html","attrs":{}}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"[2]：","attrs":{}},{"type":"link","attrs":{"href":"https://github.com/loong576/Centos7.6-install-k8s-v1.16.4-HA-cluster","title":"","type":null},"content":[{"type":"text","text":"https://github.com/loong576/Centos7.6-install-k8s-v1.16.4-HA-cluster","attrs":{}}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"[3]：","attrs":{}},{"type":"link","attrs":{"href":"https://kubernetes.io/zh/docs/setup/production-environment/tools/kubeadm/ha-topology/","title":"","type":null},"content":[{"type":"text","text":"https://kubernetes.io/zh/docs/setup/production-environment/tools/kubeadm/ha-topology/","attrs":{}}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"[4]：","attrs":{}},{"type":"link","attrs":{"href":"https://www.kubernetes.org.cn/6964.html","title":"","type":null},"content":[{"type":"text","text":"https://www.kubernetes.org.cn/6964.html","attrs":{}}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://bbs.huaweicloud.com/blogs?utm_source=infoq&utm_medium=bbs-ex&utm_campaign=other&utm_content=content","title":"","type":null},"content":[{"type":"text","text":"點擊關注，第一時間瞭解華爲雲新鮮技術~","attrs":{}}]},{"type":"text","text":"​​​​","attrs":{}}]}]}