聲明:
如果您有更好的技術與作者分享,或者商業合作;
請訪問作者個人網站 http://www.esqabc.com/view/message.html 留言給作者。
如果該案例觸犯您的專利,請在這裏:http://www.esqabc.com/view/message.html 留言給作者說明原由
作者一經查實,馬上刪除。
1、搭建前說明
a、kubernetes - master節點運行組件如下:
- kube-apiserver
- kube-scheduler
- kube-controller-manager
如沒有特殊說明,一般都在k8s-01服務器操作
前提提條件、服務器,請查看這個地址:https://blog.csdn.net/esqabc/article/details/102726771
2、部署master節點
a、下載kubernetes二進制包
[root@k8s-01 ~]# cd /opt/k8s/work
[root@k8s-01 work]# wget http://down.i4t.com/k8s1.14/kubernetes-server-linux-amd64.tar.gz
[root@k8s-01 work]# tar -xzvf kubernetes-server-linux-amd64.tar.gz
[root@k8s-01 work]# cd kubernetes
[root@k8s-01 kubernetes]# tar -xzvf kubernetes-src.tar.gz
b、分發到所有master節點
[root@k8s-01 ~]# cd /opt/k8s/work
[root@k8s-01 ~]# source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
scp kubernetes/server/bin/{apiextensions-apiserver,cloud-controller-manager,kube-apiserver,kube-controller-manager,kube-proxy,kube-scheduler,kubeadm,kubectl,kubelet,mounter} root@${node_ip}:/opt/k8s/bin/
ssh root@${node_ip} "chmod +x /opt/k8s/bin/*"
done
c、創建Kubernetes 證書和私鑰
[root@k8s-01 ~]# cd /opt/k8s/work
[root@k8s-01 work]# cat > kubernetes-csr.json <<EOF
添加下面內容:
{
"CN": "kubernetes",
"hosts": [
"127.0.0.1",
"172.26.16.249",
"172.26.16.250",
"172.26.16.251",
"172.26.16.252",
"10.254.0.1",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local."
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "4Paradigm"
}
]
}
EOF
注意:需要將集羣的所有IP都添加進去
d、生成證書和私鑰
[root@k8s-01 ~]# cd /opt/k8s/work
cfssl gencert -ca=/opt/k8s/work/ca.pem \
-ca-key=/opt/k8s/work/ca-key.pem \
-config=/opt/k8s/work/ca-config.json \
-profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes
[root@k8s-01 ~]# ls kubernetes*pem
e、分發到所有master節點
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
ssh root@${node_ip} "mkdir -p /etc/kubernetes/cert"
scp kubernetes*.pem root@${node_ip}:/etc/kubernetes/cert/
done
f、創建加密配置文件
[root@k8s-01 ~]# cd /opt/k8s/work
[root@k8s-01 ~]# cat > encryption-config.yaml <<EOF
添加下面內容
kind: EncryptionConfig
apiVersion: v1
resources:
- resources:
- secrets
providers:
- aescbc:
keys:
- name: key1
secret: ${ENCRYPTION_KEY}
- identity: {}
EOF
g、將加密配置文件拷貝到master節點的/etc/kubernetes目錄下
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
scp encryption-config.yaml root@${node_ip}:/etc/kubernetes/
done
h、創建審計策略文件
[root@k8s-01 ~]# cd /opt/k8s/work
[root@k8s-01 ~]# cat > audit-policy.yaml <<EOF
添加下面內容:
apiVersion: audit.k8s.io/v1beta1
kind: Policy
rules:
# The following requests were manually identified as high-volume and low-risk, so drop them.
- level: None
resources:
- group: ""
resources:
- endpoints
- services
- services/status
users:
- 'system:kube-proxy'
verbs:
- watch
- level: None
resources:
- group: ""
resources:
- nodes
- nodes/status
userGroups:
- 'system:nodes'
verbs:
- get
- level: None
namespaces:
- kube-system
resources:
- group: ""
resources:
- endpoints
users:
- 'system:kube-controller-manager'
- 'system:kube-scheduler'
- 'system:serviceaccount:kube-system:endpoint-controller'
verbs:
- get
- update
- level: None
resources:
- group: ""
resources:
- namespaces
- namespaces/status
- namespaces/finalize
users:
- 'system:apiserver'
verbs:
- get
# Don't log HPA fetching metrics.
- level: None
resources:
- group: metrics.k8s.io
users:
- 'system:kube-controller-manager'
verbs:
- get
- list
# Don't log these read-only URLs.
- level: None
nonResourceURLs:
- '/healthz*'
- /version
- '/swagger*'
# Don't log events requests.
- level: None
resources:
- group: ""
resources:
- events
# node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes
- level: Request
omitStages:
- RequestReceived
resources:
- group: ""
resources:
- nodes/status
- pods/status
users:
- kubelet
- 'system:node-problem-detector'
- 'system:serviceaccount:kube-system:node-problem-detector'
verbs:
- update
- patch
- level: Request
omitStages:
- RequestReceived
resources:
- group: ""
resources:
- nodes/status
- pods/status
userGroups:
- 'system:nodes'
verbs:
- update
- patch
# deletecollection calls can be large, don't log responses for expected namespace deletions
- level: Request
omitStages:
- RequestReceived
users:
- 'system:serviceaccount:kube-system:namespace-controller'
verbs:
- deletecollection
# Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data,
# so only log at the Metadata level.
- level: Metadata
omitStages:
- RequestReceived
resources:
- group: ""
resources:
- secrets
- configmaps
- group: authentication.k8s.io
resources:
- tokenreviews
# Get repsonses can be large; skip them.
- level: Request
omitStages:
- RequestReceived
resources:
- group: ""
- group: admissionregistration.k8s.io
- group: apiextensions.k8s.io
- group: apiregistration.k8s.io
- group: apps
- group: authentication.k8s.io
- group: authorization.k8s.io
- group: autoscaling
- group: batch
- group: certificates.k8s.io
- group: extensions
- group: metrics.k8s.io
- group: networking.k8s.io
- group: policy
- group: rbac.authorization.k8s.io
- group: scheduling.k8s.io
- group: settings.k8s.io
- group: storage.k8s.io
verbs:
- get
- list
- watch
# Default level for known APIs
- level: RequestResponse
omitStages:
- RequestReceived
resources:
- group: ""
- group: admissionregistration.k8s.io
- group: apiextensions.k8s.io
- group: apiregistration.k8s.io
- group: apps
- group: authentication.k8s.io
- group: authorization.k8s.io
- group: autoscaling
- group: batch
- group: certificates.k8s.io
- group: extensions
- group: metrics.k8s.io
- group: networking.k8s.io
- group: policy
- group: rbac.authorization.k8s.io
- group: scheduling.k8s.io
- group: settings.k8s.io
- group: storage.k8s.io
# Default level for all other requests.
- level: Metadata
omitStages:
- RequestReceived
EOF
i、分發審計策略文件
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
scp audit-policy.yaml root@${node_ip}:/etc/kubernetes/audit-policy.yaml
done
j、創建證書籤名請求
[root@k8s-01 ~]# cd /opt/k8s/work
[root@k8s-01 ~]# cat > proxy-client-csr.json <<EOF
添加下面內容:
{
"CN": "aggregator",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "4Paradigm"
}
]
}
EOF
k、生成證書和私鑰
[root@k8s-01 ~]# cd /opt/k8s/work
cfssl gencert -ca=/etc/kubernetes/cert/ca.pem \
-ca-key=/etc/kubernetes/cert/ca-key.pem \
-config=/etc/kubernetes/cert/ca-config.json \
-profile=kubernetes proxy-client-csr.json | cfssljson -bare proxy-client
[root@k8s-01 ~]# ls proxy-client*.pem
l、將生成的證書和私鑰文件分發到master節點
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
scp proxy-client*.pem root@${node_ip}:/etc/kubernetes/cert/
done
m、創建kube-apiserver啓動文件
[root@k8s-01 ~]# cd /opt/k8s/work
[root@k8s-01 work]# cat > kube-apiserver.service.template <<EOF
添加下面內容:
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target
[Service]
WorkingDirectory=${K8S_DIR}/kube-apiserver
ExecStart=/opt/k8s/bin/kube-apiserver \\
--advertise-address=##NODE_IP## \\
--default-not-ready-toleration-seconds=360 \\
--default-unreachable-toleration-seconds=360 \\
--feature-gates=DynamicAuditing=true \\
--max-mutating-requests-inflight=2000 \\
--max-requests-inflight=4000 \\
--default-watch-cache-size=200 \\
--delete-collection-workers=2 \\
--encryption-provider-config=/etc/kubernetes/encryption-config.yaml \\
--etcd-cafile=/etc/kubernetes/cert/ca.pem \\
--etcd-certfile=/etc/kubernetes/cert/kubernetes.pem \\
--etcd-keyfile=/etc/kubernetes/cert/kubernetes-key.pem \\
--etcd-servers=${ETCD_ENDPOINTS} \\
--bind-address=##NODE_IP## \\
--secure-port=6443 \\
--tls-cert-file=/etc/kubernetes/cert/kubernetes.pem \\
--tls-private-key-file=/etc/kubernetes/cert/kubernetes-key.pem \\
--insecure-port=0 \\
--audit-dynamic-configuration \\
--audit-log-maxage=15 \\
--audit-log-maxbackup=3 \\
--audit-log-maxsize=100 \\
--audit-log-truncate-enabled \\
--audit-log-path=${K8S_DIR}/kube-apiserver/audit.log \\
--audit-policy-file=/etc/kubernetes/audit-policy.yaml \\
--profiling \\
--anonymous-auth=false \\
--client-ca-file=/etc/kubernetes/cert/ca.pem \\
--enable-bootstrap-token-auth \\
--requestheader-allowed-names="aggregator" \\
--requestheader-client-ca-file=/etc/kubernetes/cert/ca.pem \\
--requestheader-extra-headers-prefix="X-Remote-Extra-" \\
--requestheader-group-headers=X-Remote-Group \\
--requestheader-username-headers=X-Remote-User \\
--service-account-key-file=/etc/kubernetes/cert/ca.pem \\
--authorization-mode=Node,RBAC \\
--runtime-config=api/all=true \\
--enable-admission-plugins=NodeRestriction \\
--allow-privileged=true \\
--apiserver-count=3 \\
--event-ttl=168h \\
--kubelet-certificate-authority=/etc/kubernetes/cert/ca.pem \\
--kubelet-client-certificate=/etc/kubernetes/cert/kubernetes.pem \\
--kubelet-client-key=/etc/kubernetes/cert/kubernetes-key.pem \\
--kubelet-https=true \\
--kubelet-timeout=10s \\
--proxy-client-cert-file=/etc/kubernetes/cert/proxy-client.pem \\
--proxy-client-key-file=/etc/kubernetes/cert/proxy-client-key.pem \\
--service-cluster-ip-range=${SERVICE_CIDR} \\
--service-node-port-range=${NODE_PORT_RANGE} \\
--logtostderr=true \\
--v=2
Restart=on-failure
RestartSec=10
Type=notify
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
說明一下:
- advertise-address:apiserver 對外通告的 IP(kubernetes 服務後端節點 IP);
- default-*-toleration-seconds:設置節點異常相關的閾值;
- max-*-requests-inflight:請求相關的最大閾值;
- etcd-*:訪問 etcd 的證書和 etcd 服務器地址;
- experimental-encryption-provider-config:指定用於加密 etcd 中 secret 的配置;
- bind-address: https 監聽的 IP,不能爲 127.0.0.1,否則外界不能訪問它的安全端口 6443;
- secret-port:https 監聽端口;
- insecure-port=0:關閉監聽 http 非安全端口(8080);
- tls-*-file:指定 apiserver 使用的證書、私鑰和 CA 文件;
- audit-*:配置審計策略和審計日誌文件相關的參數;
- client-ca-file:驗證 client (kue-controller-manager、kube-scheduler、kubelet、kube-proxy 等)請求所帶的證書;
- enable-bootstrap-token-auth:啓用 kubelet bootstrap 的 token 認證;
- requestheader-*:kube-apiserver 的 aggregator layer 相關的配置參數,proxy-client & HPA 需要使用;
- requestheader-client-ca-file:用於簽名 --proxy-client-cert-file 和 --proxy-client-key-file 指定的證書;在啓用了 metric aggregator 時使用;
- requestheader-allowed-names:不能爲空,值爲逗號分割的 --proxy-client-cert-file 證書的 CN 名稱,這裏設置爲 “aggregator”;
- service-account-key-file:簽名 ServiceAccount Token 的公鑰文件,kube-controller-manager 的 --service-account-private-key-file 定私鑰文件,兩者配對使用;
- runtime-config=api/all=true: 啓用所有版本的 APIs,如 autoscaling/v2alpha1;
- authorization-mode=Node,RBAC、–anonymous-auth=false: 開啓 Node 和 RBAC 授權模式,拒絕未授權的請求;
- enable-admission-plugins:啓用一些默認關閉的 plugins;
- allow-privileged:運行執行 privileged 權限的容器;
- apiserver-count=3:指定 apiserver 實例的數量;
- event-ttl:指定 events 的保存時間;
- kubelet-:如果指定,則使用 https 訪問 kubelet APIs;需要爲證書對應的用戶(上面 kubernetes.pem 證書的用戶爲 kubernetes) 用戶定義 RBAC 規則,否則訪問 kubelet API 時提示未授權;
- proxy-client-*:apiserver 訪問 metrics-server 使用的證書;
- service-cluster-ip-range: 指定 Service Cluster IP 地址段;
- service-node-port-range: 指定 NodePort 的端口範圍;
如果 kube-apiserver 機器沒有運行 kube-proxy,則還需要添加 --enable-aggregator-routing=true 參數;
n、分發kube-apiserver啓動文件
[root@k8s-01 ~]# cd /opt/k8s/work
for (( i=0; i < 3; i++ ))
do
sed -e "s/##NODE_NAME##/${MASTER_NAMES[i]}/" -e "s/##NODE_IP##/${MASTER_IPS[i]}/" kube-apiserver.service.template > kube-apiserver-${MASTER_IPS[i]}.service
done
[root@k8s-01 work]# ls kube-apiserver*.service
for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
scp kube-apiserver-${node_ip}.service root@${node_ip}:/etc/systemd/system/kube-apiserver.service
done
o、啓動apiserver
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
ssh root@${node_ip} "mkdir -p ${K8S_DIR}/kube-apiserver"
ssh root@${node_ip} "systemctl daemon-reload && systemctl enable kube-apiserver && systemctl restart kube-apiserver"
done
正常圖示:
p、檢查服務是否正常
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
ssh root@${node_ip} "systemctl status kube-apiserver |grep 'Active:'"
done
正常圖示:
r、kube-apiserver寫入etcd數據
[root@k8s-01 ~]# cd /opt/k8s/work
ETCDCTL_API=3 etcdctl \
--endpoints=${ETCD_ENDPOINTS} \
--cacert=/opt/k8s/work/ca.pem \
--cert=/opt/k8s/work/etcd.pem \
--key=/opt/k8s/work/etcd-key.pem \
get /registry/ --prefix --keys-only
s、檢查kube-apiserver監聽的端口、檢查集羣信息
(1)檢查kube-apiserver監聽的端口
[root@k8s-01 ~]# cd /opt/k8s/work
[root@k8s-01 work]# netstat -lntup|grep kube
正常圖示:
(2)檢查集羣信息
[root@k8s-01 ~]# cd /opt/k8s/work
[root@k8s-01 work]# kubectl cluster-info
正常圖示:
[root@k8s-01 work]# kubectl get all --all-namespaces
正常圖示:
[root@k8s-01 work]# kubectl get componentstatuses
正常圖示:
t、授權kube-apiserver訪問kubelet API的權限
[root@k8s-01 ~]# cd /opt/k8s/work
kubectl create clusterrolebinding kube-apiserver:kubelet-apis --clusterrole=system:kubelet-api-admin --user kubernetes
正常圖示:
3、部署高可用kube-controller-manager集羣
該集羣包含三個節點,啓動後通過競爭選舉機制產生一個leader節點,其他節點爲阻塞狀態。
當leader節點不可用時,阻塞節點將會在此選舉產生新的leader,從而保證服務的高可用。
a、創建kube-controller-manager證書和私鑰
[root@k8s-01 ~]# cd /opt/k8s/work
[root@k8s-01 ~]# cat > kube-controller-manager-csr.json <<EOF
[root@k8s-01 ~]# 添加下面內容:
{
"CN": "system:kube-controller-manager",
"key": {
"algo": "rsa",
"size": 2048
},
"hosts": [
"127.0.0.1",
"172.26.16.249",
"172.26.16.250",
"172.26.16.251"
],
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "system:kube-controller-manager",
"OU": "4Paradigm"
}
]
}
EOF
說明一下:
- host列表包含所有的kube-controller-manager節點IP
- CN和O均爲system:kube-controller-manager,kubernetes
內置的ClusterRoleBindings
system:kube-controller-manager賦予kube-controller-manager工作所需權限
b、生成證書和私鑰
[root@k8s-01 ~]# cd /opt/k8s/work
cfssl gencert -ca=/opt/k8s/work/ca.pem \
-ca-key=/opt/k8s/work/ca-key.pem \
-config=/opt/k8s/work/ca-config.json \
-profile=kubernetes kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager
[root@k8s-01 ~]# ls kube-controller-manager*pem
c、將生成的證書和私鑰分發到所有master節點
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
scp kube-controller-manager*.pem root@${node_ip}:/etc/kubernetes/cert/
done
d、創建和分發kubeconfig文件
[root@k8s-01 ~]# cd /opt/k8s/work
(1)創建
kubectl config set-cluster kubernetes \
--certificate-authority=/opt/k8s/work/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kube-controller-manager.kubeconfig
kubectl config set-credentials system:kube-controller-manager \
--client-certificate=kube-controller-manager.pem \
--client-key=kube-controller-manager-key.pem \
--embed-certs=true \
--kubeconfig=kube-controller-manager.kubeconfig
kubectl config set-context system:kube-controller-manager \
--cluster=kubernetes \
--user=system:kube-controller-manager \
--kubeconfig=kube-controller-manager.kubeconfig
kubectl config use-context system:kube-controller-manager --kubeconfig=kube-controller-manager.kubeconfig
(2)分發
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
scp kube-controller-manager.kubeconfig root@${node_ip}:/etc/kubernetes/
done
c、創建kube-controller-manager啓動文件
[root@k8s-01 ~]# cd /opt/k8s/work
[root@k8s-01 ~]# cat > kube-controller-manager.service.template <<EOF
添加下面內容:
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
[Service]
WorkingDirectory=${K8S_DIR}/kube-controller-manager
ExecStart=/opt/k8s/bin/kube-controller-manager \\
--profiling \\
--cluster-name=kubernetes \\
--controllers=*,bootstrapsigner,tokencleaner \\
--kube-api-qps=1000 \\
--kube-api-burst=2000 \\
--leader-elect \\
--use-service-account-credentials\\
--concurrent-service-syncs=2 \\
--bind-address=0.0.0.0 \\
#--secure-port=10252 \\
--tls-cert-file=/etc/kubernetes/cert/kube-controller-manager.pem \\
--tls-private-key-file=/etc/kubernetes/cert/kube-controller-manager-key.pem \\
#--port=0 \\
--authentication-kubeconfig=/etc/kubernetes/kube-controller-manager.kubeconfig \\
--client-ca-file=/etc/kubernetes/cert/ca.pem \\
--requestheader-allowed-names="" \\
--requestheader-client-ca-file=/etc/kubernetes/cert/ca.pem \\
--requestheader-extra-headers-prefix="X-Remote-Extra-" \\
--requestheader-group-headers=X-Remote-Group \\
--requestheader-username-headers=X-Remote-User \\
--authorization-kubeconfig=/etc/kubernetes/kube-controller-manager.kubeconfig \\
--cluster-signing-cert-file=/etc/kubernetes/cert/ca.pem \\
--cluster-signing-key-file=/etc/kubernetes/cert/ca-key.pem \\
--experimental-cluster-signing-duration=876000h \\
--horizontal-pod-autoscaler-sync-period=10s \\
--concurrent-deployment-syncs=10 \\
--concurrent-gc-syncs=30 \\
--node-cidr-mask-size=24 \\
--service-cluster-ip-range=${SERVICE_CIDR} \\
--pod-eviction-timeout=6m \\
--terminated-pod-gc-threshold=10000 \\
--root-ca-file=/etc/kubernetes/cert/ca.pem \\
--service-account-private-key-file=/etc/kubernetes/cert/ca-key.pem \\
--kubeconfig=/etc/kubernetes/kube-controller-manager.kubeconfig \\
--logtostderr=true \\
--v=2
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
說明一下:
- port=0:關閉監聽非安全端口(http),同時 –address 參數無效,–bind-address 參數有效;
- secure-port=10252、–bind-address=0.0.0.0: 在所有網絡接口監聽 10252 端口的 https /metrics 請求;
- kubeconfig:指定 kubeconfig 文件路徑,kube-controller-manager 使用它連接和驗證 kube-apiserver;
- authentication-kubeconfig 和 –authorization-kubeconfig:kube-controller-manager 使用它連接 apiserver,對 client 的請求進行認證和授權。kube-controller-manager 不再使用 –tls-ca-file 對請求 https metrics 的 Client 證書進行校驗。如果沒有配置這兩個 kubeconfig 參數,則 client 連接 kube-controller-manager https 端口的請求會被拒絕(提示權限不足)。
- cluster-signing-*-file:簽名 TLS Bootstrap 創建的證書;
- experimental-cluster-signing-duration:指定 TLS Bootstrap 證書的有效期;
- root-ca-file:放置到容器 ServiceAccount 中的 CA 證書,用來對 kube-apiserver 的證書進行校驗;
- service-account-private-key-file:簽名 ServiceAccount 中 Token 的私鑰文件,必須和 kube-apiserver 的 –service-account-key-file 指定的公鑰文件配對使用;
- service-cluster-ip-range :指定 Service Cluster IP 網段,必須和 kube-apiserver 中的同名參數一致;
- leader-elect=true:集羣運行模式,啓用選舉功能;被選爲 leader 的節點負責處理工作,其它節點爲阻塞狀態;
- controllers=*,bootstrapsigner,tokencleaner:啓用的控制器列表,tokencleaner 用於自動清理過期的 Bootstrap token;
- horizontal-pod-autoscaler-*:custom metrics 相關參數,支持 autoscaling/v2alpha1;
- tls-cert-file、–tls-private-key-file:使用 https 輸出 metrics 時使用的 Server 證書和祕鑰;
- use-service-account-credentials=true: kube-controller-manager 中各 controller 使用 serviceaccount 訪問 kube-apiserver;
d、替換啓動文件
[root@k8s-01 ~]# cd /opt/k8s/work
for (( i=0; i < 3; i++ ))
do
sed -e "s/##NODE_NAME##/${MASTER_NAMES[i]}/" -e "s/##NODE_IP##/${MASTER_IPS[i]}/" kube-controller-manager.service.template > kube-controller-manager-${MASTER_IPS[i]}.service
done
[root@k8s-01 work]# ls kube-controller-manager*.service
e、分發到所有master節點
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
scp kube-controller-manager-${node_ip}.service root@${node_ip}:/etc/systemd/system/kube-controller-manager.service
done
f、啓動服務
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
ssh root@${node_ip} "mkdir -p ${K8S_DIR}/kube-controller-manager"
ssh root@${node_ip} "systemctl daemon-reload && systemctl enable kube-controller-manager && systemctl restart kube-controller-manager"
done
g、檢查運行狀態、檢查服務狀態
[root@k8s-01 ~]# cd /opt/k8s/work
(1)檢查運行狀態
for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
ssh root@${node_ip} "systemctl status kube-controller-manager|grep Active"
done
正常圖示:
(2)檢查運行狀態
[root@k8s-01 ~]# netstat -lnpt | grep kube-cont
正常圖示:
4、kube-controller-manager 創建權限
a、ClusteRole system:kube-controller-manager的權限太小,
只能創建secret、serviceaccount等資源,將controller的權限分散到ClusterRole system:controller:xxx中
[root@k8s-01 ~]# cd /opt/k8s/work
[root@k8s-01 work]# kubectl describe clusterrole system:kube-controller-manager
正常圖示:
c、以 deployment controller 爲例:
[root@k8s-01 ~]# cd /opt/k8s/work
[root@k8s-01 work]# kubectl describe clusterrole system:controller:deployment-controller
正常圖示:
b、 查看當前的 leader
[root@k8s-01 ~]# cd /opt/k8s/work
[root@k8s-01 work]# kubectl get endpoints kube-controller-manager --namespace=kube-system -o yaml
正常圖示:
5、部署高可用kube-scheduler
a、創建 kube-scheduler 證書和私鑰
[root@k8s-01 ~]# cd /opt/k8s/work
[root@k8s-01 ~]# cat > kube-scheduler-csr.json <<EOF
添加下面內容:
{
"CN": "system:kube-scheduler",
"hosts": [
"127.0.0.1",
"172.26.16.249",
"172.26.16.250",
"172.26.16.251"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "system:kube-scheduler",
"OU": "4Paradigm"
}
]
}
EOF
說明一下:
- hosts 列表包含所有 kube-scheduler 節點 IP;
- CN 和 O 均爲 system:kube-scheduler,kubernetes 內置的 ClusterRoleBindings
system:kube-scheduler 將賦予 kube-scheduler 工作所需的權限;
b、生成證書和私鑰
[root@k8s-01 ~]# cd /opt/k8s/work
cfssl gencert -ca=/opt/k8s/work/ca.pem \
-ca-key=/opt/k8s/work/ca-key.pem \
-config=/opt/k8s/work/ca-config.json \
-profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler
[root@k8s-01 ~]# ls kube-scheduler*pem
c、將生成的證書和私鑰分發到所有 master 節點
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
scp kube-scheduler*.pem root@${node_ip}:/etc/kubernetes/cert/
done
d、創建和分發 kubeconfig 文件
(1)創建
[root@k8s-01 ~]# cd /opt/k8s/work
kubectl config set-cluster kubernetes \
--certificate-authority=/opt/k8s/work/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kube-scheduler.kubeconfig
kubectl config set-credentials system:kube-scheduler \
--client-certificate=kube-scheduler.pem \
--client-key=kube-scheduler-key.pem \
--embed-certs=true \
--kubeconfig=kube-scheduler.kubeconfig
kubectl config set-context system:kube-scheduler \
--cluster=kubernetes \
--user=system:kube-scheduler \
--kubeconfig=kube-scheduler.kubeconfig
kubectl config use-context system:kube-scheduler --kubeconfig=kube-scheduler.kubeconfig
(2)分發
for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
scp kube-scheduler.kubeconfig root@${node_ip}:/etc/kubernetes/
done
e、創建 kube-scheduler 配置文件
[root@k8s-01 ~]# cd /opt/k8s/work
[root@k8s-01 work]# cat >kube-scheduler.yaml.template <<EOF
添加下面內容:
apiVersion: kubescheduler.config.k8s.io/v1alpha1
kind: KubeSchedulerConfiguration
bindTimeoutSeconds: 600
clientConnection:
burst: 200
kubeconfig: "/etc/kubernetes/kube-scheduler.kubeconfig"
qps: 100
enableContentionProfiling: false
enableProfiling: true
hardPodAffinitySymmetricWeight: 1
healthzBindAddress: 127.0.0.1:10251
leaderElection:
leaderElect: true
metricsBindAddress: ##NODE_IP##:10251
EOF
說明一下:
- kubeconfig:指定 kubeconfig 文件路徑,kube-scheduler 使用它連接和驗證 kube-apiserver;
- leader-elect=true:集羣運行模式,啓用選舉功能;被選爲 leader 的節點負責處理工作,其它節點爲阻塞狀態;
f、替換模板文件中的變量
[root@k8s-01 ~]# cd /opt/k8s/work
for (( i=0; i < 3; i++ ))
do
sed -e "s/##NODE_NAME##/${NODE_NAMES[i]}/" -e "s/##NODE_IP##/${NODE_IPS[i]}/" kube-scheduler.yaml.template > kube-scheduler-${NODE_IPS[i]}.yaml
done
[root@k8s-01 ~]# ls kube-scheduler*.yaml
g、分發 kube-scheduler 配置文件到所有 master 節點
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
scp kube-scheduler-${node_ip}.yaml root@${node_ip}:/etc/kubernetes/kube-scheduler.yaml
done
h、創建kube-scheduler啓動文件
[root@k8s-01 ~]# cd /opt/k8s/work
[root@k8s-01 ~]# cat > kube-scheduler.service.template <<EOF
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
[Service]
WorkingDirectory=${K8S_DIR}/kube-scheduler
ExecStart=/opt/k8s/bin/kube-scheduler \\
--config=/etc/kubernetes/kube-scheduler.yaml \\
--bind-address=##NODE_IP## \\
--secure-port=10259 \\
--port=0 \\
--tls-cert-file=/etc/kubernetes/cert/kube-scheduler.pem \\
--tls-private-key-file=/etc/kubernetes/cert/kube-scheduler-key.pem \\
--authentication-kubeconfig=/etc/kubernetes/kube-scheduler.kubeconfig \\
--client-ca-file=/etc/kubernetes/cert/ca.pem \\
--requestheader-allowed-names="" \\
--requestheader-client-ca-file=/etc/kubernetes/cert/ca.pem \\
--requestheader-extra-headers-prefix="X-Remote-Extra-" \\
--requestheader-group-headers=X-Remote-Group \\
--requestheader-username-headers=X-Remote-User \\
--authorization-kubeconfig=/etc/kubernetes/kube-scheduler.kubeconfig \\
--logtostderr=true \\
--v=2
Restart=always
RestartSec=5
StartLimitInterval=0
[Install]
WantedBy=multi-user.target
EOF
i、分發配置文件
[root@k8s-01 ~]# cd /opt/k8s/work
for (( i=0; i < 3; i++ ))
do
sed -e "s/##NODE_NAME##/${NODE_NAMES[i]}/" -e "s/##NODE_IP##/${NODE_IPS[i]}/" kube-scheduler.service.template > kube-scheduler-${NODE_IPS[i]}.service
done
[root@k8s-01 ~]# ls kube-scheduler*.service
for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
scp kube-scheduler-${node_ip}.service root@${node_ip}:/etc/systemd/system/kube-scheduler.service
done
j、啓動kube-scheduler
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
ssh root@${node_ip} "mkdir -p ${K8S_DIR}/kube-scheduler"
ssh root@${node_ip} "systemctl daemon-reload && systemctl enable kube-scheduler && systemctl restart kube-scheduler"
done
k、檢查服務運行狀態
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
ssh root@${node_ip} "systemctl status kube-scheduler|grep Active"
done
正常圖示:
l、查看輸出的 metrics
- 注意:以下命令在 kube-scheduler 節點上執行
- kube-scheduler 監聽 10251 和 10251 端口:
- 10251:接收 http 請求,非安全端口,不需要認證授權;
- 10259:接收 https 請求,安全端口,需要認證授權;
- 兩個接口都對外提供 /metrics 和 /healthz 的訪問。
[root@k8s-01 ~]# cd /opt/k8s/work
[root@k8s-01 work]# curl -s http://172.26.16.249:10251/metrics|head
正常圖示:
e、查看當前leader
[root@k8s-01 ~]# cd /opt/k8s/work
[root@k8s-01 work~]# kubectl get endpoints kube-scheduler --namespace=kube-system -o yaml
正常圖示: