企業級K8S集羣二進制環境部署 安裝cfssl證書生成工具

集羣架構圖:

集羣架構信息

 

 

【關閉所有節點swap分區】

# swapoff -a && sysctl -w vm.swappiness=0

# sed -i '/swap/s/^/&#/' /etc/fstab

【yum環境配置】

curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo
yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

【集羣節點時間同步】

# rpm -ivh http://mirrors.wlnmp.com/centos/wlnmp-release-centos.noarch.rpm
# yum install -y ntpdate
#echo 'Asia/Shanghai' > /etc/timezone
#ntpdate time2.aliyun.com

設置時間定期執行任務

# crontab -l
*/1 * * * * ntpdate time2.aliyun.com

【設置ssh免密】

# ssh-keygen -t rsa

# for i in k8s-master01 k8s-master02 k8s-master03 k8s-node01 k8s-node02;do ssh-copy-id -i .ssh/id_rsa.pub $i ;done

#Master01下載安裝文件

#鏈接: https://pan.baidu.com/s/1zdh46AnHrk4NabaPClwn8A  密碼: 1u7n  #從網盤中下載證書以及k8s集羣所依賴的yaml文件

 #【部署ipvsadmin

yum install ipvsadm ipset sysstat conntrack libseccomp -y

# cat /etc/modules-load.d/ipvs.conf

ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip

#systemctl enable --now systemd-modules-load.service

#所有節點配置內核參數

cat <<EOF > /etc/sysctl.d/k8s.conf

net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720

net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384

EOF

#systctl --system

#reboot

#lsmod | grep --color=auto -e ip_vs -e nf_conntrack

【docker部署】

#yum install -y docker-ce-19.03.*

#systemctl start docker

#mkdir -p /etc/docker

#cat > /etc/docker/daemon.json <<EOF

{
"registry-mirrors": [
"https://registry.docker-cn.com",
"http://hub-mirror.c.163.com",
"https://docker.mirrors.ustc.edu.cn"
],
"exec-opts": ["native.cgroupdriver=systemd"]
}

#EOF

#systemctl daemon-reload && systemctl enable --now docker

【kubernetes組件以及etcd部署】

#systemctl daemon-reload && systemctl enable --now dockerwget

[[email protected] ~]#wget  https://github.com/etcd-io/etcd/releases/download/v3.4.12/etcd-v3.4.12-linux-amd64.tar.gz

[[email protected] ~]#tar -zxvf etcd-v3.4.12-linux-amd64.tar.gz --strip-components=1 -C /usr/local/bin etcd-v3.4.12-linux-amd64/etcd{,ctl}  #將etcd、etcdctl二進制文件傳送到/usr/local/bin目錄中

[[email protected] ~]#wget https://dl.k8s.io/v1.20.0/kubernetes-server-linux-amd64.tar.gz

#將kubernetes二進制文件(kubelet,kubectl,kube-apiserver,kube-controller-manager,kube-scheduler,kube-proxy)解壓到指定的/usr/local/bin目錄中;這裏的--strip-components=3表示去除3級目錄結構,只將需要的二進制文件拷貝到/usr/local/bin/目錄中

[[email protected] ~]#tar -xf kubernetes-server-linux-amd64.tar.gz  --strip-components=3 -C /usr/local/bin kubernetes/server/bin/kube{let,ctl,-apiserver,-controller-manager,-scheduler,-proxy}  

#定義環境變量,將master01節點上解壓出來的二進制文件拷貝到另外兩個master節點

[[email protected] ~]#MasterNodes='k8s-master02 k8s-master03'
[[email protected] ~]#for NODE in $MasterNodes; do echo $NODE; scp /usr/local/bin/kube{let,ctl,-apiserver,-controller-manager,-scheduler,-proxy} $NODE:/usr/local/bin/; scp /usr/local/bin/etcd* $NODE:/usr/local/bin/; done

#而對於node節點,只需要將master01節點生成的kubelet和kubelet-proxy組件拷貝到node節點即可

[[email protected] ~]#WorkNodes='k8s-node01 k8s-node02'
[[email protected] ~]#for NODE in $WorkNodes;do scp /usr/local/bin/kube{let,-proxy} $NODE:/usr/local/bin/ ; done

【生成證書】

所有節點創建/opt/cni/bin目錄

#mkdir -p /opt/cni/bin

安裝cfssl證書生成工具

#wget "https://pkg.cfssl.org/R1.2/cfssl_linux-amd64" -O /usr/local/bin/cfssl
#wget "https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64" -O /usr/local/bin/cfssljson
#chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson

在所有master節點創建etcd證書

#mkdir /etc/etcd/ssl -p

在kubernetes集羣所有節點創建kubernetes相關pki證書目錄

#mkdir -p /etc/kubernetes/kpi

Ps: https://pan.baidu.com/s/1zdh46AnHrk4NabaPClwn8A  密碼: 1u7n  #從網盤中下載的k8s-ha-install.tar.gz 壓縮文件

[[email protected] ~]# tar xvf k8s-ha-install.tar.gz --strip-components=1
[[email protected]-master01 ~]# cd k8s-ha-install/
[[email protected]-master01 k8s-ha-install]# ls /root/k8s-ha-install/pki/
admin-csr.json      ca-config.json  etcd-ca-csr.json  front-proxy-ca-csr.json      kubelet-csr.json     manager-csr.json
apiserver-csr.json  ca-csr.json     etcd-csr.json     front-proxy-client-csr.json  kube-proxy-csr.json  scheduler-csr.json

#在Master01節點上生成etcd證書,生成證書的CSR文件,主要包含證書籤名請求文件,配置了一些域名,公司,單位等信息;

[[email protected] pki]# cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare /etc/etcd/ssl/etcd-ca

[[email protected] pki]# ls /etc/etcd/ssl/
etcd-ca.csr etcd-ca-key.pem etcd-ca.pem

[[email protected] pki]# cfssl gencert \
 -ca=/etc/etcd/ssl/etcd-ca.pem \
 -ca-key=/etc/etcd/ssl/etcd-ca-key.pem \
 -config=ca-config.json \
 -hostname=127.0.0.1,k8s-master01,k8s-master02,k8s-master03,192.168.60.101,192.168.60.102,192.168.60.103 \
 -profile=kubernetes \
 etcd-csr.json | cfssljson -bare /etc/etcd/ssl/etcd

#將etcd相關的證書文件複製到其他Master節點

#MasterNodes='k8s-master02 k8s-master03'

#WorkNodes='k8s-node01 k8s-node02'

[[email protected] pki]# for NODE in $MasterNodes;do ssh $NODE "mkdir -p /etc/etcd/ssl";for FILE in etcd-ca-key.pem  etcd-ca.pem  etcd-key.pem  etcd.pem; do scp /etc/etcd/ssl/${FILE} $NODE:/etc/etcd/ssl/${FILE};done; done

【k8s組件證書】

#Master01生成kubernetes證書;

[[email protected] ~]# cd /root/k8s-ha-install/pki/

[[email protected] pki]# cfssl gencert -initca ca-csr.json | cfssljson -bare /etc/kubernetes/pki/ca
#10.96.0.1是k8s-service的網段,如果需要更改k8sservice網段,那麼久需要更改10.96.0.1,如果不是高可用集羣,那麼192.168.60.236爲master01的vip

[[email protected] pki]#cfssl gencert   -ca=/etc/kubernetes/pki/ca.pem   -ca-key=/etc/kubernetes/pki/ca-key.pem   -config=ca-config.json   -hostname=10.96.0.1,192.168.60.236,127.0.0.1,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster,kubernetes.default.svc.cluster.local,192.168.60.101,192.168.60.102,192.168.0.103   -profile=kubernetes   apiserver-csr.json | cfssljson -bare /etc/kubernetes/pki/apiserver

#生成apiserver的聚合證書;

[[email protected] pki]# cfssl gencert   -initca front-proxy-ca-csr.json | cfssljson -bare /etc/kubernetes/pki/front-proxy-ca

[[email protected] pki]# cfssl gencert   -ca=/etc/kubernetes/pki/front-proxy-ca.pem   -ca-key=/etc/kubernetes/pki/front-proxy-ca-key.pem   -config=ca-config.json   -profile=kubernetes   front-proxy-client-csr.json | cfssljson -bare /etc/kubernetes/pki/front-proxy-client

#生成controller-manage客戶端證書

[[email protected] pki]# cfssl gencert    -ca=/etc/kubernetes/pki/ca.pem    -ca-key=/etc/kubernetes/pki/ca-key.pem    -config=ca-config.json    -profile=kubernetes    manager-csr.json | cfssljson -bare /etc/kubernetes/pki/controller-manager

#設置一個集羣項,主要用於配置多個集羣

[[email protected]-master01 pki]# kubectl config set-cluster kubernetes \
> --certificate-authority=/etc/kubernetes/pki/ca.pem \
> --embed-certs=true \
> --server=https://192.168.60.236:8443 \
> --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig
Cluster "kubernetes" set.

#設置一個用戶項

[[email protected]-master01 pki]# kubectl config set-credentials system:kube-controller-manager \
> --client-certificate=/etc/kubernetes/pki/controller-manager.pem \
> --client-key=/etc/kubernetes/pki/controller-manager-key.pem \
> --embed-certs=true \
> --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig
User "system:kube-controller-manager" set.

#設置一個環境項,一個上下文,通過kube-controller-manager用戶名連接kubernetes集羣

[[email protected]-master01 pki]# kubectl config set-context system:kube-controller-[email protected] \
> --cluster=kubernetes \
> --user=system:kube-controller-manager \
> --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig
Context "system:[email protected]" created.

#使用某個環境當做默認的環境

[[email protected]-master01 pki]# kubectl config use-context system:kube-controller-[email protected] \
> --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig
Switched to context "system:[email protected]".

 #生成scheduler證書文件

[email protected] pki]# cfssl gencert \
> -ca=/etc/kubernetes/pki/ca.pem \
> -ca-key=/etc/kubernetes/pki/ca-key.pem \
> -config=ca-config.json \
> -profile=kubernetes \
> scheduler-csr.json | cfssljson -bare /etc/kubernetes/pki/scheduler
2020/12/21 14:59:32 [INFO] generate received request
2020/12/21 14:59:32 [INFO] received CSR
2020/12/21 14:59:32 [INFO] generating key: rsa-2048
2020/12/21 14:59:32 [INFO] encoded CSR
2020/12/21 14:59:32 [INFO] signed certificate with serial number 212482711725599124851040824209737958939315938395
2020/12/21 14:59:32 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
[[email protected] pki]# kubectl config set-cluster kubernetes \ --certificate-authority=/etc/kubernetes/pki/ca.pem \ --embed-certs=true \ --server=https://192.168.60.236:8443 \ --kubeconfig=/etc/kubernetes/scheduler.kubeconfig Cluster "kubernetes" set. [[email protected]-master01 pki]# kubectl config set-credentials system:kube-scheduler \ --client-certificate=/etc/kubernetes/pki/scheduler.pem \ --client-key=/etc/kubernetes/pki/scheduler-key.pem \ --embed-certs=true \ --kubeconfig=/etc/kubernetes/scheduler.kubeconfig User "system:kube-scheduler" set. [[email protected]-master01 pki]# kubectl config set-context system:kube-[email protected] \ --cluster=kubernetes \ --user=system:kube-scheduler \ --kubeconfig=/etc/kubernetes/scheduler.kubeconfig Context "system:[email protected]" created. [[email protected]-master01 pki]# kubectl config use-context system:kube-[email protected] \ --kubeconfig=/etc/kubernetes/scheduler.kubeconfig Switched to context "system:[email protected]".
#生成admin證書,主要用於管理kubernetes集羣 [[email protected]
-master01 pki]# cfssl gencert \ -ca=/etc/kubernetes/pki/ca.pem \ -ca-key=/etc/kubernetes/pki/ca-key.pem \ -config=ca-config.json \ -profile=kubernetes \ admin-csr.json | cfssljson -bare /etc/kubernetes/pki/admin 2020/12/21 15:03:35 [INFO] generate received request 2020/12/21 15:03:35 [INFO] received CSR 2020/12/21 15:03:35 [INFO] generating key: rsa-2048 2020/12/21 15:03:35 [INFO] encoded CSR 2020/12/21 15:03:35 [INFO] signed certificate with serial number 335202371700726560784659537380501229533603198548 2020/12/21 15:03:35 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for websites. For more information see the Baseline Requirements for the Issuance and Management of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org); specifically, section 10.2.3 ("Information Requirements").

[[email protected] pki]# kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --server=https://192.168.60.236:8443 --kubeconfig=/etc/kubernetes/admin.kubeconfig
Cluster "kubernetes" set.
[[email protected] pki]# kubectl config set-credentials kubernetes-admin --client-certificate=/etc/kubernetes/pki/admin.pem --client-key=/etc/kubernetes/pki/admin-key.pem --embed-certs=true --kubeconfig=/etc/kubernetes/admin.kubeconfig
User "kubernetes-admin" set.
[[email protected] pki]# kubectl config set-context [email protected] --cluster=kubernetes --user=kubernetes-admin --kubeconfig=/etc/kubernetes/admin.kubeconfig
Context "[email protected]" created.
[[email protected] pki]# kubectl config use-context [email protected] --kubeconfig=/etc/kubernetes/admin.kubeconfig
Switched to context "[email protected]".

#創建ServiceAccount key會生成一個與之綁定的secret,那麼secret會產生一個token;

[[email protected] pki]# openssl genrsa -out /etc/kubernetes/pki/sa.key 2048
Generating RSA private key, 2048 bit long modulus
.................................................................................................................................+++
......................................+++
e is 65537 (0x10001)

[[email protected] pki]# openssl rsa -in /etc/kubernetes/pki/sa.key -pubout -out /etc/kubernetes/pki/sa.pub
writing RSA key

#將生成的kubernetes的證書發送到其他master節點上

[[email protected] pki]# for NODE in k8s-master02 k8s-master03; do for FILE in $(ls /etc/kubernetes/pki/ | grep -v etcd); do scp /etc/kubernetes/pki/${FILE} $NODE:/etc/kubernetes/pki/${FILE}; done; for FILE in admin.kubeconfig controller-manager.kubeconfig scheduler.kubeconfig; do scp /etc/kubernetes/${FILE} $NODE:/etc/kubernetes/${FILE}; done; done

【Kubernetes系統組件配置】

#【etcd配置】

 k8s-master01配置

[[email protected] ~]# vim /etc/etcd/etcd.config.yml

name: 'k8s-master01'
data-dir: /var/lib/etcd
wal-dir: /var/lib/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://192.168.60.101:2380'
listen-client-urls: 'https://192.168.60.101:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://192.168.60.101:2380'
advertise-client-urls: 'https://192.168.60.101:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-master01=https://192.168.60.101:2380,k8s-master02=https://192.168.60.102:2380,k8s-master03=https://192.168.60.103:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
  cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
  key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
  client-cert-auth: true
  trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
  auto-tls: true
peer-transport-security:
  cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
  key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
  peer-client-cert-auth: true
  trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
  auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false

 #在master01節點創建etcd service文件,用於設置開機自啓

[[email protected] ~]# vim /usr/lib/systemd/system/etcd.service

[Unit]
Description=Etcd Service
Documentation=https://coreos.com/etcd/docs/latest/
After=network.target

[Service]
Type=notify
ExecStart=/usr/local/bin/etcd --config-file=/etc/etcd/etcd.config.yml
Restart=on-failure
RestartSec=10
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
Alias=etcd1.service

[[email protected] ~]#mkdir /etc/kubernetes/pki/etcd

[[email protected] ~]#ln -s /etc/etcd/ssl/* /etc/kubernetes/pki/etcd/

[[email protected] ~]#systemctl daemon-reload
[[email protected] ~]#systemctl enable --now etcd

#k8s-master02配置

[[email protected] ~]# vim /etc/etcd/etcd.config.yml

name: 'k8s-master02'
data-dir: /var/lib/etcd
wal-dir: /var/lib/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://192.168.60.102:2380'
listen-client-urls: 'https://192.168.60.102:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://192.168.60.102:2380'
advertise-client-urls: 'https://192.168.60.102:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-master01=https://192.168.60.101:2380,k8s-master02=https://192.168.60.102:2380,k8s-master03=https://192.168.60.103:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
  cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
  key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
  client-cert-auth: true
  trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
  auto-tls: true
peer-transport-security:
  cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
  key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
  peer-client-cert-auth: true
  trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
  auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false

[[email protected] ~]# vim /usr/lib/systemd/system/etcd.service

[Unit]
Description=Etcd Service
Documentation=https://coreos.com/etcd/docs/latest/
After=network.target

[Service]
Type=notify
ExecStart=/usr/local/bin/etcd --config-file=/etc/etcd/etcd.config.yml
Restart=on-failure
RestartSec=10
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
Alias=etcd2.service

[[email protected] ~]#mkdir /etc/kubernetes/pki/etcd

[[email protected] ~]#ln -s /etc/etcd/ssl/* /etc/kubernetes/pki/etcd/

[[email protected] ~]#systemctl daemon-reload
[[email protected] ~]#systemctl enable --now etcd

#k8s-master03

[[email protected] ~]# vim /etc/etcd/etcd.config.yml   #修改配置文件

name: 'k8s-master03'
data-dir: /var/lib/etcd
wal-dir: /var/lib/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://192.168.60.103:2380'
listen-client-urls: 'https://192.168.60.103:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://192.168.60.103:2380'
advertise-client-urls: 'https://192.168.60.103:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-master01=https://192.168.60.101:2380,k8s-master02=https://192.168.60.102:2380,k8s-master03=https://192.168.60.103:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
  cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
  key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
  client-cert-auth: true
  trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
  auto-tls: true
peer-transport-security:
  cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
  key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
  peer-client-cert-auth: true
  trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
  auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false

[[email protected] ~]# vim /usr/lib/systemd/system/etcd.service

[Unit]
Description=Etcd Service
Documentation=https://coreos.com/etcd/docs/latest/
After=network.target

[Service]
Type=notify
ExecStart=/usr/local/bin/etcd --config-file=/etc/etcd/etcd.config.yml
Restart=on-failure
RestartSec=10
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
Alias=etcd3.service

[[email protected] ~]#mkdir /etc/kubernetes/pki/etcd

[[email protected] ~]#ln -s /etc/etcd/ssl/* /etc/kubernetes/pki/etcd/

[[email protected] ~]#systemctl daemon-reload
[[email protected] ~]#systemctl enable --now etcd

#驗證etcd集羣是否正常運行

[[email protected] ~]# export ETCDCTL_API=3   #切換etcd API爲3版本
[[email protected] ~]# etcdctl --endpoints="192.168.60.103:2379,192.168.60.102:2379,192.168.60.101:2379" --cacert=/etc/kubernetes/pki/etcd/etcd-ca.pem --cert=/etc/kubernetes/pki/etcd/etcd.pem --key=/etc/kubernetes/pki/etcd/etcd-key.pem endpoint status --write-out=table

 

 etcd集羣成功運行,並且192.168.60.103爲etcd集羣的leader節點;

【高可用集羣配置】

主要爲三臺k8s-master節點配置一個高可用,採用haproxy+keepalived形式,實現高可用架構 

Ps:所有Master節點部署keepalived和haproxy軟件

#yum install -y keepalived haproxy

# vim /etc/haproxy/haproxy.cfg

global
  maxconn  2000
  ulimit-n  16384
  log  127.0.0.1 local0 err
  stats timeout 30s

defaults
  log global
  mode  http
  option  httplog
  timeout connect 5000
  timeout client  50000
  timeout server  50000
  timeout http-request 15s
  timeout http-keep-alive 15s

frontend k8s-master
  bind 0.0.0.0:8443
  bind 127.0.0.1:8443
  mode tcp
  option tcplog
  tcp-request inspect-delay 5s
  default_backend k8s-master

backend k8s-master
  mode tcp
  option tcplog
  option tcp-check
  balance roundrobin
  default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
  server k8s-master01    192.168.60.101:6443  check
  server k8s-master02    192.168.60.102:6443  check
  server k8s-master03    192.168.60.103:6443  check

# systemctl enable --now haproxy  #爲每個master節點上的haproxy服務設置開機自啓動

#修改三臺k8s-master節點keepliaved配置文件

[[email protected] ~]# vim /etc/keepalived/keepalived.conf

 Configuration File for keepalived
global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5
    weight -5
    fall 2
    rise 1
}
vrrp_instance VI_1 {
    state MASTER
    interface eth0
    mcast_src_ip 192.168.60.101
    virtual_router_id 51
    priority 101
    nopreempt
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {
        192.168.60.236
    }
    track_script {
      chk_apiserver
} }

 [[email protected] ~]# vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived
global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5
    weight -5
    fall 2
    rise 1

}
vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    mcast_src_ip 192.168.60.102
    virtual_router_id 51
    priority 100
    nopreempt
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {
        192.168.60.236
    }
    track_script {
      chk_apiserver
} }

 [[email protected] ~]# vim  /etc/keepalived/keepalived.conf 

! Configuration File for keepalived
global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5
    weight -5
    fall 2
    rise 1
}
vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    mcast_src_ip 192.168.60.103
    virtual_router_id 51
    priority 100
    nopreempt
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {
        192.168.60.236
    }
    track_script {
      chk_apiserver
} }

 Ps:每個k8s-master節點編寫健康檢查腳本; 

# vim /etc/keepalived/check_apiserver.sh

#!/bin/bash

err=0
for k in $(seq 1 3)
do
    check_code=$(pgrep haproxy)
    if [[ $check_code == "" ]]; then
        err=$(expr $err + 1)
        sleep 1
        continue
    else
        err=0
        break
    fi
done

if [[ $err != "0" ]]; then
    echo "systemctl stop keepalived"
    /usr/bin/systemctl stop keepalived
    exit 1
else
    exit 0
fi

 #chmod +x /etc/keepalived/check_apiserver.sh

 #systemctl enable --now keepalived

驗證高可用集羣是否正常,在任意節點通過ping或者telnet進行測試即可如

 

【kubernetes組件】

#mkdir -p /etc/kubernetes/manifests/ /etc/systemd/system/kubelet.service.d /var/lib/kubelet /var/log/kubernetes    #爲所有節點創建目錄,用於後續環境部署

部署kubernetes-apiserver組件】

#Master01配置

[[email protected] ~]# vim   /usr/lib/systemd/system/kube-apiserver.service

[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-apiserver \
      --v=2  \
      --logtostderr=true  \
      --allow-privileged=true  \
      --bind-address=0.0.0.0  \
      --secure-port=6443  \
      --insecure-port=0  \
      --advertise-address=192.168.60.101 \
      --service-cluster-ip-range=10.96.0.0/12  \
      --service-node-port-range=30000-32767  \
      --etcd-servers=https://192.168.60.101:2379,https://192.168.60.102:2379,https://192.168.60.103:2379 \
      --etcd-cafile=/etc/etcd/ssl/etcd-ca.pem  \
      --etcd-certfile=/etc/etcd/ssl/etcd.pem  \
      --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem  \
      --client-ca-file=/etc/kubernetes/pki/ca.pem  \
      --tls-cert-file=/etc/kubernetes/pki/apiserver.pem  \
      --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem  \
      --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem  \
      --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem  \
      --service-account-key-file=/etc/kubernetes/pki/sa.pub  \
      --service-account-signing-key-file=/etc/kubernetes/pki/sa.key  \
      --service-account-issuer=https://kubernetes.default.svc.cluster.local \
      --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname  \
      --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota  \
      --authorization-mode=Node,RBAC  \
      --enable-bootstrap-token-auth=true  \
      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem  \
      --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem  \
      --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem  \
      --requestheader-allowed-names=aggregator  \
      --requestheader-group-headers=X-Remote-Group  \
      --requestheader-extra-headers-prefix=X-Remote-Extra-  \
      --requestheader-username-headers=X-Remote-User
      # --token-auth-file=/etc/kubernetes/token.csv

Restart=on-failure
RestartSec=10s
LimitNOFILE=65535

[Install]
WantedBy=multi-user.target

[[email protected] ~]# systemctl enable --now kube-apiserver

#Master02

[[email protected] ~]# vim /usr/lib/systemd/system/kube-apiserver.service

[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-apiserver \
      --v=2  \
      --logtostderr=true  \
      --allow-privileged=true  \
      --bind-address=0.0.0.0  \
      --secure-port=6443  \
      --insecure-port=0  \
      --advertise-address=192.168.60.102 \
      --service-cluster-ip-range=10.96.0.0/12  \
      --service-node-port-range=30000-32767  \
      --etcd-servers=https://192.168.60.101:2379,https://192.168.60.102:2379,https://192.168.60.103:2379 \
      --etcd-cafile=/etc/etcd/ssl/etcd-ca.pem  \
      --etcd-certfile=/etc/etcd/ssl/etcd.pem  \
      --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem  \
      --client-ca-file=/etc/kubernetes/pki/ca.pem  \
      --tls-cert-file=/etc/kubernetes/pki/apiserver.pem  \
      --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem  \
      --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem  \
      --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem  \
      --service-account-key-file=/etc/kubernetes/pki/sa.pub  \
      --service-account-signing-key-file=/etc/kubernetes/pki/sa.key  \
      --service-account-issuer=https://kubernetes.default.svc.cluster.local \
      --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname  \
      --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota  \
      --authorization-mode=Node,RBAC  \
      --enable-bootstrap-token-auth=true  \
      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem  \
      --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem  \
      --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem  \
      --requestheader-allowed-names=aggregator  \
      --requestheader-group-headers=X-Remote-Group  \
      --requestheader-extra-headers-prefix=X-Remote-Extra-  \
      --requestheader-username-headers=X-Remote-User
      # --token-auth-file=/etc/kubernetes/token.csv

Restart=on-failure
RestartSec=10s
LimitNOFILE=65535

[Install]
WantedBy=multi-user.target

[[email protected] ~]# systemctl enable --now kube-apiserver

#Master03

[[email protected] ~]# vim /usr/lib/systemd/system/kube-apiserver.service

[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-apiserver \
      --v=2  \
      --logtostderr=true  \
      --allow-privileged=true  \
      --bind-address=0.0.0.0  \
      --secure-port=6443  \
      --insecure-port=0  \
      --advertise-address=192.168.60.103 \
      --service-cluster-ip-range=10.96.0.0/12  \
      --service-node-port-range=30000-32767  \
      --etcd-servers=https://192.168.60.101:2379,https://192.168.60.102:2379,https://192.168.60.103:2379 \
      --etcd-cafile=/etc/etcd/ssl/etcd-ca.pem  \
      --etcd-certfile=/etc/etcd/ssl/etcd.pem  \
      --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem  \
      --client-ca-file=/etc/kubernetes/pki/ca.pem  \
      --tls-cert-file=/etc/kubernetes/pki/apiserver.pem  \
      --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem  \
      --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem  \
      --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem  \
      --service-account-key-file=/etc/kubernetes/pki/sa.pub  \
      --service-account-signing-key-file=/etc/kubernetes/pki/sa.key  \
      --service-account-issuer=https://kubernetes.default.svc.cluster.local \
      --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname  \
      --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota  \
      --authorization-mode=Node,RBAC  \
      --enable-bootstrap-token-auth=true  \
      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem  \
      --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem  \
      --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem  \
      --requestheader-allowed-names=aggregator  \
      --requestheader-group-headers=X-Remote-Group  \
      --requestheader-extra-headers-prefix=X-Remote-Extra-  \
      --requestheader-username-headers=X-Remote-User
      # --token-auth-file=/etc/kubernetes/token.csv

Restart=on-failure
RestartSec=10s
LimitNOFILE=65535

[Install]
WantedBy=multi-user.target

[[email protected] ~]# systemctl enable --now kube-apiserver

【kube-controller-manager】

所有master節點配置kube-controller-manager-service

#vim /usr/lib/systemmd/system/kube-controller-manager.service

[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-controller-manager \
      --v=2 \
      --logtostderr=true \
      --address=127.0.0.1 \
      --root-ca-file=/etc/kubernetes/pki/ca.pem \
      --cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem \
      --cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem \
      --service-account-private-key-file=/etc/kubernetes/pki/sa.key \
      --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig \
      --leader-elect=true \
      --use-service-account-credentials=true \
      --node-monitor-grace-period=40s \
      --node-monitor-period=5s \
      --pod-eviction-timeout=2m0s \
      --controllers=*,bootstrapsigner,tokencleaner \
      --allocate-node-cidrs=true \
      --cluster-cidr=172.16.0.0/12 \
      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \
      --node-cidr-mask-size=24

Restart=always
RestartSec=10s

[Install]
WantedBy=multi-user.target

#systemctl daemon-reload 

#systemctl enable --now kube-controller-manager

【部署Scheduler組件】

 #vim  /usr/lib/systemd/system/kube-scheduler.service

[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-scheduler \
      --v=2 \
      --logtostderr=true \
      --address=127.0.0.1 \
      --leader-elect=true \
      --kubeconfig=/etc/kubernetes/scheduler.kubeconfig

Restart=always
RestartSec=10s

[Install]
WantedBy=multi-user.target

# systemctl daemon-reload

# systemctl enable --now kube-scheduler

【TSI Bootstrapping配置】

在kuberlete集羣環境中,node節點的組件kubelet和kybe-proxy需要與Master端(kube-apiserver)進行通信,爲了確保通信私密性其不受干擾,並確保集羣的每個組件都在與另一個受信任的組件通信,在這裏使用客戶端TLS證書

只需要在Master01 創建bootstrap配置文件,爲node節點生成TLS證書文件

[[email protected] ~]# cd   /root/k8s-ha-install/bootstrap/

[roo[email protected] bootstrap]# kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --server=https://192.168.60.236:8443 --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig
Cluster "kubernetes" set.
[[email protected] bootstrap]# kubectl config set-credentials tls-bootstrap-token-user --token=c8ad9c.2e4d610cf3e7426e --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig
User "tls-bootstrap-token-user" set.
[[email protected] bootstrap]# kubectl config set-context [email protected] --cluster=kubernetes --user=tls-bootstrap-token-user --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig
Context "[email protected]" modified.
[[email protected] bootstrap]# kubectl config use-context [email protected] --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig
Switched to context "[email protected]".

[[email protected] ~]# mkdir -p /root/.kube ; cp /etc/kubernetes/admin.kubeconfig /root/.kube/config

[[email protected] bootstrap]# kubectl create -f bootstrap.secret.yaml   #創建一個secret的yaml文件

【Node節點配置】

複製證書,將在kubernetes的生成的證書複製到其他master節點和node節點上

[[email protected] ~]# cd /etc/kubernetes/

[[email protected] kubernetes]# for NODE in k8s-master02 k8s-master03 k8s-node01 k8s-node02; do      ssh $NODE mkdir -p /etc/kubernetes/pki /etc/etcd/ssl /etc/etcd/ssl;      for FILE in etcd-ca.pem etcd.pem etcd-key.pem; do        scp /etc/etcd/ssl/$FILE $NODE:/etc/etcd/ssl/;      done;      for FILE in pki/ca.pem pki/ca-key.pem pki/front-proxy-ca.pem bootstrap-kubelet.kubeconfig; do        scp /etc/kubernetes/$FILE $NODE:/etc/kubernetes/${FILE};  done;  done

kubelet.service服務配置

#vim  /usr/lib/systemd/system/kubelet.servic

[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/kubernetes/kubernetes
After=docker.service
Requires=docker.service

[Service]
ExecStart=/usr/local/bin/kubelet

Restart=always
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target

# vim /etc/systemd/system/kubelet.service.d/10-kubelet.conf

[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/kubernetes/kubernetes
After=docker.service
Requires=docker.service

[Service]
ExecStart=/usr/local/bin/kubelet

Restart=always
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target
[[email protected]-node01 ~]# cat  /etc/systemd/system/kubelet.service.d/10-kubelet.conf
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig --kubeconfig=/etc/kubernetes/kubelet.kubeconfig"
Environment="KUBELET_SYSTEM_ARGS=--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
Environment="KUBELET_CONFIG_ARGS=--config=/etc/kubernetes/kubelet-conf.yml --pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google_containers/pause-amd64:3.2"
Environment="KUBELET_EXTRA_ARGS=--node-labels=node.kubernetes.io/node='' "
ExecStart=
ExecStart=/usr/local/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_SYSTEM_ARGS $KUBELET_EXTRA_ARGS

 # vim /etc/kubernetes/kubelet-conf.yml

apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
address: 0.0.0.0
port: 10250
readOnlyPort: 10255
authentication:
  anonymous:
    enabled: false
  webhook:
    cacheTTL: 2m0s
    enabled: true
  x509:
    clientCAFile: /etc/kubernetes/pki/ca.pem
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 5m0s
    cacheUnauthorizedTTL: 30s
cgroupDriver: systemd
cgroupsPerQOS: true
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local
containerLogMaxFiles: 5
containerLogMaxSize: 10Mi
contentType: application/vnd.kubernetes.protobuf
cpuCFSQuota: true
cpuManagerPolicy: none
cpuManagerReconcilePeriod: 10s
enableControllerAttachDetach: true
enableDebuggingHandlers: true
enforceNodeAllocatable:
- pods
eventBurst: 10
eventRecordQPS: 5
evictionHard:
  imagefs.available: 15%
  memory.available: 100Mi
  nodefs.available: 10%
  nodefs.inodesFree: 5%
evictionPressureTransitionPeriod: 5m0s
failSwapOn: true
fileCheckFrequency: 20s
hairpinMode: promiscuous-bridge
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 20s
imageGCHighThresholdPercent: 85
imageGCLowThresholdPercent: 80
imageMinimumGCAge: 2m0s
iptablesDropBit: 15
iptablesMasqueradeBit: 14
kubeAPIBurst: 10
kubeAPIQPS: 5
makeIPTablesUtilChains: true
maxOpenFiles: 1000000
maxPods: 110
nodeStatusUpdateFrequency: 10s
oomScoreAdj: -999
podPidsLimit: -1
registryBurst: 10
registryPullQPS: 5
resolvConf: /etc/resolv.conf
rotateCertificates: true
runtimeRequestTimeout: 2m0s
serializeImagePulls: true
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 4h0m0s
syncFrequency: 1m0s
volumeStatsAggPeriod: 1m0s

# systemctl daemon-reload

# systemctl enable --now kubelet

 #在所有Master以及node節點上配置kube-proxy

[[email protected] # cd /root/k8s-ha-install/

[[email protected] k8s-ha-install]# kubectl -n kube-system create serviceaccount kube-proxy
ubeconfig
kubectl config set-context kubernetes --cluster=kubernetes --user=kubernetes --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
kubectl config use-context kubernetes --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfigserviceaccount/kube-proxy created
[[email protected] k8s-ha-install]# kubectl create clusterrolebinding system:kube-proxy --clusterrole system:node-proxier --serviceaccount kube-system:kube-proxy
clusterrolebinding.rbac.authorization.k8s.io/system:kube-proxy created
[[email protected] k8s-ha-install]# SECRET=$(kubectl -n kube-system get sa/kube-proxy \
> --output=jsonpath='{.secrets[0].name}')
[[email protected] k8s-ha-install]# JWT_TOKEN=$(kubectl -n kube-system get secret/$SECRET \
> --output=jsonpath='{.data.token}' | base64 -d)
[[email protected] k8s-ha-install]# PKI_DIR=/etc/kubernetes/pki
[[email protected] k8s-ha-install]# K8S_DIR=/etc/kubernetes
[[email protected] k8s-ha-install]# kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --server=https://192.168.60.236:8443 --kubeconfig=${K8S_DIR}/kube-proxy.kubeconfig
Cluster "kubernetes" set.
[[email protected] k8s-ha-install]# kubectl config set-credentials kubernetes --token=${JWT_TOKEN} --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
User "kubernetes" set.
[[email protected] k8s-ha-install]# kubectl config set-context kubernetes --cluster=kubernetes --user=kubernetes --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
Context "kubernetes" created.
[[email protected] k8s-ha-install]# kubectl config use-context kubernetes --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig

[[email protected] k8s-ha-install]# vim kube-proxy/kube-proxy.conf

apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
clientConnection:
  acceptContentTypes: ""
  burst: 10
  contentType: application/vnd.kubernetes.protobuf
  kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig
  qps: 5
clusterCIDR: 172.16.0.0/12
configSyncPeriod: 15m0s
conntrack:
  max: null
  maxPerCore: 32768
  min: 131072
  tcpCloseWaitTimeout: 1h0m0s
  tcpEstablishedTimeout: 24h0m0s
enableProfiling: false
healthzBindAddress: 0.0.0.0:10256
hostnameOverride: ""
iptables:
  masqueradeAll: false
  masqueradeBit: 14
  minSyncPeriod: 0s
  syncPeriod: 30s
ipvs:
  masqueradeAll: true
  minSyncPeriod: 5s
  scheduler: "rr"
  syncPeriod: 30s
kind: KubeProxyConfiguration
metricsBindAddress: 127.0.0.1:10249
mode: "ipvs"
nodePortAddresses: null
oomScoreAdj: -999
portRange: ""
udpIdleTimeout: 250ms

 #在k8s-master01節點將kube-proxy的system.service文件發送到其他節點

[[email protected] k8s-ha-install]# for NODE in k8s-master01 k8s-master02 k8s-master03; do      scp ${K8S_DIR}/kube-proxy.kubeconfig $NODE:/etc/kubernetes/kube-proxy.kubeconfig;      scp kube-proxy/kube-proxy.conf $NODE:/etc/kubernetes/kube-proxy.conf;      scp kube-proxy/kube-proxy.service $NODE:/usr/lib/systemd/system/kube-proxy.service;  done

[[email protected] k8s-ha-install]# for NODE in k8s-node01 k8s-node02; do
> scp /etc/kubernetes/kube-proxy.kubeconfig $NODE:/etc/kubernetes/kube-proxy.kubeconfig
> scp kube-proxy/kube-proxy.conf $NODE:/etc/kubernetes/kube-proxy.conf
> scp kube-proxy/kube-proxy.service $NODE:/usr/lib/systemd/system/kube-proxy.service
> done

#啓用所有k8s node節點上的kube-proxy組件

# systemctl daemon-reload

# systemctl enable --now kube-proxy

【部署Calico 】

[[email protected] ~]# cd /root/k8s-ha-install/calico/
[[email protected] calico]# sed -i 's#etcd_endpoints: "http://<ETCD_IP>:<ETCD_PORT>"#etcd_endpoints: "https://192.168.60.101:2379,https://192.168.60.102:2379,https://192.168.60.103:2379"#g' calico-etcd.yaml
[[email protected] calico]# sed -i 's#etcd_endpoints: "http://<ETCD_IP>:<ETCD_PORT>"#etcd_endpoints: "https://192.168.60.101:2379,https://192.168.60.102:2379,https://192.168.60.103:2379"#g' calico-etcd.yaml
[[email protected] calico]# ETCD_CA=`cat /etc/kubernetes/pki/etcd/etcd-ca.pem | base64 | tr -d '\n'`
[[email protected] calico]# ETCD_CERT=`cat /etc/kubernetes/pki/etcd/etcd.pem | base64 | tr -d '\n'`
[[email protected] calico]# ETCD_KEY=`cat /etc/kubernetes/pki/etcd/etcd-key.pem | base64 | tr -d '\n'`
[[email protected] calico]# sed -i "[email protected]# etcd-key: [email protected]: ${ETCD_KEY}@g; [email protected]# etcd-cert: [email protected]: ${ETCD_CERT}@g; [email protected]# etcd-ca: [email protected]: ${ETCD_CA}@g" calico-etcd.yaml
[[email protected] calico]# sed -i 's#etcd_ca: ""#etcd_ca: "/calico-secrets/etcd-ca"#g; s#etcd_cert: ""#etcd_cert: "/calico-secrets/etcd-cert"#g; s#etcd_key: "" #etcd_key: "/calico-secrets/etcd-key" #g' calico-etcd.yaml
[[email protected] calico]# POD_SUBNET="172.16.0.0/12"
[[email protected] calico]# sed -i '[email protected]# - name: [email protected] name: [email protected]; [email protected]# value: "192.168.0.0/16"@ value: '"${POD_SUBNET}"'@g' calico-etcd.yaml

[[email protected] calico]# kubectl apply -f calico-etcd.yaml

secret/calico-etcd-secrets created
configmap/calico-config created
clusterrole.rbac.authorization.k8s.io/calico-kube-controllers created
clusterrolebinding.rbac.authorization.k8s.io/calico-kube-controllers created
clusterrole.rbac.authorization.k8s.io/calico-node created
clusterrolebinding.rbac.authorization.k8s.io/calico-node created
daemonset.apps/calico-node created
serviceaccount/calico-node created
deployment.apps/calico-kube-controllers created
serviceaccount/calico-kube-controllers created

 [[email protected] calico]# kubectl get po -n kube-system  #查看node節點狀態

 【安裝CoreDNS】

 [[email protected] ~]# cd /root/k8s-ha-install/

 [[email protected] k8s-ha-install]#vim  CoreDNS/coredns.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: coredns
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
    kubernetes.io/bootstrapping: rbac-defaults
  name: system:coredns
rules:
- apiGroups:
  - ""
  resources:
  - endpoints
  - services
  - pods
  - namespaces
  verbs:
  - list
  - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  annotations:
    rbac.authorization.kubernetes.io/autoupdate: "true"
  labels:
    kubernetes.io/bootstrapping: rbac-defaults
  name: system:coredns
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:coredns
subjects:
- kind: ServiceAccount
  name: coredns
  namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: coredns
  namespace: kube-system
data:
  Corefile: |
    .:53 {
        errors
        health {
          lameduck 5s
        }
        ready
        kubernetes cluster.local in-addr.arpa ip6.arpa {
          fallthrough in-addr.arpa ip6.arpa
        }
        prometheus :9153
        forward . /etc/resolv.conf {
          max_concurrent 1000
        }
        cache 30
        loop
        reload
        loadbalance
    }
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: coredns
  namespace: kube-system
  labels:
    k8s-app: kube-dns
    kubernetes.io/name: "CoreDNS"
spec:
  # replicas: not specified here:
  # 1. Default is 1.
  # 2. Will be tuned in real time if DNS horizontal auto-scaling is turned on.
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
  selector:
    matchLabels:
      k8s-app: kube-dns
  template:
    metadata:
      labels:
        k8s-app: kube-dns
    spec:
      priorityClassName: system-cluster-critical
      serviceAccountName: coredns
      tolerations:
        - key: "CriticalAddonsOnly"
          operator: "Exists"
      nodeSelector:
        kubernetes.io/os: linux
      affinity:
         podAntiAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
           - weight: 100
             podAffinityTerm:
               labelSelector:
                 matchExpressions:
                   - key: k8s-app
                     operator: In
                     values: ["kube-dns"]
               topologyKey: kubernetes.io/hostname
      containers:
      - name: coredns
        image: registry.cn-beijing.aliyuncs.com/dotbalo/coredns:1.7.0
        imagePullPolicy: IfNotPresent
        resources:
          limits:
            memory: 170Mi
          requests:
            cpu: 100m
            memory: 70Mi
        args: [ "-conf", "/etc/coredns/Corefile" ]
        volumeMounts:
        - name: config-volume
          mountPath: /etc/coredns
          readOnly: true
        ports:
        - containerPort: 53
          name: dns
          protocol: UDP
        - containerPort: 53
          name: dns-tcp
          protocol: TCP
        - containerPort: 9153
          name: metrics
          protocol: TCP
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            add:
            - NET_BIND_SERVICE
            drop:
            - all
          readOnlyRootFilesystem: true
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
            scheme: HTTP
          initialDelaySeconds: 60
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        readinessProbe:
          httpGet:
            path: /ready
            port: 8181
            scheme: HTTP
      dnsPolicy: Default
      volumes:
        - name: config-volume
          configMap:
            name: coredns
            items:
            - key: Corefile
              path: Corefile
---
apiVersion: v1
kind: Service
metadata:
  name: kube-dns
  namespace: kube-system
  annotations:
    prometheus.io/port: "9153"
    prometheus.io/scrape: "true"
  labels:
    k8s-app: kube-dns
    kubernetes.io/cluster-service: "true"
    kubernetes.io/name: "CoreDNS"
spec:
  selector:
    k8s-app: kube-dns
  clusterIP: 10.96.0.10
  ports:
  - name: dns
    port: 53
    protocol: UDP
  - name: dns-tcp
    port: 53
    protocol: TCP
  - name: metrics
    port: 9153
    protocol: TCP
[[email protected]-master01 k8s-ha-install]# cat  CoreDNS/coredns.yaml  | grep 10.96.0.10
  clusterIP: 10.96.0.10

 

[[email protected] k8s-ha-install]# kubectl create -f CoreDNS/coredns.yaml
serviceaccount/coredns created
clusterrole.rbac.authorization.k8s.io/system:coredns created
clusterrolebinding.rbac.authorization.k8s.io/system:coredns created
configmap/coredns created
deployment.apps/coredns created
service/kube-dns created

 【安裝Metrics Server】

在新版Kubernetes中系統資源的採集均使用Metrics-server,可以通過Metrics採集節點和Pod的內存、磁盤、CPU和網絡的使用率

[[email protected] ~]# cd /root/k8s-ha-install/metrics-server-0.4.x

[[email protected] metrics-server-0.4.x]# kubectl create -f .

【安裝dashboard 】

[[email protected] ~]# cd /root/k8s-ha-install/dashboard/
[[email protected] dashboard]# kubectl create -f .

[[email protected] dashboard]# kubectl get po -n kubernetes-dashboard

 [[email protected] dashboard]# kubectl edit svc kubernetes-dashboard -n kubernetes-dashboard   #更改dashboard的svc爲NodePort

apiVersion: v1
kind: Service
metadata:
  creationTimestamp: "2020-12-24T01:20:21Z"
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
  resourceVersion: "30932"
  uid: 141a3d84-aa0f-414f-995f-5d40a609ca22
spec:
  clusterIP: 10.96.33.35
  clusterIPs:
  - 10.96.33.35
  externalTrafficPolicy: Cluster
  ports:
  - nodePort: 30900
    port: 443
    protocol: TCP
    targetPort: 8443
  selector:
    k8s-app: kubernetes-dashboard
  sessionAffinity: None
  type: NodePort
status:
  loadBalancer: {}

將Cluster IP更改爲NodePort(Ps:如果已經爲NodePort可忽略次步驟)

[[email protected] dashboard]# kubectl get svc kubernetes-dashboard -n kubernetes-dashboard  # 查看dashboard暴露的端口號,通過任意安裝了kube-proxy宿主機或者VIP+端口即可訪問到dashboard界面

# kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}') #查看token,用於dashboard界面認證

Name:         admin-user-token-lvc4w
Namespace:    kube-system
Labels:       <none>
Annotations:  kubernetes.io/service-account.name: admin-user
              kubernetes.io/service-account.uid: 02c91803-12bc-4ca4-8107-5690f6551adb

Type:  kubernetes.io/service-account-token

Data
====
ca.crt:     1411 bytes
namespace:  11 bytes
token:      eyJhbGciOiJSUzI1NiIsImtpZCI6IkQ2LXlsQXZkSWFjQzQ5TFE4N1dQalJCeXpRNUdBQmkwOVJSamRINGVabHMifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi11c2VyLXRva2VuLWx2YzR3Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImFkbWluLXVzZXIiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiIwMmM5MTgwMy0xMmJjLTRjYTQtODEwNy01NjkwZjY1NTFhZGIiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZS1zeXN0ZW06YWRtaW4tdXNlciJ9.vszo1cC_bZMwYUoPC71xSQH7P1pbLqkxFRF4RslVj1_eKsAQF0fE7N83Uwu9AiE_Van_nQfocTWIUz6db6cu6o4HYDALLn26oyrJ8mdZZHoLbdiGo-UBo8WbAAxl_n195hxTLZLSminj-ck28bCMtKpaYUZ9DnkToo26afMla8YO-Hj7h2AVcX2P4i7wQVKkb9H2-ImR1hBik1t2an_bsL8ycU00_-50RNNlTEr7CIHxmabGXjFI9R55lPuZW-uQDH3DbPnwbwFNTdr3tV1mriv8aF0l-6-lJbXGyY7t4mlPyiu5BakpLt-Dky3ZcNfCKAcB_XyzgKYFqaPmBgcJ1Q

 #https://192.168.60.236:30900/#/login  我這裏選擇是的訪問VIP加上暴露的30900端口號;

 

 

【集羣驗證】

新建一個kubernetes po

[[email protected] ~]#  cat<<EOF | kubectl apply -f -
 apiVersion: v1
 kind: Pod
 metadata:
   name: busybox
   namespace: default
 spec:
   containers:
   - name: busybox
     image: busybox:1.28
     command:
       - sleep
       - "3600"
     imagePullPolicy: IfNotPresent
   restartPolicy: Always
 EOF

 

2、驗證pod是否能解析跨namespace的service;在這裏 busybox是在default的namespache上,而kube-dns則是在kube-system命名空間的services;

[[email protected] ~]# kubectl exec  busybox -n default -- nslookup kube-dns.kube-system
Server:    10.96.0.10
Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local

Name:      kube-dns.kube-system
Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local

 

3、驗證每個節點都必須能訪問kubernetes的kubernetes svc 443端口和kube-dns的service 53端口

[[email protected] ~]# kubectl get svc   #查看一下kubernetes services地址
NAME         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)   AGE
kubernetes   ClusterIP   10.96.0.1    <none>        443/TCP   5d1h
[[email protected]-master01 ~]# kubectl get svc -n kube-system   #
NAME             TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                  AGE
kube-dns         ClusterIP   10.96.0.10       <none>        53/UDP,53/TCP,9153/TCP   4d23h
metrics-server   ClusterIP   10.111.213.213   <none>        443/TCP                  4d23h

 

 

 4、驗證集羣pod之間能正常通信

[[email protected] ~]# kubectl get po -n kube-system -owide
NAME                                       READY   STATUS    RESTARTS   AGE     IP               NODE           NOMINATED NODE   READINESS GATES
calico-kube-controllers-5f6d4b864b-67tbv   1/1     Running   3          5d      192.168.60.105   k8s-node02     <none>           <none>
calico-node-b2jlr                          1/1     Running   3          5d      192.168.60.104   k8s-node01     <none>           <none>
calico-node-ggzsc                          1/1     Running   10         4d22h   192.168.60.102   k8s-master02   <none>           <none>
calico-node-mrmb7                          1/1     Running   3          5d      192.168.60.105   k8s-node02     <none>           <none>
calico-node-z4jgk                          1/1     Running   9          4d22h   192.168.60.103   k8s-master03   <none>           <none>
calico-node-zzbbk                          1/1     Running   4          4d22h   192.168.60.101   k8s-master01   <none>           <none>
coredns-867d46bfc6-x72tn                   1/1     Running   3          4d23h   172.17.125.8     k8s-node01     <none>           <none>
metrics-server-595f65d8d5-gw2rw            1/1     Running   3          4d23h   172.17.125.7     k8s-node01     <none>           <none>
[[email protected] ~]# kubectl exec -it calico-node-b2jlr -n kube-system -- bash #通過kubectl命令進入node01容器中

[[email protected] ~]# kubectl get pod -n default -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
busybox 1/1 Running 0 47m 172.25.92.65 k8s-master02 <none> <none>

[[email protected] ~]# kubectl exec -it busybox -n default  -- sh   #進入busybox容器中,通過ping命令檢測pod之間以及跨namespace能否正常通信

 

 

 

[email protected] ~]# kubectl create deploy nginx --image=nginx --replicas=3  #創建並部署一個nginx pod,並生成三個副本數量
deployment.apps/nginx created

[[email protected] ~]# kubectl get deploy
NAME READY UP-TO-DATE AVAILABLE AGE
nginx 3/3 3 3 47s

[[email protected] ~]# kubectl get pod -n default  -owide  #獲取default命令空間下的pod的詳細信息
NAME                     READY   STATUS    RESTARTS   AGE     IP               NODE           NOMINATED NODE   READINESS GATES
busybox                  1/1     Running   0          59m     172.25.92.65     k8s-master02   <none>           <none>
nginx                    1/1     Running   0          4m6s    172.18.195.1     k8s-master03   <none>           <none>
nginx-6799fc88d8-lhxr7   1/1     Running   0          2m19s   172.25.244.198   k8s-master01   <none>           <none>
nginx-6799fc88d8-snd7l   1/1     Running   0          2m19s   172.27.14.193    k8s-node02     <none>           <none>
nginx-6799fc88d8-wr7v8   1/1     Running   0          2m19s   172.25.244.197   k8s-master01   <none>           <none>
[[email protected] ~]# kubectl delete deploy nginx #刪除nginx的pod

 

 

 

END!

發表評論
所有評論
還沒有人評論,想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.
相關文章