CentOS 7.x下Kubernetes 1.18.1安裝
一. 配置系統環境
cat /etc/redhat-release
CentOS Linux release 7.7.1908 (Core)
uname -a
Linux k8s-master 3.10.0-1062.18.1.el7.x86_64 #1 SMP Tue Mar 17 23:49:17 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux
配置主機名：
hostnamectl set-hostname k8s-master
hostnamectl set-hostname k8s-node01
hostnamectl set-hostname k8s-node02
hostnamectl set-hostname k8s-node03
配置hosts解析:
vi /etc/hosts
192.168.0.106 k8s-master
192.168.0.107 k8s-node01
192.168.0.108 k8s-node02
192.168.0.109 k8s-node03
關閉防火牆和安全設置：
systemctl stop firewalld && systemctl disable firewalld
關閉swap，註釋swap分區：
swapoff -a
vim /etc/fstab
#
# /etc/fstab
# Created by anaconda on Mon Nov 19 11:39:58 2018
#
# Accessible filesystems, by reference, are maintained under '/dev/disk'
# See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info
#
UUID=706fda26-fada-4463-b934-512228b42f8e / ext4 defaults 1 1
UUID=f7cf7fdf-b05f-4891-a8d9-419931d0ce63 /boot ext4 defaults 1 2
#UUID=fb737556-8d56-40b9-9f35-042a8dd3c5b5 swap swap defaults 0 0
關閉selinux:
sed -i '/SELINUX/s/enforcing/disabled/' /etc/selinux/config
setenforce 0
配置免密碼登錄:
在每個節點都執行ssh-keygen產生公私鑰對，都選缺省值，一路回車即可。
[root@k8s-master ~]# ssh-keygen
用ssh-copy-id將本節點的公鑰複製到其它節點，如k8s-master節點，需要將公鑰複製到k8s-node01、k8s-node02和k8s-node03三個節點，其它節點都要類似操作。
[root@k8s-master ~]# ssh-copy-id k8s-node01
[root@k8s-master ~]# ssh-copy-id k8s-node02
[root@k8s-master ~]# ssh-copy-id k8s-node03
修改內核參數:
將橋接的IPv4流量傳遞到iptables的鏈
[root@k8s-master ~]# cat <<EOF> /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
[root@k8s-master ~]# modprobe br_netfilter
[root@k8s-master ~]# sysctl --system
#注意需要有以下兩行
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
* Applying /etc/sysctl.conf ...
二. 安裝Docker
[root@k8s-master ~]# yum install -y yum-utils device-mapper-persistent-data lvm2
[root@k8s-master ~]# yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
或者
#yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
[root@k8s-master ~]# yum list docker-ce --showduplicates | sort -r
已加載插件：fastestmirror
已安裝的軟件包
可安裝的軟件包
* updates: mirror.bit.edu.cn
Loading mirror speeds from cached hostfile
* extras: mirrors.huaweicloud.com
docker-ce.x86_64 3:19.03.8-3.el7 docker-ce-stable
docker-ce.x86_64 3:19.03.8-3.el7 docker-ce-stable
docker-ce.x86_64 3:19.03.7-3.el7 docker-ce-stable
docker-ce.x86_64 3:19.03.6-3.el7 docker-ce-stable
docker-ce.x86_64 3:19.03.5-3.el7 docker-ce-stable
docker-ce.x86_64 3:19.03.4-3.el7 docker-ce-stable
docker-ce.x86_64 3:19.03.3-3.el7 docker-ce-stable
docker-ce.x86_64 3:19.03.2-3.el7 docker-ce-stable
docker-ce.x86_64 3:19.03.1-3.el7 docker-ce-stable
docker-ce.x86_64 3:19.03.0-3.el7 docker-ce-stable
docker-ce.x86_64 3:18.09.9-3.el7 docker-ce-stable
docker-ce.x86_64 3:18.09.8-3.el7 docker-ce-stable
docker-ce.x86_64 3:18.09.7-3.el7 docker-ce-stable
docker-ce.x86_64 3:18.09.6-3.el7 docker-ce-stable
docker-ce.x86_64 3:18.09.5-3.el7 docker-ce-stable
......
安裝：
[root@k8s-master ~]# yum install -y docker-ce-19.03.8-3.el7
[root@k8s-master ~]# systemctl start docker
[root@k8s-master ~]# systemctl enable docker
[root@k8s-master ~]# docker version
Client: Docker Engine - Community
Version: 19.03.8
API version: 1.40
Go version: go1.12.17
Git commit: afacb8b
Built: Wed Mar 11 01:27:04 2020
OS/Arch: linux/amd64
Experimental: false
......
配置鏡像加速器：
針對Docker客戶端版本大於 1.10.0 的用戶
您可以通過修改daemon配置文件/etc/docker/daemon.json來使用加速器
sudo mkdir -p /etc/docker
sudo tee /etc/docker/daemon.json <<-'EOF'
{
"registry-mirrors": ["https://xxxx.mirror.aliyuncs.com"]
}
EOF
sudo systemctl daemon-reload
sudo systemctl restart docker
三.安裝Kubernetes
1.設置Kubernetes的yum源：
cat <<EOF> /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
或
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=http://packages.cloud.google.com/yum/doc/yum-key.gpg http://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
EOF
2.安裝kubeadmin、kubectl和kubelet服務：
[root@k8s-master ~]# yum list -y kubeadm --showduplicates
已加載插件：fastestmirror
Loading mirror speeds from cached hostfile
* base: mirror.bit.edu.cn
* extras: mirrors.huaweicloud.com
* updates: mirror.bit.edu.cn
kubernetes/signature | 454 B 00:00:00
從 https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg 檢索密鑰
導入 GPG key 0xA7317B0F:
用戶ID : "Google Cloud Packages Automatic Signing Key <[email protected]>"
指紋 : d0bc 747f d8ca f711 7500 d6fa 3746 c208 a731 7b0f
來自 : https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
從 https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg 檢索密鑰
kubernetes/signature | 1.4 kB 00:00:00 !!!
kubernetes/primary | 66 kB 00:00:00
kubernetes 481/481
可安裝的軟件包
kubeadm.x86_64 1.6.0-0 kubernetes
kubeadm.x86_64 1.6.1-0 kubernetes
kubeadm.x86_64 1.6.2-0 kubernetes
kubeadm.x86_64 1.6.3-0 kubernetes
......
kubeadm.x86_64 1.12.7-0 kubernetes
kubeadm.x86_64 1.12.8-0 kubernetes
kubeadm.x86_64 1.12.9-0 kubernetes
kubeadm.x86_64 1.12.10-0 kubernetes
......
kubeadm.x86_64 1.16.4-0 kubernetes
kubeadm.x86_64 1.16.5-0 kubernetes
kubeadm.x86_64 1.16.6-0 kubernetes
kubeadm.x86_64 1.16.7-0 kubernetes
kubeadm.x86_64 1.16.8-0 kubernetes
kubeadm.x86_64 1.17.0-0 kubernetes
kubeadm.x86_64 1.17.1-0 kubernetes
kubeadm.x86_64 1.17.2-0 kubernetes
kubeadm.x86_64 1.17.3-0 kubernetes
kubeadm.x86_64 1.17.4-0 kubernetes
kubeadm.x86_64 1.18.0-0 kubernetes
kubeadm.x86_64 1.18.1-0 kubernetes
[root@k8s-master ~]# yum install -y kubeadm-1.18.1-0 kubectl-1.18.1-0 kubelet-1.18.1-0
[root@k8s-master ~]# kubeadm version
kubeadm version: &version.Info{Major:"1", Minor:"18", GitVersion:"v1.18.1", GitCommit:"9e991415386e4cf155a24b1da15becaa390438d8", GitTreeState:"clean", BuildDate:"2020-03-25T14:56:30Z", GoVersion:"go1.13.8", Compiler:"gc", Platform:"linux/amd64"}
[root@k8s-master ~]# systemctl start kubelet
[root@k8s-master ~]# systemctl enable kubelet
查看Kubernetes所需要的鏡像版本，並提前下載好所需的七個鏡像:
由於kubeadm 默認從官網k8s.grc.io下載所需鏡像，國內無法訪問，因此需要通過–image-repository指定阿里雲鏡像倉庫地址.
[root@k8s-master ~]# kubeadm config images pull --image-repository=registry.aliyuncs.com/google_containers
W0408 09:53:39.571992 1651 configset.go:202] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io]
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-apiserver:v1.18.1
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-controller-manager:v1.18.1
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-scheduler:v1.18.1
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-proxy:v1.18.1
[config/images] Pulled registry.aliyuncs.com/google_containers/pause:3.2
[config/images] Pulled registry.aliyuncs.com/google_containers/etcd:3.4.3-0
[config/images] Pulled registry.aliyuncs.com/google_containers/coredns:1.6.7
[root@k8s-master ~]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
registry.aliyuncs.com/google_containers/kube-proxy v1.18.1 43940c34f24f 13 days ago 117MB
registry.aliyuncs.com/google_containers/kube-apiserver v1.18.1 74060cea7f70 13 days ago 173MB
registry.aliyuncs.com/google_containers/kube-controller-manager v1.18.1 d3e55153f52f 13 days ago 162MB
registry.aliyuncs.com/google_containers/kube-scheduler v1.18.1 a31f78c7c8ce 13 days ago 95.3MB
registry.aliyuncs.com/google_containers/pause 3.2 80d28bedfe5d 7 weeks ago 683kB
registry.aliyuncs.com/google_containers/coredns 1.6.7 67da37a9a360 2 months ago 43.8MB
registry.aliyuncs.com/google_containers/etcd 3.4.3-0 303ce5db0e90 5 months ago 288MB
以上所有的命令需要在k8s-master、node01、node02和node03四個節點都操作。以下初始化操作只需要在k8s-master節點操作。
3.初始化Master節點
# –-pod-network-cidr：用於指定Pod的網絡範圍
# –-service-cidr：用於指定service的網絡範圍；
# --image-repository: 鏡像倉庫的地址，和提前下載的鏡像倉庫應該對應上。
[root@k8s-master ~]# kubeadm init --kubernetes-version=v1.18.1 \
> --pod-network-cidr=10.244.0.0/16 \
> --service-cidr=10.1.0.0/16 \
> --image-repository=registry.aliyuncs.com/google_containers
W0414 15:04:45.582533 19360 configset.go:202] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io]
[init] Using Kubernetes version: v1.18.1
[preflight] Running pre-flight checks
[WARNING IsDockerSystemdCheck]: detected "cgroupfs" as the Docker cgroup driver. The recommended driver is "systemd". Please follow the guide at https://kubernetes.io/docs/setup/cri/
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Starting the kubelet
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "ca" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [k8s-master kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.1.0.1 192.168.0.106]
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "front-proxy-ca" certificate and key
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "etcd/ca" certificate and key
[certs] Generating "etcd/server" certificate and key
[certs] etcd/server serving cert is signed for DNS names [k8s-master localhost] and IPs [192.168.0.106 127.0.0.1 ::1]
[certs] Generating "etcd/peer" certificate and key
[certs] etcd/peer serving cert is signed for DNS names [k8s-master localhost] and IPs [192.168.0.106 127.0.0.1 ::1]
[certs] Generating "etcd/healthcheck-client" certificate and key
[certs] Generating "apiserver-etcd-client" certificate and key
[certs] Generating "sa" key and public key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
W0414 15:04:50.881738 19360 manifests.go:225] the default kube-apiserver authorization-mode is "Node,RBAC"; using "Node,RBAC"
[control-plane] Creating static Pod manifest for "kube-scheduler"
W0414 15:04:50.882749 19360 manifests.go:225] the default kube-apiserver authorization-mode is "Node,RBAC"; using "Node,RBAC"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[apiclient] All control plane components are healthy after 30.508279 seconds
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config-1.18" in namespace kube-system with the configuration for the kubelets in the cluster
[upload-certs] Skipping phase. Please see --upload-certs
[mark-control-plane] Marking the node k8s-master as control-plane by adding the label "node-role.kubernetes.io/master=''"
[mark-control-plane] Marking the node k8s-master as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule]
[bootstrap-token] Using token: 5d6tsi.8wlso84uxt7f3gim
[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to get nodes
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace
[kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy
Your Kubernetes control-plane has initialized successfully!
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
Then you can join any number of worker nodes by running the following on each as root:
kubeadm join 192.168.0.106:6443 --token 5d6tsi.8wlso84uxt7f3gim \
--discovery-token-ca-cert-hash sha256:79363215f508526295f535a62efb70544c9f0db5dc2cfdb73074c1bcac87655f
初始化完成後，需要按照提示執行以下命令。注意最後的kubeadm join命令後續會使用到，需要記錄，在加入master的node節點執行。
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
# 添加kubectl的自動補全功能
echo "source <(kubectl completion bash)" >> ~/.bashrc
4. 修改Service的nodePort缺省取值範圍
vi /etc/kubernetes/manifests/kube-apiserver.yaml
- --service-node-port-range=1-65535
systemctl daemon-reload
systemctl restart kubelet
查看Kubernetes狀態
kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master NotReady master 4m45s v1.18.0
kubectl get all -n kube-system
NAME READY STATUS RESTARTS AGE
pod/coredns-7ff77c879f-8flq5 0/1 Pending 0 5m53s
pod/coredns-7ff77c879f-xth7n 0/1 Pending 0 5m54s
pod/etcd-k8s-master 1/1 Running 0 6m3s
pod/kube-apiserver-k8s-master 1/1 Running 0 19s
pod/kube-controller-manager-k8s-master 1/1 Running 1 6m2s
pod/kube-proxy-fk7tn 1/1 Running 0 5m54s
pod/kube-scheduler-k8s-master 1/1 Running 1 6m2s
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kube-dns ClusterIP 10.1.0.10 <none> 53/UDP,53/TCP,9153/TCP 6m11s
NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE
daemonset.apps/kube-proxy 1 1 1 1 1 kubernetes.io/os=linux 6m11s
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/coredns 0/2 2 0 6m11s
NAME DESIRED CURRENT READY AGE
replicaset.apps/coredns-7ff77c879f 2 2 0 5m54s
四. kubernetes網絡
部署calico網絡
[root@master01 ~]# kubectl apply -f https://docs.projectcalico.org/manifests/calico.yaml
configmap/calico-config created
customresourcedefinition.apiextensions.k8s.io/bgpconfigurations.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/bgppeers.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/blockaffinities.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/clusterinformations.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/felixconfigurations.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/globalnetworkpolicies.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/globalnetworksets.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/hostendpoints.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ipamblocks.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ipamconfigs.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ipamhandles.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ippools.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/networkpolicies.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/networksets.crd.projectcalico.org created
clusterrole.rbac.authorization.k8s.io/calico-kube-controllers created
clusterrolebinding.rbac.authorization.k8s.io/calico-kube-controllers created
clusterrole.rbac.authorization.k8s.io/calico-node created
clusterrolebinding.rbac.authorization.k8s.io/calico-node created
daemonset.apps/calico-node created
serviceaccount/calico-node created
deployment.apps/calico-kube-controllers created
serviceaccount/calico-kube-controllers created
查看pod和node
[root@master01 ~]# kubectl get pod --all-namespaces
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system calico-kube-controllers-555fc8cc5c-k8rbk 1/1 Running 0 36s
kube-system calico-node-5km27 1/1 Running 0 36s
kube-system coredns-7ff77c879f-fsj9l 1/1 Running 0 5m22s
kube-system coredns-7ff77c879f-q5ll2 1/1 Running 0 5m22s
kube-system etcd-master01.paas.com 1/1 Running 0 5m32s
kube-system kube-apiserver-master01.paas.com 1/1 Running 0 5m32s
kube-system kube-controller-manager-master01.paas.com 1/1 Running 0 5m32s
kube-system kube-proxy-th472 1/1 Running 0 5m22s
kube-system kube-scheduler-master01.paas.com 1/1 Running 0 5m32s
[root@master01 ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
master01 Ready master 5m47s v1.18.1
[root@master01 ~]#
此時集羣狀態正常.
注：以上命令只在k8s-master節點執行。
Join node節點到Kubernetes集羣中。分別在k8s-node1、k8s-node2和k8s-node3節點執行如下join命令，並執行配置命令。
kubeadm join 192.168.1.50:6443 --token 4oxcgj.1dqz97nbu4pcf84l \
--discovery-token-ca-cert-hash sha256:2445a08ab9e210e9d3f82949ae16472d47abbc188a2b28e4d6470b02d5ddce3a
W0408 10:42:33.434915 10437 join.go:346] [preflight] WARNING: JoinControlPane.controlPlane settings will be ignored when control-plane flag is not set.
[preflight] Running pre-flight checks
[WARNING IsDockerSystemdCheck]: detected "cgroupfs" as the Docker cgroup driver. The recommended driver is "systemd". Please follow the guide at https://kubernetes.io/docs/setup/cri/
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
[kubelet-start] Downloading configuration for the kubelet from the "kubelet-config-1.18" ConfigMap in the kube-system namespace
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Starting the kubelet
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...
This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.
Run 'kubectl get nodes' on the control-plane to see this node join the cluster.
爲了在node節點上執行kubectl命令，可以如下操作。
mkdir -p $HOME/.kube
scp 192.168.0.106:/root/.kube/config $HOME/.kube/config
查看kubernetes狀態
kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master Ready master 34m v1.18.1
k8s-node1 Ready <none> 2m47s v1.18.1
k8s-node2 Ready <none> 2m45s v1.18.1
k8s-node3 Ready <none> 2m43s v1.18.1
kubectl get all -n kube-system
NAME READY STATUS RESTARTS AGE
pod/coredns-7ff77c879f-8flq5 1/1 Running 0 35m
pod/coredns-7ff77c879f-xth7n 1/1 Running 0 35m
pod/etcd-k8s-master 1/1 Running 0 35m
pod/kube-apiserver-k8s-master 1/1 Running 0 29m
pod/kube-controller-manager-k8s-master 1/1 Running 1 35m
pod/kube-flannel-ds-amd64-8k5pg 1/1 Running 0 3m41s
pod/kube-flannel-ds-amd64-8nft4 1/1 Running 0 24m
pod/kube-flannel-ds-amd64-w27q7 1/1 Running 0 3m39s
pod/kube-proxy-b6c2q 1/1 Running 0 3m39s
pod/kube-proxy-bvrhx 1/1 Running 0 3m41s
pod/kube-proxy-fk7tn 1/1 Running 0 35m
pod/kube-scheduler-k8s-master 1/1 Running 1 35m
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kube-dns ClusterIP 10.1.0.10 <none> 53/UDP,53/TCP,9153/TCP 35m
NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE
daemonset.apps/kube-flannel-ds-amd64 3 3 3 3 3 <none> 24m
daemonset.apps/kube-proxy 3 3 3 3 3 kubernetes.io/os=linux 35m
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/coredns 2/2 2 2 35m
NAME DESIRED CURRENT READY AGE
replicaset.apps/coredns-7ff77c879f 2 2 2 35m
五.kubenetes集羣中部署dashboard
Kubernetes dashboard 是Kubernetes集羣的基於Web的通用UI。
它允許用戶管理羣集中運行的應用程序並對其進行故障排除，以及管理羣集本身。
--------------------------------------------------------------------------------
1、部署
如果你的網絡很好且可以訪問國外網，那你可以直接運行下面命令部署。
kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-rc7/aio/deploy/recommended.yaml
[root@k8s-master ~]# kubectl get pod -n kubernetes-dashboard
NAME READY STATUS RESTARTS AGE
dashboard-metrics-scraper-dc6947fbf-gcqg4 1/1 Running 0 18h
kubernetes-dashboard-5d4dc8b976-t8tmm 1/1 Running 0 18h
[root@k8s-master ~]#
--------------------------------------------------------------------------------
2、本地訪問
如果你打算本地訪問，可以直接運行： kubectl proxy 。
然後訪問即可：http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/
--------------------------------------------------------------------------------
3、登錄
查看Token，然後使用token登錄。
kubectl -n kubernetes-dashboard describe secret $(kubectl -n kubernetes-dashboard get secret | grep kubernetes-dashboard-token | awk '{print $1}')
然後你會發現無法查看到系統資源，這是因爲默認創建的token是kubernetes-dashboard命名空間，
所以我們需要在kube-system命名空間創建一個用戶，然後使用對應token登錄即可。
apiVersion: v1
kind: ServiceAccount
metadata:
name: admin-user
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: admin-user
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: admin-user
namespace: kube-system
--------------------------------------------------------------------------------
4、開放端口
很多時候我們需要提供給多個人訪問，比如研發、測試環境，這時候我們可以開啓nodeport對外暴露一個端口。
[root@k8s-master ~]# kubectl edit svc kubernetes-dashboard -n kubernetes-dashboard
--------------------------------------------------------------------------------
5、一步到位：
當然你也可以直接用我修改好的，直接部署，其中已對鏡像源做了更改，添加了系統用戶。
[root@k8s-master ~]# wget https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-rc7/aio/deploy/recommended.yaml
[root@k8s-master ~]# vim recommended.yaml
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: v1
kind: Namespace
metadata:
name: kubernetes-dashboard
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kubernetes-dashboard
---
kind: Service
apiVersion: v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kubernetes-dashboard
spec:
type: NodePort
ports:
- port: 443
targetPort: 8443
nodePort: 30001
selector:
k8s-app: kubernetes-dashboard
---
apiVersion: v1
kind: Secret
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-certs
namespace: kubernetes-dashboard
type: Opaque
---
apiVersion: v1
kind: Secret
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-csrf
namespace: kubernetes-dashboard
type: Opaque
data:
csrf: ""
---
apiVersion: v1
kind: Secret
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-key-holder
namespace: kubernetes-dashboard
type: Opaque
---
kind: ConfigMap
apiVersion: v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-settings
namespace: kubernetes-dashboard
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kubernetes-dashboard
rules:
# Allow Dashboard to get, update and delete Dashboard exclusive secrets.
- apiGroups: [""]
resources: ["secrets"]
resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]
verbs: ["get", "update", "delete"]
# Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
- apiGroups: [""]
resources: ["configmaps"]
resourceNames: ["kubernetes-dashboard-settings"]
verbs: ["get", "update"]
# Allow Dashboard to get metrics.
- apiGroups: [""]
resources: ["services"]
resourceNames: ["heapster", "dashboard-metrics-scraper"]
verbs: ["proxy"]
- apiGroups: [""]
resources: ["services/proxy"]
resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"]
verbs: ["get"]
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
rules:
# Allow Metrics Scraper to get metrics from the Metrics server
- apiGroups: ["metrics.k8s.io"]
resources: ["pods", "nodes"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kubernetes-dashboard
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: kubernetes-dashboard
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard
namespace: kubernetes-dashboard
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kubernetes-dashboard
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: kubernetes-dashboard
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard
namespace: kubernetes-dashboard
---
kind: Deployment
apiVersion: apps/v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kubernetes-dashboard
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
k8s-app: kubernetes-dashboard
template:
metadata:
labels:
k8s-app: kubernetes-dashboard
spec:
containers:
- name: kubernetes-dashboard
image: kubernetesui/dashboard:v2.0.0-rc7
imagePullPolicy: Always
ports:
- containerPort: 8443
protocol: TCP
args:
- --auto-generate-certificates
- --namespace=kubernetes-dashboard
# Uncomment the following line to manually specify Kubernetes API server Host
# If not specified, Dashboard will attempt to auto discover the API server and connect
# to it. Uncomment only if the default does not work.
# - --apiserver-host=http://my-address:port
volumeMounts:
- name: kubernetes-dashboard-certs
mountPath: /certs
# Create on-disk volume to store exec logs
- mountPath: /tmp
name: tmp-volume
livenessProbe:
httpGet:
scheme: HTTPS
path: /
port: 8443
initialDelaySeconds: 30
timeoutSeconds: 30
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
runAsUser: 1001
runAsGroup: 2001
volumes:
- name: kubernetes-dashboard-certs
secret:
secretName: kubernetes-dashboard-certs
- name: tmp-volume
emptyDir: {}
serviceAccountName: kubernetes-dashboard
nodeSelector:
"beta.kubernetes.io/os": linux
# Comment the following tolerations if Dashboard must not be deployed on master
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
---
kind: Service
apiVersion: v1
metadata:
labels:
k8s-app: dashboard-metrics-scraper
name: dashboard-metrics-scraper
namespace: kubernetes-dashboard
spec:
ports:
- port: 8000
targetPort: 8000
selector:
k8s-app: dashboard-metrics-scraper
---
kind: Deployment
apiVersion: apps/v1
metadata:
labels:
k8s-app: dashboard-metrics-scraper
name: dashboard-metrics-scraper
namespace: kubernetes-dashboard
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
k8s-app: dashboard-metrics-scraper
template:
metadata:
labels:
k8s-app: dashboard-metrics-scraper
annotations:
seccomp.security.alpha.kubernetes.io/pod: 'runtime/default'
spec:
containers:
- name: dashboard-metrics-scraper
image: kubernetesui/metrics-scraper:v1.0.4
ports:
- containerPort: 8000
protocol: TCP
livenessProbe:
httpGet:
scheme: HTTP
path: /
port: 8000
initialDelaySeconds: 30
timeoutSeconds: 30
volumeMounts:
- mountPath: /tmp
name: tmp-volume
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
runAsUser: 1001
runAsGroup: 2001
serviceAccountName: kubernetes-dashboard
nodeSelector:
"beta.kubernetes.io/os": linux
# Comment the following tolerations if Dashboard must not be deployed on master
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
volumes:
- name: tmp-volume
emptyDir: {}
[[root@k8s-master ~]# kubectl create -f recommended.yaml
或者 kubectl apply -f recommended.yaml
namespace/kubernetes-dashboard created
serviceaccount/kubernetes-dashboard created
service/kubernetes-dashboard created
secret/kubernetes-dashboard-certs created
secret/kubernetes-dashboard-csrf created
secret/kubernetes-dashboard-key-holder created
configmap/kubernetes-dashboard-settings created
role.rbac.authorization.k8s.io/kubernetes-dashboard created
clusterrole.rbac.authorization.k8s.io/kubernetes-dashboard created
rolebinding.rbac.authorization.k8s.io/kubernetes-dashboard created
clusterrolebinding.rbac.authorization.k8s.io/kubernetes-dashboard created
deployment.apps/kubernetes-dashboard created
service/dashboard-metrics-scraper created
deployment.apps/dashboard-metrics-scraper created
查看pod，service：
[root@k8s-master ~]# kubectl get pod -n kubernetes-dashboard
NAME READY STATUS RESTARTS AGE
dashboard-metrics-scraper-dc6947fbf-gcqg4 1/1 Running 0 18h
kubernetes-dashboard-5d4dc8b976-t8tmm 1/1 Running 0 18h
[root@k8s-master ~]# kubectl get svc -n kubernetes-dashboard
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
dashboard-metrics-scraper ClusterIP 10.1.85.176 <none> 8000/TCP 18h
kubernetes-dashboard NodePort 10.1.134.194 <none> 443:30001/TCP 18h
[root@k8s-master ~]#
查看token：
[root@k8s-master ~]# kubectl get secret -o wide -n kubernetes-dashboard
NAME TYPE DATA AGE
dashboard-admin-token-rhgbh kubernetes.io/service-account-token 3 18h
default-token-z7qxp kubernetes.io/service-account-token 3 18h
kubernetes-dashboard-certs Opaque 0 18h
kubernetes-dashboard-csrf Opaque 1 18h
kubernetes-dashboard-key-holder Opaque 2 18h
kubernetes-dashboard-token-fgxgw kubernetes.io/service-account-token 3 18h
[root@k8s-master ~]#
[root@k8s-master ~]# kubectl describe secret kubernetes-dashboard-token-fgxgw -n kubernetes-dashboard
Name: kubernetes-dashboard-token-fgxgw
Namespace: kubernetes-dashboard
Labels: <none>
Annotations: kubernetes.io/service-account.name: kubernetes-dashboard
kubernetes.io/service-account.uid: 5c6e3efe-53e3-4bde-8440-a006ccd51540
Type: kubernetes.io/service-account-token
Data
====
ca.crt: 1025 bytes
namespace: 20 bytes
token: eyJhbGciOiJSUzI1NiIsImtpZCI6InVlLXVwZW1WNnYtYkdPSzZ2ZjNlclRpaHM2UzMwSTFwZEc3TG9ZR2swZjQifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJrdWJlcm5ldGVzLWRhc2hib2FyZC10b2tlbi1mZ3hndyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjVjNmUzZWZlLTUzZTMtNGJkZS04NDQwLWEwMDZjY2Q1MTU0MCIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlcm5ldGVzLWRhc2hib2FyZDprdWJlcm5ldGVzLWRhc2hib2FyZCJ9.ApKr-LoiNlzxk4mLC5_TsrpPWGYVwQJ919SP_1z5t5Rs3i-Qr4nyHcW4S5Pcp8P5P6HYQWbjYtWCk1aAf_w3apBr7oAxEPjCStLPVrXssSGzmTYRXaYoMV3cp2YG7uWR39wFEGRlvxSLB_C1zEQAy_EOjofWioopS9TZEP8zkHaO_-7gA6lp87R40tcA_SQrOX3RM-cFeaMxDKAvhAU_rlGrwq7FLsICwswfcHKQUbPcOyevQEEybFnxDfwfXemrtTKQzQXuLoPIEnHU1LUpbLsYP4G_DMNqdpU_O8xFmoB7GVCEuDipOwOF316ykBLVoPZaAdE9Xm0Sf-B4Dp_J2Q
通過頁面訪問，推薦使用firefox瀏覽器
https://192.168.0.106:30001/#/login
登錄後如下展示，如果沒有namespace可選，並且提示找不到資源 ，那麼就是權限問題
查看dashboard日誌：
[root@k8s-master ~]# kubectl logs -f -n kubernetes-dashboard kubernetes-dashboard-5d4dc8b976-t8tmm
查看日誌有如下報錯：
Non-critical error occurred during resource retrieval: pods is forbidden: User "system:serviceaccount:kube-dashboard:kubernetes-dashboard" cannot list resource "pods" in API group "" in the namespace "kube-dashboard"
解決方法
[root@master01 ~]# kubectl create clusterrolebinding serviceaccounts-cluster-admin --clusterrole=cluster-admin --group=system:serviceaccounts
clusterrolebinding.rbac.authorization.k8s.io/serviceaccount-cluster-admin created
或者
創建一個如下的yaml文件：
[root@k8s-master ~]# vim k8s-admin.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: dashboard-admin
namespace: kubernetes-dashboard
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: dashboard-admin
subjects:
- kind: ServiceAccount
name: dashboard-admin
namespace: kubernetes-dashboard
roleRef:
kind: ClusterRole
name: cluster-admin
apiGroup: rbac.authorization.k8s.io
[root@k8s-master ~]#
創建用戶：
[root@k8s-master ~]# kubectl create -f k8s-admin.yaml
刪除：
[root@k8s-master ~]# kubectl delete -f k8s-admin.yaml
然後拿到這個新的secret的token即可
[root@k8s-master ~]# kubectl describe secret kubernetes-dashboard-token-fgxgw -n kubernetes-dashboard
Name: kubernetes-dashboard-token-fgxgw
Namespace: kubernetes-dashboard
Labels: <none>
Annotations: kubernetes.io/service-account.name: kubernetes-dashboard
kubernetes.io/service-account.uid: 5c6e3efe-53e3-4bde-8440-a006ccd51540
Type: kubernetes.io/service-account-token
Data
====
ca.crt: 1025 bytes
namespace: 20 bytes
token: eyJhbGciOiJSUzI1NiIsImtpZCI6InVlLXVwZW1WNnYtYkdPSzZ2ZjNlclRpaHM2UzMwSTFwZEc3TG9ZR2swZjQifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJrdWJlcm5ldGVzLWRhc2hib2FyZC10b2tlbi1mZ3hndyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjVjNmUzZWZlLTUzZTMtNGJkZS04NDQwLWEwMDZjY2Q1MTU0MCIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlcm5ldGVzLWRhc2hib2FyZDprdWJlcm5ldGVzLWRhc2hib2FyZCJ9.ApKr-LoiNlzxk4mLC5_TsrpPWGYVwQJ919SP_1z5t5Rs3i-Qr4nyHcW4S5Pcp8P5P6HYQWbjYtWCk1aAf_w3apBr7oAxEPjCStLPVrXssSGzmTYRXaYoMV3cp2YG7uWR39wFEGRlvxSLB_C1zEQAy_EOjofWioopS9TZEP8zkHaO_-7gA6lp87R40tcA_SQrOX3RM-cFeaMxDKAvhAU_rlGrwq7FLsICwswfcHKQUbPcOyevQEEybFnxDfwfXemrtTKQzQXuLoPIEnHU1LUpbLsYP4G_DMNqdpU_O8xFmoB7GVCEuDipOwOF316ykBLVoPZaAdE9Xm0Sf-B4Dp_J2Q
通過頁面訪問，推薦使用firefox瀏覽器
https://192.168.0.106:30001/#/login
