系統版本:CentOS7.5
Kubernetes版本:v1.15.0
docker版本:18.06
k8s-vip | k8s-m1 | k8s-m2 | k8s-m3 |
192.168.1.200 | 192.168.1.201 | 192.168.1.202 | 192.168.1.203 |
配置主機名
cat /etc/hosts 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 192.168.1.201 k8s-m1 192.168.1.202 k8s-m2 192.168.1.203 k8s-m3 hostnamectl set-hostname k8s-m1/2/3
ssh免祕鑰登錄
ssh-keygen ssh-copy-id [email protected]/202/203
初始化腳本
cat init_env.sh #!/bin/bash #關閉防火牆 SELINUX systemctl stop firewalld && systemctl disable firewalld setenforce 0 sed -i 's/^SELINUX=enforcing$/SELINUX=disable/' /etc/selinux/config #關閉swap swapoff -a && sysctl -w vm.swappiness=0 sed -i 's/.*swap.*/#&/g' /etc/fstab #設置Docker所需參數 cat > /etc/sysctl.d/k8s.conf << EOF net.ipv4.ip_forward = 1 net.bridge.bridge-nf-call-ip6tables = 1 net.bridge.bridge-nf-call-iptables = 1 EOF modprobe br_netfilter sysctl -p /etc/sysctl.d/k8s.conf #加載ip_vs模塊 cat > /etc/sysconfig/modules/ipvs.modules <<EOF #!/bin/bash modprobe -- ip_vs modprobe -- ip_vs_rr modprobe -- ip_vs_wrr modprobe -- ip_vs_sh modprobe -- nf_conntrack_ipv4 EOF chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4 #安裝docker18.06版本 yum -y install yum-utils device-mapper-persistent-data lvm2 wget epel-release ipvsadm vim ntpdate yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo yum install -y docker-ce-18.06.1.ce-3.el7 cat > /etc/docker/daemon.json <<EOF { "exec-opts": ["native.cgroupdriver=systemd"], "registry-mirrors": ["https://gco4rcsp.mirror.aliyuncs.com"], "storage-driver": "overlay2", "storage-opts": [ "overlay2.override_kernel_check=true" ], "log-driver": "json-file", "log-opts": { "max-size": "100m", "max-file": "3" } } EOF systemctl enable docker && systemctl daemon-reload && systemctl restart docker #安裝kube組件 cat > /etc/yum.repos.d/kubernetes.repo <<EOF [kubernetes] name=Kubernetes baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/ enabled=1 gpgcheck=0 repo_gpgcheck=0 gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg EOF yum install -y kubelet kubeadm kubectl --disableexcludes=kubernetes
#所有節點時間同步計劃任務 0 * * * * ntpdate 202.112.10.36
#設置環境變量 export ETCD_version=v3.3.12 export ETCD_SSL_DIR=/etc/etcd/ssl export APISERVER_IP=192.168.1.200 export SYSTEM_SERVICE_DIR=/usr/lib/systemd/system
#cfssl下載 wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -O /usr/local/bin/cfssl wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -O /usr/local/bin/cfssljson chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson
#ETCD下載 wget https://github.com/etcd-io/etcd/releases/download/${ETCD_version}/etcd-${ETCD_version}-linux-amd64.tar.gz tar -zxvf etcd-${ETCD_version}-linux-amd64.tar.gz && cd etcd-${ETCD_version}-linux-amd64 #複製可執行文件到其餘master節點 for node in k8s-m{1,2,3};do scp etcd* root@$node:/usr/local/bin/;done
#生成etcd CA、etcd證書 mkdir -p ${ETCD_SSL_DIR} && cd ${ETCD_SSL_DIR} cat > ca-config.json <<EOF {"signing":{"default":{"expiry":"87600h"},"profiles":{"kubernetes":{"usages":["signing","key encipherment","server auth","client auth"],"expiry":"87600h"}}}} EOF cat > etcd-ca-csr.json <<EOF {"CN":"etcd","key":{"algo":"rsa","size":2048},"names":[{"C":"CN","ST":"BeiJing","L":"BeiJing","O":"etcd","OU":"etcd"}]} EOF cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare etcd-ca cfssl gencert -ca=etcd-ca.pem -ca-key=etcd-ca-key.pem -config=ca-config.json -hostname=127.0.0.1,192.168.1.201,192.168.1.202,192.168.1.203 -profile=kubernetes etcd-ca-csr.json | cfssljson -bare etcd rm -rf *.json *.csr for node in k8s-m{2,3};do ssh root@$node mkdir -p ${ETCD_SSL_DIR} /var/lib/etcd;scp * root@$node:${ETCD_SSL_DIR};done
#ETCD配置文件 cat /etc/etcd/config #[Member] ETCD_NAME="etcd01" ETCD_DATA_DIR="/var/lib/etcd/default.etcd" ETCD_LISTEN_PEER_URLS="https://192.168.1.201:2380" ETCD_LISTEN_CLIENT_URLS="https://192.168.1.201:2379" #[Clustering] ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.201:2380" ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.201:2379" ETCD_INITIAL_CLUSTER="etcd01=https://192.168.1.201:2380,etcd02=https://192.168.1.202:2380,etcd03=https://192.168.1.203:2380" ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster" ETCD_INITIAL_CLUSTER_STATE="new" cat ${SYSTEM_SERVICE_DIR}/etcd.service [Unit] Description=Etcd Server After=neCNork.target After=network-online.target Wants=network-online.target [Service] Type=notify EnvironmentFile=/etc/etcd/config ExecStart=/usr/local/bin/etcd \ --name=${ETCD_NAME} \ --data-dir=${ETCD_DATA_DIR} \ --listen-peer-urls=${ETCD_LISTEN_PEER_URLS} \ --listen-client-urls=${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \ --advertise-client-urls=${ETCD_ADVERTISE_CLIENT_URLS} \ --initial-advertise-peer-urls=${ETCD_INITIAL_ADVERTISE_PEER_URLS} \ --initial-cluster=${ETCD_INITIAL_CLUSTER} \ --initial-cluster-token=${ETCD_INITIAL_CLUSTER_TOKEN} \ --initial-cluster-state=new \ --cert-file=/etc/etcd/ssl/etcd.pem \ --key-file=/etc/etcd/ssl/etcd-key.pem \ --peer-cert-file=/etc/etcd/ssl/etcd.pem \ --peer-key-file=/etc/etcd/ssl/etcd-key.pem \ --trusted-ca-file=/etc/etcd/ssl/etcd-ca.pem \ --peer-trusted-ca-file=/etc/etcd/ssl/etcd-ca.pem Restart=on-failure LimitNOFILE=65536 [Install] WantedBy=multi-user.target
#複製配置文件及啓動文件到其餘主節點 for node in k8s-m{2,3};do scp /etc/etcd/config root@$node:/etc/etcd/;scp ${SYSTEM_SERVICE_DIR}/etcd.service root@$node:${SYSTEM_SERVICE_DIR};done
#修改配置文件後,主節點分別啓動etcd systemctl enable --now etcd
#檢查etcd集羣狀態 etcdctl \ --ca-file=/etc/etcd/ssl/etcd-ca.pem \ --cert-file=/etc/etcd/ssl/etcd.pem \ --key-file=/etc/etcd/ssl/etcd-key.pem \ --endpoints="https://192.168.1.201:2379,\ https://192.168.1.202:2379,\ https://192.168.1.203:2379" cluster-health
#apiserver高可用部署 Haproxy+Keepalived for node in k8s-m{1,2,3};do ssh root@$node yum -y install haproxy keepalived;done #keepalived配置文件,其餘節點修改state爲BACKUP,priority小於主節點即可;檢查網卡名稱並修改 cat > /etc/keepalived/keepalived.conf << EOF vrrp_script check_haproxy { script "/etc/keepalived/check_haproxy.sh" interval 3 } vrrp_instance VI_1 { state MASTER interface ens192 virtual_router_id 51 priority 100 advert_int 1 authentication { auth_type PASS auth_pass 1111 } virtual_ipaddress { ${APISERVER_IP} } track_script { check_haproxy } } EOF cat > /etc/keepalived/check_haproxy.sh <<EOF #!/bin/bash systemctl status haproxy > /dev/null if [[ \$? != 0 ]];then echo "haproxy is down,close the keepalived" systemctl stop keepalived fi EOF cat > /etc/haproxy/haproxy.cfg << EOF global log 127.0.0.1 local2 chroot /var/lib/haproxy pidfile /var/run/haproxy.pid maxconn 4000 user haproxy group haproxy daemon stats socket /var/lib/haproxy/stats #--------------------------------------------------------------------- defaults mode http log global option httplog option dontlognull option http-server-close option forwardfor except 127.0.0.0/8 option redispatch retries 3 timeout http-request 10s timeout queue 1m timeout connect 10s timeout client 1m timeout server 1m timeout http-keep-alive 10s timeout check 10s maxconn 3000 #--------------------------------------------------------------------- frontend k8s-api bind *:8443 mode tcp default_backend apiserver #--------------------------------------------------------------------- backend apiserver balance roundrobin mode tcp server k8s-m1 192.168.1.201:6443 check weight 1 maxconn 2000 check inter 2000 rise 2 fall 3 server k8s-m2 192.168.1.202:6443 check weight 1 maxconn 2000 check inter 2000 rise 2 fall 3 server k8s-m3 192.168.1.203:6443 check weight 1 maxconn 2000 check inter 2000 rise 2 fall 3 EOF
#複製配置文件到其餘主節點 for node in k8s-m{2,3};do scp /etc/keepalived/* root@$node:/etc/keepalived;scp /etc/haproxy/haproxy.cfg root@$node:/etc/haproxy;done
#修改keepalived.conf文件並啓動服務 #修改過程省略 for node in k8s-m{1,2,3};do ssh root@$node systemctl enable --now keepalived haproxy;done #查看VIP是否工作正常ping ${APISERVER_IP} -c 3
#kubeadm初始化
#查看需要拉取的容器鏡像 kubeadm config images list
images=(kube-apiserver:v1.15.0 kube-controller-manager:v1.15.0 kube-scheduler:v1.15.0 kube-proxy:v1.15.0 pause:3.1 etcd:3.3.10 coredns:1.3.1) for image in ${images[@]};do docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/${image};docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/${image} k8s.gcr.io/${image};docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/${image};done
cat kubeadm-config.yaml apiVersion: kubeadm.k8s.io/v1beta2 kind: ClusterConfiguration kubernetesVersion: v1.15.0 controlPlaneEndpoint: "192.168.1.200:8443" etcd: external: endpoints: - https://192.168.1.201:2379 - https://192.168.1.202:2379 - https://192.168.1.203:2379 caFile: /etc/etcd/ssl/etcd-ca.pem certFile: /etc/etcd/ssl/etcd.pem keyFile: /etc/etcd/ssl/etcd-key.pem networking: podSubnet: 10.244.0.0/16 --- apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration mode: ipvs kubeadm init kubeadm-config.yaml --upload-certs
完成會生成加入集羣命令
在其餘節點執行加入集羣命令
查看是否爲IPVS模式
#部署flannel網絡 kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/62e44c867a2846fefb68bd5f178daf4da3095ccb/Documentation/kube-flannel.yml
驗證dns
參考:
https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/
https://blog.51cto.com/13740724/2393698
https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2