官方提供的三種部署方式
- minikube
minikube是一個工具,可以在本地快速運行一個單點的kubernetes,僅用於嘗試K8S或日常開發的測試環境使用
部署地址:https://kubernetes.io/docs/setup/minkube/
- kubeadm
kubeadm也是一個工具,提供kubeadm init和kubeadm join,用於快速部署kubernetes集羣
部署地址:https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm/
- 二進制包
從官方下載發行版的二進制包,手動部署每個組件,組成kubernetes集羣
下載地址:https://github.com/kubernetes/kubernetes/releases
https://github.com/kubernetes/kubernetes/releases?after=v1.13.1
服務器
初步環境部署
1、關閉網絡管理器,清空iptabels,關閉核心防護,編輯主機名
master01:192.168.49.205
[root@localhost ~]# hostnamectl set-hostname master1
[root@localhost ~]# su
[root@master1 ~]# systemctl stop NetworkManager
[root@master1 ~]# systemctl disable NetworkManager
Removed symlink /etc/systemd/system/multi-user.target.wants/NetworkManager.service.
Removed symlink /etc/systemd/system/dbus-org.freedesktop.NetworkManager.service.
Removed symlink /etc/systemd/system/dbus-org.freedesktop.nm-dispatcher.service.
[root@master1 ~]# setenforce 0
[root@master1 ~]# sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
[root@master1 ~]# iptables -F
node01:192.168.49.129
[root@node01 ~]# systemctl stop NetworkManager
[root@node01 ~]# systemctl disable NetworkManager
Removed symlink /etc/systemd/system/multi-user.target.wants/NetworkManager.service.
Removed symlink /etc/systemd/system/dbus-org.freedesktop.NetworkManager.service.
Removed symlink /etc/systemd/system/dbus-org.freedesktop.nm-dispatcher.service.
[root@node01 ~]# setenforce 0
[root@node01 ~]# sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
[root@node01 ~]# iptables -F
node02:192.168.49.130
[root@localhost ~]# hostnamectl set-hostname node02
[root@localhost ~]# su
[root@node02 ~]# systemctl stop NetworkManager
[root@node02 ~]# systemctl disable NetworkManager
Removed symlink /etc/systemd/system/multi-user.target.wants/NetworkManager.service.
Removed symlink /etc/systemd/system/dbus-org.freedesktop.NetworkManager.service.
Removed symlink /etc/systemd/system/dbus-org.freedesktop.nm-dispatcher.service.
Removed symlink /etc/systemd/system/network-online.target.wants/NetworkManager-wait-online.service.
[root@node02 ~]# setenforce 0
[root@node02 ~]# sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
[root@node02 ~]# iptables -F
2、創建ca證書,各組件之間的通訊必須有ca證書
創建臨時目錄
[root@master1 k8s]# mkdir /abc
[root@master1 k8s]# mount -t cifs //192.168.56.1/anzhuangbao/ /abc -o username=anonymous,vers=2.0
Password for anonymous@//192.168.56.1/anzhuangbao/:
[root@master1 k8s]# cp /abc/k8s/etcd* .
######3、etcd-cert.sh用來創建關於etcd的CA證書
expiry 有效期10年
使用密鑰驗證 key encipherment
cat > ca-config.json <<EOF
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"www": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
EOF
cat > ca-csr.json <<EOF
{
"CN": "etcd CA",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "Beijing",
"ST": "Beijing"
}
]
}
EOF
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
#-----------------------
cat > server-csr.json <<EOF
{
"CN": "etcd",
"hosts": [
"192.168.49.205",
"192.168.49.129",
"192.168.49.130"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing"
}
]
}
EOF
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server
4、etcd.sh用來創建啓動腳本和配置文件
2380是etcd之間進行通訊的端口
2379是etcd對外提供的端口
cat etcd.sh
#!/bin/bash
# example: ./etcd.sh etcd01 192.168.1.10 etcd02=https://192.168.1.11:2380,etcd03=https://192.168.1.12:2380
ETCD_NAME=$1
ETCD_IP=$2
ETCD_CLUSTER=$3
WORK_DIR=/opt/etcd
cat <<EOF >$WORK_DIR/cfg/etcd
#[Member]
ETCD_NAME="${ETCD_NAME}"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://${ETCD_IP}:2380"
ETCD_LISTEN_CLIENT_URLS="https://${ETCD_IP}:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://${ETCD_IP}:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://${ETCD_IP}:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://${ETCD_IP}:2380,${ETCD_CLUSTER}"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF
cat <<EOF >/usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=${WORK_DIR}/cfg/etcd
ExecStart=${WORK_DIR}/bin/etcd \
--name=\${ETCD_NAME} \
--data-dir=\${ETCD_DATA_DIR} \
--listen-peer-urls=\${ETCD_LISTEN_PEER_URLS} \
--listen-client-urls=\${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \
--advertise-client-urls=\${ETCD_ADVERTISE_CLIENT_URLS} \
--initial-advertise-peer-urls=\${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
--initial-cluster=\${ETCD_INITIAL_CLUSTER} \
--initial-cluster-token=\${ETCD_INITIAL_CLUSTER_TOKEN} \
--initial-cluster-state=new \
--cert-file=${WORK_DIR}/ssl/server.pem \
--key-file=${WORK_DIR}/ssl/server-key.pem \
--peer-cert-file=${WORK_DIR}/ssl/server.pem \
--peer-key-file=${WORK_DIR}/ssl/server-key.pem \
--trusted-ca-file=${WORK_DIR}/ssl/ca.pem \
--peer-trusted-ca-file=${WORK_DIR}/ssl/ca.pem
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable etcd
systemctl restart etcd
5、下載cfssl官方腳本包
cfssl 生成證書工具
cfssljson 通過傳入json文件生成證書
cfssl-certinfo 查看證書信息
-o 導出
vim cfss.sh
curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o /usr/local/bin/cfssl
curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o /usr/local/bin/cfssljson
curl -L https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -o /usr/local/bin/cfssl-certinfo
chmod +x /usr/local/bin/cfssl*
查看驗證
6、創建etcd組件證書臨時目錄
[root@master1 k8s]# mkdir etcd-cert
[root@master1 k8s]# mv etcd-cert.sh etcd-cert
7、定義ca證書配置
[root@master1 k8s]# cd etcd-cert/
[root@master1 etcd-cert]# ls
etcd-cert.sh
[root@master1 etcd-cert]# cat > ca-config.json <<EOF
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"www": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
EOF
[root@master1 etcd-cert]# ls
ca-config.json etcd-cert.sh
8、實現ca證書籤名
[root@master1 etcd-cert]# cat > ca-csr.json <<EOF
{
"CN": "etcd CA",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "Beijing",
"ST": "Beijing"
}
]
}
EOF
[root@master1 etcd-cert]# ls
ca-config.json ca-csr.json etcd-cert.sh
9、生產證書,生成ca-key.pem ca.pem這兩個文件
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
[root@master1 etcd-cert]# ls
ca-config.json ca.csr ca-csr.json ca-key.pem ca.pem etcd-cert.sh
10、指定etcd三個節點之間的通信驗證
cat > server-csr.json <<EOF
{
"CN": "etcd",
"hosts": [
"192.168.247.149",
"192.168.247.143",
"192.168.247.144"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing"
}
]
}
EOF
[root@master1 etcd-cert]# ls
ca-config.json ca.csr ca-csr.json ca-key.pem ca.pem etcd-cert.sh server-csr.json
11、生成etcd的server證書和密鑰
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server
12、Etcd數據庫集羣部署
https://github.com/etcd-io/etcd/releases
我下載好直接拉到本地
[root@master1 etcd-cert]# cp /abc/k8s/etcd-v3.3.10-linux-amd64.tar.gz /root/k8s/
[root@master1 etcd-cert]# cd ..
[root@master1 k8s]# pwd
/root/k8s
[root@master1 k8s]# ls
etcd-cert etcd.sh etcd-v3.3.10-linux-amd64.tar.gz
[root@master1 k8s]# tar xf etcd-v3.3.10-linux-amd64.tar.gz
[root@master1 k8s]# ls
etcd-cert etcd.sh etcd-v3.3.10-linux-amd64 etcd-v3.3.10-linux-amd64.tar.gz
[root@master1 k8s]# cd etcd-v3.3.10-linux-amd64/
[root@master1 etcd-v3.3.10-linux-amd64]# ls
Documentation etcd etcdctl README-etcdctl.md README.md READMEv2-etcdctl.md
13、創建etcd的工作目錄,下面還有配置文件cfg,命令bin,證書ssl的目錄
[root@master1 etcd-v3.3.10-linux-amd64]# mkdir /k8s/etcd/{cfg,bin,ssl} -p
[root@master1 etcd-v3.3.10-linux-amd64]# cd /k8s
[root@master1 k8s]# tree .
.
└── etcd
├── bin
├── cfg
└── ssl
4 directories, 0 files
14、將證書文件和命令文件複製過來
[root@master1 k8s]# mv /root/k8s/etcd-v3.3.10-linux-amd64/etcd* /k8s/etcd/bin/
[root@master1 k8s]# cp /root/k8s/etcd-cert/*.pem /k8s/etcd/ssl/
[root@master1 k8s]# tree .
15、編輯etcd的配置文件和啓動腳本
[root@master1 ~]# cd /k8s/
[root@master1 k8s]# tree .
vim etcd.sh
#!/bin/bash
# example: ./etcd.sh etcd01 192.168.49.205 etcd02=https://192.168.49.129:2380,etcd03=https://192.168.49.130:2380
ETCD_NAME=$1
ETCD_IP=$2
ETCD_CLUSTER=$3
WORK_DIR=/k8s/etcd
cat <<EOF >$WORK_DIR/cfg/etcd
#[Member]
ETCD_NAME="${ETCD_NAME}"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://${ETCD_IP}:2380"
ETCD_LISTEN_CLIENT_URLS="https://${ETCD_IP}:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://${ETCD_IP}:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://${ETCD_IP}:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://${ETCD_IP}:2380,${ETCD_CLUSTER}"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF
cat <<EOF >/usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=${WORK_DIR}/cfg/etcd
ExecStart=${WORK_DIR}/bin/etcd \
--name=\${ETCD_NAME} \
--data-dir=\${ETCD_DATA_DIR} \
--listen-peer-urls=\${ETCD_LISTEN_PEER_URLS} \
--listen-client-urls=\${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \
--advertise-client-urls=\${ETCD_ADVERTISE_CLIENT_URLS} \
--initial-advertise-peer-urls=\${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
--initial-cluster=\${ETCD_INITIAL_CLUSTER} \
--initial-cluster-token=\${ETCD_INITIAL_CLUSTER_TOKEN} \
--initial-cluster-state=new \
--cert-file=${WORK_DIR}/ssl/server.pem \
--key-file=${WORK_DIR}/ssl/server-key.pem \
--peer-cert-file=${WORK_DIR}/ssl/server.pem \
--peer-key-file=${WORK_DIR}/ssl/server-key.pem \
--trusted-ca-file=${WORK_DIR}/ssl/ca.pem \
--peer-trusted-ca-file=${WORK_DIR}/ssl/ca.pem
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable etcd
systemctl restart etcd
chmod +x etcd.sh
sh etcd.sh
執行會報錯,不管繼續做後面,因爲還沒有做後面兩個節點
cd /k8s
tree .
[root@master1 k8s]# ll /usr/lib/systemd/system/ | grep etcd
-rw-r--r--. 1 root root 923 4月 30 10:07 etcd.service
16、第一步產生配置文件,啓動腳本生成到systemd下
端口,2379是提供給外部端口,2380是內部集羣通訊端口,最多65536端口
此時進入一個等待狀態,查找別的etcd集羣節點,查找不到過5分鐘默認退出
bash etcd.sh etcd01 192.168.49.205 etcd02=https://192.168.49.129:2380,etcd03=https://192.168.49.130:2380
拷貝證書去另外兩個節點
[root@master1 ~]# cd /root/k8s/
[root@master1 k8s]# ls
cfss.sh etcd- etcd-cert etcd.sh etcd-v3.3.10-linux-amd64 etcd-v3.3.10-linux-amd64.tar.gz
[root@master1 k8s]# pwd
/root/k8s
[root@master1 k8s]# ./etcd.sh etcd01 192.168.49.205 etcd02=https://192.168.49.129:2380,etcd03=https://192.168.49.130:2380
查看端口
17、拷貝證書去另外兩個節點
到node01節點去查看驗證
18、啓動腳本
scp /usr/lib/systemd/system/etcd.service [email protected]:/usr/lib/systemd/system/
scp /usr/lib/systemd/system/etcd.service [email protected]:/usr/lib/systemd/system/
19、相關文件複製過去了,/k8s/etcd/cfg/etcd配置文件中有些參數需要修改
node01節點
node02節點
20、開啓節點服務
[root@node01 k8s]# vim /k8s/etcd/cfg/etcd
[root@node01 k8s]# systemctl start etcd
[root@node01 k8s]# systemctl status etcd
21、此時在主節點master1上重新執行腳本命令
./etcd.sh etcd01 192.168.49.205 etcd02=https://192.168.49.129:2380,etcd03=https://192.168.49.130:2380
可以檢查下集羣狀態
/k8s/etcd/bin/etcdctl \
--ca-file=/k8s/etcd/ssl/ca.pem \
--cert-file=/k8s/etcd/ssl/server.pem --key-file=/k8s/etcd/ssl/server-key.pem \
--endpoints="https://192.168.49.205:2379,https://192.168.49.129:2379,https://192.168.49.130:2379" \
cluster-health
查看2379端口
node安裝docker
22、我這裏只演示node01,另外node02一樣操作
sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
setenforce 0
23 、安裝docker-ce
[root@node01 ~]# yum install -y yum-utils device-mapper-persistent-data lvm2
[root@node01 ~]# cd /etc/yum.repos.d/ && yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
[root@node01 yum.repos.d]# ls
CentOS-Base.repo CentOS-Debuginfo.repo CentOS-Media.repo CentOS-Vault.repo
CentOS-CR.repo CentOS-fasttrack.repo CentOS-Sources.repo docker-ce.repo
[root@node01 yum.repos.d]# yum install -y docker-ce
[root@node01 yum.repos.d]# systemctl start docker
[root@node01 yum.repos.d]# systemctl enable docker
[root@node01 yum.repos.d]# tee /etc/docker/daemon.json <<-'EOF'
{
"registry-mirrors": ["https://fk2yrsh1.mirror.aliyuncs.com"]
}
EOF
[root@node01 yum.repos.d]# echo "net.ipv4.ip_forward=1" >> /etc/sysctl.conf
[root@node01 yum.repos.d]# sysctl -p
net.ipv4.ip_forward = 1
[root@node01 yum.repos.d]# systemctl restart network
[root@node01 yum.repos.d]# systemctl restart docker
k8s/etcd/bin/etcdctl \
--ca-file=/k8s/etcd/ssl/ca.pem \
--cert-file=/k8s/etcd/ssl/server.pem --key-file=/k8s/etcd/ssl/server-key.pem \
--endpoints="https://192.168.49.205:2379,https://192.168.49.129:2379,https://192.168.49.130:2379" \
set /coreos.com/network/config '{ "network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}'
安裝flannel
flannel網絡組件,還有一個是calico,calico支持bgp
overlay network:覆蓋網絡,在基礎網絡上疊加的一種虛擬網絡技術模式,該網絡中的主機通過虛擬鏈路tunnmel連接起來
vxlan:將原數據包封裝到UDP協議中,並使用基礎網絡的IP/mac作爲外層報文頭進行封裝,然後在以太網二層鏈路上傳輸,到達目的地後由隧道端點解封裝並將數據發送給目標地址
24、 寫入分配的子網段到etcd中,共flannel使用
[root@master1 k8s]# /k8s/etcd/bin/etcdctl --ca-file=/k8s/etcd/ssl/ca.pem --cert-file=/k8s/etcd/ssl/server.pem --key-file=/k8s/etcd/ssl/server-key.pem --endpoints="https://192.168.49.205:2379,https://192.168.49.129:2379,https://192.168.49.130:2379" set /coreos.com/network/config '{ "network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}'
{ "network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}
25、查看寫入信息,別的節點也能查看到
/k8s/etcd/bin/etcdctl \
--ca-file=/k8s/etcd/ssl/ca.pem \
--cert-file=/k8s/etcd/ssl/server.pem --key-file=/k8s/etcd/ssl/server-key.pem \
--endpoints="https://192.168.49.205:2379,https://192.168.49.129:2379,https://192.168.49.130:2379" \
get /coreos.com/network/config
26、導入二進制包,flannel安裝在node節點
誰需要跑業務資源,設就需要安裝fannel
[root@master1 /]# cp /abc/k8s/flannel-v0.10.0-linux-amd64.tar.gz /root/k8s/
[root@master1 /]# cd /root/k8s/
[root@master1 k8s]# ls
cfss.sh etcd-cert etcd-v3.3.10-linux-amd64 flannel-v0.10.0-linux-amd64.tar.gz
etcd- etcd.sh etcd-v3.3.10-linux-amd64.tar.gz
[root@master1 k8s]# scp flannel-v0.10.0-linux-amd64.tar.gz [email protected]:/opt/
[email protected]'s password:
flannel-v0.10.0-linux-amd64.tar.gz 100% 9479KB 39.7MB/s 00:00
[root@master1 k8s]# scp flannel-v0.10.0-linux-amd64.tar.gz [email protected]:/opt/
[email protected]'s password:
flannel-v0.10.0-linux-amd64.tar.gz
27、部署與配置flannel,編輯flannel啓動腳本,加入到systemd中
node01節點爲例
[root@node01 yum.repos.d]# cd /opt
[root@node01 opt]# tar xf flannel-v0.10.0-linux-amd64.tar.gz
[root@node01 opt]# ls
containerd flanneld flannel-v0.10.0-linux-amd64.tar.gz mk-docker-opts.sh README.md rh
創建flannel工作目錄
[root@node01 opt]# mkdir /k8s/flannel/{cfg,bin,ssl} -p
[root@node01 opt]# mv mk-docker-opts.sh /k8s/flannel/bin/
[root@node01 opt]# mv flanneld /k8s/flannel/bin/
每個node節點都要做
vim flannel.sh
#!/bin/bash
ETCD_ENDPOINTS=${1:-"http://127.0.0.1:2379"}
cat <<EOF >/k8s/flannel/cfg/flanneld
FLANNEL_OPTIONS="--etcd-endpoints=${ETCD_ENDPOINTS} \
-etcd-cafile=/k8s/etcd/ssl/ca.pem \
-etcd-certfile=/k8s/etcd/ssl/server.pem \
-etcd-keyfile=/k8s/etcd/ssl/server-key.pem"
EOF
cat <<EOF >/usr/lib/systemd/system/flanneld.service
[Unit]
Description=Flanneld overlay address etcd agent
After=network-online.target network.target
Before=docker.service
[Service]
Type=notify
EnvironmentFile=/k8s/flannel/cfg/flanneld
ExecStart=/k8s/flannel/bin/flanneld --ip-masq \$FLANNEL_OPTIONS
ExecStartPost=/k8s/flannel/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/subnet.env
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable flanneld
systemctl restart flanneld
28、開啓flannel網絡功能,指定etcdIP:端口
bash flannel.sh https://192.168.49.205:2379,https://192.168.49.129:2379,https://192.168.49.130:2379
查看狀態
29、配置docker,以使用flannel生成的子網
以node01爲例,別的節點也要做
讓docker連接flannel的網段
vim /usr/lib/systemd/system/docker.service
13行下插入
EnvironmentFile=/run/flannel/subnet.env
修改15行
ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS -H fd:// --containerd=/run/containerd/contain erd.sock
30、啓動flannel
[root@node01 flannel]# systemctl daemon-reload
[root@node01 flannel]# systemctl restart docker
31、查看node01節點分配的flannelIP地址
查看flannel網絡
32、此時便可以讓不同node間的容器互聯互通
測試一下,兩個node各創建容器測試ping
node01 ping node02
node02 ping node01