rancher ha

环境

IP	别名	配置	应用
172.19.129.68	op	2U4G	harbor / pg / nginx
172.19.201.241	m1	2U8G	k3s master
172.19.201.242	m2	2U8G	k3s master
172.19.201.243	m3	2U8G	k3s master

Harbor安装

harbor离线docker安装

软件

K3s(1.18.2)

https://github.com/rancher/k3s/releases/download/v1.18.2%2Bk3s1/k3s
https://github.com/rancher/k3s/releases/download/v1.18.2%2Bk3s1/k3s-airgap-images-amd64.tar
curl -sfL https://get.k3s.io -o install.sh

Rancher

rancher-images.txt
rancher-load-images.sh
rancher-save-images.sh

打包images

chmod a+x ./rancher-save-images.sh
./rancher-save-images.sh --image-list ./rancher-images.txt

推送images到私有仓库

harbor仓库中创建rancher项目

docker login -u admin -p Harbor12345 harbor.${domain}
# harbor中创建项目： rancher
sh ./rancher-load-images.sh --image-list ./rancher-images.txt --registry harbor.${domain}

安装K3s集群

挂载光盘源(centos8)

mkdir -p /media/cdrom
mount /dev/sr0 -t iso9660 /media/cdrom

rm -f /etc/yum.repos.d/*.repo
cat > /etc/yum.repos.d/CentOS-Media.repo << EOF
[c8-media-BaseOS]
name=CentOS-BaseOS-$releasever - Media
baseurl=file:///media/cdrom/BaseOS
gpgcheck=0
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial

[c8-media-AppStream]
name=CentOS-AppStream-$releasever - Media
baseurl=file:///media/cdrom/AppStream
gpgcheck=0
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial
EOF

## 重建cache
yum clean all
yum makecache
## 开机有效
echo "/dev/sr0    /media/cdrom     auto    defaults    0  0" >> /etc/fstab

所有K3s节点

脚本变量

# ip
export do=172.19.129.68
export m1=172.19.201.241
export m2=172.19.201.242
export m3=172.19.201.243
export domain=do.io

脚本

# 修改提示符
echo "export PS1='[\u@\h \t \w]\\$ '" >> /etc/profile
# 系统语言
echo 'LANG="en_US.UTF-8"' >> /etc/profile
# harbor认证
echo "export domain=${domain}" >> /etc/profile
echo "${do} harbor.${domain}" >> /etc/hosts
echo "${do} do" >> /etc/hosts
echo "${m1} m1" >> /etc/hosts
echo "${m2} m2" >> /etc/hosts
echo "${m3} m3" >> /etc/hosts
source /etc/profile
# 优化OS
cat > /etc/sysctl.conf << EOF
net.bridge.bridge-nf-call-ip6tables=1
net.bridge.bridge-nf-call-iptables=1
net.ipv4.ip_forward=1
net.ipv4.conf.all.forwarding=1
net.ipv4.neigh.default.gc_thresh1=4096
net.ipv4.neigh.default.gc_thresh2=6144
net.ipv4.neigh.default.gc_thresh3=8192
net.ipv4.neigh.default.gc_interval=60
net.ipv4.neigh.default.gc_stale_time=120

# 参考 https://github.com/prometheus/node_exporter#disabled-by-default
kernel.perf_event_paranoid=-1

#sysctls for k8s node config
net.ipv4.tcp_slow_start_after_idle=0
net.core.rmem_max=16777216
fs.inotify.max_user_watches=524288
kernel.softlockup_all_cpu_backtrace=1

kernel.softlockup_panic=0

kernel.watchdog_thresh=30
fs.file-max=2097152
fs.inotify.max_user_instances=8192
fs.inotify.max_queued_events=16384
vm.max_map_count=262144
fs.may_detach_mounts=1
net.core.netdev_max_backlog=16384
net.ipv4.tcp_wmem=4096 12582912 16777216
net.core.wmem_max=16777216
net.core.somaxconn=32768
net.ipv4.ip_forward=1
net.ipv4.tcp_max_syn_backlog=8096
net.ipv4.tcp_rmem=4096 12582912 16777216

net.ipv6.conf.all.disable_ipv6=1
net.ipv6.conf.default.disable_ipv6=1
net.ipv6.conf.lo.disable_ipv6=1

kernel.yama.ptrace_scope=0
vm.swappiness=0

# 可以控制core文件的文件名中是否添加pid作为扩展。
kernel.core_uses_pid=1

# Do not accept source routing
net.ipv4.conf.default.accept_source_route=0
net.ipv4.conf.all.accept_source_route=0

# Promote secondary addresses when the primary address is removed
net.ipv4.conf.default.promote_secondaries=1
net.ipv4.conf.all.promote_secondaries=1

# Enable hard and soft link protection
fs.protected_hardlinks=1
fs.protected_symlinks=1

# 源路由验证
# see details in https://help.aliyun.com/knowledge_detail/39428.html
net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.default.rp_filter=0
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.lo.arp_announce=2
net.ipv4.conf.all.arp_announce=2

# see details in https://help.aliyun.com/knowledge_detail/41334.html
net.ipv4.tcp_max_tw_buckets=5000
net.ipv4.tcp_syncookies=1
net.ipv4.tcp_fin_timeout=30
net.ipv4.tcp_synack_retries=2
kernel.sysrq=1
EOF

cat > /etc/security/limits.conf << EOF
root soft nofile 65535
root hard nofile 65535
* soft nofile 65535
* hard nofile 65535
EOF
sysctl -p

# 关闭selinux
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
setenforce 0
# 关闭防火墙
systemctl stop firewalld.service && systemctl disable firewalld.service
# 修改时区
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
# 关闭swap
swapoff -a && sysctl -w vm.swappiness=0 && sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
# 重启
reboot

准备镜像

cd /home/k3s_1.18.2
mkdir -p /var/lib/rancher/k3s/agent/images/
\cp ./k3s-airgap-images-amd64.tar /var/lib/rancher/k3s/agent/images/
chmod a+x ./k3s ./install.sh

使用 Containerd

# docker
echo "alias docker='k3s crictl'" >> /etc/profile
## harbor认证 
\cp /home/certs/ca.crt /etc/ssl/certs/
## 配置 /etc/rancher/k3s/registries.yaml
mkdir -p /etc/rancher/k3s
cat > /etc/rancher/k3s/registries.yaml << EOF
mirrors:
  docker.io:  # 用docker.io遇到问题时,可尝试改为harbor.do.io.
    endpoint:
      - "https://harbor.${domain}"
configs:
  docker.io: 
    auth:
      username: admin # 镜像仓库用户名
      password: Harbor12345 # 镜像仓库密码
    tls:
      cert_file: /home/certs/${domain}.cert #<镜像仓库所用的客户端证书文件路径>
      key_file: /home/certs/${domain}.key #<镜像仓库所用的客户端密钥文件路径>
      ca_file: /home/certs/ca.key # <镜像仓库所用的ca文件路径>
EOF
## 检查
cat /etc/rancher/k3s/registries.yaml

Dqlit or PG

DQlite

Master-1

export INSTALL_K3S_SKIP_DOWNLOAD=true
export INSTALL_K3S_EXEC="server --cluster-init --write-kubeconfig ~/.kube/config --write-kubeconfig-mode 666"
\cp ./k3s /usr/local/bin/ && ./install.sh
cat /var/lib/rancher/k3s/server/node-token

Master-N

export K3S_TOKEN=

export INSTALL_K3S_SKIP_DOWNLOAD=true
export INSTALL_K3S_EXEC="server --server https://m1:6443 --write-kubeconfig ~/.kube/config --write-kubeconfig-mode 666"
\cp ./k3s /usr/local/bin/ && ./install.sh

Postgres 11.8

docker-compose.yml

version: '3.1'

services:

  db:
    image: postgres:11.8
    container_name: pg
    restart: always
    environment:
      - POSTGRES_PASSWORD: 12345
    ports:
      - "5432:5432"
    volumes:
      - /home/pgdata:/var/lib/postgresql/data

  adminer:
    image: adminer
    container_name: adminer
    restart: always
    ports:
      - 8888:8080

Master-1-N

export INSTALL_K3S_SKIP_DOWNLOAD=true
export INSTALL_K3S_EXEC="server --datastore-endpoint=postgres://postgres:12345@harbor.${domain}:5432/kubernetes?sslmode=disable
 --write-kubeconfig ~/.kube/config --write-kubeconfig-mode 666"
\cp ./k3s /usr/local/bin/ && ./install.sh

检查

# 检查
echo $INSTALL_K3S_EXEC# 镜像仓库是否为harbor
cat /var/lib/rancher/k3s/agent/etc/containerd/config.toml
kubectl get nodes
kubectl get pods --all-namespaces
# 修改kubeconfig指向ranchder.do.io 实现负载均衡
## sed -i "s#server: https://127.0.0.1:6443#server: https://rancher.${domain}:6443#g"  /etc/rancher/k3s/k3s.yaml

Worker-N

变量

cat /var/lib/rancher/k3s/server/node-token # master1上取token执行
export K3S_TOKEN=

脚本

export INSTALL_K3S_SKIP_DOWNLOAD=true
export K3S_URL=https://do:6443
export INSTALL_K3S_EXEC="agent"
chmod a+x ./k3s ./install.sh 
cp ./k3s /usr/local/bin/ && ./install.sh

安装rancher

公网取数据包

# 添加Helm Chart仓库
helm repo add rancher-stable https://releases.rancher.com/server-charts/stable
#rancher.2.4.3.tgz落到本地
helm fetch rancher-stable/rancher

安装脚本

Master-1

cd /home/rancher243
# 安装Helm 3.2.1
chmod a+x ./helm
\cp ./helm /usr/local/bin/
# 2. 自签证书渲染 rancher helm模板
helm template rancher ./rancher-2.4.3.tgz --output-dir . \
--namespace cattle-system \
--set hostname=rancher.${domain} \
--set rancherImage=harbor.${domain}/rancher/rancher \
--set privateCA=true \
--set ingress.tls.source=secret \
--set systemDefaultRegistry=harbor.${domain} \
--set useBundledSystemChart=true

# 3. 安装
kubectl create namespace cattle-system
# #TLS密文
kubectl -n cattle-system create secret tls tls-rancher-ingress --cert=/home/certs/do.io.crt --key=/home/certs/do.io.key
## 私有签发
\cp /home/certs/do.io.crt ./cacerts.pem
kubectl -n cattle-system create secret generic tls-ca --from-file=cacerts.pem
## 安装 rancher
kubectl -n cattle-system apply -R -f ./rancher

检查

### 查看pod事件
kubectl -n cattle-system describe pod
### rancher是否运行
kubectl -n cattle-system get pods
## 访问
https://rancher.do.io

Nginx

docker-compose.yml

services:
  nginx:
    image: nginx
    container_name: nginx
    restart: always
    ports:
      - "80:80"
      - "6443:6443"
      - "8443:8443"
    volumes:
      - /home/nginx.conf:/etc/nginx/nginx.conf:ro

nginx.conf

worker_processes 4;
worker_rlimit_nofile 40000;

events {
worker_connections 8192;
}

stream {
    upstream rancher_servers_http {
        least_conn;
        server 172.19.201.241:80 max_fails=3 fail_timeout=5s;
        server 172.19.201.242:80 max_fails=3 fail_timeout=5s;
    }
    server {
        listen 80;
        proxy_pass rancher_servers_http;
    }

    upstream rancher_servers_https {
        least_conn;
        server 172.19.201.241:443 max_fails=3 fail_timeout=5s;
        server 172.19.201.242:443 max_fails=3 fail_timeout=5s;
    }
    server {
        listen     8443;
        proxy_pass rancher_servers_https;
    }

    upstream apiserver_https {
        least_conn;
        server 172.19.201.241:6443 max_fails=3 fail_timeout=5s;
        server 172.19.201.242:6443 max_fails=3 fail_timeout=5s;
    }
    server {
        listen     6443;
        proxy_pass apiserver_https;
    }
}

websocker错误

nginx.conf

worker_processes 4;
worker_rlimit_nofile 40000;

events {
    worker_connections 8192;
}

http {
    include ./conf.d/*.conf;
}

./conf.d/rancher.conf

upstream rancher {
    least_conn;
    server 10.10.10.10:443 max_fails=3 fail_timeout=5s;
    server 10.10.10.11:443 max_fails=3 fail_timeout=5s;
    server 10.10.10.12:443 max_fails=3 fail_timeout=5s;
}

server {
    listen 80;
    server_name rancher.do.io;
    return 301 https://$server_name$request_uri;
}

server {
    listen 443 ssl ;
    server_name rancher.do.io;

    ssl on;
    ssl_certificate /home/certs/cert.pem;
    ssl_certificate_key /home/certs/cert.key;

    location / {
        proxy_pass https://rancher;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "Upgrade";
    }
}