[原創] k8s 部署文檔

k8s note, support all cni plugin.

master first

install docker

curl -fsSL https://get.docker.com -o get-docker.sh
bash get-docker.sh
systemctl enable docker && systemctl start docker
docker ps

cat > /etc/docker/daemon.json <<EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2"
}
EOF

# apt-get install -y libseccomp2 # if you nead
mkdir -p /etc/systemd/system/docker.service.d
systemctl daemon-reload
systemctl enable docker && systemctl restart docker

install CRI-O

# k8s new version has create CRI sockets, maybe you dont nead this
modprobe overlay
modprobe br_netfilter

cat > /etc/sysctl.d/99-kubernetes-cri.conf <<EOF
net.bridge.bridge-nf-call-iptables  = 1
net.ipv4.ip_forward                 = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF

sysctl --system

apt-get update
apt-get install software-properties-common
add-apt-repository ppa:projectatomic/ppa
apt-get update
apt-get install cri-o-1.11 -y

systemctl enable crio && systemctl start crio

install haproxy

# if you just one master node, ignore this
add-apt-repository ppa:vbernat/haproxy-1.9
apt update
apt install haproxy

# edit haproxy.cfg
cat >> /etc/haproxy/haproxy.cfg <<EOF
listen  admin_stats
    bind 0.0.0.0:10080
    mode http
    log 127.0.0.1 local0 err
    stats refresh 30s
    stats uri /status
    stats realm welcome login\ Haproxy
    stats auth admin_name:admin_passwd
    stats hide-version
    stats admin if TRUE
listen kube-master
    bind 0.0.0.0:8443
    mode tcp
    option tcplog
    balance source
    server MASTER-01 MASTER-02-IP:6443 check inter 2000 fall 2 rise 2 weight 1
    server MASTER-02 MASTER-02-IP:6443 check inter 2000 fall 2 rise 2 weight 1
EOF
systemctl  enable haproxy && systemctl  restart haproxy

install k8s and kubeadm

# install k8s
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list
apt update -y
apt-get install -y kubelet kubeadm kubectl
apt-mark hold kubelet kubeadm kubectl

cat > /etc/sysctl.d/99-k8s <<EOF
net.ipv4.ip_forward                 = 1
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF

systemctl daemon-reload
systemctl restart kubelet
systemctl enable kubelet && systemctl start kubelet
systemctl enable docker && systemctl start docker

install docker-compose

# apt install -y py-pip  python-dev  libffi-dev openssl-dev gcc libc-dev make
curl -L "https://github.com/docker/compose/releases/download/1.24.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
chmod +x /usr/local/bin/docker-compose
docker-compose --version

install kompose

covert docker-compose to k8s format, if you need

curl -L https://github.com/kubernetes/kompose/releases/download/v1.18.0/kompose-linux-amd64 -o kompose
chmod +x kompose
mv ./kompose /usr/local/bin/kompose

config and init k8s

cat >> kubeadm-config.yaml <<EOF
apiServer:
  timeoutForControlPlane: 4m0s
  extraArgs:
    enable-admission-plugins: NamespaceLifecycle,LimitRanger,ServiceAccount,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,PersistentVolumeClaimResize,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota
    runtime-config: "api/all=true"
apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration
kubernetesVersion: stable
controlPlaneEndpoint: "HAPROXY_IP:8443"
#controlPlaneEndpoint: "LOCAL_SUB_IP:6443"
dns:
  type: "CoreDNS"
networking:
  dnsDomain: cluster.local
  podSubnet: "10.244.0.0/16"
EOF

# if not install or config haproxy server, controlPlaneEndpoint set local subnet ip, like this `192.168.8.181:6443`. don't change port 6443 if you not custom endpoint port

kubeadm reset --force
kubeadm init --config=kubeadm-config.yaml --experimental-upload-certs
# kubeadm init --config=kubeadm-config.yaml --experimental-upload-certs --ignore-preflight-errors=all

## here will print join k8s command for master(control-plane) node and worker node.
## copy command for master node, like this
## kubeadm join MASTER-01:8443 --token mepvx0.7db40f5sgr33af0s --discovery-token-ca-cert-hash sha256:0ba3d8aea484c03acb29c867a36e0d9ecbb281406db665fe47e0abd931f48718 --experimental-control-plane --certificate-key aba1fb9973c7122ed4bfb2c8dbf946494cc26dbca01444a58ac1b2c7b1384e84
## copy command for worker node, like this
## kubeadm join MASTER-01:8443 --token mepvx0.7db40f5sgr33af0s --discovery-token-ca-cert-hash sha256:0ba3d8aea484c03acb29c867a36e0d9ecbb281406db665fe47e0abd931f48718

# save kube admin config
mkdir -p $HOME/.kube
cp /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config
kubectl get nodes
# NAME              STATUS     ROLES    AGE    VERSION
# master-01   NotReady   master   152m   v1.14.1

docker registry server

# this is test, so run it on master
docker run -d --name registry --publish published=5000,target=5000 registry:2

init cni plugin

Not support bandwidth for now.

# use calico cni
kubectl apply -f https://docs.projectcalico.org/v3.7/manifests/canal.yaml
# 2019-06-24 v3.8 can use bandwidth plugin
kubectl apply -f https://docs.projectcalico.org/v3.8/manifests/canal.yaml

# use standard flannel
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/v0.11.0/Documentation/kube-flannel.yml

Support bandwidth

# clone project
git clone https://github.com/Ox0400/cni-plugin
cd cni-plugin

# build and push images
REGISTRY="YOUREGISTRYHOSTIP:5000"
sed -i -e "s?REGISTRYHOST?$REGISTRY?g" start.sh
bash start.sh

# apply cni plugin
sed -i -e "s?REGISTRY?$REGISTRY?g" example-cni-calico.yml
kubelet apply -f example-cni-calico.yml

2019-06-24 補充: v3.8 已經發布, 內置了 bandwidth https://docs.projectcalico.org/v3.8/getting-started/kubernetes/installation/flannel#installing-with-the-kubernetes-api-datastore-recommended

test apply app

# myweb.yaml
# ------------------- APP Deployment ------------------- #

kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    app: myweb
  name: myweb
spec:
  replicas: 1
  revisionHistoryLimit: 1
  selector:
    matchLabels:
      app: myweb
  template:
    metadata:
      labels:
        app: myweb
      #annotations:
      #  ingress-bandwidth and egress-bandwidth is difference with the test results. 10Mi just is 1kb. unit is bytes always? 10MB maybe set value is 10000Mi or 10Gi?
      #  kubernetes.io/ingress-bandwidth: 10Mi
      #  kubernetes.io/egress-bandwidth: 10Mi
    spec:
      hostAliases:
      - ip: "127.0.0.1"
        hostnames:
        - "redis"
      dnsConfig:
        nameservers:
          - 8.8.8.8
          - 8.8.4.4
      containers:
      - name: api
        image: nginx
        imagePullPolicy: Always
        ports:
        - containerPort: 80
          protocol: TCP
        resources:
          limits:
            cpu: 0.5
            memory: "256Mi"
            ephemeral-storage: "100Mi"
          requests:
            cpu: 0.25
            memory: "64Mi"
            ephemeral-storage: "100Mi"
        volumeMounts:
        - mountPath: /etc/localtime
          name: localtime
      - name: redis
        image: redis:alpine
        ports:
        - containerPort: 6379
          protocol: TCP
        resources:
          limits:
            cpu: 0.25
            memory: "64Mi"
            ephemeral-storage: "100Mi"
          requests:
            cpu: 0.20
            memory: "32Mi"
            ephemeral-storage: "100Mi"
        volumeMounts:
        - mountPath: /etc/localtime
          name: localtime
      volumes:
      - name: localtime
        hostPath:
          path: /etc/localtime
          type: FileOrCreate
         
---
# ------------------- APP Service ------------------- #

kind: Service
apiVersion: v1
metadata:
  labels:
    app: myweb
  name: myweb
spec:
  #type: ClusterIP
  type: NodePort
  ports:
    - port: 80
      targetPort: 80
  selector:
    app: myweb

# deploy app 
kubectl apply -f myweb.yaml

# show app info
app=myweb
kubectl get pods,svc,deployments -o wide -l app=$app

# test connect nginx use HEAD
curl -I "`kubectl get pods -o wide -l app=$app | grep Running | awk '{print $7}'`:`kubectl get svc -o wide -l app=$app | grep NodePort | grep -P '\d\d\d\d+'  -o`/"

# Spawns a shell within the container
kubectl exec `kubectl get pods -o wide -l app=$app | grep Running | awk '{print $1}'` -c api -it sh

## test storage limit
while true; do dd if=/dev/zero of=$(date '+%s').out count=1 bs=20MB 2> /dev/null; du -sh . ;sleep 1; done 

## test bandwidth limit
pip install speedtest-cli && speedtest-cli

wget --output-document=/dev/null http://speedtest.wdc01.softlayer.com/downloads/test500.zip

master second

install docker

see master first

install cri

see master first

install haproxy

see master first

install k8s

see master first

install kompose

see master first

join k8s master node

# join master nodes use the master first init output command
kubeadm join MASTER-01-token mepvx0.7db40f5sgr33af0s --discovery-token-ca-cert-hash sha256:0ba3d8aea484c03acb29c867a36e0d9ecbb281406db665fe47e0abd931f48718 --experimental-control-plane --certificate-key aba1fb9973c7122ed4bfb2c8dbf946494cc26dbca01444a58ac1b2c7b1384e84

# save kube admin config
mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config

worker

install docker

see master first

install cri

see master first

install k8s

see master first

join k8s worker node

kubeadm join 172.31.42.98:8443 --token mepvx0.7db40f5sgr33af0s --discovery-token-ca-cert-hash sha256:0ba3d8aea484c03acb29c867a36e0d9ecbb281406db665fe47e0abd931f48718
docker ps

QA

WARNING: Ignoring APKINDEX.xxxx.tar.gz: No such file or directory

set dns for docker daemon. /etc/docker/daemon.json

{ 
  "dns": [own_dns,"8.8.8.8","8.8.8.4"] 
}

Pod status: ImagePullBackOff

set insecure-registries for docker daemon

# show log
kubectl describe pods POD_NAME

{
  "insecure-registries":["YOUREGISHOST:5000"]
}

kompose convert
mkdir k8s && mv *yaml k8s
kubectl apply -f k8s/
kubectl delete -f k8s/
kubectl get services gr1-api

Container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized

1. remove cni: /var/lib/kubelet/kubeadm-flags.env
2. install cni: https://docs.docker.com/ee/ucp/kubernetes/install-cni-plugin/

kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

NetworkPlugin cni failed to set up pod “v1-58cb8ccd5c-w2wlm_default” network: failed to set bridge addr: “cni0” already has an IP address different from 10.244.3.1/24"

## delete cni* interface
ifconfig cni0 down
ip link delete cni0

https://github.com/kubernetes/kubernetes/issues/39557#issuecomment-271944481

https://github.com/kubernetes/kubernetes/issues/39557#issuecomment-342604193

E0517 08:27:19.936024 1 main.go:289] Error registering network: failed to acquire lease: node “node-01” pod cidr not assigned

# edit /etc/kubernetes/manifests/kube-controller-manager.yaml
ifconfig cni0 down
ip link delete cni0

    - --allocate-node-cidrs=true
    - --cluster-cidr=10.244.0.0/16

if use kubeadm, please set podSubnet: “10.244.0.0/16”

https://github.com/coreos/flannel/blob/master/Documentation/kubernetes.md

kubelet version 1.9, yaml config selectorarea must set matchLabels

Pod can’t connect internet

cat > kube-dns.yml <<EOF
apiVersion: v1
kind: ConfigMap
metadata:
  name: kube-dns
  namespace: kube-system
data:
  upstreamNameservers: |
    ["8.8.8.8", "8.8.4.4"]
EOF

kubectl apply -f kube-dns.yml
kubectl get pod  -n kube-system -o wide -l k8s-app=kube-dns
kubectl delete pod  -n kube-system  -l k8s-app=kube-dns

https://medium.com/@boruah.rajen/dns-resolution-works-in-host-but-not-from-kubernetes-pod-226dcbeccc13

error: unable to recognize “lxcfs-initializer.yaml”: no matches for kind “InitializerConfiguration” in version “admissionregistration.k8s.io/v1alpha1”

https://github.com/kubernetes/community/blob/b3349d5b1354df814b67bbdee6890477f3c250cb/contributors/design-proposals/api-machinery/admission-control-webhooks.md

https://zhuanlan.zhihu.com/p/27700061

https://docs.bitnami.com/kubernetes/how-to/configure-autoscaling-custom-metrics/#cluster-configuration

https://github.com/kubernetes/website/pull/8852

https://github.com/kubernetes/kubernetes/pull/58428

list all apiversion

kubectl api-versions

remove ip link datapath

curl -L git.io/weave -o /usr/local/bin/weave
chmod a+x /usr/local/bin/weave
systemctl restart kubelet && systemctl restart docker
/usr/local/bin/weave reset
ifconfig -a

set nameserver for pod

      dnsConfig:
        nameservers:
          - 8.8.8.8
          - 8.8.4.4

https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/

set hosts for pod

      hostAliases:
      - ip: "127.0.0.1"
        hostnames:
        - "redis"

limit bandwidth

see cni plugin

reset k8s cluters

## RESET k8s
kubeadm reset --force
systemctl stop kubelet && systemctl stop docker
iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X

# reset lxcfs
fusermount -u /var/lib/lxcfs
rm -rf /var/lib/lxcfs
# delete kubelet config
rm -rf /var/lib/kubelet/*

# reset cni
rm -rf /var/lib/cni/
rm -rf /etc/cni/
rm -rf /opt/cni

# reset interfaces
modprobe -r ipip

# system default interface is eth0 and lo, keep both.
ifconfig -a | grep "Link " |grep -e eth0 -e lo -v| awk '{print "ifconfig " $1 " down; ip link delete "$1}' | bash

# reset services
systemctl start kubelet && systemctl start docker
ifconfig -a