使用SSL訪問kubernetes-dashboard

一、環境說明

主機名	內網地址	外網地址	類型
k8s-master	172.16.32.10	134.175.200.121	masters
k8s-node1	172.16.32.14	123.207.123.159	nodes
k8s-node2	172.16.32.17	182.254.164.35	nodes

二、環境檢查

SSL證書申請

[root@k8s-master ~]# ll
total 36
-rw-r--r-- 1 root root 1675 Apr  2 12:02 3701103_dashboard.mydoyou.cn.key
-rw-r--r-- 1 root root 3671 Apr  2 12:02 3701103_dashboard.mydoyou.cn.pem

[root@k8s-master ~]# kubectl get pod,svc,ingress,secret,deploy --all-namespaces
NAMESPACE     NAME                                     READY   STATUS    RESTARTS   AGE
kube-system   pod/coredns-58cc8c89f4-lhzn8             1/1     Running   0          4h8m
kube-system   pod/coredns-58cc8c89f4-rvvrr             1/1     Running   0          4h8m
kube-system   pod/etcd-k8s-master                      1/1     Running   0          4h7m
kube-system   pod/kube-apiserver-k8s-master            1/1     Running   0          4h7m
kube-system   pod/kube-controller-manager-k8s-master   1/1     Running   0          4h6m
kube-system   pod/kube-flannel-ds-amd64-4d698          1/1     Running   0          4h4m
kube-system   pod/kube-flannel-ds-amd64-dsb9n          1/1     Running   0          4h4m
kube-system   pod/kube-flannel-ds-amd64-ndvtk          1/1     Running   0          4h4m
kube-system   pod/kube-proxy-fxlgv                     1/1     Running   0          4h4m
kube-system   pod/kube-proxy-l5hq5                     1/1     Running   0          4h8m
kube-system   pod/kube-proxy-vxq6v                     1/1     Running   0          4h6m
kube-system   pod/kube-scheduler-k8s-master            1/1     Running   0          4h7m

NAMESPACE     NAME                 TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)                  AGE
default       service/kubernetes   ClusterIP   10.1.0.1     <none>        443/TCP                  4h8m
kube-system   service/kube-dns     ClusterIP   10.1.0.10    <none>        53/UDP,53/TCP,9153/TCP   4h8m

NAMESPACE         NAME                                                    TYPE                                  DATA   AGE
default           secret/default-token-49rqp                              kubernetes.io/service-account-token   3      4h8m
kube-node-lease   secret/default-token-x6sg4                              kubernetes.io/service-account-token   3      4h8m
kube-public       secret/default-token-f6pkm                              kubernetes.io/service-account-token   3      4h8m
kube-system       secret/attachdetach-controller-token-drzzq              kubernetes.io/service-account-token   3      4h8m
kube-system       secret/bootstrap-signer-token-k56c8                     kubernetes.io/service-account-token   3      4h8m
kube-system       secret/bootstrap-token-89o9sw                           bootstrap.kubernetes.io/token         7      4h8m
kube-system       secret/certificate-controller-token-hlgml               kubernetes.io/service-account-token   3      4h8m
kube-system       secret/clusterrole-aggregation-controller-token-jj5j2   kubernetes.io/service-account-token   3      4h8m
kube-system       secret/coredns-token-86fth                              kubernetes.io/service-account-token   3      4h8m
kube-system       secret/cronjob-controller-token-t8ljw                   kubernetes.io/service-account-token   3      4h8m
kube-system       secret/daemon-set-controller-token-fsnrh                kubernetes.io/service-account-token   3      4h8m
kube-system       secret/default-token-kj268                              kubernetes.io/service-account-token   3      4h8m
kube-system       secret/deployment-controller-token-jh5wl                kubernetes.io/service-account-token   3      4h8m
kube-system       secret/disruption-controller-token-rckf8                kubernetes.io/service-account-token   3      4h8m
kube-system       secret/endpoint-controller-token-57nzj                  kubernetes.io/service-account-token   3      4h8m
kube-system       secret/expand-controller-token-4zx2s                    kubernetes.io/service-account-token   3      4h8m
kube-system       secret/flannel-token-g7tb5                              kubernetes.io/service-account-token   3      4h4m
kube-system       secret/generic-garbage-collector-token-t7gl6            kubernetes.io/service-account-token   3      4h8m
kube-system       secret/horizontal-pod-autoscaler-token-l9v85            kubernetes.io/service-account-token   3      4h8m
kube-system       secret/job-controller-token-q5g4x                       kubernetes.io/service-account-token   3      4h8m
kube-system       secret/kube-proxy-token-frmvx                           kubernetes.io/service-account-token   3      4h8m
kube-system       secret/namespace-controller-token-bdcm4                 kubernetes.io/service-account-token   3      4h8m
kube-system       secret/node-controller-token-vbzpr                      kubernetes.io/service-account-token   3      4h8m
kube-system       secret/persistent-volume-binder-token-7fpnx             kubernetes.io/service-account-token   3      4h8m
kube-system       secret/pod-garbage-collector-token-wvhs7                kubernetes.io/service-account-token   3      4h8m
kube-system       secret/pv-protection-controller-token-mqqk9             kubernetes.io/service-account-token   3      4h8m
kube-system       secret/pvc-protection-controller-token-dtknk            kubernetes.io/service-account-token   3      4h8m
kube-system       secret/replicaset-controller-token-gx6k7                kubernetes.io/service-account-token   3      4h8m
kube-system       secret/replication-controller-token-kwwdh               kubernetes.io/service-account-token   3      4h8m
kube-system       secret/resourcequota-controller-token-bhfb7             kubernetes.io/service-account-token   3      4h8m
kube-system       secret/service-account-controller-token-xzfl6           kubernetes.io/service-account-token   3      4h8m
kube-system       secret/service-controller-token-jr9sb                   kubernetes.io/service-account-token   3      4h8m
kube-system       secret/statefulset-controller-token-6bpmb               kubernetes.io/service-account-token   3      4h8m
kube-system       secret/token-cleaner-token-x9jrx                        kubernetes.io/service-account-token   3      4h8m
kube-system       secret/ttl-controller-token-zmgqh                       kubernetes.io/service-account-token   3      4h8m

NAMESPACE     NAME                      READY   UP-TO-DATE   AVAILABLE   AGE
kube-system   deployment.apps/coredns   2/2     2            2           4h8m

三、部署kubernetes-dashboard控制面板

[root@k8s-master ~]# wget https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta6/aio/deploy/recommended.yaml
[root@k8s-master ~]# kubectl apply -f recommended.yaml 
namespace/kubernetes-dashboard created
serviceaccount/kubernetes-dashboard created
service/kubernetes-dashboard created
secret/kubernetes-dashboard-certs created
secret/kubernetes-dashboard-csrf created
secret/kubernetes-dashboard-key-holder created
configmap/kubernetes-dashboard-settings created
role.rbac.authorization.k8s.io/kubernetes-dashboard created
clusterrole.rbac.authorization.k8s.io/kubernetes-dashboard created
rolebinding.rbac.authorization.k8s.io/kubernetes-dashboard created
clusterrolebinding.rbac.authorization.k8s.io/kubernetes-dashboard created
deployment.apps/kubernetes-dashboard created
service/dashboard-metrics-scraper created
deployment.apps/dashboard-metrics-scraper created

## 創建訪問的token
[root@k8s-master ~]# kubectl create serviceaccount dashboard-admin -n kubernetes-dashboard
[root@k8s-master ~]# kubectl create clusterrolebinding dashboard-admin --clusterrole=cluster-admin --serviceaccount=kubernetes-dashboard:dashboard-admin
[root@k8s-master ~]# kubectl describe secrets -n kubernetes-dashboard $(kubectl -n kubernetes-dashboard get secret | awk '/dashboard-admin/{print $1}')
Name:         dashboard-admin-token-dgp56
Namespace:    kubernetes-dashboard
Labels:       <none>
Annotations:  kubernetes.io/service-account.name: dashboard-admin
              kubernetes.io/service-account.uid: a887e34d-cad2-47ee-912d-404b811b9b81

Type:  kubernetes.io/service-account-token

Data
====
ca.crt:     1025 bytes
namespace:  20 bytes
token:      eyJhbGciOiJSUzI1NiIsImtpZCI6IktrcVhCOEtDdWhZaGtLNWp2d3hwN1VnWHJ6ZHp3V1J3bFFNVEVvVGd2WDQifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkYXNoYm9hcmQtYWRtaW4tdG9rZW4tZGdwNTYiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoiZGFzaGJvYXJkLWFkbWluIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQudWlkIjoiYTg4N2UzNGQtY2FkMi00N2VlLTkxMmQtNDA0YjgxMWI5YjgxIiwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50Omt1YmVybmV0ZXMtZGFzaGJvYXJkOmRhc2hib2FyZC1hZG1pbiJ9.BcNwHy0zgCPGKj5ZFrPTxwx-Md6VRRXz0YbH3zQNxEP-hE5hJ10a1R6ralSNt6SHdkfXL50y5kJAOYP9_ohqqJvuptq2znfOamHZ5sLrgpHaE-Zd8vR_FlwruPf4ltJFMKKjM06O-zZEMkCcUctDgkxrjRJgU7AcK8fLpCM7oJZXCI6BUJz6iTHjVw1fH1cbbrZupHCSJ32892XOy3M2h6FOlWhBgwO6pI9MqM8ZWjxunCE7wg9GKlfWWt-GnI2EE-Ul2gkOqRpveis86JQPLON2MtVWwGxAVd0thl5OalP7bz9cDpHMZITBmfymTPpthlpx_1aFogDuhYLMecc6pA

## 創建tls
[root@k8s-master ~]# kubectl -n kubernetes-dashboard  create secret tls k8s-dashboard-secret  --key ./3701103_dashboard.mydoyou.cn.key --cert ./3701103_dashboard.mydoyou.cn.pem  -o yaml --dry-run > k8s-dashboard-secret.yaml
[root@k8s-master ~]# kubectl apply -f k8s-dashboard-secret.yaml 
secret/k8s-dashboard-secret created

四、部署ingress-nginx，提供外部訪問。

[root@k8s-master ~]# wget  wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/nginx-0.30.0/deploy/static/mandatory.yaml
## 修改三處地方，完整的文件如下
[root@k8s-master ~]# cat mandatory.yaml 
apiVersion: v1
kind: Namespace
metadata:
  name: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---

kind: ConfigMap
apiVersion: v1
metadata:
  name: nginx-configuration
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---
kind: ConfigMap
apiVersion: v1
metadata:
  name: tcp-services
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---
kind: ConfigMap
apiVersion: v1
metadata:
  name: udp-services
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: nginx-ingress-serviceaccount
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: nginx-ingress-clusterrole
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
rules:
  - apiGroups:
      - ""
    resources:
      - configmaps
      - endpoints
      - nodes
      - pods
      - secrets
    verbs:
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - nodes
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - services
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - events
    verbs:
      - create
      - patch
  - apiGroups:
      - "extensions"
      - "networking.k8s.io"
    resources:
      - ingresses
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - "extensions"
      - "networking.k8s.io"
    resources:
      - ingresses/status
    verbs:
      - update

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
  name: nginx-ingress-role
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
rules:
  - apiGroups:
      - ""
    resources:
      - configmaps
      - pods
      - secrets
      - namespaces
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - configmaps
    resourceNames:
      # Defaults to "<election-id>-<ingress-class>"
      # Here: "<ingress-controller-leader>-<nginx>"
      # This has to be adapted if you change either parameter
      # when launching the nginx-ingress-controller.
      - "ingress-controller-leader-nginx"
    verbs:
      - get
      - update
  - apiGroups:
      - ""
    resources:
      - configmaps
    verbs:
      - create
  - apiGroups:
      - ""
    resources:
      - endpoints
    verbs:
      - get

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
  name: nginx-ingress-role-nisa-binding
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: nginx-ingress-role
subjects:
  - kind: ServiceAccount
    name: nginx-ingress-serviceaccount
    namespace: ingress-nginx

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: nginx-ingress-clusterrole-nisa-binding
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: nginx-ingress-clusterrole
subjects:
  - kind: ServiceAccount
    name: nginx-ingress-serviceaccount
    namespace: ingress-nginx

---

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: nginx-ingress-controller
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
spec:
#  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/part-of: ingress-nginx
  template:
    metadata:
      labels:
        app.kubernetes.io/name: ingress-nginx
        app.kubernetes.io/part-of: ingress-nginx
      annotations:
        prometheus.io/port: "10254"
        prometheus.io/scrape: "true"
    spec:
      hostNetwork: true
      # wait up to five minutes for the drain of connections
      terminationGracePeriodSeconds: 300
      serviceAccountName: nginx-ingress-serviceaccount
      nodeSelector:
        kubernetes.io/os: linux
      containers:
        - name: nginx-ingress-controller
          image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.30.0
          args:
            - /nginx-ingress-controller
            - --configmap=$(POD_NAMESPACE)/nginx-configuration
            - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
            - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
            - --publish-service=$(POD_NAMESPACE)/ingress-nginx
            - --annotations-prefix=nginx.ingress.kubernetes.io
          securityContext:
            allowPrivilegeEscalation: true
            capabilities:
              drop:
                - ALL
              add:
                - NET_BIND_SERVICE
            # www-data -> 101
            runAsUser: 101
          env:
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
          ports:
            - name: http
              containerPort: 80
              protocol: TCP
            - name: https
              containerPort: 443
              protocol: TCP
          livenessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            initialDelaySeconds: 10
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 10
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 10
          lifecycle:
            preStop:
              exec:
                command:
                  - /wait-shutdown

---

apiVersion: v1
kind: LimitRange
metadata:
  name: ingress-nginx
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
spec:
  limits:
  - min:
      memory: 90Mi
      cpu: 100m
    type: Container
[root@k8s-master ~]# cat dashboard-ingress.yaml 
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: kubernetes-dashboard-ingress
  namespace: kubernetes-dashboard
  annotations:
    kubernetes.io/ingress.class: "nginx"
    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
    nginx.ingress.kubernetes.io/rewrite-target: /
    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
spec:
  tls:
   - hosts:
     - dashboard.mydoyou.cn
     secretName: k8s-dashboard-secret
  rules:
  - host: dashboard.mydoyou.cn
    http:
      paths:
      - path: 
        backend:
          serviceName: kubernetes-dashboard
          servicePort: 443

[root@k8s-master ~]# kubectl apply -f mandatory.yaml 
namespace/ingress-nginx created
configmap/nginx-configuration created
configmap/tcp-services created
configmap/udp-services created
serviceaccount/nginx-ingress-serviceaccount created
clusterrole.rbac.authorization.k8s.io/nginx-ingress-clusterrole created
role.rbac.authorization.k8s.io/nginx-ingress-role created
rolebinding.rbac.authorization.k8s.io/nginx-ingress-role-nisa-binding created
clusterrolebinding.rbac.authorization.k8s.io/nginx-ingress-clusterrole-nisa-binding created
daemonset.apps/nginx-ingress-controller created
limitrange/ingress-nginx created
[root@k8s-master ~]# kubectl apply -f dashboard-ingress.yaml
ingress.extensions/kubernetes-dashboard-ingress created

五、查看kubernetes-dashboard命名空間的資源

## 查看kubernetes-dashboard命名空間的資源
[root@k8s-master ~]# kubectl get pod,svc,ingress,secret -n kubernetes-dashboard
NAME                                             READY   STATUS    RESTARTS   AGE
pod/dashboard-metrics-scraper-76585494d8-l9vh5   1/1     Running   0          10m
pod/kubernetes-dashboard-b65488c4-xs8xq          1/1     Running   0          10m

NAME                                TYPE        CLUSTER-IP    EXTERNAL-IP   PORT(S)    AGE
service/dashboard-metrics-scraper   ClusterIP   10.1.21.2     <none>        8000/TCP   10m
service/kubernetes-dashboard        ClusterIP   10.1.119.43   <none>        443/TCP    10m

NAME                                              HOSTS                  ADDRESS   PORTS     AGE
ingress.extensions/kubernetes-dashboard-ingress   dashboard.mydoyou.cn             80, 443   29s

NAME                                      TYPE                                  DATA   AGE
secret/dashboard-admin-token-dgp56        kubernetes.io/service-account-token   3      10m
secret/default-token-nxq49                kubernetes.io/service-account-token   3      10m
secret/k8s-dashboard-secret               kubernetes.io/tls                     2      4m45s
secret/kubernetes-dashboard-certs         Opaque                                0      10m
secret/kubernetes-dashboard-csrf          Opaque                                1      10m
secret/kubernetes-dashboard-key-holder    Opaque                                2      10m
secret/kubernetes-dashboard-token-qrflq   kubernetes.io/service-account-token   3      10m

六、使用token驗證

## 獲取token
[root@k8s-master ~]# kubectl describe secrets -n kubernetes-dashboard $(kubectl -n kubernetes-dashboard get secret | awk '/dashboard-admin/{print $1}') 
Name:         dashboard-admin-token-dgp56
Namespace:    kubernetes-dashboard
Labels:       <none>
Annotations:  kubernetes.io/service-account.name: dashboard-admin
              kubernetes.io/service-account.uid: a887e34d-cad2-47ee-912d-404b811b9b81

Type:  kubernetes.io/service-account-token

Data
====
ca.crt:     1025 bytes
namespace:  20 bytes
token:      eyJhbGciOiJSUzI1NiIsImtpZCI6IktrcVhCOEtDdWhZaGtLNWp2d3hwN1VnWHJ6ZHp3V1J3bFFNVEVvVGd2WDQifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkYXNoYm9hcmQtYWRtaW4tdG9rZW4tZGdwNTYiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoiZGFzaGJvYXJkLWFkbWluIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQudWlkIjoiYTg4N2UzNGQtY2FkMi00N2VlLTkxMmQtNDA0YjgxMWI5YjgxIiwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50Omt1YmVybmV0ZXMtZGFzaGJvYXJkOmRhc2hib2FyZC1hZG1pbiJ9.BcNwHy0zgCPGKj5ZFrPTxwx-Md6VRRXz0YbH3zQNxEP-hE5hJ10a1R6ralSNt6SHdkfXL50y5kJAOYP9_ohqqJvuptq2znfOamHZ5sLrgpHaE-Zd8vR_FlwruPf4ltJFMKKjM06O-zZEMkCcUctDgkxrjRJgU7AcK8fLpCM7oJZXCI6BUJz6iTHjVw1fH1cbbrZupHCSJ32892XOy3M2h6FOlWhBgwO6pI9MqM8ZWjxunCE7wg9GKlfWWt-GnI2EE-Ul2gkOqRpveis86JQPLON2MtVWwGxAVd0thl5OalP7bz9cDpHMZITBmfymTPpthlpx_1aFogDuhYLMecc6pA