基於Ingress 實現集羣服務訪問

一、前言

Ingress與Ingress-controller

ingress對象：

指的是k8s中的一個api對象，一般用於yaml配置，如果集成了kubesphere，可以直接在UI界面上進行創建，其作用就是定義請求如何轉發到service的規則，可以理解爲配置模版

Ingress-controller

具體實現反向代理以及負載均衡的程序，對ingress定義的規則進行解析，根據配置的規則來實現請求轉發，簡單來說，ingress-controller纔是負責具體轉發的組建，通過各種方式將它暴露在集羣入口，外部對集羣的請求流量會先到ingress-controller，而ingress對象是用來告訴ingress-controller該如何轉發請求，比如哪些哪些域名path路徑要轉發哪些服務等等

 

本章採用DaemonSet結合NodeSelector來部署Ingress-controller到特定的node節點上，然後使用HostNetwork直接 把該pod與宿主機node節點的網路打通，直接使用宿主機的80/443端口就能訪問服務

此時，ingress-controller所在的node機器就很類似於傳統架構的邊緣節點，比如機房入口的nginx服務器，該方式整個請求鏈路都很簡單，性能相對NodePort模式更好，缺點就是由於直接利用宿主機節點的網絡和端口，一個node只能部署一個ingress-controller pod，比較適合大併發的生產環境使用

首先給指定的node節點打上標籤，讓我們的ingress-controller根據標籤運行在指定的節點上，在這裏我們選擇k8s-node02節點

[root@k8s-master01 ingress]# kubectl label node k8s-node02 ingress=true

[root@k8s-master01 ingress]# kubectl get nodes --show-labels

 

 

部署nginx-ingress-controller

該mandatory.yaml文件中包含了很多資源的創建，例如namespace、Configmap、role、ServiceAccount等等所有部署ingress-controller需要的資源

#wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/nginx-0.25.0/deploy/static/mandatory.yaml 

對nginx-ingress-controller的yaml文件根據業務情況進行修改，在這改動的是ClusterRole資源配置、修改控制器類型爲DaemonSet，並開啓hostNetwork網絡

#vim /opt/ingress/mandatory.yaml 

apiVersion: v1
kind: Namespace
metadata:
  name: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---

kind: ConfigMap
apiVersion: v1
metadata:
  name: nginx-configuration
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---
kind: ConfigMap
apiVersion: v1
metadata:
  name: tcp-services
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---
kind: ConfigMap
apiVersion: v1
metadata:
  name: udp-services
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: nginx-ingress-serviceaccount
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: nginx-ingress-clusterrole
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
rules:
  - apiGroups:
      - ""
    resources:
      - configmaps
      - endpoints
      - nodes
      - pods
      - secrets
    verbs:
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - nodes
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - services
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - "extensions"
      - "networking.k8s.io"　　#nginx-ingress-controller:0.25版本增加network.k8s.io Ingress資源api
    resources:
      - ingresses
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - events
    verbs:
      - create
      - patch
  - apiGroups:
      - "extensions"
      - "networking.k8s.io"　　##nginx-ingress-controller:0.25版本增加network.k8s.io Ingress資源api
    resources:
      - ingresses/status
    verbs:
      - update

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
  name: nginx-ingress-role
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
rules:
  - apiGroups:
      - ""
    resources:
      - configmaps
      - pods
      - secrets
      - namespaces
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - configmaps
    resourceNames:
      # Defaults to "<election-id>-<ingress-class>"
      # Here: "<ingress-controller-leader>-<nginx>"
      # This has to be adapted if you change either parameter
      # when launching the nginx-ingress-controller.
      - "ingress-controller-leader-nginx"
    verbs:
      - get
      - update
  - apiGroups:
      - ""
    resources:
      - configmaps
    verbs:
      - create
  - apiGroups:
      - ""
    resources:
      - endpoints
    verbs:
      - get

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
  name: nginx-ingress-role-nisa-binding
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: nginx-ingress-role
subjects:
  - kind: ServiceAccount
    name: nginx-ingress-serviceaccount
    namespace: ingress-nginx

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: nginx-ingress-clusterrole-nisa-binding
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: nginx-ingress-clusterrole
subjects:
  - kind: ServiceAccount
    name: nginx-ingress-serviceaccount
    namespace: ingress-nginx

---

apiVersion: apps/v1
#kind: Deployment
kind: DaemonSet　　#將nginx-ingress-controller部署爲Daemonset
metadata:
  name: nginx-ingress-controller
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
spec:
#  replicas: 1　　#因爲DaemonSet沒有副本機制的概念，因此這裏需要將其註釋
  selector:
    matchLabels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/part-of: ingress-nginx
  template:
    metadata:
      labels:
        app.kubernetes.io/name: ingress-nginx
        app.kubernetes.io/part-of: ingress-nginx
      annotations:
        prometheus.io/port: "10254"
        prometheus.io/scrape: "true"
    spec:
      hostNetwork: true　　#開啓hostNetwork網絡，讓其使用宿主機節點的端口
      nodeSelector:　　　　#配置nodeSelector標籤選擇器
        ingress: "true"
      serviceAccountName: nginx-ingress-serviceaccount
      containers:
        - name: nginx-ingress-controller
          image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.0
          args:
            - /nginx-ingress-controller
            - --configmap=$(POD_NAMESPACE)/nginx-configuration
            - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
            - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
            - --publish-service=$(POD_NAMESPACE)/ingress-nginx
            - --annotations-prefix=nginx.ingress.kubernetes.io
          securityContext:
            allowPrivilegeEscalation: true
            capabilities:
              drop:
                - ALL
              add:
                - NET_BIND_SERVICE
            # www-data -> 33
            runAsUser: 33
          env:
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
          ports:
            - name: http
              containerPort: 80
            - name: https
              containerPort: 443
          livenessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            initialDelaySeconds: 10
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 10
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 10

---

 

[root@k8s-master01 ingress]# kubectl apply -f mandatory.yaml

[root@k8s-master01 ~]# kubectl get pod -n ingress-nginx -o wide
NAME                             READY   STATUS    RESTARTS   AGE   IP                NODE         NOMINATED NODE   READINESS GATES
nginx-ingress-controller-j4wq8   1/1     Running   0          20m   192.168.126.135   k8s-node02   <none>           <none>
[root@k8s-master01 ~]#  kubectl get cm,daemonset -n ingress-nginx -o wide
NAME                                        DATA   AGE
configmap/ingress-controller-leader-nginx   0      17m
configmap/kube-root-ca.crt                  1      20m
configmap/nginx-configuration               0      20m
configmap/tcp-services                      0      20m
configmap/udp-services                      0      20m

NAME                                      DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR   AGE   CONTAINERS                 IMAGES                                                                  SELECTOR
daemonset.apps/nginx-ingress-controller   1         1         1       1            1           ingress=true    20m   nginx-ingress-controller   quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.0   app.kubernetes.io/name=ingress-nginx,app.kubernetes.io/part-of=ingress-nginx

 在ingres-ingress-controller所在的node2節點上查看監聽的端口以及容器服務

[root@k8s-node02 ~]# netstat -antp | grep nginx

 

 創建Ingress規則

這裏我們啓動了兩個nginx pod作爲後端的應用服務，以及指定暴露出service

[root@k8s-master01 ingress]# vim service-nginx.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-app
spec:
  replicas: 3
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
        - name: nginx
          image: nginx
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  name: nginx-app-svc
spec:
  type: ClusterIP
  ports:
  - protocol: TCP
    port: 80
    targetPort: 80
  selector:
    app: nginx

#[root@k8s-master01 ingress]# kubectl get -f service-nginx.yaml

創建Ingress服務，

#vim ingressp-app.yaml

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: nginx-app-ingress
spec:
  rules:
  - host: www.test.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: nginx-app-svc　　#指定svc名稱
            port:
              number: 80

#kubectl apply -f  ingressp-app.yaml

 驗證測試

在本地PC端配置一個host解析，測試訪問

 

 假設我們需要給指定的域名加上https認證，那麼該如何實現呢？

首先我們需要導入tsl證書，創建類型爲tsl的secret, 將我們的證書和密鑰導入進去

創建Secret

[root@k8s-master01 tsl]# kubectl create secret tls jiicm-com --key server.key --cert  jiicm.com.crt

查看詳情

 配置ingres

#vim ingressp-app.yaml

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: nginx-app-ingress
spec:
  tls:
   - hosts:
       - ts.jiicm.com　　#指定https域名
     secretName: jiicm-com　　#指定類型爲tls的secret名稱（也就是上面我們創建的ssl證書）
  rules:
  - host: ts.jiicm.com
    http:
      paths:
      - path: /
        pathType: Prefix  #路徑的匹配方式，目前有implementationSpecific、Exact和Prefix方式
        backend:
          service:
            name: nginx-app-svc　　#指定後端nginx服務的service名稱
            port:
              number: 80

 

#kubectl replace -f  ingressp-app.yaml

 

 END！