k8s實戰案例之運行dubbo微服務

1、dubbo微服務架構圖

通過上述架構可以瞭解到，生產者通過註冊中心，將服務註冊至註冊中心，消費者通過註冊中心找到生產者，從而實現消費者拿到生產者的實際地址，然後直接和生產者通信；管理端通過註冊中心發現生產者和消費者，通過svc來管理生產者和消費者；集羣外部客戶端通過負載均衡器來訪問集羣內部服務；dubbo產生的數據可以通過pv/pvc將數據放置外部存儲，如ceph,nas，nfs等；

2、部署provider服務（生產者）

2.1、準備provider鏡像

2.1.1、provider鏡像數據目錄文件

2.1.2、構建provider鏡像Dockerfile

root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/provider# cat Dockerfile 
#Dubbo provider
FROM harbor.ik8s.cc/pub-images/jdk-base:v8.212 

RUN yum install file nc -y
RUN mkdir -p /apps/dubbo/provider
ADD dubbo-demo-provider-2.1.5/  /apps/dubbo/provider
ADD run_java.sh /apps/dubbo/provider/bin 
RUN chown nginx.nginx /apps -R
RUN chmod a+x /apps/dubbo/provider/bin/*.sh

CMD ["/apps/dubbo/provider/bin/run_java.sh"]
root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/provider# 

2.1.3、運行provider鏡像腳本

root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/provider# cat run_java.sh 
#!/bin/bash
#echo "nameserver 223.6.6.6" > /etc/resolv.conf
#/usr/share/filebeat/bin/filebeat -c /etc/filebeat/filebeat.yml -path.home /usr/share/filebeat -path.config /etc/filebeat -path.data /var/lib/filebeat -path.logs /var/log/filebeat  &
su - nginx -c "/apps/dubbo/provider/bin/start.sh"
tail -f /etc/hosts
root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/provider# 

2.1.4、構建provider鏡像腳本

root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/provider# cat build-command.sh 
#!/bin/bash
#docker build -t harbor.ik8s.cc/magedu/dubbo-demo-provider:v1  .
#sleep 3
#docker push harbor.ik8s.cc/magedu/dubbo-demo-provider:v1

nerdctl build -t harbor.ik8s.cc/magedu/dubbo-demo-provider:v1  .
nerdctl push harbor.ik8s.cc/magedu/dubbo-demo-provider:v1
root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/provider# 

修改腳本權限

root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/provider# chmod a+x dubbo-demo-provider-2.1.5/bin/*.sh
root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/provider# ll dubbo-demo-provider-2.1.5/bin/*.sh
-rwxr-xr-x 1 root root 2165 Jun 22  2021 dubbo-demo-provider-2.1.5/bin/dump.sh*
-rwxr-xr-x 1 root root   49 Jun 22  2021 dubbo-demo-provider-2.1.5/bin/restart.sh*
-rwxr-xr-x 1 root root  413 Jun 22  2021 dubbo-demo-provider-2.1.5/bin/server.sh*
-rwxr-xr-x 1 root root 3227 Jun 22  2021 dubbo-demo-provider-2.1.5/bin/start.sh*
-rwxr-xr-x 1 root root  807 Jun 22  2021 dubbo-demo-provider-2.1.5/bin/stop.sh*
root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/provider# 

修改註冊中心zookeeper地址

這個註冊中心地址我們寫zk節點的svc名稱即可，如果是在同一名稱空間下，直接寫svc名稱加端口即可，不在同一名稱空間下，需要寫svc名稱.名稱空間後面跟上端口即可；

構建provider鏡像

2.2、運⾏provider服務

運行provider配置清單

kind: Deployment
#apiVersion: extensions/v1beta1
apiVersion: apps/v1
metadata:
  labels:
    app: magedu-provider
  name: magedu-provider-deployment
  namespace: magedu
spec:
  replicas: 1
  selector:
    matchLabels:
      app: magedu-provider
  template:
    metadata:
      labels:
        app: magedu-provider
    spec:
      containers:
      - name: magedu-provider-container
        image: harbor.ik8s.cc/magedu/dubbo-demo-provider:v1
        #command: ["/apps/tomcat/bin/run_tomcat.sh"]
        #imagePullPolicy: IfNotPresent
        imagePullPolicy: Always
        ports:
        - containerPort: 20880
          protocol: TCP
          name: http

---
kind: Service
apiVersion: v1
metadata:
  labels:
    app: magedu-provider
  name: magedu-provider-spec
  namespace: magedu
spec:
  type: NodePort
  ports:
  - name: http
    port: 80
    protocol: TCP
    targetPort: 20880
    #nodePort: 30001
  selector:
    app: magedu-provider

應用配置清單

root@k8s-master01:~/k8s-data/yaml/magedu/dubbo/provider# kubectl apply -f provider.yaml
deployment.apps/magedu-provider-deployment created
service/magedu-provider-spec created
root@k8s-master01:~/k8s-data/yaml/magedu/dubbo/provider# 

2.3、驗證provider服務

2.3.1、驗證pod是否正常running？

2.3.2、zookeeper驗證provider註冊

3、部署consumer服務（消費者）

3.1、準備consumer鏡像

3.1.1、consumer鏡像數據目錄文件

3.1.2、構建consumer鏡像Dockerfile

root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/consumer# cat Dockerfile
#Dubbo consumer
FROM harbor.ik8s.cc/pub-images/jdk-base:v8.212 

RUN yum install file -y
RUN mkdir -p /apps/dubbo/consumer 
ADD dubbo-demo-consumer-2.1.5  /apps/dubbo/consumer
ADD run_java.sh /apps/dubbo/consumer/bin 
RUN chown nginx.nginx /apps -R
RUN chmod a+x /apps/dubbo/consumer/bin/*.sh

CMD ["/apps/dubbo/consumer/bin/run_java.sh"]
root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/consumer# 

3.1.3、運行consumer鏡像腳本

root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/consumer# cat run_java.sh 
#!/bin/bash
#echo "nameserver 223.6.6.6" > /etc/resolv.conf
#/usr/share/filebeat/bin/filebeat -c /etc/filebeat/filebeat.yml -path.home /usr/share/filebeat -path.config /etc/filebeat -path.data /var/lib/filebeat -path.logs /var/log/filebeat  &
su - nginx -c "/apps/dubbo/consumer/bin/start.sh"
tail -f /etc/hosts
root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/consumer# 

3.1.4、構建consumer鏡像腳本

root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/consumer# cat build-command.sh
#!/bin/bash
#docker build -t harbor.ik8s.cc/magedu/dubbo-demo-consumer:v1  .
#sleep 3
#docker push harbor.ik8s.cc/magedu/dubbo-demo-consumer:v1

nerdctl build -t harbor.ik8s.cc/magedu/dubbo-demo-consumer:v1  .
nerdctl push harbor.ik8s.cc/magedu/dubbo-demo-consumer:v1
root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/consumer# 

修改腳本權限

root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/consumer# chmod a+x dubbo-demo-consumer-2.1.5/bin/*.sh
root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/consumer# ll dubbo-demo-consumer-2.1.5/bin/*.sh
-rwxr-xr-x 1 root root 2165 Jun 22  2021 dubbo-demo-consumer-2.1.5/bin/dump.sh*
-rwxr-xr-x 1 root root   49 Jun 22  2021 dubbo-demo-consumer-2.1.5/bin/restart.sh*
-rwxr-xr-x 1 root root  413 Jun 22  2021 dubbo-demo-consumer-2.1.5/bin/server.sh*
-rwxr-xr-x 1 root root 3227 Jun 22  2021 dubbo-demo-consumer-2.1.5/bin/start.sh*
-rwxr-xr-x 1 root root  807 Jun 22  2021 dubbo-demo-consumer-2.1.5/bin/stop.sh*
root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/consumer# 

修改註冊中心zookeeper地址

構建consumer鏡像

3.2、運行consumer服務

運行comsumer配置清單

kind: Deployment
#apiVersion: extensions/v1beta1
apiVersion: apps/v1
metadata:
  labels:
    app: magedu-consumer
  name: magedu-consumer-deployment
  namespace: magedu
spec:
  replicas: 1
  selector:
    matchLabels:
      app: magedu-consumer
  template:
    metadata:
      labels:
        app: magedu-consumer
    spec:
      containers:
      - name: magedu-consumer-container
        image: harbor.ik8s.cc/magedu/dubbo-demo-consumer:v1 
        #command: ["/apps/tomcat/bin/run_tomcat.sh"]
        #imagePullPolicy: IfNotPresent
        imagePullPolicy: Always
        ports:
        - containerPort: 80
          protocol: TCP
          name: http

---
kind: Service
apiVersion: v1
metadata:
  labels:
    app: magedu-consumer
  name: magedu-consumer-server
  namespace: magedu
spec:
  type: NodePort
  ports:
  - name: http
    port: 80
    protocol: TCP
    targetPort: 80
    #nodePort: 30001
  selector:
    app: magedu-consumer

應用配置清單

root@k8s-master01:~/k8s-data/yaml/magedu/dubbo/consumer# kubectl apply -f consumer.yaml 
deployment.apps/magedu-consumer-deployment created
service/magedu-consumer-server created
root@k8s-master01:~/k8s-data/yaml/magedu/dubbo/consumer# 

3.3、驗證consumer服務

3.3.1、驗證pod是否正常running？

3.3.2、zookeeper驗證consumer註冊

3.3.3、驗證consumer調用provider

可以看到consumer通過註冊中心找到了provider；

3.3.4、增加consumer副本數量看看對應consumer是否能夠正常找到provide？

增加consumer副本

應用配置清單

root@k8s-master01:~/k8s-data/yaml/magedu/dubbo/consumer# kubectl apply -f consumer.yaml 
deployment.apps/magedu-consumer-deployment configured
service/magedu-consumer-server unchanged
root@k8s-master01:~/k8s-data/yaml/magedu/dubbo/consumer# kubectl get pods -n magedu 
NAME                                          READY   STATUS      RESTARTS      AGE
magedu-consumer-deployment-798c7d785b-fp4b9   1/1     Running     0             26s
magedu-consumer-deployment-798c7d785b-wmv9p   1/1     Running     0             18m
magedu-consumer-deployment-798c7d785b-zqm74   1/1     Running     0             26s
magedu-provider-deployment-6fccc6d9f5-nl4zd   1/1     Running     0             73m
mysql-0                                       2/2     Running     6 (97m ago)   56d
mysql-1                                       2/2     Running     6 (97m ago)   56d
mysql-2                                       2/2     Running     6 (97m ago)   56d
redis-0                                       1/1     Running     5 (97m ago)   65d
redis-1                                       1/1     Running     5 (97m ago)   65d
redis-2                                       1/1     Running     5 (97m ago)   65d
redis-3                                       1/1     Running     5 (97m ago)   65d
redis-4                                       1/1     Running     5 (97m ago)   65d
redis-5                                       1/1     Running     5 (97m ago)   65d
ubuntu1804                                    0/1     Completed   0             65d
zookeeper1-675c5477cb-vmwwq                   1/1     Running     7 (97m ago)   67d
zookeeper2-759fb6c6f-7jktr                    1/1     Running     7 (97m ago)   67d
zookeeper3-5c78bb5974-vxpbh                   1/1     Running     7 (97m ago)   67d
root@k8s-master01:~/k8s-data/yaml/magedu/dubbo/consumer# 

查看provider日誌

在provider中看日誌，可以看到有3個consumer在調用provider,這說明consumer通過註冊中興可以正常發現provider，然後和provider正常通信；

3.3.5、增加provider副本，看看consumer是否能夠找到多個provider?

增加provider副本

應用配置清單

查看任意一個consumer日誌

可以看到provider副本增加，consumer也可以通過註冊中心發現provider；

在註冊中心zookeeper查看consumer和provider情況

可以看到多副本的provider和多副本consumer都正常註冊到zookeeper；

4、部署dubbo admin

4.1、dubbo admin鏡像準備

4.1.1、dubbo admin鏡像數據目錄文件

4.1.2、構建dubbo admin鏡像Docerfile

root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/dubboadmin# cat Dockerfile
#Dubbo dubboadmin
#FROM harbor.ik8s.cc/pub-images/tomcat-base:v8.5.43 
FROM harbor.ik8s.cc/pub-images/tomcat-base:v8.5.43

RUN yum install unzip -y  
ADD server.xml /apps/tomcat/conf/server.xml
ADD logging.properties /apps/tomcat/conf/logging.properties
ADD catalina.sh /apps/tomcat/bin/catalina.sh
ADD run_tomcat.sh /apps/tomcat/bin/run_tomcat.sh
ADD dubboadmin.war  /data/tomcat/webapps/dubboadmin.war
RUN cd /data/tomcat/webapps && unzip dubboadmin.war && rm -rf dubboadmin.war && chown -R nginx.nginx /data /apps

EXPOSE 8080 8443

CMD ["/apps/tomcat/bin/run_tomcat.sh"]
root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/dubboadmin# 

4.1.3、運行dubbo admin鏡像腳本

root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/dubboadmin# cat run_tomcat.sh 
#!/bin/bash

su - nginx -c "/apps/tomcat/bin/catalina.sh start"
su - nginx -c "tail -f /etc/hosts"
root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/dubboadmin# 

4.1.4、構建dubbo admin鏡像腳本

root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/dubboadmin# cat build-command.sh 
#!/bin/bash
TAG=$1
#docker build -t harbor.ik8s.cc/magedu/dubboadmin:${TAG}  .
#sleep 3
#docker push  harbor.ik8s.cc/magedu/dubboadmin:${TAG}

nerdctl build -t  harbor.ik8s.cc/magedu/dubboadmin:${TAG}  .
nerdctl push harbor.ik8s.cc/magedu/dubboadmin:${TAG}
root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/dubboadmin# 

修改註冊中心zookeeper地址

打包dubboadmin文件夾爲dubboadmin.war

root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/dubboadmin# zip -rmq dubboadmin.war dubboadmin/*
root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/dubboadmin# ll
total 54312
drwxr-xr-x 2 root root     4096 Aug 11 05:32 ./
drwxr-xr-x 5 root root     4096 Aug  9  2022 ../
-rw-r--r-- 1 root root      575 Aug 11 05:17 Dockerfile
-rwxr-xr-x 1 root root      257 Aug 11 05:20 build-command.sh*
-rwxr-xr-x 1 root root    22201 Jun 22  2021 catalina.sh*
-rw-r--r-- 1 root root 27778001 Aug 11 05:32 dubboadmin.war
-rw-r--r-- 1 root root 27777984 Jun 22  2021 dubboadmin.war.bak
-rw-r--r-- 1 root root     3436 Jun 22  2021 logging.properties
-rwxr-xr-x 1 root root       99 Jun 22  2021 run_tomcat.sh*
-rw-r--r-- 1 root root     6427 Jun 22  2021 server.xml
root@k8s-master01:~/k8s-data/dockerfile/web/magedu/dubbo/dubboadmin# 

運行構建腳本構建dubboadmin鏡像

4.2、運行dubbo admin

dubboadmin配置清單

kind: Deployment
#apiVersion: extensions/v1beta1
apiVersion: apps/v1
metadata:
  labels:
    app: magedu-dubboadmin
  name: magedu-dubboadmin-deployment
  namespace: magedu
spec:
  replicas: 1
  selector:
    matchLabels:
      app: magedu-dubboadmin
  template:
    metadata:
      labels:
        app: magedu-dubboadmin
    spec:
      containers:
      - name: magedu-dubboadmin-container
        image: harbor.ik8s.cc/magedu/dubboadmin:v1 
        #imagePullPolicy: IfNotPresent
        imagePullPolicy: Always
        ports:
        - containerPort: 8080
          protocol: TCP
          name: http

---
kind: Service
apiVersion: v1
metadata:
  labels:
    app: magedu-dubboadmin
  name: magedu-dubboadmin-service
  namespace: magedu
spec:
  type: NodePort
  ports:
  - name: http
    port: 80
    protocol: TCP
    targetPort: 8080
    nodePort: 31080
  selector:
    app: magedu-dubboadmin

應用配置清單

root@k8s-master01:~/k8s-data/yaml/magedu/dubbo/dubboadmin# kubectl apply -f dubboadmin.yaml 
deployment.apps/magedu-dubboadmin-deployment created
service/magedu-dubboadmin-service created
root@k8s-master01:~/k8s-data/yaml/magedu/dubbo/dubboadmin# kubectl get pods -n magedu 
NAME                                            READY   STATUS      RESTARTS       AGE
magedu-consumer-deployment-798c7d785b-fp4b9     1/1     Running     0              41m
magedu-consumer-deployment-798c7d785b-wmv9p     1/1     Running     0              60m
magedu-consumer-deployment-798c7d785b-zqm74     1/1     Running     0              41m
magedu-dubboadmin-deployment-798c4dfdd8-kvfvh   1/1     Running     0              49s
magedu-provider-deployment-6fccc6d9f5-k6z7m     1/1     Running     0              33m
magedu-provider-deployment-6fccc6d9f5-nl4zd     1/1     Running     0              115m
magedu-provider-deployment-6fccc6d9f5-p94rb     1/1     Running     0              33m
mysql-0                                         2/2     Running     6 (138m ago)   56d
mysql-1                                         2/2     Running     6 (138m ago)   56d
mysql-2                                         2/2     Running     6 (138m ago)   56d
redis-0                                         1/1     Running     5 (138m ago)   65d
redis-1                                         1/1     Running     5 (138m ago)   65d
redis-2                                         1/1     Running     5 (138m ago)   65d
redis-3                                         1/1     Running     5 (138m ago)   65d
redis-4                                         1/1     Running     5 (138m ago)   65d
redis-5                                         1/1     Running     5 (138m ago)   65d
ubuntu1804                                      0/1     Completed   0              65d
zookeeper1-675c5477cb-vmwwq                     1/1     Running     7 (138m ago)   67d
zookeeper2-759fb6c6f-7jktr                      1/1     Running     7 (138m ago)   67d
zookeeper3-5c78bb5974-vxpbh                     1/1     Running     7 (138m ago)   67d
root@k8s-master01:~/k8s-data/yaml/magedu/dubbo/dubboadmin# kubectl get svc -n magedu 
NAME                        TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                                        AGE
magedu-consumer-server      NodePort    10.100.208.121   <none>        80:49630/TCP                                   61m
magedu-dubboadmin-service   NodePort    10.100.244.92    <none>        80:31080/TCP                                   2m11s
magedu-provider-spec        NodePort    10.100.187.168   <none>        80:44873/TCP                                   116m
mysql                       ClusterIP   None             <none>        3306/TCP                                       56d
mysql-read                  ClusterIP   10.100.15.127    <none>        3306/TCP                                       56d
redis                       ClusterIP   None             <none>        6379/TCP                                       65d
redis-access                NodePort    10.100.117.185   <none>        6379:36379/TCP                                 65d
zookeeper                   ClusterIP   10.100.237.95    <none>        2181/TCP                                       67d
zookeeper1                  NodePort    10.100.63.118    <none>        2181:32181/TCP,2888:30541/TCP,3888:31200/TCP   67d
zookeeper2                  NodePort    10.100.199.43    <none>        2181:32182/TCP,2888:32670/TCP,3888:32264/TCP   67d
zookeeper3                  NodePort    10.100.41.9      <none>        2181:32183/TCP,2888:31329/TCP,3888:32546/TCP   67d
root@k8s-master01:~/k8s-data/yaml/magedu/dubbo/dubboadmin# 

4.3、在dubbo admin web驗證provider和consumer

登錄dubboadmin web


在dubboadmin web查看provider和consumer