生產項目容器化改造(二)使用Jenkins任務部署Maven項目

構建jre1.8 tomcat8 基礎鏡像：

• 登陸harbor主機，構建tomcat基礎鏡像，並上傳到harbor

cd /tmp
wget --no-check-certificate https://wget.XXXXXXX.com:10194/jre/server-jre-8u241-linux-x64.tar.gz
wget --no-check-certificate https://wget.XXXXXXX.com:10194/tomcat/apache-tomcat-8.5.51.tar.gz

vi Dockerfile
FROM centos:6
LABEL description="jdk1.8 and tomcat8"
# 時區調整
RUN /bin/cp -av /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
# 適應中文
ENV LC_ALL en_US.UTF-8
# 安裝unzip包和git包
RUN sed -i 's/gpgcheck=1/gpgcheck=0/g' /etc/yum.repos.d/*.repo && \
  yum -y install unzip git
# 部署jdk1.8
ENV JDK_TAR   server-jre-8u241-linux-x64.tar.gz
ENV JDK_FILE  jdk1.8.0_241
ENV JAVA_HOME /usr/local/java/java_1.8
ENV CLASSPATH ${JAVA_HOME}/lib:${JAVA_HOME}/jre/lib
ENV PATH      ${JAVA_HOME}/bin:${JAVA_HOME}/jre/bin:${PATH}
WORKDIR /usr/local/java
ADD ${JDK_TAR} .
RUN ln -s ${JDK_FILE} java_1.8 && \
    chown root: -R .
# 部署tomcat8
ENV TOM_TAR apache-tomcat-8.5.51.tar.gz
ENV TOM_FILE apache-tomcat-8.5.51
WORKDIR /usr/local/tomcat
ADD ${TOM_TAR} .
RUN ln -s ${TOM_FILE} tomcat8 && \
    chown root: -R .
# tomcat8 簡單優化調整
ENV OPTS "-Djava.security.egd=file:/dev/./urandom -Xms1256m -Xmx1512m"
ENV OPTS=${OPTS}" -XX:PermSize=50m -XX:MaxPermSize=70m"
ENV OPTS=${OPTS}" -XX:-UseGCOverheadLimit"
ENV OPTS2=${OPTS}" -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp"
RUN sed -i "s|cygwin=false|JAVA_OPTS=\"${OPTS}\"\n&|g" tomcat8/bin/catalina.sh && \
    sed -i "s|cygwin=false|# JAVA_OPTS=\"${OPTS2}\"\n&|g" tomcat8/bin/catalina.sh && \
    sed -i 's/Connector port="8080" protocol="HTTP\/1.1"/&\n \
               maxThreads="800" acceptCount="1000"\n \
               compression="on"\n \
               compressionMinSize="2048"\n \
               noCompressionUserAgents="gozilla,traviata"\n \
               compressableMimeType="text\/html,text\/xml,text\/javascript,text\/css,text\/plain"/g' tomcat8/conf/server.xml && \
    sed -i 's|${catalina.base}/logs|/web/logs|g' tomcat8/conf/logging.properties && \
    sed -i 's|appBase="webapps"|appBase="/web/project"|g' tomcat8/conf/server.xml && \
    sed -i 's|directory="logs"|directory="/web/logs/access"|g' tomcat8/conf/server.xml && \
    mkdir -pv /web/{profile,project/ROOT} && \
    echo "$(hostname) $(date +%F_%T)">/web/project/ROOT/index.html
EXPOSE 8080
CMD tomcat8/bin/catalina.sh run &> /web/logs/catalina.out

docker build -t harbor.vincent.com/library/tomcat8-j8 .
docker push harbor.vincent.com/library/tomcat8-j8

部署Jenkins模版任務，複製模版部署測試：

• 網頁登陸jenkins，創建一個模版任務deploy-templete，類型爲pipeline， 丟棄舊構建選擇保留2個，添加groovy腳本如下：

#!groovy
pipeline{
  agent any
  environment{
    PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin"
    GitLab="gitlab"
    GitLabCredentialsId="gitlabcredentialsid"
    GitNS="gitns"
    GitName="gitname"
    GitSubPro="gitsubpro"
    GitSubProPort="gitsubproport"
    IPs="ips"
    Nginx="nginx"
    Harbor="harbor"
    LbNginx="lbnginx"
    LbNginxConf='lbnginxconf'
  }
  parameters{
    choice(
      name: 'Option',
      choices: 'Update\nRepublish\nRollback',
      description: '功能選擇，升級發佈、項目再發布，回滾發佈'
    )
    choice(
      name: 'Range',
      choices: 'Canary\nExcept\nALL\nips',
      description: '發佈範圍：單機發布，剩餘主機發布，全量發佈，單機發布'
    )
    choice(
      name: 'Interval',
      choices: '15\n30\n60\n120',
      description: '多機發布時的發佈間隔，單位秒'
    )
    /* 參數使用情形舉例：
    // ------- 金絲雀升級發佈，測試後，再發布到剩餘主機
    // Option=Update，Range=Canary，金絲雀升級發佈，默認發佈到第一個IP
    // Option=Republish，Range=Except，項目再發布到第一個IP之外的其他主機
    // ------- 全量升級發佈，不進行測試，直接發佈到全局主機
    // Option=Update，Range=ALL
    // ------- 指定IP升級發佈，測試，再發布到剩餘主機
    // Option=Update，Range=IP2，當指定的IP是第一個時，既是金絲雀發佈
    // Option=Republish，Range=IP3/IP4...，指定IP的再發布
    // ------- 指定IP再發布，當某個主機的項目包損壞時或者新添加了主機時
    // Option=Republish，Range=IP5
    // ------- 全量回滾
    // Option=Rollback，Range=ALL
    // ------- 金絲雀回滾發佈，測試後，剩餘主機回滾
    // Option=Rollback，Range=Canary
    // Option=Rollback，Range=Except
    // ------- 指定IP回滾
    // Option=Rollback，Range=IP2...
    // ------- 當Range=Except/ALL時，Interval 參數生效
    */
  }
  stages{
    stage("GitClone"){
      when { environment name: "Option", value: "Update"}
      steps{
        // 代碼下載
        sh 'git config --global http.sslVerify false'
        dir("${env.WORKSPACE}"){
          git branch: 'develop',
          // git branch: 'master',
          credentialsId: "${GitLabCredentialsId}",
          url:"${GitLab}/${GitNS}/${GitName}.git"
        }
      }
    }
    stage("ProjectBuild"){
      when { environment name: "Option", value: "Update"}
      steps{
        // 項目構建
        dir("${env.WORKSPACE}"){
          sh "source /etc/profile && mvn -Dmaven.test.skip=true clean package"
        }
      }
    }
    stage("ImageBuild"){
      when { environment name: "Option", value: "Update"}
      steps{
        // 項目包上傳包共享服務器，構建鏡像並存入harbor鏡像共享服務器
        sh "bash ${JENKINS_HOME}/script/warMoveAndClear.sh ${WORKSPACE} ${GitSubPro} ${Nginx} ${Harbor}"
      }
    }
    stage("PublishProject"){
      steps{
        sh "bash ${JENKINS_HOME}/script/mutiPublish.sh ${Option} ${env.Range} '${IPs}' ${Interval} ${GitSubPro} ${GitSubProPort} ${Harbor} ${LbNginx} ${LbNginxConf}"
        
      }
    }
  }
}

• 在jenkins主機上部署任務所調用的腳本

su - deploy
mkdir -pv /var/lib/jenkins/script && cd /var/lib/jenkins/script
vi warMoveAndClear.sh
#!/bin/bash
source ~/.bash_profile
warPath=${1}
warProName=${2}
imageName=$(echo ${warProName}| tr '[A-Z]' '[a-z]')
Release=$(date +%Y%m%d.%H%M%S)
WarFile=${warProName}-${Release}.war
nginxRootPath=/usr/share/nginx/html
Nginx=${3}
Harbor=${4}
# 創建nginx相應目錄並存儲當前構建的war包
mkdir -p ${nginxRootPath}/${warProName}
cd ${warPath}/${warProName}/target/
find . -name "*.war" -type f -exec cp -a {} ${nginxRootPath}/${warProName}/${WarFile} \;
# 對歷史war包進行清理，保留3個war包
WarFileNumber=$(ls ${nginxRootPath}/${warProName}/*.war|wc -l)
if [ "${WarFileNumber}" -ge 3 ]
then
  StandFile=$(ls -t ${nginxRootPath}/${warProName}/*.war|head -3|tail -1)
  find ${nginxRootPath}/${warProName}/ -type f -name "*.war" -not -newer ${StandFile} -exec rm -f {} \;
fi
echo "[+] ############## $(date +%F_%T) 項目包${WarFile}存儲到nginx完成"
# 操縱harbor，構建鏡像並上傳
ssh root@${Harbor} "rm -rf /opt/${warProName} && mkdir -p /opt/${warProName}"
cat >/tmp/Dockerfile-${warProName}-${Release}<<EOF
FROM harbor.vincent.com/library/tomcat8-j8
WORKDIR /web/profile
ADD ${warProName}.tar.gz .
# git clone ...
WORKDIR /web/project
RUN rm -rf *
COPY ${WarFile} ROOT.war
EXPOSE 8080
CMD /usr/local/tomcat/tomcat8/bin/catalina.sh run &> /web/logs/catalina.out
EOF
echo "[+] ############## $(date +%F_%T) 同步Dockerfile到harbor"
scp /tmp/Dockerfile-${warProName}-${Release} root@${Harbor}:/opt/${warProName}/Dockerfile
rm -rf /tmp/Dockerfile-${warProName}-${Release}
echo "[+] ############## $(date +%F_%T) 下載profile"
ssh root@${Harbor} "cd /opt/${warProName} && wget ${Nginx}/profile/${warProName}.tar.gz -o /dev/null"
echo "[+] ############## $(date +%F_%T) 下載war包"
ssh root@${Harbor} "cd /opt/${warProName} && wget ${Nginx}/${warProName}/${WarFile} -o /dev/null"
echo "[+] ############## $(date +%F_%T) 構建鏡像"
ssh root@${Harbor} "cd /opt/${warProName} && docker build -t harbor.vincent.com/library/${imageName}:${Release} ."
echo "[+] ############## $(date +%F_%T) 鏡像上傳到harbor"
ssh root@${Harbor} "docker push harbor.vincent.com/library/${imageName}:${Release}"
echo "[+] ############## $(date +%F_%T) 本地鏡像清除"
ssh root@${Harbor} "docker rmi harbor.vincent.com/library/${imageName}:${Release}"
echo "[+] ############## $(date +%F_%T) 鏡像 harbor.vincent.com/library/${imageName}:${Release}存儲到harbor完成"

vi mutiPublish.sh
#!/bin/bash
source ~/.bash_profile
Option=${1}
Range=${2}
IPs=${3}
Interval=${4}
GitSubPro=${5}
imageName=$(echo ${GitSubPro}| tr '[A-Z]' '[a-z]')
GitSubProPort=${6}
Harbor=${7}
LbNginx=${8}
LbNginxConf=${9}
ImagePath="harbor.vincent.com/library"
# 獲取項目的鏡像tag列表
TagInfo=$(curl -s -k -X GET "https://${Harbor}/api/repositories/library%2F${imageName}/tags" \
  -H "accept: application/json" -H "X-Xsrftoken: GUXirgNTqmudkEPhJtOtl520yTRJz6RW" \
   | jq '.[].name'|sed 's/"//g')
Tag0=$(echo ${TagInfo}|awk '{print $(NF-0)}')
Tag1=$(echo ${TagInfo}|awk '{print $(NF-1)}')
Tag2=$(echo ${TagInfo}|awk '{print $(NF-2)}')
# 發佈類型判斷
case ${Option} in
  "Update")
    Image=${imageName}:${Tag0}
    ;;
  "Republish")
    Image=${imageName}:${Tag0}
    ;;
  "Rollback")
    Image=${imageName}:${Tag1}
    ;;
esac
# 發佈範圍判斷
case ${Range} in
  "Canary")
    ip=$(echo ${IPs}|awk '{print $1}')
    ;;
  "Except")
    ip=$(echo ${IPs}|awk '{for(i=2;i<=NF;i++) printf("%s ",$i)}')
    ;;
  "ALL")
    ip=${IPs}
    ;;
  *)
    ip=${Range}
    ;;
esac
# 最終發佈
echo "[+] ############## $(date +%F_%T) ${Option}發佈，使用鏡像：${ImagePath}/${Image}，發佈IP爲：${ip}"
for i in ${ip}
do
  ssh root@${i} "echo -ne \"[+] ############## $(date +%F_%T) ${i} 發佈開始 \n\""
  # 需要配置deploy到LbNginx的root用戶的免密登陸
  # ssh root@${LbNginx} "sed -i 's/${i}:${GitSubProPort}/# ${i}:${GitSubProPort}/g' ${LbNginxConf}"
  # ssh root@${LbNginx} "systemctl reload nginx"
  # echo "[+] ############## $(date +%F_%T) ${GitSubPro}-${i}-${GitSubProPort} 負載下線，${Interval}秒後部署開始"
  # curl -s 'https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=xxxxxxxxx' \
  #   -H "Content-Type: application/json" -d "{\"msgtype\": \"text\", \
  #        \"text\": {\"content\": \"${GitSubPro}-${i}-${GitSubProPort}:\n$(date +%F_%T) 負載下線，${Interval}秒後部署開始\"}}"
  # sleep ${Interval}
  ssh root@${i} "echo -ne \"[+] ############## $(date +%F_%T) 容器關閉 \" && docker stop ${imageName}-${i}-${GitSubProPort}"
  ssh root@${i} "echo -ne \"[+] ############## $(date +%F_%T) 容器刪除 \" && docker rm ${imageName}-${i}-${GitSubProPort}"
  ssh root@${i} "echo -ne \"[+] ############## $(date +%F_%T) 鏡像刪除 \n\" && for i in \$(docker image ls|grep ${imageName}|awk '{print \$3}');do docker rmi \$i &>/dev/null;done"
  ssh root@${i} "echo -ne \"[+] ############## $(date +%F_%T) 鏡像拉取 \n\" && docker pull ${ImagePath}/${Image} &>/dev/null"
  ssh root@${i} "echo -ne \"[+] ############## $(date +%F_%T) 容器啓動 \" && docker run -d --name=${imageName}-${i}-${GitSubProPort} -p ${GitSubProPort}:8080 -v /opt/logs/${imageName}-${i}-${GitSubProPort}:/web/logs -m 1512M -it ${ImagePath}/${Image}"
  echo "[+] ############## $(date +%F_%T) ${imageName}-${i}-${GitSubProPort} ${Option}發佈成功" #，${Interval}秒後負載上線"
  curl -s 'https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=xxxxxxxxx' \
    -H "Content-Type: application/json" -d "{\"msgtype\": \"text\", \
         \"text\": {\"content\": \"${GitSubPro}-${i}-${GitSubProPort}:\n$(date +%F_%T) ${Option}發佈成功\"}}" #，${Interval}秒後負載上線\"}}"
  sleep ${Interval}
  # ssh root@${LbNginx} "sed -i 's/# ${i}:${GitSubProPort}/${i}:${GitSubProPort}/g' ${LbNginxConf}"
  # ssh root@${LbNginx} "systemctl reload nginx"
  # echo "[+] ############## $(date +%F_%T) ${GitSubPro}-${i} 負載上線"
  # curl -s 'https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=xxxxxxxxx' \
  #   -H "Content-Type: application/json" -d "{\"msgtype\": \"text\", \
  #        \"text\": {\"content\": \"${GitSubPro}-${i}-${GitSubProPort}:\n$(date +%F_%T) 負載上線\"}}"
done

• 模版複製創建任務，構建發佈測試

su - deploy
gitns='vincent'
gitname='test'
gitsubpro='test-soaport'
gitsubproport='8081'
ips='192.168.1.171 192.168.1.172 192.168.1.173 192.168.1.174 192.168.1.175'
ipS='192.168.1.171\\n192.168.1.172\\n192.168.1.173\\n192.168.1.174\\n192.168.1.175'
gitlab='https://gitlab.vincent.com'
gitlabcredentialsid='https.gitlab.root.pass'
nginx='http://dockerjenkins:8080'
harbor='harbor.vincent.com'
lbnginx='lbnginx'
lbnginxconf='lbnginxconf'
cd /var/lib/jenkins/jobs/
cp -av deploy-templete ${gitsubpro}
sed -i "s|GitLab=\"gitlab\"|GitLab=\"${gitlab}\"|g" ${gitsubpro}/config.xml
sed -i "s|GitLabCredentialsId=\"gitlabcredentialsid\"|GitLabCredentialsId=\"${gitlabcredentialsid}\"|g" ${gitsubpro}/config.xml
sed -i "s|GitNS=\"gitns\"|GitNS=\"${gitns}\"|g" ${gitsubpro}/config.xml
sed -i "s|GitName=\"gitname\"|GitName=\"${gitname}\"|g" ${gitsubpro}/config.xml
sed -i "s|GitSubPro=\"gitsubpro\"|GitSubPro=\"${gitsubpro}\"|g" ${gitsubpro}/config.xml
sed -i "s|GitSubProPort=\"gitsubproport\"|GitSubProPort=\"${gitsubproport}\"|g" ${gitsubpro}/config.xml
sed -i "s|IPs=\"ips\"|IPs=\"${ips}\"|g" ${gitsubpro}/config.xml
sed -i "s|Nginx=\"nginx\"|Nginx=\"${nginx}\"|g" ${gitsubpro}/config.xml
sed -i "s|Harbor=\"harbor\"|Harbor=\"${harbor}\"|g" ${gitsubpro}/config.xml
sed -i "s|LbNginx=\"lbnginx\"|LbNginx=\"${lbnginx}\"|g" ${gitsubpro}/config.xml
sed -i "s|LbNginxConf=\'lbnginxconf\'|LbNginxConf=\'${lbnginxconf}\'|g" ${gitsubpro}/config.xml
sed -i "s|ips\&apos|${ipS}\&apos|g" ${gitsubpro}/config.xml
su - root /etc/init.d/jenkins restart

• 項目做過配置分離，需要將相應的配置目錄profile打包上傳到nginx對應目錄：

cd /tmp
tar -czf test-soaport.tar.gz profile
mkdir -pv /usr/share/nginx/html/profile
mv -v test-soaport.tar.gz /usr/share/nginx/html/profile

• 網頁登陸Jenkins，使用任務進行構建部署測試

[TOC]