Docker CI: Jenkins Pipeline 介紹和實戰
一、概述
基於 Docker 集成 CI 環境。涉及技術：Linux（Ubuntu 14.04）, Docker, Jenkins, Git/Gitlab, Web/Httpbin, Python/Pytest, UI/Selenium, Robotframework, Grid Server, Appium 等。
架構圖如下：
二、什麼是Pipeline？
Jenkins Pipeline是一套插件，支持將連續輸送Pipeline實施和整合到Jenkins。Pipeline提供了一組可擴展的工具，用於通過PipelineDSL爲代碼創建簡單到複雜的傳送Pipeline。
通常，此“Pipeline代碼”將被寫入 Jenkinsfile項目的源代碼控制存儲庫，例如：
pipeline {
agent any (1)
stages {
stage('build') { (2)
steps { (3)
sh 'make' (4)
}
}
stage('sita'){
steps {
sh 'python script.py'
junit 'reports/**/*.xml' (5)
}
}
}
}
- agent 表示Jenkins應該爲Pipeline的這一部分分配一個執行者和工作區。
- stage 描述了這條Pipeline的一個階段。
- steps 描述了要在其中運行的步驟 stage
- sh 執行給定的shell命令
- junit是由JUnit插件提供的 用於聚合測試報告的Pipeline步驟。
三、爲什麼使用Pipeline？
Jenkins從根本上講是一種支持多種自動化模式的自動化引擎。Pipeline在Jenkins上添加了一套強大的自動化工具，支持從簡單的連續集成到全面的連續輸送Pipeline的用例。通過建模一系列相關任務，用戶可以利用Pipeline 的許多功能。
Pipeline的功能和優點：
- 持久性：在jenkins的master按計劃和非計劃的重啓後，pipeline的job仍然能夠工作，不受影響。其實理解起來也很簡單，jenkins的master和agent通過ssh連接，如果你知道nohup或disown的話，就可以理解爲啥master的重啓不會影響agent上的job繼續運行。
- 可暫停性：pipeline基於groovy可以實現job的暫停和等待用戶的輸入或批准然後繼續執行。
- 更靈活的並行執行，更強的依賴控制，通過groovy腳本可以實現step，stage間的並行執行，和更復雜的相互依賴關係。
- 可擴展性：通過groovy的編程更容易的擴展插件。
- 設計Pipeline = 設計代碼，很優雅
- As Code：集中管理CI腳本、用代碼庫來管理腳本、從代碼庫直接讀取腳本，從而可以將項目CI迅速拉起來！
四、Pipeline 實現 build & scan
一、conf 配置Pipeline各個需要執行的參數: qa.xxx.com-b-master.yaml
credentials_id: github-xxx-jenkins
git_base: [email protected]
# If the container is running on a host who needs to resolve qa.xindong.com
# differently, you may need to pass a host:ip mapping for the container to
# resolve qa.xindong.com correctly. This could be done by setting a global
# environment variable 'ADD_HOST'. e.g. qa.xindong.com:10.10.10.100
add_host:
branch: master
stages:
checkout_src: true
build_tools: true
build: true
test: false
pack: false
check: false
scan: true
sita: false
email:
recipients: qabuilder
always: false # Always send emails regardless task status.
failed: false # Send emails when the task fails.
changed: false # Send emails only when the task status changes.
image:
proj: xxx-server
user: qa
tag: bmas
sonar:
url: https://qa.xxx.com/sonar/
# To use influxdb, a target must be configured in Jenkins global configure
influxdb:
target: qa.xxx.com
sita:
env: sita
- Jenkins 憑據：credentials_id
- 下載代碼地址：git_base
- 容器到主機的映射：add_host
- 服務器代碼分支：branch
- Jenkinsfile 執行階段：stages
- 完成測試發送郵件：email
- 服務器 Docker 鏡像：image
- SonarQube 掃描結果：sonar
- 數據庫存儲：influxdb
- 測試參數：sita
注：stages中，若 stage 爲 true，則 Jnekinsfile 執行對應stage，反之，跳過。
二、Jenkinsfile 實現 build 和 scan
因爲 conf 文件 qa.xxx.com-b-master.yaml 中 stages 決定是否執行。以下四個 stages 爲 true，即，
- checkout_src
- build_tools
- build
- scan
#!/usr/bin/env groovy
/**
* Pipeline to build and test the project, and pack the deliverables.
*/
// Shared library for the pipeline.
// @Library('ci-shared-library') _
// Global configuration parameters for the pipeline.
def conf = [:]
pipeline {
agent any //將在Jenkins master 執行
// agent {label 'slave'} //將在Jenkins slave節點執行
// Stages could ben enabled/disabled by configuration parameters.
stages {
// Load parameters from configuration file.
stage("setup") { // 讀取yaml文件內容，參考安裝配置Jenkins文檔中的“參數化構建過程”
steps {
script {
conf = readYaml(file: "conf/" + (params.conf ?: "default") + ".yaml")
}
}
} // setup
// Checkout project source code.
stage("checkout-src") { //git|svn 下載源代碼
when {
beforeAgent true
expression { conf.stages.checkout_src } //conf 中 stages 參數
}
steps {
// Checkout source code.
checkout([
$class: "GitSCM",
branches: [[name: "*/${conf.branch}"]], //conf 中 branch 參數
extensions: [], // 默認存儲到 workspace 目錄下
submoduleCfg: [],
userRemoteConfigs: [[
credentialsId: "${conf.credentials_id}",
//conf 中 credentials_id 參數
url: "${conf.git_base}:xxx/xxx-server.git"
//conf 中 git_base 參數
]]
])
// Checkout proto.
checkout([
$class: "GitSCM",
branches: [[name: "*/${conf.branch}"]],
doGenerateSubmoduleConfigurations: false,
extensions: [[
$class: "RelativeTargetDirectory",
relativeTargetDir: "xxx-proto"
//存儲到 worksapce 下的 proto 目錄
]],
submoduleCfg: [],
userRemoteConfigs: [[
credentialsId: "${conf.credentials_id}",
url: "${conf.git_base}:xxx/xxx-proto.git"
]]
])
// Checkout resource.
checkout([
$class: "GitSCM",
branches: [[name: "*/${conf.branch}"]],
doGenerateSubmoduleConfigurations: false,
extensions: [[
$class: "RelativeTargetDirectory",
relativeTargetDir: "bin/Debug/DesignerWork"
//存儲到 worksapce 下的 bin/Debug/DesignerWork 目錄
]],
submoduleCfg: [],
userRemoteConfigs: [[
credentialsId: "${conf.credentials_id}",
url: "${conf.git_base}:xxx/DesignerWork.git"
]]
])
// Checkout atelier-ci.
checkout([
$class: "GitSCM",
branches: [[name: "*/master"]],
doGenerateSubmoduleConfigurations: false,
extensions: [[
$class: "RelativeTargetDirectory",
relativeTargetDir: "ci"
//存儲到 worksapce 下的 ci 目錄
]],
submoduleCfg: [],
userRemoteConfigs: [[
credentialsId: "${conf.credentials_id}",
url: "${conf.git_base}:xxx/xxx-ci.git"
]]
])
}
} // stage checkout-src
// Build docker images for tools.
stage("build-tools") { //構建鏡像
when {
beforeAgent true
expression { conf.stages.build_tools }
}
steps {
// Build all tools for the toolchain.
script {
def add_host = conf.add_host ? "--add-host ${conf.add_host}" : ""
//用於主機地址到容器的映射
dir ("ci/toolchain") { //$ cd ci/toolchain
sh ( // build-tools.sh 構建鏡像：builder tester scanner sita
""" \
./build-tools.sh \
--user ${conf.image.user} \
--version ${conf.image.tag}-latest \
${add_host} \
${conf.image.proj} \
"""
)
}
}
}
} // stage build-tools
// Build source.
stage("build") { // build 服務器
when {
beforeAgent true
expression { conf.stages.build }
}
agent {
docker { // 生成服務器容器
image "${conf.image.user}/${conf.image.proj}-builder:${conf.image.tag}-latest"
args "-v /etc/hosts:/etc/hosts"
reuseNode true
}
}
steps {
script { // 構建服務器
sh "/opt/script/start-build.sh -w ${WORKSPACE}"
}
}
} // stage build
// Run unit tests.
stage("test") { // 跳過
when {
beforeAgent true
expression { conf.stages.test }
}
agent {
docker {
image "${conf.image.user}/${conf.image.proj}-tester:${conf.image.tag}-latest"
args "-v /etc/hosts:/etc/hosts"
reuseNode true
}
}
steps {
script {
sh "/opt/script/start-test.sh -w ${WORKSPACE}/src"
}
}
} // stage test
// Check source code by static anlaysis tools.
stage("check") { // 跳過
when {
beforeAgent true
expression { conf.stages.check }
}
agent {
docker {
image "${conf.image.user}/${conf.image.proj}-scanner:${conf.image.tag}-latest"
args "-v /etc/hosts:/etc/hosts"
reuseNode true
}
}
steps {
script {
sh("/opt/script/start-cppcheck.sh -w ${WORKSPACE}/src")
}
}
} // stage check
// Scan source code by sonar.
stage("scan") { // sonar 掃描
when {
beforeAgent true
expression { conf.stages.scan }
}
agent {
docker { // 生成 sonar 容器
image "${conf.image.user}/${conf.image.proj}-scanner:${conf.image.tag}-latest"
args "-v /etc/hosts:/etc/hosts"
reuseNode true
}
}
steps {
script { // 掃描 workspace 下的 src 目錄
sh(
"""\
/opt/script/start-scan.sh \
-w ${WORKSPACE}/src \
-Dsonar.host.url=${conf.sonar.url} \
-Dsonar.branch.name=${conf.branch}
"""
)
}
}
} // stage scan
// 發送郵件
changed {
script {
// Send notification emails.
if (conf.email.changed) {
emailext (
recipientProviders: [
[$class: 'DevelopersRecipientProvider'],
[$class: 'CulpritsRecipientProvider'],
[$class: 'RequesterRecipientProvider']],
mimeType: 'text/html',
to: "${conf.email.recipients}",
subject: '$PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS!',
body: "Check ${BUILD_URL} to view the results.",
attachLog: true
)
}
}
}
failure {
script {
// Send notification emails.
if (conf.email.failed) {
emailext (
recipientProviders: [
[$class: 'DevelopersRecipientProvider'],
[$class: 'CulpritsRecipientProvider'],
[$class: 'RequesterRecipientProvider']],
mimeType: 'text/html',
to: "${conf.email.recipients}",
subject: '$PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS!',
body: "Check ${BUILD_URL} to view the results.",
attachLog: true
)
}
}
}
} // post
} // pipeline