Docker CI: Jenkins Pipeline 介绍和实战
一、概述
基于 Docker 集成 CI 环境。涉及技术：Linux（Ubuntu 14.04）, Docker, Jenkins, Git/Gitlab, Web/Httpbin, Python/Pytest, UI/Selenium, Robotframework, Grid Server, Appium 等。
架构图如下：
二、什么是Pipeline？
Jenkins Pipeline是一套插件，支持将连续输送Pipeline实施和整合到Jenkins。Pipeline提供了一组可扩展的工具，用于通过PipelineDSL为代码创建简单到复杂的传送Pipeline。
通常，此“Pipeline代码”将被写入 Jenkinsfile项目的源代码控制存储库，例如：
pipeline {
agent any (1)
stages {
stage('build') { (2)
steps { (3)
sh 'make' (4)
}
}
stage('sita'){
steps {
sh 'python script.py'
junit 'reports/**/*.xml' (5)
}
}
}
}
- agent 表示Jenkins应该为Pipeline的这一部分分配一个执行者和工作区。
- stage 描述了这条Pipeline的一个阶段。
- steps 描述了要在其中运行的步骤 stage
- sh 执行给定的shell命令
- junit是由JUnit插件提供的 用于聚合测试报告的Pipeline步骤。
三、为什么使用Pipeline？
Jenkins从根本上讲是一种支持多种自动化模式的自动化引擎。Pipeline在Jenkins上添加了一套强大的自动化工具，支持从简单的连续集成到全面的连续输送Pipeline的用例。通过建模一系列相关任务，用户可以利用Pipeline 的许多功能。
Pipeline的功能和优点：
- 持久性：在jenkins的master按计划和非计划的重启后，pipeline的job仍然能够工作，不受影响。其实理解起来也很简单，jenkins的master和agent通过ssh连接，如果你知道nohup或disown的话，就可以理解为啥master的重启不会影响agent上的job继续运行。
- 可暂停性：pipeline基于groovy可以实现job的暂停和等待用户的输入或批准然后继续执行。
- 更灵活的并行执行，更强的依赖控制，通过groovy脚本可以实现step，stage间的并行执行，和更复杂的相互依赖关系。
- 可扩展性：通过groovy的编程更容易的扩展插件。
- 设计Pipeline = 设计代码，很优雅
- As Code：集中管理CI脚本、用代码库来管理脚本、从代码库直接读取脚本，从而可以将项目CI迅速拉起来！
四、Pipeline 实现 build & scan
一、conf 配置Pipeline各个需要执行的参数: qa.xxx.com-b-master.yaml
credentials_id: github-xxx-jenkins
git_base: [email protected]
# If the container is running on a host who needs to resolve qa.xindong.com
# differently, you may need to pass a host:ip mapping for the container to
# resolve qa.xindong.com correctly. This could be done by setting a global
# environment variable 'ADD_HOST'. e.g. qa.xindong.com:10.10.10.100
add_host:
branch: master
stages:
checkout_src: true
build_tools: true
build: true
test: false
pack: false
check: false
scan: true
sita: false
email:
recipients: qabuilder
always: false # Always send emails regardless task status.
failed: false # Send emails when the task fails.
changed: false # Send emails only when the task status changes.
image:
proj: xxx-server
user: qa
tag: bmas
sonar:
url: https://qa.xxx.com/sonar/
# To use influxdb, a target must be configured in Jenkins global configure
influxdb:
target: qa.xxx.com
sita:
env: sita
- Jenkins 凭据：credentials_id
- 下载代码地址：git_base
- 容器到主机的映射：add_host
- 服务器代码分支：branch
- Jenkinsfile 执行阶段：stages
- 完成测试发送邮件：email
- 服务器 Docker 镜像：image
- SonarQube 扫描结果：sonar
- 数据库存储：influxdb
- 测试参数：sita
注：stages中，若 stage 为 true，则 Jnekinsfile 执行对应stage，反之，跳过。
二、Jenkinsfile 实现 build 和 scan
因为 conf 文件 qa.xxx.com-b-master.yaml 中 stages 决定是否执行。以下四个 stages 为 true，即，
- checkout_src
- build_tools
- build
- scan
#!/usr/bin/env groovy
/**
* Pipeline to build and test the project, and pack the deliverables.
*/
// Shared library for the pipeline.
// @Library('ci-shared-library') _
// Global configuration parameters for the pipeline.
def conf = [:]
pipeline {
agent any //将在Jenkins master 执行
// agent {label 'slave'} //将在Jenkins slave节点执行
// Stages could ben enabled/disabled by configuration parameters.
stages {
// Load parameters from configuration file.
stage("setup") { // 读取yaml文件内容，参考安装配置Jenkins文档中的“参数化构建过程”
steps {
script {
conf = readYaml(file: "conf/" + (params.conf ?: "default") + ".yaml")
}
}
} // setup
// Checkout project source code.
stage("checkout-src") { //git|svn 下载源代码
when {
beforeAgent true
expression { conf.stages.checkout_src } //conf 中 stages 参数
}
steps {
// Checkout source code.
checkout([
$class: "GitSCM",
branches: [[name: "*/${conf.branch}"]], //conf 中 branch 参数
extensions: [], // 默认存储到 workspace 目录下
submoduleCfg: [],
userRemoteConfigs: [[
credentialsId: "${conf.credentials_id}",
//conf 中 credentials_id 参数
url: "${conf.git_base}:xxx/xxx-server.git"
//conf 中 git_base 参数
]]
])
// Checkout proto.
checkout([
$class: "GitSCM",
branches: [[name: "*/${conf.branch}"]],
doGenerateSubmoduleConfigurations: false,
extensions: [[
$class: "RelativeTargetDirectory",
relativeTargetDir: "xxx-proto"
//存储到 worksapce 下的 proto 目录
]],
submoduleCfg: [],
userRemoteConfigs: [[
credentialsId: "${conf.credentials_id}",
url: "${conf.git_base}:xxx/xxx-proto.git"
]]
])
// Checkout resource.
checkout([
$class: "GitSCM",
branches: [[name: "*/${conf.branch}"]],
doGenerateSubmoduleConfigurations: false,
extensions: [[
$class: "RelativeTargetDirectory",
relativeTargetDir: "bin/Debug/DesignerWork"
//存储到 worksapce 下的 bin/Debug/DesignerWork 目录
]],
submoduleCfg: [],
userRemoteConfigs: [[
credentialsId: "${conf.credentials_id}",
url: "${conf.git_base}:xxx/DesignerWork.git"
]]
])
// Checkout atelier-ci.
checkout([
$class: "GitSCM",
branches: [[name: "*/master"]],
doGenerateSubmoduleConfigurations: false,
extensions: [[
$class: "RelativeTargetDirectory",
relativeTargetDir: "ci"
//存储到 worksapce 下的 ci 目录
]],
submoduleCfg: [],
userRemoteConfigs: [[
credentialsId: "${conf.credentials_id}",
url: "${conf.git_base}:xxx/xxx-ci.git"
]]
])
}
} // stage checkout-src
// Build docker images for tools.
stage("build-tools") { //构建镜像
when {
beforeAgent true
expression { conf.stages.build_tools }
}
steps {
// Build all tools for the toolchain.
script {
def add_host = conf.add_host ? "--add-host ${conf.add_host}" : ""
//用于主机地址到容器的映射
dir ("ci/toolchain") { //$ cd ci/toolchain
sh ( // build-tools.sh 构建镜像：builder tester scanner sita
""" \
./build-tools.sh \
--user ${conf.image.user} \
--version ${conf.image.tag}-latest \
${add_host} \
${conf.image.proj} \
"""
)
}
}
}
} // stage build-tools
// Build source.
stage("build") { // build 服务器
when {
beforeAgent true
expression { conf.stages.build }
}
agent {
docker { // 生成服务器容器
image "${conf.image.user}/${conf.image.proj}-builder:${conf.image.tag}-latest"
args "-v /etc/hosts:/etc/hosts"
reuseNode true
}
}
steps {
script { // 构建服务器
sh "/opt/script/start-build.sh -w ${WORKSPACE}"
}
}
} // stage build
// Run unit tests.
stage("test") { // 跳过
when {
beforeAgent true
expression { conf.stages.test }
}
agent {
docker {
image "${conf.image.user}/${conf.image.proj}-tester:${conf.image.tag}-latest"
args "-v /etc/hosts:/etc/hosts"
reuseNode true
}
}
steps {
script {
sh "/opt/script/start-test.sh -w ${WORKSPACE}/src"
}
}
} // stage test
// Check source code by static anlaysis tools.
stage("check") { // 跳过
when {
beforeAgent true
expression { conf.stages.check }
}
agent {
docker {
image "${conf.image.user}/${conf.image.proj}-scanner:${conf.image.tag}-latest"
args "-v /etc/hosts:/etc/hosts"
reuseNode true
}
}
steps {
script {
sh("/opt/script/start-cppcheck.sh -w ${WORKSPACE}/src")
}
}
} // stage check
// Scan source code by sonar.
stage("scan") { // sonar 扫描
when {
beforeAgent true
expression { conf.stages.scan }
}
agent {
docker { // 生成 sonar 容器
image "${conf.image.user}/${conf.image.proj}-scanner:${conf.image.tag}-latest"
args "-v /etc/hosts:/etc/hosts"
reuseNode true
}
}
steps {
script { // 扫描 workspace 下的 src 目录
sh(
"""\
/opt/script/start-scan.sh \
-w ${WORKSPACE}/src \
-Dsonar.host.url=${conf.sonar.url} \
-Dsonar.branch.name=${conf.branch}
"""
)
}
}
} // stage scan
// 发送邮件
changed {
script {
// Send notification emails.
if (conf.email.changed) {
emailext (
recipientProviders: [
[$class: 'DevelopersRecipientProvider'],
[$class: 'CulpritsRecipientProvider'],
[$class: 'RequesterRecipientProvider']],
mimeType: 'text/html',
to: "${conf.email.recipients}",
subject: '$PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS!',
body: "Check ${BUILD_URL} to view the results.",
attachLog: true
)
}
}
}
failure {
script {
// Send notification emails.
if (conf.email.failed) {
emailext (
recipientProviders: [
[$class: 'DevelopersRecipientProvider'],
[$class: 'CulpritsRecipientProvider'],
[$class: 'RequesterRecipientProvider']],
mimeType: 'text/html',
to: "${conf.email.recipients}",
subject: '$PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS!',
body: "Check ${BUILD_URL} to view the results.",
attachLog: true
)
}
}
}
} // post
} // pipeline