前言
为了体验CI/CD实现过程并深入理解CI、CD流水线上的各组件的作用；
我自己搭建了1个GitLab+Jenkins+maven；
一、GitLab实现CI
GitLab服务器用于存储代码资源，可以实现代码的持续集成（CI），可以借助NFS、Ceph等存储系统，对数据进行高可用存储；
1.安装GitLab依赖
安装GitLab依赖，并设置开机启动
yum -y install curl policycoreutils openssh-server openssh-clents libsemanage-static libsemanage-devel
yum install curlpolicycoreutils openssh-server openssh-clients
systemctl enablesshd
systemctl enable sshd
systemctl start sshd
yum install postfix
systemctl enable postfix
systemctl start postfix
firewall-cmd --permanent --add-service=http
systemctl reload firewalld
2.安装GitLab服务
2.1.yum仓库配置
配置yum仓库
[root@hecs-83208 ~]# curl -fsSL https://packages.gitlab.cn/repository/raw/scripts/setup.sh | /bin/bash ==> Detected OS centos ==> Add yum repo file to /etc/yum.repos.d/gitlab-jh.repo [gitlab-jh] name=JiHu GitLab baseurl=https://packages.gitlab.cn/repository/el/$releasever/ gpgcheck=0 gpgkey=https://packages.gitlab.cn/repository/raw/gpg/public.gpg.key priority=1 enabled=1 ==> Generate yum cache for gitlab-jh ==> Successfully added gitlab-jh repo. To install JiHu GitLab, run "sudo yum/dnf install gitlab-jh".
2.2.安装GitLab
EXTERNAL_URL="http://114.115.128.169" yum -y install gitlab-jh
3.配置GitLab服务
3.1.访问URL设置
vim /etc/gitlab/gitlab.rb
EXTERNAL_URL="http://114.115.128.169"
3.2.存储路径设置
vim /etc/gitlab/gitlab.rb
"default" => { "path" => "/data/git-data", "failure_count_threshold" => 10, "failure_wait_time" => 30, "failure_reset_time" => 1800, "failure_timeout" => 30 } })
3.3.GitLab初始化密码
查看GitLab的初始化密码
[root@hecs-83208 gitlab]# pwd /etc/gitlab [root@hecs-83208 gitlab]# cat initial_root_password
3.4.设置root用户密码
设置新的root密码
4.管理GitLab服务
gitlab-ctl start #启动所有gitlab组件 gitlab-ctl stop #停止所有gitlab组件 gitlab-ctl restart #重启所有gitlab组件 gitlab-ctl status #查看所有gitlab组件的运行状态 gitlab-ctl reconfigure #刷新gitlab配置通常和gitlab-ctl restart组合使用，否则配置不生效
5.使用GitLab
5.1.GitLab账户添加SSHKey
$ cd ~/.ssh/ $ git config --global user.name "root" $ git config --global user.email "root@le.com" $ ssh-keygen -t rsa -C "root@le.com" $ ls id_rsa id_rsa.pub known_hosts $ cat id_rsa.pub $ git clone git@114.115.128.169:root/java-project.git Cloning into 'java-project'... remote: Enumerating objects: 3, done. remote: Counting objects: 100% (3/3), done. remote: Compressing objects: 100% (2/2), done. remote: Total 3 (delta 0), reused 0 (delta 0), pack-reused 0 Receiving objects: 100% (3/3), done.
GitLab
6.GitLab设置系统钩子
当GitLab的某1个仓库发生Push、Merge..事件时，调用Jenkins的WebHool，触发Jenkins的自动构建；
6.1.GitLab添加系统钩子
http://192.168.56.20:8081/buildByToken/build?job=build-maven-java-project&token=123.com
6.2.GitLab允许出站请求
只有GitLab允许出站请求之后，GitLab才可以去请求Jenkins的WebHook；
6.3.测试系统钩子的可用性
GitLab会向Jenkins的WebHook发起HTTP请求；
7.Python调用GitlabAPI
使用Python调用GitLab的API
from public import RecordLoggre import gitlab class GitlabApi: def __init__(self): self.GitlabUrl = 'https://git.apuscn.com:8443' # jenkins用户 private_token = '2icW6gSLdrX9ZGoPLAEs' self.ApiVersion = '4' self.GitClent = gitlab.Gitlab(self.GitlabUrl, private_token=private_token) # 记录日志 self.record_log = RecordLoggre() class ProjectApi(GitlabApi): def __init__(self, project_url): super(ProjectApi, self).__init__() self.project_url = project_url self.project = self.get_project() def get_project(self): """通过项目url获取项目实例""" project_name = self.project_url.split('/')[-1].split('.')[0] project_list = self.GitClent.projects.list(search=project_name) fit_fileter = [] for item in project_list: if (self.project_url == item.ssh_url_to_repo) or ( self.project_url == item.http_url_to_repo) or ( self.project_url == item.web_url) or ( f'git@{self.project_url}' == item.ssh_url_to_repo): fit_fileter.append(item) if len(fit_fileter) == 1: return fit_fileter[0] else: self.record_log.error(f'获取gitlab项目数量为{len(fit_fileter)},{str(fit_fileter)}') return None def get_branch(self, branch): """获取分支信息""" branch_data = {} try: branch_data = self.project.branches.get(branch) except Exception as Error: self.record_log.error(str(Error), exc_info=True) return branch_data def get_tag(self, tag): """获取标签信息""" tag_data = {} try: tag_data = self.project.tags.get(id=tag) except Exception as Error: self.record_log.error(str(Error), exc_info=True) return tag_data def get_version(self, version): """通过version获取分支或tag的信息""" try: data = self.project.branches.get(version) except Exception: try: data = self.project.tags.get(id=version) except Exception: data = None return data def list_branch(self): """列出项目的所有分支""" try: branch_list = self.project.branches.list(per_page=1000) for item in branch_list: yield item except Exception as Error: raise Exception(f'{self.project_url}获取分支失败，{str(Error)}') def list_tags(self): """列出项目的所有标签""" try: branch_list = self.project.tags.list(per_page=1000) for item in branch_list: yield item except Exception as Error: raise Exception(f'{self.project_url}获取标签失败，{str(Error)}') def find_merge_branch(self, tag): """查找tag从那个分支合并来的代码""" branch_commit_id = [] try: tag_obj = self.get_tag(tag) parent_ids = tag_obj.commit.get('parent_ids') for item in self.list_branch(): commit_id = item.commit.get('id') branch_name = item.name if commit_id in parent_ids: branch_commit_id.append((commit_id, branch_name)) except Exception as Error: self.record_log.error(f'{str(Error)}', exc_info=True) else: if len(branch_commit_id) == 1: return branch_commit_id[0] else: raise Exception(f'{self.project_url}获取合并前的分支失败，{str(branch_commit_id)}') if __name__ == '__main__': a = ProjectApi('git@git.apuscn.com:sa/ci-cd.git') print(a.find_merge_branch('v2.0.2')) # a.project.commits.get('e3d5a71b') # print(a.get_version('dev')) # print(dir(branch_data)) # print(branch_data.commit.get('id'))
二、Jenkins自动构建Maven项目
Jenkins支持master/agent模式，agent听从Master的任务调度，分摊Matser之上的的CICD工作量，使持续继承和部署的效率更高；
Jekins不能直接去Gitlab拉取代码，需要在Jekins上配置Gitlab的秘钥。
1.Jenkins服务器准备
Jenkins可以通过安装采用插件的方式，完成对Python、Java等不同类型的项目的CI、CD；
1.1.启动Jenkins
java -Dhudson.model.DownloadService.noSignatureCheck=true -jar jenkins.war --httpPort=8081
yum -y install java-1.8.0-openjdk*
Jenkins的插件的文件后缀名有两种格式 .jpi和.hpi，本质是被编译打包压缩之后的.clss文件；
jpi中的j就是Jenkins , hpi中的h是Hudson，Jenkins项目的前身是Hudson。
Publish Over SSH插件
Build Authorization Token Root插件
允许GitLab或者其他第三方程序免登录，通过Token参数调用Jenkins的WebHook；
buildByToken/build?job=NAME&token=SECRET
http://192.168.56.20:8081/buildByToken/build?job=build-maven-java-project&token=123.com
1.4.1.安装Git
yum -i install git
1.4.2.安装Maven
配置仓库路径和阿里镜像源，不在赘述
1.5.构建触发器
构建触发器是可以被外部程序调用的触发器，一旦触发器被触发就Jenkins就可以开始自动构建项目；
Jenkins支持以下几种方式进行自动构建
A、快照依赖构建/Build whenever a SNAPSHOT dependency is built
当依赖的快照被构建时执行本job
B、触发远程构建(例如,使用脚本)。远程调用本job的restapi时执行本jobjob依赖构建/Build after other projects are built
当依赖的job被构建时执行本job
C、定时构建/Build periodically。使用cron表达式定时构建本iob
D、向GitHub提交代码时触发Jenkins自动构建/GitHub hook trigger for GITScm pollingo Github-WebHook出发时构建本iob
E、定期检查代码变更/PollSCM
使用cron表达式定时检查代码变更，变更后构建本iob!0
1.5.1.webHook
通常由GitLab触发，一旦该URL被访问，Jenkins就会自动构建；
http://192.168.56.20:8081/job/build-maven-java-project/build?token=123.com
2.1.Maven配置
2.2.Git配置
2.3.指定分支
设置Jenkins去GitLab哪1个仓库的哪1个分支去拉取代码；
2.4.指定项目的根pom文件
设置Maven打包时去项目的什么路径下找到.pom文件，每次打包之前一定清理maven缓存，否则不更新！
Maven的Assembly插件的主要作用是：允许用户将项目输出与它的依赖项、模块、站点文档、和其他文件一起组装成1个可分发的归档文件（tar/zip）。
在Java项目Pom文件中指定源码打包之后是归档为jar/war/tar包，也指定了target目录；
Maven仅仅是通过读取Pom文件内容进行项目打包的打包工具。
/var/lib/jenkins/tools/maven/bin/mvn
-B
-f /var/lib/jenkins/workspace/common-service/sg-2/xxl-job-admin__runner/pom.xml
clean install -Dmaven.test.skip=true
-Pprod
实际执行的命令如所示。
2.5.Pre Step设置
2.5.1./root/clean.sh脚本开发
如果clean.sh的语法正确，但执行老报错，原因是脚本从Windos平台Copy到Unix平台中，Shell脚本的格式发生了肉眼不可见的变化；
直接使用Vim编辑即可！
#!/bin/bash echo "cenling........" #删除历史数据 rm -rf ./jar #获取传入的参数 appName=$1 echo "参数是$1" #获取正在运行的jar包的Pid pid=`ps -ef | grep $1 | grep 'java -jar' | awk '{printf $2}' ` echo $pid #如果Pid为空提示 if [ -z $pid ]; then echo "$appName not started" else kill -9 $pid && echo "$appName stoping" fi
2.5.2.设置执行/root/clean.sh脚本
2.6.配置部署服务器
把Jenkins服务器调用Maven编译打包之后，Jar包应该传输到哪1台服务器上去部署？
在Configure System菜单下
2.7.Post Step设置
当Jar包传输到部署服务器之后就是启动jar包！
启动jar包不能使用 java -jar直接启动，因为该命令没有正确退出，会延迟Jenkins的构建时间；
nohup java -jar /root/jar/CI-CD*.jar --server.port=8001 > mylog.log 2>&1 &
2.8.构建成功效果
切记观察Transferred file的数量，确定Jar包传输到了部署服务器上；
3.Python调用JenkinAPI
python-jenkins==1.7.0
在Python使用python-jenkins这个第三方包，来实现Python和Jenkins的交互。
from public.utils import record_loggre from config import OpsConfig from public.hash_ring import HashRing import jenkins import time class JenkinsApi: def __init__(self): self.jenkins_server = OpsConfig.Jenkins.server self.user_name = OpsConfig.Jenkins.user self.password = OpsConfig.Jenkins.password self.server_list = self.jenkins_server.split(',') # 记录日志 self.record_log = record_loggre() def select_server(self, key): ring = HashRing(self.server_list) return jenkins.Jenkins(ring.get_node(key), username=self.user_name, password=self.password) def server_iter(self): for item in self.server_list: yield jenkins.Jenkins(item, username=self.user_name, password=self.password) def request(self, client, action, *args, **kwargs): """根据传来的客户端和动作发起请求""" method = getattr(client, action, None) if method: retry, sign = 0, False while True: try: result = method(*args, **kwargs) except Exception as Error: self.record_log.error(f'{str(Error)}，重试{retry}次', exc_info=True) retry += 1 if retry >= 3: result = str(Error) break time.sleep(3) else: sign = True break return sign, result else: return False, None def request_iter(self, action, *args, **kwargs): """请求所有服务器，使所有服务器保持一致""" for client in self.server_iter(): self.request(client, action, *args, **kwargs) class JenkinsBuild(JenkinsApi): """Jenkins 构建相关""" def __init__(self, job_name): super(JenkinsBuild, self).__init__() self.job_name = job_name self.Jenkins = self.select_server(self.job_name) def get_build_console(self, build_number): """获取构建任务的日志输出""" sign, build_console = self.request(self.Jenkins, 'get_build_console_output', self.job_name, build_number) return sign, build_console def get_build_status(self, build_number): """获取构建状态""" sign, build_status = self.request(self.Jenkins, 'get_build_info', self.job_name, build_number) if sign: building, result = build_status.get('building'), build_status.get('result') else: building, result = None, None return building, result def last_test_build(self): """获取动作为test最后一次构建""" job_info = self.Jenkins.get_job_info(self.job_name) job_builds, build_number = job_info.get('builds'), None for item in job_builds: build_number = item.get('number') build_info = self.Jenkins.get_build_info(self.job_name, build_number) parameters = build_info.get('actions')[0].get('parameters') if parameters[0].get('value') == 'test': break return build_number def get_job_config(self, name): config = self.Jenkins.get_job_config(name) print(config) # self.Jenkins.create_folder('App-Dev-test/ali-cn-zj1') # a = self.Jenkins.get_job_config('App-Dev-test/ali-cn-zj1') # print(a) if __name__ == '__main__': JenkinsBuild('App-Dev/user-growth/hermes-admin1').get_job_config('App-Dev/user-growth/hermes-admin1') JenkinsApi().select_server('App-Dev/user-growth/hermes22-admin12') a = JenkinsBuild('App-Dev/ali-cn-zjk/middle-platform-ad_adx-bss-service').get_build_status(7) print(a) # func = getattr(JenkinsApi, 'request', None) # print(func)
三、Jenkins自动构建Docker项目
自动化的前提是标准化，CD的难点是部署环境的异构性，如果部署环境统一采用Docker/K8s，就为CD的实现提供了基础；
我的生产环境通常是Docker或者K8s，这样更容易实现CD；
1.外挂目录（Docker）
2.Jar包直接打包到镜像（Docker）
3.Build生成新镜像推送到Habor私服（K8s）
参考
