K8S - Jenkins在K8S下的持续集成
准备nfs网络存储
[root@master ~]# yum -y install nfs-utils rpcbind [root@master ~]# systemctl start rpcbind && systemctl enable rpcbind && systemctl status rpcbind [root@master lv]# mkdir -p /nfsdata/data/jenkins [root@master ~]# vim /etc/exports/data/jenkins /nfsdata/data/jenkins *(rw,sync,no_root_squash) [root@master ~]# systemctl start nfs-server && systemctl enable nfs-server && systemctl status nfs-server [root@master ~]# showmount -e Export list for master: /nfsdata/data/jenkins *
安装Jenkins服务到K8S集群
使用Dockerfile制作Jenkins镜像
下载war包放到Dockerfile文件同一个目录下,Dockerfile如下,war包下载地址:https://mirrors.tuna.tsinghua.edu.cn/jenkins/
yaml文件资源下载地址:
https://gitee.com/fdd-39969/cicd.git git clone https://gitee.com/fdd-39969/cicd.git
cd cicd/jenkins
ls cicd/jenkins/
Dockerfile jenkins-pv.yaml jenkins-svc.yaml rbac.yaml
jenkins-deploy.yaml jenkins-slave jenkins.war
创建Dockerfile文件
FROM java:8 RUN echo 'hello docker, start build image' RUN mkdir -p /app WORKDIR /app RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime RUN echo "Asia/Shanghai" > /etc/timezone COPY jenkins.war . CMD ["java" ,"-Xms1024m","-Xmx1024m", "-jar","/app/jenkins.war"]
制作镜像
docker build -t 192.168.166.132/library/jenkins-2.273 .
docker push 192.168.166.132/library/jenkins-2.273
K8S安装Jenkins应用
在k8s集群内创建Jenkins工作的namespace,我这边统一放在devops这个ns底下;
# 创建命名空间 kubectl create ns devops
我这里把Jenkins工作目录单独挂载到PVC,需要先创建pv-pvc,挂载点是使用的nfs服务,请先创建好服务,jenkins-pv-pvc.yaml如下:
kind: PersistentVolume apiVersion: v1 metadata: name: jenkins-home-pv namespace: devops spec: accessModes: - ReadWriteOnce #访问模式定义为只能以读写的方式挂载到单个节点 capacity: storage: 10Gi persistentVolumeReclaimPolicy: Retain storageClassName: nfs nfs: path: /nfsdata/data/jenkins server: 192.168.166.128 --- kind: PersistentVolumeClaim apiVersion: v1 metadata: name: jenkins-home-pvc namespace: devops spec: accessModes: - ReadWriteOnce resources: requests: storage: 10Gi storageClassName: nfs #这里指定关联的PV名称
为Jenkins创建单独的ServiceAccount,这里的ClusterRole直接使用的cluster-admin,jenkins-serveraccount.yaml如下;
apiVersion: v1 kind: ServiceAccount metadata: labels: app: jenkins name: jenkins-admin namespace: devops --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: name: jenkins-admin labels: app: jenkins subjects: - kind: ServiceAccount name: jenkins-admin namespace: devops roleRef: kind: ClusterRole name: cluster-admin apiGroup: rbac.authorization.k8s.io
jenkins-deployment.yaml如下:
注:如果需要指定在某个节点上运行,需要自行打污点设置 (格式: kubectl label nodes k8s-node1 app.k8s.icj1/devops=)
apiVersion: apps/v1 kind: Deployment metadata: name: jenkins namespace: devops labels: app: jenkins spec: replicas: 1 selector: matchLabels: app: jenkins template: metadata: labels: app: jenkins spec: serviceAccountName: jenkins-admin imagePullSecrets: - name: ram-secret #affinity: #nodeAffinity: # requiredDuringSchedulingIgnoredDuringExecution: #nodeSelectorTerms: #- matchExpressions: #- key: apps.k8s.icjl/devops # operator: Exists containers: - name: jenkins image: 192.168.166.132/library/jenkins-2.273 imagePullPolicy: IfNotPresent volumeMounts: - name: jenkins-home mountPath: /root/.jenkins readOnly: false ports: - containerPort: 8080 - containerPort: 50000 volumes: - name: jenkins-home persistentVolumeClaim: claimName: jenkins-home-pvc # 关联PVC卷
创建service,这边使用了NodePort,jenkins-service.yaml如下;
apiVersion: v1 kind: Service metadata: labels: app: jenkins name: jenkins namespace: devops annotations: prometheus.io/scrape: 'true' spec: type: NodePort ports: - name: jenkins-web port: 8080 targetPort: 8080 nodePort: 30008 # jenkins 地址端口 - name: jenkins-agent port: 50000 targetPort: 50000 nodePort: 30005 # jenkins 通道端口 selector: app: jenkins
也可以使用ingress暴露的方式,jenkins-ingress.yaml如下:(我这里没有使用ingress)
apiVersion: extensions/v1beta1 kind: Ingress metadata: name: jenkins labels: name: jenkins namespace: devops spec: rules: - host: jenkins.hiningmeng.cn http: paths: - path: / backend: serviceName: jenkins servicePort: 8080
执行yaml
kubectl apply -f jenkins-pv-pvc.yaml kubectl apply -f jenkins-serveraccount.yaml kubectl apply -f jenkins-deployment.yaml kubectl apply -f jenkins-service.yaml kubectl apply -f jenkins-ingress.yaml
查看pod状态
[root@k8s-master jenkins]# kubectl get pv,pvc,svc -n devops NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/jenkins-home-pv 10Gi RWO Retain Bound devops/jenkins-home-pvc nfs 44m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/jenkins-home-pvc Bound jenkins-home-pv 10Gi RWO nfs 44m NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE service/jenkins NodePort 10.111.96.4 <none> 8080:30008/TCP,50000:30005/TCP 44m [root@k8s-master jenkins]# kubectl get ep -n devops NAME ENDPOINTS AGE jenkins 10.244.169.132:50000,10.244.169.132:8080 44m [root@k8s-master jenkins]# kubectl get pods -n devops NAME READY STATUS RESTARTS AGE jenkins-74f44f658-rjfb2 1/1 Running 0 30m [root@k8s-master jenkins]# [root@k8s-master jenkins]# kubectl describe pods jenkins-74f44f658-rjfb2 -n devops --- Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal Scheduled 31m default-scheduler Successfully assigned devops/jenkins-74f44f658-rjfb2 to k8s-node2 Normal Pulled 31m kubelet, k8s-node2 Container image "192.168.166.132/library/jenkins-2.273" already present on machine Normal Created 31m kubelet, k8s-node2 Created container jenkins Normal Started 31m kubelet, k8s-node2 Started container jenkins # 看到以上这些信息说明启动成功
查看pod日志
kubectl logs jenkins-74f44f658-rjfb2 -n devops --- # 复制管理员密码
访问Jenkins,如果没有做ingress 使用NodePort+暴露的端口
![](https://img2020.cnblogs.com/blog/2174174/202104/2174174-20210425160406047-2134033688.png)
![](https://img2020.cnblogs.com/blog/2174174/202104/2174174-20210425161134292-1314172238.png)
![](https://img2020.cnblogs.com/blog/2174174/202104/2174174-20210425161211349-1196737326.png)
安装插件
# 进入到nfs共享目录 cd /nfsdata/data/jenkins/updates/ sed -i 's/https:\/\/updates.jenkins.io\/download/https:\/\/mirrors.tuna.tsinghua.edu.cn\/jenkins/g' default.json sed -i 's/http:\/\/www.google.com/https:\/\/www.baidu.com/g' default.json # 删除pod重建,pod名称改成你实际的 kubectl delete pod jenkins-dccd449c7-vx6sj -n devops # 或者 xxx.yaml 是启动Jenkins pod的yaml文件 kubectl delete -f xxx.yaml kubectl apply -f xxx.yaml
管理Jenkins->系统配置-->管理插件-->分别搜索Git Parameter/Git/Pipeline/kubernetes/Config
配置Kubernetes云信息
在系统管理 --> 系统设置 ,最后面有个Cloud设置,Add a new cloud
![](https://img2020.cnblogs.com/blog/2174174/202104/2174174-20210425162119590-723017501.png)
添加具体的Kubernetes信息,K8S服务器可以是Jenkins本身所在的服务器,也可以是其他集群(需要配置证书),这里以本身所在集群为例。
- 名称 :用于pipeline调用云名称
- Kubernetes地址:可以通过kubectl cluster-info命令获取
- Kubernetes 服务证书 key:本身所在的集群因为我们通过sa所以不需要
- Kubernetes 命名空间:Jenkins的nodePod节点启动的namespace
- Jenkins 地址:主节点8080端口通过nodeport暴露出来的,地址:端口
- Jenkins 通道:主节点50000端口通过nodeport暴露出来的,地址:端口
构建Slave镜像
-
课件目录里涉及四个文件:Dockerfile:构建镜像jenkins-slave:shell脚本启动slave.jarsettings.xml:修改maven官方源为阿里云源slave.jar:agent程序,接受master下发的任务
slave镜像资源下载地址:
https://gitee.com/fdd-39969/cicd.git
git clone https://gitee.com/fdd-39969/cicd.git
Dockerfile文件内容如下:
[root@k8s-master jenkins]# cd jenkins-slave/ [root@k8s-master jenkins-slave]# ls Dockerfile jenkins-slave kubectl settings.xml slave.jar [root@k8s-master jenkins-slave]# [root@k8s-master jenkins-slave]# [root@k8s-master jenkins-slave]# cat Dockerfile FROM centos:7 LABEL maintainer fengyuanfei RUN yum install -y java-1.8.0-openjdk maven curl git libtool-ltdl-devel && \ yum clean all && \ rm -rf /var/cache/yum/* && \ mkdir -p /usr/share/jenkins COPY slave.jar /usr/share/jenkins/slave.jar COPY jenkins-slave /usr/bin/jenkins-slave COPY settings.xml /etc/maven/settings.xml RUN chmod +x /usr/bin/jenkins-slave COPY kubectl /usr/bin/ ENTRYPOINT ["jenkins-slave"]
构建并推送到镜像仓库:
docker build -t 192.168.166.132/library/jenkins-slave-jdk .
docker push 192.168.166.132/library/enkins-slave-jdk
war包可以通过添加 new新节点 获取下载
编写Pipeline脚本
// 公共 def registry = "reg.ctnrs.com" // 项目 def project = "dev" def app_name = "java-demo" def image_name = "${registry}/${project}/${app_name}:${BUILD_NUMBER}" def git_address = "http://192.168.31.70:9999/root/java-demo.git" // 认证 def secret_name = "registry-pull-secret" def docker_registry_auth = "c333c561-73af-43a7-bad8-83c9b9433916" def git_auth = "d946c462-bec9-4b5b-9b77-9af538dd1776" def k8s_auth = "d2374738-3c6f-4077-aa55-186b74c50002" pipeline { agent { kubernetes { label "jenkins-slave" yaml """ kind: Pod metadata: name: jenkins-slave spec: containers: - name: jnlp image: "${registry}/library/jenkins-slave-jdk:1.8" imagePullPolicy: Always volumeMounts: - name: docker-cmd mountPath: /usr/bin/docker - name: docker-sock mountPath: /var/run/docker.sock - name: maven-cache mountPath: /root/.m2 volumes: - name: docker-cmd hostPath: path: /usr/bin/docker - name: docker-sock hostPath: path: /var/run/docker.sock - name: maven-cache hostPath: path: /tmp/m2 """ } } parameters { gitParameter branch: '', branchFilter: '.*', defaultValue: 'master', description: '选择发布的分支', name: 'Branch', quickFilterEnabled: false, selectedValue: 'NONE', sortMode: 'NONE', tagFilter: '*', type: 'PT_BRANCH' choice (choices: ['1', '3', '5', '7'], description: '副本数', name: 'ReplicaCount') choice (choices: ['dev','test','prod'], description: '命名空间', name: 'Namespace') } stages { stage('拉取代码'){ steps { checkout([$class: 'GitSCM', branches: [[name: "${params.Branch}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${git_auth}", url: "${git_address}"]] ]) } } stage('代码编译'){ steps { sh """ mvn clean package -Dmaven.test.skip=true """ } } stage('构建镜像'){ steps { withCredentials([usernamePassword(credentialsId: "${docker_registry_auth}", passwordVariable: 'password', usernameVariable: 'username')]) { sh """ echo ' FROM ${registry}/library/tomcat:v1 LABEL maitainer lizhenliang RUN rm -rf /usr/local/tomcat/webapps/* ADD target/*.war /usr/local/tomcat/webapps/ROOT.war ' > Dockerfile docker build -t ${image_name} . docker login -u ${username} -p '${password}' ${registry} docker push ${image_name} """ } } } stage('部署到K8S平台'){ steps { configFileProvider([configFile(fileId: "${k8s_auth}", targetLocation: "admin.kubeconfig")]){ sh """ sed -i 's#IMAGE_NAME#${image_name}#' deploy.yaml sed -i 's#SECRET_NAME#${secret_name}#' deploy.yaml sed -i 's#REPLICAS#${ReplicaCount}#' deploy.yaml kubectl apply -f deploy.yaml -n ${Namespace} --kubeconfig=admin.kubeconfig """ } } } } }
构建测试
第一步测试:
pipeline { agent { kubernetes { label "jenkins-slave" yaml ''' apiVersion: v1 kind: Pod metadata: name: jenkins-slave spec: containers: - name: jnlp image: 192.168.166.132/library/jenkins-slave-jdk:1.8 ''' } } stages { stage('第一步测试') { steps { sh 'hostname' } } } }
\未完。。。待续。。。