实战项目部署应用到kubernetes流程(jenkins+docker+k8s)
说明
通过jenkins构建java应用程序发布到k8s集群中
本文已一个大数据的java项目来演示构建部署过程
支持发布和回滚
支持一套模板应用不同项目
k8s基础准备
创建项目名称空间
[root@k8s-master ~]# kubectl create ns architect namespace/architect created [root@k8s-master ~]# kubectl label namespace architect istio-injection=enabled namespace/architect labeled [root@k8s-master ~]# kubectl get namespace -L istio-injection NAME STATUS AGE ISTIO-INJECTION architect Active 87s enabled default Active 39d enabled istio-system Active 7d17h kube-public Active 39d kube-system Active 39d [root@k8s-master ~]#
创建全局配置文件
＃创建大数据配置文件 [root@k8s-master ~]# ls bigdata-counter core-site.xml dmp_operator1.keytab hbase-site.xml hdfs-site.xml krb5.conf [root@k8s-master ~]# [root@k8s-master ~]# kubectl create configmap bigdata-config --from-file=/root/bigdata-counter/ -n architect configmap/bigdata-config created [root@k8s-master ~]# #全局环境变量 [root@k8s-master ~]# cat yaml/global-config.yaml apiVersion: v1 kind: ConfigMap metadata: name: global-config namespace: architect data: CONFIG_SERVER_URL: http://config.ms.xx.com CONFIG_SERVER_USER: myconfig ILOG_BOOTSTRAP_SERVERS: bootstrap.servers=192.168.1.122:9092 CONFIG_SERVER_PWDS: mc123456 [root@k8s-master ~]# [root@k8s-master ~]# kubectl apply -f yaml/global-config.yaml configmap/global-config created [root@k8s-master ~]# kubectl get configmap -n architect NAME DATA AGE bigdata-config 4 2m33s global-config 4 3s [root@k8s-master ~]#
创建habror认证信息
[root@k8s-master ~]# kubectl create secret docker-registry registry-secret --docker-server=dev-hub.xx.net --docker-username=admin --docker-password=Harbor12345 --docker-email=admin@dev-hub.xx.net -n architect secret/registry-secret created [root@k8s-master ~]# kubectl get secret -n architect NAME TYPE DATA AGE default-token-5wsxl kubernetes.io/service-account-token 3 2m24s istio.default istio.io/key-and-cert 3 2m24s registry-secret kubernetes.io/dockerconfigjson 1 10s
项目deployment文件
可以做成模本，通过变量替换形式．后面有讲到
[root@k8s-master ~]# cat bigdata-interface-deploy.yaml apiVersion: extensions/v1beta1 kind: Deployment metadata: labels: app: bigdata-interface version: default name: bigdata-interface namespace: architect spec: replicas: 1 selector: matchLabels: app: bigdata-interface version: default template: metadata: labels: app: bigdata-interface version: default spec: containers: - env: - name: izone - name: MY_SERVICE_NAME value: bigdata-interface - name: ilogEnvs value: izone,buildNumber - name: ILOG_BOOTSTRAP_SERVERS valueFrom: configMapKeyRef: key: ILOG_BOOTSTRAP_SERVERS name: global-config - name: CONFIG_SERVER_URL valueFrom: configMapKeyRef: key: CONFIG_SERVER_URL name: global-config - name: CONFIG_SERVER_USER valueFrom: configMapKeyRef: key: CONFIG_SERVER_USER name: global-config - name: CONFIG_SERVER_PWDS valueFrom: configMapKeyRef: key: CONFIG_SERVER_PWDS name: global-config - name: CONFIG_SERVER_PWDS valueFrom: configMapKeyRef: key: CONFIG_SERVER_PWDS name: global-config - name: JAR_BOOT_ARG value: --spring.cloud.config.fail-fast=true - name: JDK_HEAP_OPTS value: -Xms256m -Xmx256m image: dev-hub.jiatuiyun.net/architect/bigdata-interface:architect_b63 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 2 httpGet: path: /actuator/info port: 80 scheme: HTTP initialDelaySeconds: 300 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 30 name: bigdata-interface ports: - containerPort: 80 name: http protocol: TCP readinessProbe: failureThreshold: 3 httpGet: path: /actuator/info port: 80 scheme: HTTP initialDelaySeconds: 60 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 30 volumeMounts: - mountPath: /opt/jdk name: jdk-path - mountPath: /srv/applogs name: applogs imagePullSecrets: - name: registry-secret restartPolicy: Always terminationGracePeriodSeconds: 30 volumes: - hostPath: path: /usr/local/jdk/default type: "" name: jdk-path - hostPath: path: /data/applogs type: "" name: applogs [root@k8s-master ~]#
jenkins pipeline 模板文件
node { //项目名称 String APPNAME='bigdata-interface' //名称空间 String NAMESPACE='architect' String HarborUrl='dev-hub.jiatuiyun.net' String HarborUser='admin' String HarborPasswd='Harbor12345' String KubernetHost = 'root@192.168.100.101' properties([ parameters([ string(name:'GIT_BRANCH_TAG', defaultValue: 'master',description:'', trim: false) ])]) stage('git clone') { checkout([$class: 'GitSCM', branches: [[name: params.GIT_BRANCH_TAG]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'b6a484b7-cbab-4215-9e93-2271d7a333bc', url: 'git@git.jiatuiyun.net:bigdata-center/bigdata-interface.git']]]) } stage('maven build') { sh 'mvn clean package -U' } stage('docker build && push habror'){ sh "cd ${WORKSPACE}" restapi = '`ls -d *-restapi||echo .`' sh "ls -l ${restapi}/target/" sh "echo ${restapi}" sh "curl -o ${restapi}/target/Dockerfile http://192.168.1.55:8089/files/Dockerfile-java.v1" sh "docker build -t ${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} ${restapi}/target" sh "docker tag ${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} ${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER}" sh "docker login ${HarborUrl} -u ${HarborUser} -p ${HarborPasswd}" sh "docker push ${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER}" } stage('deploy'){ echo "${params.GIT_BRANCH_TAG}" echo "${BUILD_NUMBER}" //kubectl set image deployment/bigdata-interface bigdata-interface=dev-hub.jiatuiyun.net/architect/bigdata-interface:architect_b48 -n architect updateImage="kubectl set image deployment/${APPNAME} --namespace=${NAMESPACE} ${APPNAME}=${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} --record" sh "ssh -p 22 ${KubernetHost} ${updateImage}" //test } }
进一步改造pipeline
node { String HarborUrl='dev-hub.jiatuiyun.net' String HarborUser='admin' String HarborPasswd='Harbor12345' String KubernetHost = 'root@192.168.100.101' properties([ parameters([ string(defaultValue: 'architect', description: 'git组', name: 'XBUILD_TEAM', trim: false), string(defaultValue: 'bigdata-interface', description: '项目名', name: 'XBUILD_DEPLOY', trim: false), string(defaultValue: 'git@git.jiatuiyun.net:bigdata-center/bigdata-interface.git', description: 'git仓库地址', name: 'XBUILD_REPO', trim: false), string(defaultValue: 'master', description: '版本名称', name: 'XBUILD_BRANCHE', trim: false) ])]) stage('git clone') { checkout([$class: 'GitSCM', branches: [[name: params.XBUILD_BRANCHE]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'b6a484b7-cbab-4215-9e93-2271d7a333bc', url: params.XBUILD_REPO]]]) } stage('maven build') { sh 'mvn clean package -U' } stage('docker build && push habror'){ sh "cd ${WORKSPACE}" restapi = '`ls -d *-restapi||echo .`' sh "ls -l ${restapi}/target/" sh "echo ${restapi}" sh "curl -o ${restapi}/target/Dockerfile http://192.168.1.55:8089/files/Dockerfile-java.v1" NAMESPACE = params.XBUILD_TEAM APPNAME = params.XBUILD_DEPLOY sh "docker build -t ${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} ${restapi}/target" sh "docker tag ${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} ${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER}" sh "docker login ${HarborUrl} -u ${HarborUser} -p ${HarborPasswd}" sh "docker push ${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER}" } stage('deploy'){ NAMESPACE = params.XBUILD_TEAM APPNAME = params.XBUILD_DEPLOY echo "${params.GIT_BRANCH_TAG}" echo "${BUILD_NUMBER}" //kubectl set image deployment/bigdata-interface bigdata-interface=dev-hub.jiatuiyun.net/architect/bigdata-interface:architect_b48 -n architect //updateImage="kubectl set image deployment/${APPNAME} --namespace=${NAMESPACE} ${APPNAME}=${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} --record" //sh "ssh -p 22 ${KubernetHost} ${updateImage}" //test } }
jenkins上创建自由风格项目
效果如下 支持参数化构建。但每项目单独JOB
查看镜像仓库已经推送正常
项目回滚
node { properties([ parameters([ string(defaultValue: 'architect', description: 'git组', name: 'XBUILD_TEAM', trim: false), string(defaultValue: 'bigdata-interface', description: '项目名', name: 'XBUILD_DEPLOY', trim: false), string(defaultValue: '1', description: '镜像标签', name: 'XBUILD_NUMBER', trim: false) ])]) stage('deploy'){ echo "${BUILD_NUMBER}" NAMESPACE = params.XBUILD_TEAM APPNAME = params.XBUILD_DEPLOY BUILD_NUMBER = params.XBUILD_NUMBER echo "----${params.XBUILD_NUMBER}---${params.XBUILD_DEPLOY} ----${params.XBUILD_TEAM} " //kubectl set image deployment/bigdata-interface bigdata-interface=dev-hub.jiatuiyun.net/architect/bigdata-interface:architect_b48 -n architect //updateImage="kubectl set image deployment/${APPNAME} --namespace=${NAMESPACE} ${APPNAME}=${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} --record" //sh "ssh -p 22 ${KubernetHost} ${updateImage}" //test } }
jenkins效果如下：输入namespace和deployment 名称和上次构建版本
查看k8s集群
可以灵活的在已知版本回滚操作
[root@k8s-master ~]# kubectl get deploy -n architect -o wide NAME READY UP-TO-DATE AVAILABLE AGE CONTAINERS IMAGES SELECTOR bigdata-interface 0/1 1 0 3h12m bigdata-interface dev-hub.jiatuiyun.net/architect/bigdata-interface:architect_b1 app=bigdata-interface,version=default [root@k8s-master ~]# kubectl get pods -n architect NAME READY STATUS RESTARTS AGE bigdata-interface-6bf7b5646d-sf9j5 1/2 Running 0 24s [root@k8s-master ~]#
通过以上实践，可以将deployment做成模板
deployment模板文件
-bash-4.2$ cat k8s-deployment.tpl apiVersion: extensions/v1beta1 kind: Deployment metadata: labels: app: {APPNAME} version: default name: {APPNAME} namespace: {NAMESPACE} spec: replicas: 1 selector: matchLabels: app: {APPNAME} version: default template: metadata: labels: app: {APPNAME} version: default spec: containers: - env: - name: izone - name: MY_SERVICE_NAME value: {APPNAME} - name: ilogEnvs value: izone,buildNumber - name: ILOG_BOOTSTRAP_SERVERS valueFrom: configMapKeyRef: key: ILOG_BOOTSTRAP_SERVERS name: global-config - name: CONFIG_SERVER_URL valueFrom: configMapKeyRef: key: CONFIG_SERVER_URL name: global-config - name: CONFIG_SERVER_USER valueFrom: configMapKeyRef: key: CONFIG_SERVER_USER name: global-config - name: CONFIG_SERVER_PWDS valueFrom: configMapKeyRef: key: CONFIG_SERVER_PWDS name: global-config - name: CONFIG_SERVER_PWDS valueFrom: configMapKeyRef: key: CONFIG_SERVER_PWDS name: global-config - name: JAR_BOOT_ARG value: --spring.cloud.config.fail-fast=true - name: JDK_HEAP_OPTS value: -Xms256m -Xmx256m image: {HarborUrl}/{NAMESPACE}/{APPNAME}:{NAMESPACE}_b{BUILD_NUMBER} imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 2 httpGet: path: /actuator/info port: 80 scheme: HTTP initialDelaySeconds: 300 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 30 name: {APPNAME} ports: - containerPort: 80 name: http protocol: TCP readinessProbe: failureThreshold: 3 httpGet: path: /actuator/info port: 80 scheme: HTTP initialDelaySeconds: 60 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 30 volumeMounts: - mountPath: /opt/jdk name: jdk-path - mountPath: /srv/applogs name: applogs imagePullSecrets: - name: registry-secret restartPolicy: Always terminationGracePeriodSeconds: 30 volumes: - hostPath: path: /usr/local/jdk/default type: "" name: jdk-path - hostPath: path: /data/applogs type: "" name: applogs -bash-4.2$
通过模板动态生成项目的deploy文件．进行升级回滚．
比如下面的：
node { String HarborUrl='dev-hub.jiatuiyun.net' String HarborUser='admin' String HarborPasswd='Harbor12345' String KubernetHost = 'root@192.168.100.101' properties([ parameters([ string(defaultValue: 'architect', description: 'git组', name: 'XBUILD_TEAM', trim: false), string(defaultValue: 'bigdata-interface', description: '项目名', name: 'XBUILD_DEPLOY', trim: false), string(defaultValue: '1', description: '镜像标签', name: 'XBUILD_NUMBER', trim: false) ])]) stage('deploy'){ echo "${BUILD_NUMBER}" NAMESPACE = params.XBUILD_TEAM APPNAME = params.XBUILD_DEPLOY BUILD_NUMBER = params.XBUILD_NUMBER echo "----${params.XBUILD_NUMBER}---${params.XBUILD_DEPLOY} ----${params.XBUILD_TEAM} " //kubectl set image deployment/bigdata-interface bigdata-interface=dev-hub.jiatuiyun.net/architect/bigdata-interface:architect_b48 -n architect updateImage="kubectl set image deployment/${APPNAME} --namespace=${NAMESPACE} ${APPNAME}=${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} --record" command="sed -e 's#{APPNAME}#${APPNAME}#g;s#{NAMESPACE}#${NAMESPACE}#g;s#{HarborUrl}#${HarborUrl}#g;s#{BUILD_NUMBER}#${BUILD_NUMBER}#g' /data/jenkins_slave/k8s-deployment.tpl > /data/jenkins_slave/${APPNAME}-deploy.yaml" sh "${command}" sh "scp -P 22 /data/jenkins_slave/${APPNAME}-deploy.yaml ${KubernetHost}:/root/ " //sh "ssh -p 22 ${KubernetHost} ${updateImage}" sh "ssh -p 22 ${KubernetHost} kubectl apply -f /root/${APPNAME}-deploy.yaml" //test } }
pipeline参数化构建
示例：
pipeline {
agent any
parameters {
gitParameter branch: '', branchFilter: '.*', defaultValue: '', description: '', name: 'XBUILD_BRANCHE', quickFilterEnabled: false, selectedValue: 'NONE', sortMode: 'NONE', tagFilter: '*', type: 'PT_BRANCH_TAG'
string defaultValue: 'git@git.xx.net:company/xx-xx.git', description: '', name: 'XBUILD_REPO', trim: false
}
stages {
stage('Example') {
steps {
checkout([$class: 'GitSCM', branches: [[name: params.XBUILD_BRANCHE]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'b6a484b7-cbab-4215-9e93-2271d7a333bb', url: params.XBUILD_REPO]]])
}
}
}
}
构建效果如下：
