jenkins容器化
1、使用Jenkins连接k8s
第一步：创建admin-csr.json
cat > admin-csr.json << EOF
{
"CN":"admin",
"key":{
"algo":"rsa",
"size":2048
},
"names":[
{
"C":"CN",
"L":"BeiJing",
"ST":"BeiJing",
"O":"system:masters",
"OU":"System"
}
]
}
EOF
第二步：创建证书和私钥
1、kubeadm安装
cfssl gencert -ca=/etc/kubernetes/pki/ca.crt -ca-key=/etc/kubernetes/pki/ca.key --profile=kubernetes admin-csr.json | cfssljson -bare admin
2、二进制安装
cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem -ca-key=/etc/kubernetes/ssl/ca-key.pem --profile=kubernetes admin-csr.json | cfssljson -bare admin
第三步：配置证书
openssl pkcs12 -export -out ./jenkins-admin.pfx -inkey ./admin-key.pem -in ./admin.pem -passout pass:123456
[root@k8s-master-01 jenkins]# sz jenkins-admin.pfx #将生成的证书下载下来，后面配置集群需要用到
[root@h-k8s-master-01 ~]# cat /etc/kubernetes/ssl/ca.pem -----BEGIN CERTIFICATE----- MIIDYDCCAkigAwIBAgIUeJXV2IqSgZKX9aPRMqPkpjA6P+gwDQYJKoZIhvcNAQEL BQAwSDELMAkGA1UEBhMCQ04xETAPBgNVBAgTCFNoYW5nSGFpMREwDwYDVQQHEwhT aGFuZ0hhaTETMBEGA1UEAxMKa3ViZXJuZXRlczAeFw0yMjAxMDUwMjM1MDBaFw0y NzAxMDQwMjM1MDBaMEgxCzAJBgNVBAYTAkNOMREwDwYDVQQIEwhTaGFuZ0hhaTER MA8GA1UEBxMIU2hhbmdIYWkxEzARBgNVBAMTCmt1YmVybmV0ZXMwggEiMA0GCSqG SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDzWaYksqYeSAd6gYOHQa86Gdpt2Ygdn+ZL mP2AZgubgCq0yrOc81OzOr4WgVNNNO25kYkbjKSiv1PzhcIONhQteanucqLgtWh+ KkO9IXJCF29jcXy5KDFGxorEibLJcUGELBvYICWfbI0Mb/guXUHi2v715XDW674O d3caIqdiPSqlJr7OBaJkiLA6p3/2i0wr1IMy8747+GK0Ts+3Cpzz3XbRMOMldNk+ YOQa0G6UPDojEkxbsjq6oSKkFu/IJjXFT0JhjFgBVqv69+AzEfjwop2sBB9zIs/r KuU7genmNXEtceGBK4mdVertzn4HmxjDhiurEtARu/3CMWuWDKzfAgMBAAGjQjBA MA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRCgr7M P/t0zthgCWGsvFsqTShGDjANBgkqhkiG9w0BAQsFAAOCAQEAON7w+JWjkJ1arfIO INT4jT3t9zjK4fD1FuYH1dd3chuGSQtXeu4QTQtyM8QHF4MflaYKyAeZiqNRJ6JB cU0I3lkDhfKIBbDWCWLjm4Dk/BTOOPyDKnhED/VTq7pxMxXpJWNasLlHzlg0ncz1 zBce50Vm8YXp/LVXjJzU4WibQZyL+dQatNtSIuToOJ6jN1HZ3Mu+1qU2NWJZbqrd UkWqOMh8AMuQFYSKbOcYC5kvdtrgEQvOI4YewX11Nl4NBj+49qHbH73zYYUcmRbs LPrTAggKF+w7vZCOgHTifb5R+t8pOuir1GWrYgAmN8EjX9ZkISRaRaPGo4fy/7hx MSUskA== -----END CERTIFICATE-----
下图生成凭证，用于上面的选择
2、Jenkins调用k8s
1、容器化部署Django
1、基础环境(写配置清单)
---
kind: Deployment
apiVersion: apps/v1
metadata:
name: django
spec:
selector:
matchLabels:
app: django
template:
metadata:
labels:
app: django
spec:
imagePullSecrets:
- name: aliyun-registry-key
containers:
- name: django
image: nginx
---
kind: Service
apiVersion: v1
metadata:
name: django
spec:
ports:
- port: 80
targetPort: 80
selector:
app: django
---
kind: Ingress
apiVersion: extensions/v1beta1
metadata:
name: django
spec:
rules:
- host: www.django.com
http:
paths:
- backend:
serviceName: django
servicePort: 80
path: /
[root@k8s-master-01 jenkins]# vim django.yaml
[root@k8s-master-01 jenkins]# kubectl apply -f django.yaml
[root@k8s-master-01 jenkins]# kubectl get ingress
NAME CLASS HOSTS ADDRESS PORTS AGE
django <none> www.django.com 192.168.15.104 80 4d4h
[root@k8s-master-01 jenkins]# kubectl get svc -n ingress-nginx
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
ingress-nginx-controller NodePort 10.97.151.84 <none> 80:30769/TCP,443:30885/TCP 6d18h
2、下载代码
保存
因为jenkinsfile需要获取分支，所以需要配下面的参数
3、打包镜像(在容器里面打包镜像)
构建镜像在容器里面执行，需要执行下面的三条命令，username和password是docker仓库的username和password,如何获取呢？从凭证里获取，在下面的withCredentials里面有凭证的id,用户名，密码
下面的意思是设置kubernetes环境，主要有测试集群有哪些节点，secret是从仓库下载内容的时候要有一个秘钥，先删除，然后重新创建一份，让docker更好的登录，config是连接kubernets的秘钥，在/etc/kubernetes/cfg/admin.kubeconfig里面
上面有secre,所以需要创建secret
将下面的jenkins流水线复制到下面
部署（kubernetes_name可以部署多个集群，所以在这里添加选项参数）
[root@k8s-master-01 jenkins]# kubectl create secret generic kubeconfig --from-file=/etc/kubernetes/cfg/admin.kubeconfig
secret/kubeconfig created
选择不同的环境，相当于构建在不同的k8s集群中，构建在不同的环境中，相当于实现一条流水线三个环境
jenkindfile中要用到仓库的用户名，密码，需要创建仓库的凭证
可以阿里云建一个私有仓库
4、上传镜像
5、部署到django-k8s
此时会报错，原因是没有dockerfile文件，在pycharm中加入，然后推送，推送之后可以再k8s容器里面下载docker pull ，构建（docker build），运行（docker run）docker ps， django运行成功之后再部署
dockerfile
FROM python:3.6 RUN mkdir /opt/linux/ ADD ./ /opt/linux/ RUN pip install django==2.2.2 -i https://pypi.douban.com/simple/ RUN pip install uwsgi -i https://pypi.douban.com/simple/ WORKDIR /opt/linux/ EXPOSE 8000 CMD /usr/local/bin/uwsgi myweb.ini
django.yaml（需要在原先的上面加上ConfigMap,需要将配置文件挂在到nginx里面去）
--- kind: ConfigMap apiVersion: v1 metadata: name: django data: default.conf: | server { # 监听的端口 listen 80; # 配置域名 server_name www.django.com; # 配置路径 location / { # 加载Nginx代理uwsgi的配置项 include uwsgi_params; # 指定uwsgi的访问地址 uwsgi_pass 127.0.0.1:8000; # 连接uwsgi的超时时间 uwsgi_read_timeout 2; # 自定义uwsgi代理项目的路径及配置项 uwsgi_param UWSGI_SCRIPT linux.wsgi; # 指定python项目的路径 uwsgi_param UWSGI_CHDIR /opt/linux; # 索引文件 index index.html index.htm; # 客户端上传文件的最大值 client_max_body_size 35m; } } --- kind: Deployment apiVersion: apps/v1 metadata: name: django spec: selector: matchLabels: app: django template: metadata: labels: app: django spec: imagePullSecrets: - name: aliyun-registry-key containers: - name: django image: nginx - name: nginx image: nginx volumeMounts: - mountPath: /etc/nginx/conf.d/ name: nginx-config volumes: - name: nginx-config configMap: name: django items: - key: default.conf path: default.conf --- kind: Service apiVersion: v1 metadata: name: django spec: ports: - port: 80 targetPort: 80 selector: app: django --- kind: Ingress apiVersion: extensions/v1beta1 metadata: name: django spec: rules: - host: www.django.com http: paths: - backend: serviceName: django servicePort: 80 path: /
6、发邮件通知
#jenkinsfilepipeline { agent { kubernetes { cloud "${KUBERNETES_NAME}" slaveConnectTimeout 1200 yaml ''' apiVersion: v1 kind: Pod spec: containers: - name: jnlp image: jenkins/inbound-agent:4.7-1-jdk11 args: [\'$(JENKINS_SECRET)\', \'$(JENKINS_NAME)\'] imagePullPolicy: IfNotPresent volumeMounts: - mountPath: "/etc/localtime" name: "volume-2" readOnly: false - name: docker image: docker:19.03.15-git imagePullPolicy: IfNotPresent tty: true volumeMounts: - mountPath: "/etc/localtime" name: "volume-2" readOnly: false - mountPath: "/var/run/docker.sock" name: "volume-docker" readOnly: false - mountPath: "/etc/hosts" name: "volume-hosts" readOnly: false - name: kubectl image: registry.cn-beijing.aliyuncs.com/citools/kubectl:1.17.4 imagePullPolicy: IfNotPresent tty: true command: - "cat" volumeMounts: - mountPath: "/etc/localtime" name: "volume-2" readOnly: false - mountPath: "/var/run/docker.sock" name: "volume-docker" readOnly: false - mountPath: "/root/.kube" name: "kubeconfig" readOnly: false - mountPath: "/.kube" name: "kubeconfig" readOnly: false volumes: - name: volume-maven-repo emptyDir: {} - name: volume-2 hostPath: path: "/usr/share/zoneinfo/Asia/Shanghai" - name: kubeconfig secret: secretName: kubeconfig items: - key: config path: config - name: volume-docker hostPath: path: "/var/run/docker.sock" - name: volume-hosts hostPath: path: /etc/hosts ''' } } stages { stage('Pull Code') { parallel { stage('Pull Code') { steps { git(url: 'git@192.168.11.8:linux20/django.git', branch: 'master', credentialsId: '707054f2-5cbf-403b-a9a4-29ff8696e278') } } stage('check env') { steps { sleep(unit: 'SECONDS', time: 20) } } } } stage('Checkout Breanch') { steps { sh """ git checkout $GIT_TAG """ } } stage('Make Code') { steps { sh 'echo "make code"' } } stage('编译及构建') { parallel { stage('初始化操作系统及生成镜像Tag') { steps { script { CommitID = sh(returnStdout: true, script: "git log -n 1 --pretty=format:'%h'").trim() CommitMessage = sh(returnStdout: true, script: "git log -1 --pretty=format:'%h : %an %s'").trim() def curDate = sh(script: "date '+%Y%m%d-%H%M%S'", returnStdout: true).trim() TAG = curDate[0..14] + "-" + CommitID + "-$GIT_TAG" } } } } } stage('构建镜像及检查kubernetes环境') { parallel { stage('构建镜像') { steps { withCredentials([usernamePassword(credentialsId: "${DOCKER_REPOSITORY_CREDENTIAL_ID}", passwordVariable: "Password", usernameVariable: "Username")]) { container(name: 'docker') { sh """ docker build -t ${HARBOR_HOST}/${NAMESPACE_NAME}/${REPOSITORY_NAME}:${TAG} . docker login ${HARBOR_HOST} --username=${Username} --password=${Password} docker push ${HARBOR_HOST}/${NAMESPACE_NAME}/${REPOSITORY_NAME}:${TAG} """ } } } } stage('设置K8s基础环境') { steps { withCredentials([usernamePassword(credentialsId: "${DOCKER_REPOSITORY_CREDENTIAL_ID}", passwordVariable: "Password", usernameVariable: "Username")]) { container(name: 'kubectl') { sh """ kubectl get nodes --kubeconfig=/.kube/config kubectl delete secret aliyun-registry-key -n ${K8S_NAMESPACE_NAME} kubectl create secret docker-registry aliyun-registry-key -n ${K8S_NAMESPACE_NAME} --docker-server=${HARBOR_HOST} --docker-username=${Username} --docker-password=${Password} --kubeconfig=/.kube/config """ } } } } } } stage('Deploy Image to Kubernetes') { steps { container(name: 'kubectl') { sh """ kubectl set image ${CONTROLLER_TYPE} ${CONTROLLER_NAME} -n ${K8S_NAMESPACE_NAME} ${CONTAINER_NAME}=${HARBOR_HOST}/${NAMESPACE_NAME}/${REPOSITORY_NAME}:${TAG} --kubeconfig=/.kube/config """ } } } stage('Send Email') { steps { sh 'echo "Send Email"' } } } }
3、发送邮件
下载Email插件：Email Extension
