经过网上视频的学习整理如下:
环境规划
10.211.55.11 k8s-master1,nfs
10.211.55.12 k8s-master2,k8s-node1
10.211.55.13 k8s-master3,k8s-node2
10.211.55.14 Harbor仓库,git仓库,其他公共的服务
上面设计到的服务,可以查看我前面写过的文章
项目下载地址
https://github.com/erlonglong/k8s-test/tree/master/devops
https://www.daocloud.io/mirror 镜像加速
[root@wan src]# cat /etc/docker/daemon.json
{
"insecure-registries":["hub.k8s.com","10.211.55.14"],
"registry-mirrors": ["http://f1361db2.m.daocloud.io"]
}
各节点docker的加速和私有仓库的配置,域名需要绑定
hub.k8s.com 10.211.55.14
使用K8S 部署jenkins
采用jenkins-master + jenkins-slave 的模式来实现CI&CD
https://github.com/jenkinsci/kubernetes-plugin/tree/master/src/main/kubernetes
wget https://raw.githubusercontent.com/jenkinsci/kubernetes-plugin/master/src/main/kubernetes/jenkins.yml
这里面我们用不到ingress 可以把相关的资源去掉
vim jenkins-master.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: jenkins
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
name: jenkins
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
resources: ["pods/exec"]
verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
resources: ["pods/log"]
verbs: ["get","list","watch"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
name: jenkins
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: jenkins
subjects:
- kind: ServiceAccount
name: jenkins
---
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: jenkins
labels:
name: jenkins
spec:
serviceName: jenkins
replicas: 1
updateStrategy:
type: RollingUpdate
template:
metadata:
name: jenkins
labels:
name: jenkins
spec:
terminationGracePeriodSeconds: 10
serviceAccountName: jenkins
containers:
- name: jenkins
image: jenkins/jenkins:lts-alpine
imagePullPolicy: Always
ports:
- containerPort: 8080
- containerPort: 50000
resources:
limits:
cpu: 1
memory: 1Gi
requests:
cpu: 0.5
memory: 500Mi
env:
- name: LIMITS_MEMORY
valueFrom:
resourceFieldRef:
resource: limits.memory
divisor: 1Mi
- name: JAVA_OPTS
value: -Xmx$(LIMITS_MEMORY)m -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
volumeMounts:
- name: jenkins-home
mountPath: /var/jenkins_home
livenessProbe:
httpGet:
path: /login
port: 8080
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 12 # ~2 minutes
readinessProbe:
httpGet:
path: /login
port: 8080
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 12 # ~2 minutes
volumeClaimTemplates:
- metadata:
name: jenkins-home
spec:
storageClassName: "managed-nfs-storage"
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 1Gi
---
apiVersion: v1
kind: Service
metadata:
name: jenkins
spec:
type: NodePort
selector:
name: jenkins
ports:
-
name: http
port: 80
targetPort: 8080
protocol: TCP
nodePort: 30008
-
name: agent
port: 50000
protocol: TCP
配置文件关键点是上面标注的位置:
PV 的问题,需要动态创建PV ,所以需要创建一个NFS 动态分配的PV 资源,当然也可以创建静态的PV,这块内容不多介绍了
关键看一下动态PV 的创建
nfs-server
vim /etc/exports
/data/volumes/ 10.0.0.0/8(rw,no_root_squash)
vim nfs-pv.yaml
kind: ServiceAccount
apiVersion: v1
metadata:
name: nfs-client-provisioner
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-client-provisioner-runner
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: run-nfs-client-provisioner
subjects:
- kind: ServiceAccount
name: nfs-client-provisioner
namespace: default
roleRef:
kind: ClusterRole
name: nfs-client-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-client-provisioner
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-client-provisioner
subjects:
- kind: ServiceAccount
name: nfs-client-provisioner
# replace with namespace where provisioner is deployed
namespace: default
roleRef:
kind: Role
name: leader-locking-nfs-client-provisioner
apiGroup: rbac.authorization.k8s.io
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: managed-nfs-storage
provisioner: fuseim.pri/ifs # or choose another name, must match deployment's env PROVISIONER_NAME'
parameters:
archiveOnDelete: "true"
---
kind: Deployment
apiVersion: apps/v1
metadata:
name: nfs-client-provisioner
spec:
replicas: 1
selector:
matchLabels:
app: nfs-client-provisioner
strategy:
type: Recreate
template:
metadata:
labels:
app: nfs-client-provisioner
spec:
serviceAccountName: nfs-client-provisioner
containers:
- name: nfs-client-provisioner
image: doudou007/nfs-pv:latest
volumeMounts:
- name: nfs-client-root
mountPath: /persistentvolumes
env:
- name: PROVISIONER_NAME
value: fuseim.pri/ifs
- name: NFS_SERVER
value: 10.211.55.11
- name: NFS_PATH
value: /data/volumes
volumes:
- name: nfs-client-root
nfs:
server: 10.211.55.11
path: /data/volumes
还有一点需要注意,给node节点需要安装支持NFS挂载 客户端的软件包
yum install nfs-utils -y
kubectl apply -f nfs-pv.yaml
kubectl apply -f jenkins-master.yaml
[root@k8s-master1 jenkins-new]# kubectl get pods |egrep jenkins
jenkins-0 1/1 Running 0 2m52s
![](https://img2018.cnblogs.com/blog/1344156/201906/1344156-20190626090251748-1476276603.png)
这俩说明一下,需要安装的插件
重点是需要安装
pipe 插件 git 插件 kubernetes插件 kubernetes deploy 这几个插件:
![](https://img2018.cnblogs.com/blog/1344156/201906/1344156-20190626090553949-550632385.png)
![](https://img2018.cnblogs.com/blog/1344156/201906/1344156-20190626090635458-1792154378.png)
![](https://img2018.cnblogs.com/blog/1344156/201906/1344156-20190626091059546-1041549964.png)
接下来配置jenkins-slave
通过k8s 来动态创建jenkins-slave ,所以需要提供jenkins-slave 相关的镜像
FROM centos:7
LABEL maintainer doudou
RUN yum install -y java-1.8.0-openjdk maven curl git libtool-ltdl-devel && \
yum clean all && \
rm -rf /var/cache/yum/* && \
mkdir -p /usr/share/jenkins
COPY slave.jar /usr/share/jenkins/slave.jar
COPY jenkins-slave /usr/bin/jenkins-slave
COPY settings.xml /etc/maven/settings.xml
RUN chmod +x /usr/bin/jenkins-slave
ENTRYPOINT ["jenkins-slave"]
这个镜像制作涉及到的几个文件
jenkins-slave: wget https://github.com/jenkinsci/docker-jnlp-slave/blob/master/jenkins-slave
slave.jar :通过master 获取wget http://10.211.55.13:30008/jnlpJars/slave.jar
maven:settings.xml:
文件没什么变化关键是加入了一个阿里云的仓库,提高下载包的速度:
<mirror>
<id>central</id>
<mirrorOf>central</mirrorOf>
<name>aliyun maven</name>
<url>https://maven.aliyun.com/repository/public</url>
</mirror>
</mirrors>
[root@k8s-master1 jenkins-slave]# ls
Dockerfile jenkins-slave settings.xml slave.jar
docker build -t hub.k8s.com/dev/jenkins-slave-jdk:1.8 ./
说明一下,这个是打包成Harbor推送的tag
hub.k8s.com 访问路径
dev 仓库名
jenkins-slave-jdk:1.8 镜像名和版本
docker login -u admin -p'Harbor12345'
docker push hub.k8s.com/dev/jenkins-slave-jdk:1.8
登陆Harbor可以看到相关的镜像
下面通过jenkins来实现CI 和CD
配置参数化构建:默认master用于多分支发布
![](https://img2018.cnblogs.com/blog/1344156/201906/1344156-20190626101944586-341758217.png)
![](https://img2018.cnblogs.com/blog/1344156/201906/1344156-20190626102201820-746921568.png)
下面是pipe 的语法:
https://jenkins.io/zh/doc/book/pipeline/syntax/
http://10.211.55.13:30008/me/my-views/view/all/job/demo/pipeline-syntax/
官方链接:
https://jenkins.io/zh/doc/book/pipeline/syntax/
![](https://img2018.cnblogs.com/blog/1344156/201906/1344156-20190626102636508-1566540291.png)
涉及到几个认证的ID,需要我们提前生成
下面认证用的ID 是需要在jenkins上面增加相关凭据才能生成ID,这样做是为了保护隐私的密码和用户名
def docker_registry_auth = "898bbf50-9f94-480e-861d-187f5633f26c"
def git_auth = "71bd0d12-3b2b-4acb-94cb-38f7aa9c6817"
def k8s_auth = "f3b86b4f-68a7-4913-8f2d-c7683ecaa235"
![](https://img2018.cnblogs.com/blog/1344156/201906/1344156-20190626102750637-1895193918.png)
![](https://img2018.cnblogs.com/blog/1344156/201906/1344156-20190626103007901-1564694670.png)
可以通过选项生成自己需要的语法,下面是配置好K8S ,在系统配置里面,增加一个云,配置即可
![](https://img2018.cnblogs.com/blog/1344156/201906/1344156-20190626104036368-1116282866.png)
因为我们jenkins 是通过POD 安装的,所以我们用内部提供的cluster 连接就可以,默认POD 都有权限访问的
如果需要配置外部的
cat /root/.kube/config
echo certificate-authority-data | base64 -d > ca.crt
echo client-certificate-data | base64 -d > client.crt
echo client-key-data | base64 -d > client.key
openssl pkcs12 -export -out cert.pfx -inkey client.key -in client.crt -certfile ca.crt
key 拷贝ca.crt
cert.pfx 上传上去
查看文章:https://blog.csdn.net/diantun00/article/details/81180694
k8s-deploy的配置
![](https://img2018.cnblogs.com/blog/1344156/201906/1344156-20190626104850769-1680835644.png)
提取出核心的代码
kubernetesDeploy configs: 'deploy.yml', kubeconfigId: "${k8s_auth}"
deploy.yml 这个文件是在当前的仓库里面查找,所有我们需要把这个文件也git 提交到仓库
同样,我们可以吧Jenkinsfile 的文件也提交到GIT 仓库
![](https://img2018.cnblogs.com/blog/1344156/201906/1344156-20190626111347324-2064507415.png)
[root@k8s-master1 java-demo]# ls
db deploy.yml Jenkinsfile LICENSE pom.xml README.md src
还有一点需要注意,因为是在POD 里面构建镜像,所以需要通过secret来认证拉取镜像的信息
def secret_name = "registry-pull-secret"
kubectl create secret docker-registry registry-pull-secret \
--docker-username=admin --docker-password=Harbor12345 \
--docker-email=doudou@k8s.com --docker-server=hub.k8s.com
下面是具体的流水线代码
def registry = "hub.k8s.com"
def project = "dev"
def app_name = "demo"
def image_name = "${registry}/${project}/${app_name}:${BUILD_NUMBER}"
def git_address = "git@10.211.55.14:/home/git/java-demo.git"
def secret_name = "registry-pull-secret"
def docker_registry_auth = "898bbf50-9f94-480e-861d-187f5633f26c"
def git_auth = "71bd0d12-3b2b-4acb-94cb-38f7aa9c6817"
def k8s_auth = "f3b86b4f-68a7-4913-8f2d-c7683ecaa235"
podTemplate(label: 'jenkins-slave', cloud: 'kubernetes', containers: [
containerTemplate(
name: 'jnlp',
image: "${registry}/dev/jenkins-slave-jdk:1.8"
),
],
volumes: [
hostPathVolume(mountPath: '/var/run/docker.sock', hostPath: '/var/run/docker.sock'),
hostPathVolume(mountPath: '/usr/bin/docker', hostPath: '/usr/bin/docker')
],
)
{
node("jenkins-slave"){
// 第一步
stage('拉取代码'){
checkout([$class: 'GitSCM', branches: [[name: '${Branch}']], userRemoteConfigs: [[credentialsId: "${git_auth}", url: "${git_address}"]]])
}
// 第二步
stage('代码编译'){
sh "mvn clean package -Dmaven.test.skip=true"
}
// 第三步
stage('构建镜像'){
withCredentials([usernamePassword(credentialsId: "${docker_registry_auth}", passwordVariable: 'password', usernameVariable: 'username')]) {
sh """
echo '
FROM doudou007/tomcat
RUN rm -rf /usr/local/tomcat/webapps/*
ADD target/*.war /usr/local/tomcat/webapps/ROOT.war
' > Dockerfile
ls
ls target
docker build -t ${image_name} .
echo "${password}"|docker login -u ${username} --password-stdin ${registry}
docker push ${image_name}
"""
}
}
// 第四步
stage('部署到K8S平台'){
sh """
sed -i 's#\$IMAGE_NAME#${image_name}#' deploy.yml
sed -i 's#\$SECRET_NAME#${secret_name}#' deploy.yml
"""
kubernetesDeploy configs: 'deploy.yml', kubeconfigId: "${k8s_auth}"
}
}
}
部署的yaml文件
vim deploy.yml
apiVersion: apps/v1
kind: Deployment
metadata:
name: web
spec:
replicas: 2
selector:
matchLabels:
app: java-demo
template:
metadata:
labels:
app: java-demo
spec:
imagePullSecrets:
- name: $SECRET_NAME
containers:
- name: tomcat
image: $IMAGE_NAME
ports:
- containerPort: 8080
name: web
livenessProbe:
httpGet:
path: /
port: 8080
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 12
readinessProbe:
httpGet:
path: /
port: 8080
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 12
---
apiVersion: v1
kind: Service
metadata:
name: web
spec:
type: NodePort
selector:
app: java-demo
ports:
- protocol: TCP
port: 80
targetPort: 8080 ![](https://img2018.cnblogs.com/blog/1344156/201906/1344156-20190626113346832-1067616732.png)
![](https://img2018.cnblogs.com/blog/1344156/201906/1344156-20190626113417567-1322481269.png)
查看控制台日志输出:
![](https://img2018.cnblogs.com/blog/1344156/201906/1344156-20190626113507387-38117384.png)
![](https://img2018.cnblogs.com/blog/1344156/201906/1344156-20190626113522286-291451717.png)
![](https://img2018.cnblogs.com/blog/1344156/201906/1344156-20190626140210530-1983228353.png)
推送上的镜像
[root@k8s-master1 demo]# kubectl get pods|egrep web
web-5b7c77d664-ddwbp 1/1 Running 1 3h2m
web-5b7c77d664-rd7lr 1/1 Running 1 3h2m
然后通过节点访问,就可以看到访问的程序了
说明整个流程是完成了,这块需要加强学习的地方是jenkins 流水线发布的学习
核心就是jenkins的配置,这里方便演示,只用到GIT,后续可以扩展成gitlab
下面是通过gitlab自动构建的文章:
https://www.cnblogs.com/yinzhengjie/p/9613270.html
还有一种方式应用k8s
kubernetes cli插件
这个插件功能需要在pod里面执行kubectl 提前把kubectl 打包到jenkins的镜像里面
然后生成需要的凭据
添加kubernetes凭证
找一个有集群管理员角色的的secrets
kubectl describe secrets -n kube-system kubernetes-dashboard-token-fr9cz |egrep token:
或者
kubectl -n kube-system get secrets kubernetes-dashboard-token-fr9cz -o go-template --template '{{index .data "token"}}' | base64 -d
![](https://img2020.cnblogs.com/blog/1344156/202007/1344156-20200728150642765-1804867434.png)
然后在jenkins界面添加凭证
![](http://idcsec.com/wp-content/uploads/2019/03/image-8-1024x475.png)
![](https://img2020.cnblogs.com/blog/1344156/202007/1344156-20200728145630868-183540659.png)
![](https://img2020.cnblogs.com/blog/1344156/202007/1344156-20200728150016036-165460713.png)
![](https://img2020.cnblogs.com/blog/1344156/202007/1344156-20200728145700313-532439686.png)
【推荐】编程新体验,更懂你的AI,立即体验豆包MarsCode编程助手
【推荐】凌霞软件回馈社区,博客园 & 1Panel & Halo 联合会员上线
【推荐】抖音旗下AI助手豆包,你的智能百科全书,全免费不限次数
【推荐】博客园社区专享云产品让利特惠,阿里云新客6.5折上折
【推荐】轻量又高性能的 SSH 工具 IShell:AI 加持,快人一步