【西天取经】10、自动化部署-【从提交code开始自动化部署到k8s里面】
【西天取经】10、自动化部署-【从提交code开始自动化部署到k8s里面】
早就想写一篇关于自动化部署的文章了，但是由于自己是k8s初学者掌握的还不是很好，写这样的文章最自己来说挑战很大，也不知道最后能不能达到自己的满意。既然决定要写，早晚都需要开始动手去写，于是给自己打打气试着写写看。下面这个链接是一篇很好的参考文章
图文详解k8s自动化持续集成之GitLab CI/CD
如下图所示，我们的自动化包括从生成文件版本号到部署k8s里面全部由gitlab的runner实现的。由于全部是在k8s里面实现的，所以对于自己的挑战来说真的是很大。
helm的参考例子：https://gitlab.com/gitlab-org/charts/gitlab-runner
部署在k8s里面的gitlab-runner
apiVersion: apps/v1
kind: Deployment
metadata:
annotations:
deployment.kubernetes.io/revision: "3"
meta.helm.sh/release-name: gitlab-runner
meta.helm.sh/release-namespace: gitlab-managed-apps
creationTimestamp: "2020-09-03T09:50:43Z"
generation: 3
labels:
app: gitlab-runner-gitlab-runner
app.kubernetes.io/managed-by: Helm
chart: gitlab-runner-0.22.0
heritage: Helm
release: gitlab-runner
name: gitlab-runner-gitlab-runner
namespace: gitlab-managed-apps
resourceVersion: "126342124"
selfLink: /apis/apps/v1/namespaces/gitlab-managed-apps/deployments/gitlab-runner-gitlab-runner
uid: 46c9dceb-6f67-4aab-9102-326baca2d689
spec:
progressDeadlineSeconds: 600
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
app: gitlab-runner-gitlab-runner
strategy:
rollingUpdate:
maxSurge: 25%
maxUnavailable: 25%
type: RollingUpdate
template:
metadata:
annotations:
checksum/configmap: c2df9d47952753570f28484a7f9c58922ef8572a7fb4dbdf354759e59cd6cdb3
checksum/secrets: ef9159bc884121cfa71e30a14e7a7959eddd24d251ed651f6c5b5d1699b04bd9
prometheus.io/port: "9252"
prometheus.io/scrape: "true"
creationTimestamp: null
labels:
app: gitlab-runner-gitlab-runner
chart: gitlab-runner-0.22.0
heritage: Helm
release: gitlab-runner
spec:
containers:
- command:
- /bin/bash
- /scripts/entrypoint
env:
- name: CI_SERVER_URL
value: https://git.runshopstore.com/
- name: CLONE_URL
- name: RUNNER_REQUEST_CONCURRENCY
value: "1"
- name: RUNNER_EXECUTOR
value: kubernetes
- name: REGISTER_LOCKED
value: "true"
- name: RUNNER_TAG_LIST
value: docker,company
- name: RUNNER_OUTPUT_LIMIT
value: "4096"
- name: KUBERNETES_IMAGE
value: ubuntu:18.04
- name: KUBERNETES_PRIVILEGED
value: "true"
- name: KUBERNETES_NAMESPACE
value: gitlab-managed-apps
- name: KUBERNETES_POLL_TIMEOUT
value: "180"
- name: KUBERNETES_CPU_LIMIT
- name: KUBERNETES_CPU_LIMIT_OVERWRITE_MAX_ALLOWED
- name: KUBERNETES_MEMORY_LIMIT
- name: KUBERNETES_MEMORY_LIMIT_OVERWRITE_MAX_ALLOWED
- name: KUBERNETES_CPU_REQUEST
value: 2000m
- name: KUBERNETES_CPU_REQUEST_OVERWRITE_MAX_ALLOWED
- name: KUBERNETES_MEMORY_REQUEST
value: 1Gi
- name: KUBERNETES_MEMORY_REQUEST_OVERWRITE_MAX_ALLOWED
- name: KUBERNETES_SERVICE_ACCOUNT
- name: KUBERNETES_SERVICE_CPU_LIMIT
- name: KUBERNETES_SERVICE_MEMORY_LIMIT
- name: KUBERNETES_SERVICE_CPU_REQUEST
value: 2000m
- name: KUBERNETES_SERVICE_MEMORY_REQUEST
value: 1Gi
- name: KUBERNETES_HELPER_CPU_LIMIT
- name: KUBERNETES_HELPER_MEMORY_LIMIT
- name: KUBERNETES_HELPER_CPU_REQUEST
value: 2000m
- name: KUBERNETES_HELPER_MEMORY_REQUEST
value: 1Gi
- name: KUBERNETES_HELPER_IMAGE
- name: KUBERNETES_PULL_POLICY
- name: CACHE_TYPE
value: s3
- name: CACHE_PATH
- name: CACHE_SHARED
value: "true"
- name: CACHE_S3_SERVER_ADDRESS
value: minio.gitlab-managed-apps.svc.cluster.local:9000
- name: CACHE_S3_BUCKET_NAME
value: meshop-gitlab-runner
- name: CACHE_S3_BUCKET_LOCATION
- name: CACHE_S3_INSECURE
value: "true"
image: gitlab/gitlab-runner:alpine-v13.5.0
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
exec:
command:
- /entrypoint
- unregister
- --all-runners
livenessProbe:
exec:
command:
- /bin/bash
- /scripts/check-live
failureThreshold: 3
initialDelaySeconds: 60
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
name: gitlab-runner-gitlab-runner
ports:
- containerPort: 9252
name: metrics
protocol: TCP
readinessProbe:
exec:
command:
- /usr/bin/pgrep
- gitlab.*runner
failureThreshold: 3
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
resources: {}
securityContext:
allowPrivilegeEscalation: false
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /secrets
name: runner-secrets
- mountPath: /home/gitlab-runner/.gitlab-runner
name: etc-gitlab-runner
- mountPath: /scripts
name: scripts
dnsPolicy: ClusterFirst
initContainers:
- command:
- sh
- /config/configure
env:
- name: CI_SERVER_URL
value: https://git.runshopstore.com/
- name: CLONE_URL
- name: RUNNER_REQUEST_CONCURRENCY
value: "1"
- name: RUNNER_EXECUTOR
value: kubernetes
- name: REGISTER_LOCKED
value: "true"
- name: RUNNER_TAG_LIST
value: docker,company
- name: RUNNER_OUTPUT_LIMIT
value: "4096"
- name: KUBERNETES_IMAGE
value: ubuntu:18.04
- name: KUBERNETES_PRIVILEGED
value: "true"
- name: KUBERNETES_NAMESPACE
value: gitlab-managed-apps
- name: KUBERNETES_POLL_TIMEOUT
value: "180"
- name: KUBERNETES_CPU_LIMIT
- name: KUBERNETES_CPU_LIMIT_OVERWRITE_MAX_ALLOWED
- name: KUBERNETES_MEMORY_LIMIT
- name: KUBERNETES_MEMORY_LIMIT_OVERWRITE_MAX_ALLOWED
- name: KUBERNETES_CPU_REQUEST
value: 2000m
- name: KUBERNETES_CPU_REQUEST_OVERWRITE_MAX_ALLOWED
- name: KUBERNETES_MEMORY_REQUEST
value: 1Gi
- name: KUBERNETES_MEMORY_REQUEST_OVERWRITE_MAX_ALLOWED
- name: KUBERNETES_SERVICE_ACCOUNT
- name: KUBERNETES_SERVICE_CPU_LIMIT
- name: KUBERNETES_SERVICE_MEMORY_LIMIT
- name: KUBERNETES_SERVICE_CPU_REQUEST
value: 2000m
- name: KUBERNETES_SERVICE_MEMORY_REQUEST
value: 1Gi
- name: KUBERNETES_HELPER_CPU_LIMIT
- name: KUBERNETES_HELPER_MEMORY_LIMIT
- name: KUBERNETES_HELPER_CPU_REQUEST
value: 2000m
- name: KUBERNETES_HELPER_MEMORY_REQUEST
value: 1Gi
- name: KUBERNETES_HELPER_IMAGE
- name: KUBERNETES_PULL_POLICY
- name: CACHE_TYPE
value: s3
- name: CACHE_PATH
- name: CACHE_SHARED
value: "true"
- name: CACHE_S3_SERVER_ADDRESS
value: minio.gitlab-managed-apps.svc.cluster.local:9000
- name: CACHE_S3_BUCKET_NAME
value: meshop-gitlab-runner
- name: CACHE_S3_BUCKET_LOCATION
- name: CACHE_S3_INSECURE
value: "true"
image: gitlab/gitlab-runner:alpine-v13.5.0
imagePullPolicy: IfNotPresent
name: configure
resources: {}
securityContext:
allowPrivilegeEscalation: false
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /secrets
name: runner-secrets
- mountPath: /config
name: scripts
readOnly: true
- mountPath: /init-secrets
name: init-runner-secrets
readOnly: true
restartPolicy: Always
schedulerName: default-scheduler
securityContext:
fsGroup: 65533
runAsUser: 100
serviceAccount: gitlab-runner-gitlab-runner
serviceAccountName: gitlab-runner-gitlab-runner
terminationGracePeriodSeconds: 3600
volumes:
- emptyDir:
medium: Memory
name: runner-secrets
- emptyDir:
medium: Memory
name: etc-gitlab-runner
- name: init-runner-secrets
projected:
defaultMode: 420
sources:
- secret:
name: minio
- secret:
items:
- key: runner-registration-token
path: runner-registration-token
- key: runner-token
path: runner-token
name: gitlab-runner-gitlab-runner
- configMap:
defaultMode: 420
name: gitlab-runner-gitlab-runner
name: scripts
status:
availableReplicas: 1
conditions:
- lastTransitionTime: "2020-10-22T13:01:34Z"
lastUpdateTime: "2020-10-22T13:01:34Z"
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: "True"
type: Available
- lastTransitionTime: "2020-09-03T09:50:43Z"
lastUpdateTime: "2020-10-31T02:33:52Z"
message: ReplicaSet "gitlab-runner-gitlab-runner-576df999df" has successfully
progressed.
reason: NewReplicaSetAvailable
status: "True"
type: Progressing
observedGeneration: 3
readyReplicas: 1
replicas: 1
updatedReplicas: 1
yml文件不好看懂，可以看看rancher里面工作负载的信息，比较直观
apiVersion: v1
data:
check-live: |
#!/bin/bash
if /usr/bin/pgrep -f .*register-the-runner; then
exit 0
elif /usr/bin/pgrep gitlab.*runner; then
exit 0
else
exit 1
fi
config.toml: |
concurrent = 10
check_interval = 5
log_level = "info"
listen_address = ':9252'
configure: |
set -e
cp /init-secrets/* /secrets
entrypoint: |
#!/bin/bash
set -e
mkdir -p /home/gitlab-runner/.gitlab-runner/
cp /scripts/config.toml /home/gitlab-runner/.gitlab-runner/
# Register the runner
if [[ -f /secrets/accesskey && -f /secrets/secretkey ]]; then
export CACHE_S3_ACCESS_KEY=$(cat /secrets/accesskey)
export CACHE_S3_SECRET_KEY=$(cat /secrets/secretkey)
fi
if [[ -f /secrets/gcs-applicaton-credentials-file ]]; then
export GOOGLE_APPLICATION_CREDENTIALS="/secrets/gcs-applicaton-credentials-file"
elif [[ -f /secrets/gcs-application-credentials-file ]]; then
export GOOGLE_APPLICATION_CREDENTIALS="/secrets/gcs-application-credentials-file"
else
if [[ -f /secrets/gcs-access-id && -f /secrets/gcs-private-key ]]; then
export CACHE_GCS_ACCESS_ID=$(cat /secrets/gcs-access-id)
# echo -e used to make private key multiline (in google json auth key private key is oneline with \n)
export CACHE_GCS_PRIVATE_KEY=$(echo -e $(cat /secrets/gcs-private-key))
fi
fi
if [[ -f /secrets/runner-registration-token ]]; then
export REGISTRATION_TOKEN=$(cat /secrets/runner-registration-token)
fi
if [[ -f /secrets/runner-token ]]; then
export CI_SERVER_TOKEN=$(cat /secrets/runner-token)
fi
if ! sh /scripts/register-the-runner; then
exit 1
fi
# Run pre-entrypoint-script
if ! bash /scripts/pre-entrypoint-script; then
exit 1
fi
# Start the runner
exec /entrypoint run --user=gitlab-runner \
--working-directory=/home/gitlab-runner
pre-entrypoint-script: ""
register-the-runner: |
#!/bin/bash
MAX_REGISTER_ATTEMPTS=30
for i in $(seq 1 "${MAX_REGISTER_ATTEMPTS}"); do
echo "Registration attempt ${i} of ${MAX_REGISTER_ATTEMPTS}"
/entrypoint register \
--non-interactive
retval=$?
if [ ${retval} = 0 ]; then
break
elif [ ${i} = ${MAX_REGISTER_ATTEMPTS} ]; then
exit 1
fi
sleep 5
done
exit 0
kind: ConfigMap
metadata:
annotations:
meta.helm.sh/release-name: gitlab-runner
meta.helm.sh/release-namespace: gitlab-managed-apps
creationTimestamp: "2020-09-03T09:50:43Z"
labels:
app: gitlab-runner-gitlab-runner
app.kubernetes.io/managed-by: Helm
chart: gitlab-runner-0.22.0
heritage: Helm
release: gitlab-runner
name: gitlab-runner-gitlab-runner
namespace: gitlab-managed-apps
resourceVersion: "126341496"
selfLink: /api/v1/namespaces/gitlab-managed-apps/configmaps/gitlab-runner-gitlab-runner
uid: 087d2130-6d3e-4938-8abc-fa98fab2f299
配置映射里面的信息
密文yml：
apiVersion: v1
data:
runner-registration-token: M015NVhVR1ZyUmtzQVU1dkszLVQ=
runner-token: ""
kind: Secret
metadata:
annotations:
meta.helm.sh/release-name: gitlab-runner
meta.helm.sh/release-namespace: gitlab-managed-apps
creationTimestamp: "2020-09-03T09:50:43Z"
labels:
app: gitlab-runner-gitlab-runner
app.kubernetes.io/managed-by: Helm
chart: gitlab-runner-0.22.0
heritage: Helm
release: gitlab-runner
name: gitlab-runner-gitlab-runner
namespace: gitlab-managed-apps
resourceVersion: "126341494"
selfLink: /api/v1/namespaces/gitlab-managed-apps/secrets/gitlab-runner-gitlab-runner
uid: 11a6b6d8-6c5a-469a-a50a-72b7321234e6
type: Opaque
密文yml：
apiVersion: v1
data:
runner-registration-token: ""
runner-token: R1B2enh2b2ZMSnRzbmhITS1wV2s=
kind: Secret
metadata:
creationTimestamp: "2020-03-31T06:09:51Z"
labels:
app: runner-gitlab-runner
chart: gitlab-runner-0.8.0
heritage: Tiller
release: runner
name: runner-gitlab-runner
namespace: gitlab-managed-apps
resourceVersion: "48351692"
selfLink: /api/v1/namespaces/gitlab-managed-apps/secrets/runner-gitlab-runner
uid: 7a527ac0-d01e-48b8-9d00-eacab32b9f1c
type: Opaque
因为我们的gitlab-ci还用到了cache，所以还需要minio：
minio的helm参考链接：https://github.com/minio/charts
minio的工作负载：
apiVersion: apps/v1
kind: Deployment
metadata:
annotations:
deployment.kubernetes.io/revision: "1"
field.cattle.io/publicEndpoints: '[{"addresses":["192.168.0.116"],"port":443,"protocol":"HTTPS","serviceName":"gitlab-managed-apps:minio","ingressName":"gitlab-managed-apps:minio","hostname":"minio.tidebuy.net","path":"/","allNodes":true}]'
creationTimestamp: "2020-09-03T10:16:45Z"
generation: 7
labels:
app: minio
chart: minio-5.0.11
heritage: Tiller
io.cattle.field/appId: minio
release: minio
name: minio
namespace: gitlab-managed-apps
resourceVersion: "131790225"
selfLink: /apis/apps/v1/namespaces/gitlab-managed-apps/deployments/minio
uid: efb93058-000e-4d7c-921e-8145ff3910db
spec:
progressDeadlineSeconds: 600
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
app: minio
release: minio
strategy:
rollingUpdate:
maxSurge: 100%
maxUnavailable: 0
type: RollingUpdate
template:
metadata:
annotations:
checksum/config: 998ce9f304563b05446f4b36a8947f27f5b830ec70fe11de96eadc64e9dd080b
checksum/secrets: 7aa1d920bfd796647076374d34d46c4676f42fd0cda6ed74af8075e0046ec15e
creationTimestamp: null
labels:
app: minio
release: minio
name: minio
spec:
containers:
- command:
- /bin/sh
- -ce
- /usr/bin/docker-entrypoint.sh minio -S /etc/minio/certs/ server /export
env:
- name: MINIO_ACCESS_KEY
valueFrom:
secretKeyRef:
key: accesskey
name: minio
- name: MINIO_SECRET_KEY
valueFrom:
secretKeyRef:
key: secretkey
name: minio
- name: MINIO_BROWSER
value: "on"
image: minio/minio:RELEASE.2020-02-07T23-28-16Z
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /minio/health/live
port: http
scheme: HTTP
initialDelaySeconds: 5
periodSeconds: 30
successThreshold: 1
timeoutSeconds: 1
name: minio
ports:
- containerPort: 9000
name: http
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
path: /minio/health/ready
port: http
scheme: HTTP
initialDelaySeconds: 60
periodSeconds: 15
successThreshold: 1
timeoutSeconds: 1
resources:
requests:
cpu: "1"
memory: 4Gi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /export
name: export
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: minio
serviceAccountName: minio
terminationGracePeriodSeconds: 30
volumes:
- name: export
persistentVolumeClaim:
claimName: minio
- name: minio-user
secret:
defaultMode: 420
secretName: minio
status:
availableReplicas: 1
conditions:
- lastTransitionTime: "2020-09-03T10:16:45Z"
lastUpdateTime: "2020-09-03T10:18:16Z"
message: ReplicaSet "minio-757d968495" has successfully progressed.
reason: NewReplicaSetAvailable
status: "True"
type: Progressing
- lastTransitionTime: "2020-10-24T04:15:36Z"
lastUpdateTime: "2020-10-24T04:15:36Z"
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: "True"
type: Available
observedGeneration: 7
readyReplicas: 1
replicas: 1
updatedReplicas: 1
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
annotations:
field.cattle.io/ingressState: '{"bWluaW8vZ2l0bGFiLW1hbmFnZWQtYXBwcy9taW5pby50aWRlYnV5Lm5ldC8vLzkwMDA=":"","dGlkZWJ1eS1uZXQ=":"p-npw62:tidebuy-net"}'
field.cattle.io/publicEndpoints: '[{"addresses":["192.168.0.116"],"port":443,"protocol":"HTTPS","serviceName":"gitlab-managed-apps:minio","ingressName":"gitlab-managed-apps:minio","hostname":"minio.tidebuy.net","path":"/","allNodes":true}]'
ingress.kubernetes.io/proxy-body-size: 500m
nginx.ingress.kubernetes.io/proxy-body-size: 500m
nginx.org/client-max-body-size: 500m
creationTimestamp: "2020-04-22T03:08:33Z"
generation: 2
labels:
app: minio
chart: minio-5.0.11
heritage: Tiller
io.cattle.field/appId: minio
release: minio
name: minio
namespace: gitlab-managed-apps
resourceVersion: "131790217"
selfLink: /apis/extensions/v1beta1/namespaces/gitlab-managed-apps/ingresses/minio
uid: 9c6c3413-e5d0-469c-a85f-953f02d7a3f0
spec:
rules:
- host: minio.tidebuy.net
http:
paths:
- backend:
serviceName: minio
servicePort: 9000
path: /
pathType: ImplementationSpecific
tls:
- hosts:
- minio.tidebuy.net
secretName: tidebuy-net
status:
loadBalancer:
ingress:
- ip: 192.168.0.116
- ip: 192.168.0.117
- ip: 192.168.0.118
- ip: 192.168.0.119
- ip: 192.168.0.121
- ip: 192.168.0.122
apiVersion: v1
kind: Service
metadata:
creationTimestamp: "2020-04-21T12:51:30Z"
labels:
app: minio
chart: minio-5.0.11
heritage: Tiller
io.cattle.field/appId: minio
release: minio
name: minio
namespace: gitlab-managed-apps
resourceVersion: "54370547"
selfLink: /api/v1/namespaces/gitlab-managed-apps/services/minio
uid: b717c5b7-3d98-4276-8640-8b11b8c62247
spec:
clusterIP: 10.43.107.52
ports:
- name: http
port: 9000
protocol: TCP
targetPort: 9000
selector:
app: minio
release: minio
sessionAffinity: None
type: ClusterIP
status:
loadBalancer: {}
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
annotations:
pv.kubernetes.io/bind-completed: "yes"
pv.kubernetes.io/bound-by-controller: "yes"
volume.beta.kubernetes.io/storage-provisioner: cephfs.csi.ceph.com
creationTimestamp: "2020-09-03T10:16:45Z"
finalizers:
- kubernetes.io/pvc-protection
labels:
app: minio
chart: minio-5.0.11
heritage: Tiller
io.cattle.field/appId: minio
release: minio
name: minio
namespace: gitlab-managed-apps
resourceVersion: "99350396"
selfLink: /api/v1/namespaces/gitlab-managed-apps/persistentvolumeclaims/minio
uid: 63419d1b-f70d-4c17-87fd-4f6f93bbe505
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 500Gi
storageClassName: csi-cephfs-sc
volumeMode: Filesystem
volumeName: pvc-63419d1b-f70d-4c17-87fd-4f6f93bbe505
status:
accessModes:
- ReadWriteOnce
capacity:
storage: 500Gi
phase: Bound
apiVersion: v1
data:
accesskey: QUtJQUlPU0ZPRE5ON0VYQU1QTEU=
secretkey: d0phbHJYVXRuRkVNSS9LN01ERU5HL2JQeFJmaUNZRVhBTVBMRUtFWQ==
kind: Secret
metadata:
creationTimestamp: "2020-04-21T12:51:30Z"
labels:
app: minio
chart: minio-5.0.11
heritage: Tiller
io.cattle.field/appId: minio
release: minio
name: minio
namespace: gitlab-managed-apps
resourceVersion: "54370538"
selfLink: /api/v1/namespaces/gitlab-managed-apps/secrets/minio
uid: 18bbf298-0ec8-4b1b-89dc-c48513219dd2
type: Opaque
有了它之后，还需要搭建本地私有仓库，制作生成文件版本号的Docker镜像，写gitlab-ci的yml脚本文件，最后才可以使用gitlab-ci实现项目提交之后的自动化。
东西太多，未完待续。
