gitlab-runner helm 安装问题记录

 

 

首先说说安装步骤

1、安装版本
gitlab-runner-0.36.1.tgz

 

 

cat ./gitlab-runner/values.yaml |grep -v "#"|sed -e '/^$/d'
imagePullPolicy: IfNotPresent
gitlabUrl: http://101.43.196.155:32080/
runnerRegistrationToken: "Woq_Drxy-SSy1kQzJBZT"
terminationGracePeriodSeconds: 3600
concurrent: 10
checkInterval: 10
rbac:
  create: true
  resources: ["pods", "pods/exec", "secrets", "configmaps", "pods/attach"]
  verbs: ["get", "list", "watch", "create", "patch", "delete", "update"]
  rules: []
  clusterWideAccess: false
  podSecurityPolicy:
    enabled: false
    resourceNames:
    - gitlab-runner
metrics:
  enabled: true
  portName: metrics
  port: 9252
  serviceMonitor:
    enabled: false
service:
  enabled: false
  type: ClusterIP
runners:
  config: |
    [[runners]]
      [runners.kubernetes]
        namespace = "{{.Release.Namespace}}"
        image = "ubuntu:16.04"
  tags: "k8s-runner,k8s"
  privileged: true
  cache: {}
  builds: {}
  services: {}
  helpers: {}
securityContext:
  runAsUser: 100
  fsGroup: 65533
resources: {}
affinity: {}
nodeSelector: {}
tolerations: []
hostAliases: []
podAnnotations: {}
podLabels: {}
secrets: []
configMaps: {}
config: |
  [[runners]]
    url = "https://gitlab.com/"
    executor = "docker"
    privileged = true
    [runners.docker]
      tls_verify = false
      image = "docker:24.0.7"
      privileged = true
      disable_cache = false
      volumes = ["/var/run/docker.sock:/var/run/docker.sock","/cache"]
    [runners.cache]
      Insecure = false

helm install  gitlab-runner --namespace kube-ops ./gitlab-runner

 

这个问题经过排查 发现 gitlab-runner 无法调用docker.sock

需要修改gitlab.toml 配置文件

有两个方法可以修改

1、gitlab-runner 创建后 登录 gitlab-runner 修改 config.toml 配置文件 

[root@master gitlab-runner]# kubectl get po -n kube-ops
NAME                                           READY   STATUS    RESTARTS   AGE
gitlab-runner-gitlab-runner-5fb9cfff9c-kfbkb   1/1     Running   0          19m
[root@master gitlab-runner]# kubectl exec -it -n kube-ops gitlab-runner-gitlab-runner-5fb9cfff9c-kfbkb -- /bin/bash
Defaulted container "gitlab-runner-gitlab-runner" out of: gitlab-runner-gitlab-runner, configure (init)
bash-5.0$ cat /home/gitlab-runner/.gitlab-runner/config.toml 
listen_address = ":9252"
concurrent = 10
check_interval = 10
log_level = "info"

[session_server]
  session_timeout = 1800

[[runners]]
  name = "gitlab-runner-gitlab-runner-5fb9cfff9c-kfbkb"
  url = "http://101.43.196.155:32080/"
  token = "FbCFy9T5anUR23tMbJRQ"
  executor = "kubernetes"
  [runners.custom_build_dir]
  [runners.cache]
    [runners.cache.s3]
    [runners.cache.gcs]
    [runners.cache.azure]
  [runners.kubernetes]
    host = ""
    bearer_token_overwrite_allowed = false
    image = "ubuntu:16.04"
    namespace = "kube-ops"
    namespace_overwrite_allowed = ""
    privileged = true
    service_account_overwrite_allowed = ""
    pod_annotations_overwrite_allowed = ""
    [runners.kubernetes.affinity]
    [runners.kubernetes.pod_security_context]
    [runners.kubernetes.build_container_security_context]
      [runners.kubernetes.build_container_security_context.capabilities]
    [runners.kubernetes.helper_container_security_context]
      [runners.kubernetes.helper_container_security_context.capabilities]
    [runners.kubernetes.service_container_security_context]
      [runners.kubernetes.service_container_security_context.capabilities]
    [runners.kubernetes.volumes]
    [runners.kubernetes.dns_config]
    [runners.kubernetes.container_lifecycle]
  [[runners.kubernetes.volumes.host_path]]
        name = "docker"
        mount_path = "/var/run/docker.sock"

 

第二个方法

解压 gitlab-runner-0.36.1.tgz 加压安装包 在当前目录会生成 gitlab-runner

在 gitlab-runner 目录的 gitlab-runner/templates/configmap.yaml 文件内调整如下

然后重新安装

 helm install  gitlab-runner --namespace kube-ops ./gitlab-runner

 

[root@master gitlab-runner]# cat ./gitlab-runner/templates/configmap.yaml 
apiVersion: v1
kind: ConfigMap
metadata:
  name: {{ include "gitlab-runner.fullname" . }}
  labels:
    app: {{ include "gitlab-runner.fullname" . }}
    chart: {{ include "gitlab-runner.chart" . }}
    release: "{{ .Release.Name }}"
    heritage: "{{ .Release.Service }}"
data:
  entrypoint: |
    #!/bin/bash
    set -e
    mkdir -p /home/gitlab-runner/.gitlab-runner/
    cp /configmaps/config.toml /home/gitlab-runner/.gitlab-runner/

    # Set up environment variables for cache
    if [[ -f /secrets/accesskey && -f /secrets/secretkey ]]; then
      export CACHE_S3_ACCESS_KEY=$(cat /secrets/accesskey)
      export CACHE_S3_SECRET_KEY=$(cat /secrets/secretkey)
    fi

    if [[ -f /secrets/gcs-applicaton-credentials-file ]]; then
      export GOOGLE_APPLICATION_CREDENTIALS="/secrets/gcs-applicaton-credentials-file"
    elif [[ -f /secrets/gcs-application-credentials-file ]]; then
      export GOOGLE_APPLICATION_CREDENTIALS="/secrets/gcs-application-credentials-file"
    else
      if [[ -f /secrets/gcs-access-id && -f /secrets/gcs-private-key ]]; then
        export CACHE_GCS_ACCESS_ID=$(cat /secrets/gcs-access-id)
        # echo -e used to make private key multiline (in google json auth key private key is oneline with \n)
        export CACHE_GCS_PRIVATE_KEY=$(echo -e $(cat /secrets/gcs-private-key))
      fi
    fi

    if [[ -f /secrets/azure-account-name && -f /secrets/azure-account-key ]]; then
      export CACHE_AZURE_ACCOUNT_NAME=$(cat /secrets/azure-account-name)
      export CACHE_AZURE_ACCOUNT_KEY=$(cat /secrets/azure-account-key)
    fi

    if [[ -f /secrets/runner-registration-token ]]; then
      export REGISTRATION_TOKEN=$(cat /secrets/runner-registration-token)
    fi

    if [[ -f /secrets/runner-token ]]; then
      export CI_SERVER_TOKEN=$(cat /secrets/runner-token)
    fi

    {{- if and (not (empty .Values.runnerToken)) (ne "1" ((default "1" .Values.replicas) | toString)) }}
    {{- fail "Using a runner token with more than 1 replica is not supported." }}
    {{- end }}

    # Validate this also at runtime in case the user has set a custom secret
    if [[ ! -z "$CI_SERVER_TOKEN" && "{{ default 1 .Values.replicas }}" -ne "1" ]]; then
      echo "Using a runner token with more than 1 replica is not supported."
      exit 1
    fi

    # Register the runner
    if ! sh /configmaps/register-the-runner; then
      exit 1
    fi

    # Run pre-entrypoint-script
    if ! bash /configmaps/pre-entrypoint-script; then
      exit 1
    fi
    # add volume config
    cat >>/home/gitlab-runner/.gitlab-runner/config.toml <<EOF
      [[runners.kubernetes.volumes.host_path]]
            name = "docker"
            mount_path = "/var/run/docker.sock"
    EOF

    # Start the runner
    exec /entrypoint run --user=gitlab-runner \
      --working-directory=/home/gitlab-runner

  config.toml: |
    concurrent = {{ .Values.concurrent }}
    check_interval = {{ .Values.checkInterval }}
    log_level = {{ default "info" .Values.logLevel | quote }}
    {{- if .Values.logFormat }}
    log_format = {{ .Values.logFormat | quote }}
    {{- end }}
    {{- if .Values.metrics.enabled }}
    listen_address = ':9252'
    {{- end }}
    {{- if .Values.sentryDsn }}
    sentry_dsn = "{{ .Values.sentryDsn }}"
    {{- end }}

  {{ if .Values.runners.config }}
  config.template.toml: {{ tpl (toYaml .Values.runners.config) $ | indent 2 }}
  {{ end }}

  configure: |
    set -e
    cp /init-secrets/* /secrets
  register-the-runner: |
    #!/bin/bash
    MAX_REGISTER_ATTEMPTS=30

    for i in $(seq 1 "${MAX_REGISTER_ATTEMPTS}"); do
      echo "Registration attempt ${i} of ${MAX_REGISTER_ATTEMPTS}"
      /entrypoint register \
        {{- range .Values.runners.imagePullSecrets }}
        --kubernetes-image-pull-secrets {{ . | quote }} \
        {{- end }}
        {{- range $key, $val := .Values.runners.nodeSelector }}
        --kubernetes-node-selector {{ $key | quote }}:{{ $val | quote }} \
        {{- end }}
        {{- range .Values.runners.nodeTolerations }}
        {{-   $keyValue := .key }}
        {{-   if eq (.operator | default "Equal") "Equal"  }}
        {{-     $keyValue = print $keyValue "=" (.value | default "" ) }}
        {{-   end }}
        --kubernetes-node-tolerations {{ $keyValue }}:{{ .effect | quote }} \
        {{- end }}
        {{- range $key, $value := .Values.runners.podLabels }}
        --kubernetes-pod-labels {{ $key | quote }}:{{ $value | quote }} \
        {{- end }}
        {{- range $key, $val := .Values.runners.podAnnotations }}
        --kubernetes-pod-annotations {{ $key | quote }}:{{ $val | quote }} \
        {{- end }}
        {{- if and (hasKey .Values.runners "name") .Values.runners.name }}
        --name={{ .Values.runners.name | quote -}} \
        {{- end }}
        {{- range $key, $value := .Values.runners.env }}
        --env {{ $key | quote -}} = {{- $value | quote }} \
        {{- end }}
        {{- if and (hasKey .Values.runners "runUntagged") .Values.runners.runUntagged }}
        --run-untagged=true \
        {{- end }}
        {{- if and (hasKey .Values.runners "protected") .Values.runners.protected }}
        --access-level="ref_protected" \
        {{- end }}
        {{- if .Values.runners.pod_security_context }}
        {{-   if .Values.runners.pod_security_context.supplemental_groups }}
        {{-     range $gid := .Values.runners.pod_security_context.supplemental_groups }}
        --kubernetes-pod-security-context-supplemental-groups {{ $gid | quote }} \
        {{-     end }}
        {{-   end }}
        {{- end }}
        {{- if .Values.runners.config }}
        --template-config /configmaps/config.template.toml \
        {{- end }}
        --non-interactive

      retval=$?

      if [ ${retval} = 0 ]; then
        break
      elif [ ${i} = ${MAX_REGISTER_ATTEMPTS} ]; then
        exit 1
      fi

      sleep 5
    done

    exit 0

  check-live: |
    #!/bin/bash
    if /usr/bin/pgrep -f .*register-the-runner; then
      exit 0
    elif /usr/bin/pgrep gitlab.*runner; then
      exit 0
    else
      exit 1
    fi

  pre-entrypoint-script: |
{{ .Values.preEntrypointScript | default "" | indent 4 }}

{{ if not (empty .Values.configMaps) }}{{ toYaml .Values.configMaps | indent 2 }}{{ end }}