kubesphere 的 流水线maven 模板缺少 kubectl解决
最开始解决方案是maven 的pod 里通过 在线下载kubectl 命令
发现每次构建后端服务,都去官网下载kubectl 命令相当慢。
既然用到maven 模板,遂将master 节点的kubectl 命令通过hostpath挂载到maven的 pod模板里面。问题解决。
agent模板cm配置【jenkins-casc-config】在【kubesphere-devops-system】的namespace下,在节点data.jenkins_user.yml下添加自定义的agent模板配置,kubesphere已内置了一些模板(base、nodejs、maven、go、python)等,根据需要选择是否使用。每个模板需要有“jnlp”jenkins的agent来实现与jenkins通信,,jenkins-cacs-config 该配置文件详细如下,在maven模板里面增加kubectl 的 hostpath挂载。
kind: ConfigMap apiVersion: v1 metadata: name: jenkins-casc-config namespace: kubesphere-devops-system labels: app.kubernetes.io/managed-by: Helm annotations: devops.kubesphere.io/jenkins-config-customized: 'true' devops.kubesphere.io/jenkins-config-formula: custom meta.helm.sh/release-name: devops meta.helm.sh/release-namespace: kubesphere-devops-system data: jenkins.yaml: | jenkins: mode: EXCLUSIVE numExecutors: 0 scmCheckoutRetryCount: 2 disableRememberMe: true clouds: - kubernetes: name: "kubernetes" serverUrl: "https://kubernetes.default" skipTlsVerify: true namespace: "kubesphere-devops-worker" credentialsId: "k8s-service-account" jenkinsUrl: "http://devops-jenkins.kubesphere-devops-system:80" jenkinsTunnel: "devops-jenkins-agent.kubesphere-devops-system:50000" containerCapStr: "10" connectTimeout: "60" readTimeout: "60" maxRequestsPerHostStr: "32" templates: - name: "base" namespace: "kubesphere-devops-worker" label: "base" nodeUsageMode: "NORMAL" idleMinutes: 0 containers: - name: "base" image: "kubesphere/builder-base:v3.2.2" command: "cat" args: "" ttyEnabled: true privileged: false resourceRequestCpu: "100m" resourceLimitCpu: "4000m" resourceRequestMemory: "100Mi" resourceLimitMemory: "8192Mi" - name: "jnlp" image: "jenkins/inbound-agent:4.10-2" args: "^${computer.jnlpmac} ^${computer.name}" resourceRequestCpu: "50m" resourceLimitCpu: "500m" resourceRequestMemory: "400Mi" resourceLimitMemory: "1536Mi" workspaceVolume: emptyDirWorkspaceVolume: memory: false volumes: - hostPathVolume: hostPath: "/var/run/docker.sock" mountPath: "/var/run/docker.sock" - hostPathVolume: hostPath: "/var/data/jenkins_sonar_cache" mountPath: "/root/.sonar/cache" yaml: | spec: affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 1 preference: matchExpressions: - key: node-role.kubernetes.io/worker operator: In values: - ci tolerations: - key: "node.kubernetes.io/ci" operator: "Exists" effect: "NoSchedule" - key: "node.kubernetes.io/ci" operator: "Exists" effect: "PreferNoSchedule" containers: - name: "base" resources: requests: ephemeral-storage: "1Gi" limits: ephemeral-storage: "10Gi" securityContext: fsGroup: 1000 - name: "nodejs" namespace: "kubesphere-devops-worker" label: "nodejs" nodeUsageMode: "EXCLUSIVE" idleMinutes: 0 containers: - name: "nodejs" image: "ccr.ccs.tencentyun.com/huanghuanhui/node:16.17.0-alpine" command: "cat" args: "" ttyEnabled: true privileged: false resourceRequestCpu: "100m" resourceLimitCpu: "4000m" resourceRequestMemory: "100Mi" resourceLimitMemory: "8192Mi" - name: "jnlp" image: "jenkins/inbound-agent:4.10-2" args: "^${computer.jnlpmac} ^${computer.name}" resourceRequestCpu: "50m" resourceLimitCpu: "500m" resourceRequestMemory: "400Mi" resourceLimitMemory: "1536Mi" workspaceVolume: emptyDirWorkspaceVolume: memory: false volumes: - hostPathVolume: hostPath: "/var/run/docker.sock" mountPath: "/var/run/docker.sock" - hostPathVolume: hostPath: "/var/data/jenkins_nodejs_yarn_cache" mountPath: "/root/.yarn" - hostPathVolume: hostPath: "/var/data/jenkins_nodejs_npm_cache" mountPath: "/root/.npm" - hostPathVolume: hostPath: "/var/data/jenkins_sonar_cache" mountPath: "/root/.sonar/cache" yaml: | spec: affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 1 preference: matchExpressions: - key: node-role.kubernetes.io/worker operator: In values: - ci tolerations: - key: "node.kubernetes.io/ci" operator: "Exists" effect: "NoSchedule" - key: "node.kubernetes.io/ci" operator: "Exists" effect: "PreferNoSchedule" containers: - name: "nodejs" resources: requests: ephemeral-storage: "1Gi" limits: ephemeral-storage: "10Gi" securityContext: fsGroup: 1000 - name: "maven" namespace: "kubesphere-devops-worker" label: "maven" nodeUsageMode: "EXCLUSIVE" idleMinutes: 0 containers: - name: "maven" image: "kubesphere/builder-maven:v3.2.0" command: "cat" args: "" ttyEnabled: true privileged: false resourceRequestCpu: "100m" resourceLimitCpu: "4000m" resourceRequestMemory: "100Mi" resourceLimitMemory: "8192Mi" - name: "jnlp" image: "jenkins/inbound-agent:4.10-2" args: "^${computer.jnlpmac} ^${computer.name}" resourceRequestCpu: "50m" resourceLimitCpu: "500m" resourceRequestMemory: "400Mi" resourceLimitMemory: "1536Mi" workspaceVolume: emptyDirWorkspaceVolume: memory: false volumes: - hostPathVolume: hostPath: "/var/run/docker.sock" mountPath: "/var/run/docker.sock" - hostPathVolume: hostPath: "/usr/local/bin/kubectl" mountPath: "/usr/local/bin/kubectl" - hostPathVolume: hostPath: "/var/data/jenkins_maven_cache" mountPath: "/root/.m2" - hostPathVolume: hostPath: "/var/data/jenkins_sonar_cache" mountPath: "/root/.sonar/cache" yaml: | spec: affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 1 preference: matchExpressions: - key: node-role.kubernetes.io/worker operator: In values: - ci tolerations: - key: "node.kubernetes.io/ci" operator: "Exists" effect: "NoSchedule" - key: "node.kubernetes.io/ci" operator: "Exists" effect: "PreferNoSchedule" containers: - name: "maven" resources: requests: ephemeral-storage: "1Gi" limits: ephemeral-storage: "10Gi" volumeMounts: - name: config-volume mountPath: /opt/apache-maven-3.5.3/conf/settings.xml subPath: settings.xml volumes: - name: config-volume configMap: name: ks-devops-agent items: - key: MavenSetting path: settings.xml securityContext: fsGroup: 1000 - name: "mavenjdk11" label: "jdk11 maven java" inheritFrom: "maven" containers: - name: "maven" image: "kubesphere/builder-maven:v3.2.1-jdk11" - name: "go" namespace: "kubesphere-devops-worker" label: "go" nodeUsageMode: "EXCLUSIVE" idleMinutes: 0 containers: - name: "go" image: "kubesphere/builder-go:v3.2.0" command: "cat" args: "" ttyEnabled: true privileged: false resourceRequestCpu: "100m" resourceLimitCpu: "4000m" resourceRequestMemory: "100Mi" resourceLimitMemory: "8192Mi" - name: "jnlp" image: "jenkins/inbound-agent:4.10-2" args: "^${computer.jnlpmac} ^${computer.name}" resourceRequestCpu: "50m" resourceLimitCpu: "500m" resourceRequestMemory: "400Mi" resourceLimitMemory: "1536Mi" workspaceVolume: emptyDirWorkspaceVolume: memory: false volumes: - hostPathVolume: hostPath: "/var/run/docker.sock" mountPath: "/var/run/docker.sock" - hostPathVolume: hostPath: "/var/data/jenkins_go_cache" mountPath: "/home/jenkins/go/pkg" - hostPathVolume: hostPath: "/var/data/jenkins_sonar_cache" mountPath: "/root/.sonar/cache" yaml: | spec: affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 1 preference: matchExpressions: - key: node-role.kubernetes.io/worker operator: In values: - ci tolerations: - key: "node.kubernetes.io/ci" operator: "Exists" effect: "NoSchedule" - key: "node.kubernetes.io/ci" operator: "Exists" effect: "PreferNoSchedule" containers: - name: "go" resources: requests: ephemeral-storage: "1Gi" limits: ephemeral-storage: "10Gi" securityContext: fsGroup: 1000 - name: "go16" label: "go16" inheritFrom: "go" containers: - name: "go" image: "kubesphere/builder-go:v3.2.2-1.16" - name: "go17" label: "go17" inheritFrom: "go" containers: - name: "go" image: "kubesphere/builder-go:v3.2.2-1.17" - name: "go18" label: "go18" inheritFrom: "go" containers: - name: "go" image: "kubesphere/builder-go:v3.2.2-1.18" - name: "python" namespace: "kubesphere-devops-worker" label: "python" nodeUsageMode: "EXCLUSIVE" idleMinutes: 0 containers: - name: "python" image: "kubesphere/builder-python:v3.2.0" command: "cat" args: "" ttyEnabled: true privileged: false resourceRequestCpu: "100m" resourceLimitCpu: "4000m" resourceRequestMemory: "100Mi" resourceLimitMemory: "8192Mi" - name: "jnlp" image: "jenkins/inbound-agent:4.10-2" args: "^${computer.jnlpmac} ^${computer.name}" resourceRequestCpu: "50m" resourceLimitCpu: "500m" resourceRequestMemory: "400Mi" resourceLimitMemory: "1536Mi" workspaceVolume: emptyDirWorkspaceVolume: memory: false volumes: - hostPathVolume: hostPath: "/var/run/docker.sock" mountPath: "/var/run/docker.sock" - hostPathVolume: hostPath: "/var/data/jenkins_python_pip_cache" mountPath: "/root/.cache/pip" - hostPathVolume: hostPath: "/var/data/jenkins_python_pipenv_cache" mountPath: "/root/.local/share/virtualenvs" - hostPathVolume: hostPath: "/var/data/jenkins_sonar_cache" mountPath: "/root/.sonar/cache" yaml: | spec: affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 1 preference: matchExpressions: - key: node-role.kubernetes.io/worker operator: In values: - ci tolerations: - key: "node.kubernetes.io/ci" operator: "Exists" effect: "NoSchedule" - key: "node.kubernetes.io/ci" operator: "Exists" effect: "PreferNoSchedule" containers: - name: "python" resources: requests: ephemeral-storage: "1Gi" limits: ephemeral-storage: "10Gi" securityContext: fsGroup: 1000 securityRealm: ldap: configurations: - displayNameAttributeName: "uid" mailAddressAttributeName: "mail" inhibitInferRootDN: false managerDN: "cn=admin,dc=kubesphere,dc=io" managerPasswordSecret: "admin" rootDN: "dc=kubesphere,dc=io" userSearchBase: "ou=Users" userSearch: "(&(objectClass=inetOrgPerson)(|(uid={0})(mail={0})))" groupSearchBase: "ou=Groups" groupSearchFilter: "(&(objectClass=posixGroup)(cn={0}))" server: "ldap://openldap.kubesphere-system.svc:389" disableMailAddressResolver: false disableRolePrefixing: true unclassified: location: url: "http://jenkins.devops.kubesphere.local" kubespheretokenauthglobalconfiguration: cacheConfiguration: size: 20 ttl: 300 enabled: true server: "http://devops-apiserver.kubesphere-devops-system:9090/" eventDispatcher: receiver: "http://devops-apiserver.kubesphere-devops-system:9090/v1alpha3/webhooks/jenkins" gitLabServers: servers: - name: "https://gitlab.com" serverUrl: "https://gitlab.com" jenkins_user.yaml: | jenkins: mode: EXCLUSIVE numExecutors: 0 scmCheckoutRetryCount: 2 disableRememberMe: true clouds: - kubernetes: name: "kubernetes" serverUrl: "https://kubernetes.default" skipTlsVerify: true namespace: "kubesphere-devops-worker" credentialsId: "k8s-service-account" jenkinsUrl: "http://devops-jenkins.kubesphere-devops-system:80" jenkinsTunnel: "devops-jenkins-agent.kubesphere-devops-system:50000" containerCapStr: "10" connectTimeout: "60" readTimeout: "60" maxRequestsPerHostStr: "32" templates: - name: "base" namespace: "kubesphere-devops-worker" label: "base" nodeUsageMode: "NORMAL" idleMinutes: 0 containers: - name: "base" image: "kubesphere/builder-base:v3.2.2" command: "cat" args: "" ttyEnabled: true privileged: false resourceRequestCpu: "100m" resourceLimitCpu: "4000m" resourceRequestMemory: "100Mi" resourceLimitMemory: "8192Mi" - name: "jnlp" image: "jenkins/inbound-agent:4.10-2" args: "^${computer.jnlpmac} ^${computer.name}" resourceRequestCpu: "50m" resourceLimitCpu: "500m" resourceRequestMemory: "400Mi" resourceLimitMemory: "1536Mi" workspaceVolume: emptyDirWorkspaceVolume: memory: false volumes: - hostPathVolume: hostPath: "/var/run/docker.sock" mountPath: "/var/run/docker.sock" - hostPathVolume: hostPath: "/var/data/jenkins_sonar_cache" mountPath: "/root/.sonar/cache" yaml: | spec: affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 1 preference: matchExpressions: - key: node-role.kubernetes.io/worker operator: In values: - ci tolerations: - key: "node.kubernetes.io/ci" operator: "Exists" effect: "NoSchedule" - key: "node.kubernetes.io/ci" operator: "Exists" effect: "PreferNoSchedule" containers: - name: "base" resources: requests: ephemeral-storage: "1Gi" limits: ephemeral-storage: "10Gi" securityContext: fsGroup: 1000 - name: "nodejs" namespace: "kubesphere-devops-worker" label: "nodejs" nodeUsageMode: "EXCLUSIVE" idleMinutes: 0 containers: - name: "nodejs" image: "ccr.ccs.tencentyun.com/huanghuanhui/node:16.17.0-alpine" command: "cat" args: "" ttyEnabled: true privileged: false resourceRequestCpu: "100m" resourceLimitCpu: "4000m" resourceRequestMemory: "100Mi" resourceLimitMemory: "8192Mi" - name: "jnlp" image: "jenkins/inbound-agent:4.10-2" args: "^${computer.jnlpmac} ^${computer.name}" resourceRequestCpu: "50m" resourceLimitCpu: "500m" resourceRequestMemory: "400Mi" resourceLimitMemory: "1536Mi" workspaceVolume: emptyDirWorkspaceVolume: memory: false volumes: - hostPathVolume: hostPath: "/var/run/docker.sock" mountPath: "/var/run/docker.sock" - hostPathVolume: hostPath: "/var/data/jenkins_nodejs_yarn_cache" mountPath: "/root/.yarn" - hostPathVolume: hostPath: "/var/data/jenkins_nodejs_npm_cache" mountPath: "/root/.npm" - hostPathVolume: hostPath: "/var/data/jenkins_sonar_cache" mountPath: "/root/.sonar/cache" yaml: | spec: affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 1 preference: matchExpressions: - key: node-role.kubernetes.io/worker operator: In values: - ci tolerations: - key: "node.kubernetes.io/ci" operator: "Exists" effect: "NoSchedule" - key: "node.kubernetes.io/ci" operator: "Exists" effect: "PreferNoSchedule" containers: - name: "nodejs" resources: requests: ephemeral-storage: "1Gi" limits: ephemeral-storage: "10Gi" securityContext: fsGroup: 1000 - name: "maven" namespace: "kubesphere-devops-worker" label: "maven" nodeUsageMode: "EXCLUSIVE" idleMinutes: 0 containers: - name: "maven" image: "kubesphere/builder-maven:v3.2.0" command: "cat" args: "" ttyEnabled: true privileged: false resourceRequestCpu: "100m" resourceLimitCpu: "4000m" resourceRequestMemory: "100Mi" resourceLimitMemory: "8192Mi" - name: "jnlp" image: "jenkins/inbound-agent:4.10-2" args: "^${computer.jnlpmac} ^${computer.name}" resourceRequestCpu: "50m" resourceLimitCpu: "500m" resourceRequestMemory: "400Mi" resourceLimitMemory: "1536Mi" workspaceVolume: emptyDirWorkspaceVolume: memory: false volumes: - hostPathVolume: hostPath: "/var/run/docker.sock" mountPath: "/var/run/docker.sock" - hostPathVolume: hostPath: "/usr/local/bin/kubectl" mountPath: "/usr/local/bin/kubectl" - hostPathVolume: hostPath: "/var/data/jenkins_maven_cache" mountPath: "/root/.m2" - hostPathVolume: hostPath: "/var/data/jenkins_sonar_cache" mountPath: "/root/.sonar/cache" yaml: | spec: affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 1 preference: matchExpressions: - key: node-role.kubernetes.io/worker operator: In values: - ci tolerations: - key: "node.kubernetes.io/ci" operator: "Exists" effect: "NoSchedule" - key: "node.kubernetes.io/ci" operator: "Exists" effect: "PreferNoSchedule" containers: - name: "maven" resources: requests: ephemeral-storage: "1Gi" limits: ephemeral-storage: "10Gi" volumeMounts: - name: config-volume mountPath: /opt/apache-maven-3.5.3/conf/settings.xml subPath: settings.xml volumes: - name: config-volume configMap: name: ks-devops-agent items: - key: MavenSetting path: settings.xml securityContext: fsGroup: 1000 - name: "mavenjdk11" label: "jdk11 maven java" inheritFrom: "maven" containers: - name: "maven" image: "kubesphere/builder-maven:v3.2.1-jdk11" - name: "go" namespace: "kubesphere-devops-worker" label: "go" nodeUsageMode: "EXCLUSIVE" idleMinutes: 0 containers: - name: "go" image: "kubesphere/builder-go:v3.2.0" command: "cat" args: "" ttyEnabled: true privileged: false resourceRequestCpu: "100m" resourceLimitCpu: "4000m" resourceRequestMemory: "100Mi" resourceLimitMemory: "8192Mi" - name: "jnlp" image: "jenkins/inbound-agent:4.10-2" args: "^${computer.jnlpmac} ^${computer.name}" resourceRequestCpu: "50m" resourceLimitCpu: "500m" resourceRequestMemory: "400Mi" resourceLimitMemory: "1536Mi" workspaceVolume: emptyDirWorkspaceVolume: memory: false volumes: - hostPathVolume: hostPath: "/var/run/docker.sock" mountPath: "/var/run/docker.sock" - hostPathVolume: hostPath: "/var/data/jenkins_go_cache" mountPath: "/home/jenkins/go/pkg" - hostPathVolume: hostPath: "/var/data/jenkins_sonar_cache" mountPath: "/root/.sonar/cache" yaml: | spec: affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 1 preference: matchExpressions: - key: node-role.kubernetes.io/worker operator: In values: - ci tolerations: - key: "node.kubernetes.io/ci" operator: "Exists" effect: "NoSchedule" - key: "node.kubernetes.io/ci" operator: "Exists" effect: "PreferNoSchedule" containers: - name: "go" resources: requests: ephemeral-storage: "1Gi" limits: ephemeral-storage: "10Gi" securityContext: fsGroup: 1000 - name: "go16" label: "go16" inheritFrom: "go" containers: - name: "go" image: "kubesphere/builder-go:v3.2.2-1.16" - name: "go17" label: "go17" inheritFrom: "go" containers: - name: "go" image: "kubesphere/builder-go:v3.2.2-1.17" - name: "go18" label: "go18" inheritFrom: "go" containers: - name: "go" image: "kubesphere/builder-go:v3.2.2-1.18" - name: "python" namespace: "kubesphere-devops-worker" label: "python" nodeUsageMode: "EXCLUSIVE" idleMinutes: 0 containers: - name: "python" image: "kubesphere/builder-python:v3.2.0" command: "cat" args: "" ttyEnabled: true privileged: false resourceRequestCpu: "100m" resourceLimitCpu: "4000m" resourceRequestMemory: "100Mi" resourceLimitMemory: "8192Mi" - name: "jnlp" image: "jenkins/inbound-agent:4.10-2" args: "^${computer.jnlpmac} ^${computer.name}" resourceRequestCpu: "50m" resourceLimitCpu: "500m" resourceRequestMemory: "400Mi" resourceLimitMemory: "1536Mi" workspaceVolume: emptyDirWorkspaceVolume: memory: false volumes: - hostPathVolume: hostPath: "/var/run/docker.sock" mountPath: "/var/run/docker.sock" - hostPathVolume: hostPath: "/var/data/jenkins_python_pip_cache" mountPath: "/root/.cache/pip" - hostPathVolume: hostPath: "/var/data/jenkins_python_pipenv_cache" mountPath: "/root/.local/share/virtualenvs" - hostPathVolume: hostPath: "/var/data/jenkins_sonar_cache" mountPath: "/root/.sonar/cache" yaml: | spec: affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 1 preference: matchExpressions: - key: node-role.kubernetes.io/worker operator: In values: - ci tolerations: - key: "node.kubernetes.io/ci" operator: "Exists" effect: "NoSchedule" - key: "node.kubernetes.io/ci" operator: "Exists" effect: "PreferNoSchedule" containers: - name: "python" resources: requests: ephemeral-storage: "1Gi" limits: ephemeral-storage: "10Gi" securityContext: fsGroup: 1000 securityRealm: ldap: configurations: - displayNameAttributeName: "uid" mailAddressAttributeName: "mail" inhibitInferRootDN: false managerDN: "cn=admin,dc=kubesphere,dc=io" managerPasswordSecret: "admin" rootDN: "dc=kubesphere,dc=io" userSearchBase: "ou=Users" userSearch: "(&(objectClass=inetOrgPerson)(|(uid={0})(mail={0})))" groupSearchBase: "ou=Groups" groupSearchFilter: "(&(objectClass=posixGroup)(cn={0}))" server: "ldap://openldap.kubesphere-system.svc:389" disableMailAddressResolver: false disableRolePrefixing: true unclassified: location: url: "http://jenkins.devops.kubesphere.local" kubespheretokenauthglobalconfiguration: cacheConfiguration: size: 20 ttl: 300 enabled: true server: "http://devops-apiserver.kubesphere-devops-system:9090/" eventDispatcher: receiver: "http://devops-apiserver.kubesphere-devops-system:9090/v1alpha3/webhooks/jenkins" gitLabServers: servers: - name: "https://gitlab.com" serverUrl: "https://gitlab.com"
maven的cm配置【ks-devops-agent】在【kubesphere-devops-worker的namespace】下,该如下配置
或者整个使用能正常使用的setting.xml文件做cm