KubeSphere 开启DevOps

首先感谢这份博客 https://my.oschina.net/u/4197945/blog/15510668   作者：运维有术星主

 

参考KubeSphere官网文档：https://www.kubesphere.io/zh/docs/v3.4/devops-user-guide/how-to-use/pipelines/create-a-pipeline-using-graphical-editing-panel/

 

此份文档记录配置过程的注意事项

1.集群添加节点，配置config文件，最好是用初始化时候的配置文件。

  注意点：官方文档，添加新节点  https://www.kubesphere.io/zh/docs/v3.4/installing-on-linux/cluster-operation/add-new-nodes/ 

  用此方法生产的配置文档，少了一些配置参数，比如是kubernetes运行时是docker, 还是containerd，默认会安装docker的运行时

  所以我推荐使用初始化安装时候的配置文件

apiVersion: kubekey.kubesphere.io/v1alpha2
kind: Cluster
metadata:
  name: sample
spec:
  hosts:
  - {name: ksmaster01, address: 192.168.30.204, internalAddress: 192.168.30.204, user: root, password: "YnPJ+Wa9f9X8HMQ6qVc7"}
  - {name: ksmaster02, address: 192.168.30.205, internalAddress: 192.168.30.205, user: root, password: "YnPJ+Wa9f9X8HMQ6qVc8"}
  - {name: ksmaster03, address: 192.168.30.206, internalAddress: 192.168.30.206, user: root, password: "YnPJ+Wa9f9X8HMQ6qVc9"}
  - {name: ksworker04, address: 192.168.30.212, internalAddress: 192.168.30.212, user: root, password: "YnPJ+Wa9f9X8HMQ6qVc13"}  
  roleGroups:
    etcd:
    - ksmaster01
    - ksmaster02
    - ksmaster03
    control-plane: 
    - ksmaster01
    - ksmaster02
    - ksmaster03
    worker:
    - ksworker04
  controlPlaneEndpoint:
    ## Internal loadbalancer for apiservers 
    # internalLoadbalancer: haproxy

    domain: lb.kubesphere.local
    address: "192.168.30.203"
    port: 6443
  kubernetes:
    version: v1.23.17
    clusterName: cluster.local
    autoRenewCerts: true
    containerManager: containerd
  etcd:
    type: kubekey
  network:
    plugin: calico
    kubePodsCIDR: 10.233.64.0/18
    kubeServiceCIDR: 10.233.0.0/18
    ## multus support. https://github.com/k8snetworkplumbingwg/multus-cni
    multusCNI:
      enabled: false
  registry:
    auths:
      "harbor.emergen.cn":
        username: admin
        password: nt5Hw7T+FpAkQ4za/vBb
        skipTLSVerify: false
        plainHTTP: false
    privateRegistry: "harbor.emergen.cn/kubesphereio"
    namespaceOverride: ""
    registryMirrors: ["hub.deeprobe.online"]
    insecureRegistries: []
  addons: []



---
apiVersion: installer.kubesphere.io/v1alpha1
kind: ClusterConfiguration
metadata:
  name: ks-installer
  namespace: kubesphere-system
  labels:
    version: v3.4.1
spec:
  persistence:
    storageClass: ""
  authentication:
    jwtSecret: ""
  local_registry: ""
  # dev_tag: ""
  etcd:
    monitoring: false
    endpointIps: localhost
    port: 2379
    tlsEnable: true
  common:
    core:
      console:
        enableMultiLogin: true
        port: 30880
        type: NodePort
    # apiserver:
    #  resources: {}
    # controllerManager:
    #  resources: {}
    redis:
      enabled: false
      enableHA: false
      volumeSize: 2Gi
    openldap:
      enabled: false
      volumeSize: 2Gi
    minio:
      volumeSize: 20Gi
    monitoring:
      # type: external
      endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090
      GPUMonitoring:
        enabled: false
    gpu:
      kinds:
      - resourceName: "nvidia.com/gpu"
        resourceType: "GPU"
        default: true
    es:
      # master:
      #   volumeSize: 4Gi
      #   replicas: 1
      #   resources: {}
      # data:
      #   volumeSize: 20Gi
      #   replicas: 1
      #   resources: {}
      enabled: false
      logMaxAge: 7
      elkPrefix: logstash
      basicAuth:
        enabled: false
        username: ""
        password: ""
      externalElasticsearchHost: ""
      externalElasticsearchPort: ""
    opensearch:
      # master:
      #   volumeSize: 4Gi
      #   replicas: 1
      #   resources: {}
      # data:
      #   volumeSize: 20Gi
      #   replicas: 1
      #   resources: {}
      enabled: true
      logMaxAge: 7
      opensearchPrefix: whizard
      basicAuth:
        enabled: true
        username: "admin"
        password: "admin"
      externalOpensearchHost: ""
      externalOpensearchPort: ""
      dashboard:
        enabled: true
  alerting:
    enabled: true
    # thanosruler:
    #   replicas: 1
    #   resources: {}
  auditing:
    enabled: true
    # operator:
    #   resources: {}
    # webhook:
    #   resources: {}
  devops:
    enabled: false
    jenkinsCpuReq: 0.5
    jenkinsCpuLim: 1
    jenkinsMemoryReq: 4Gi
    jenkinsMemoryLim: 4Gi
    jenkinsVolumeSize: 16Gi
  events:
    enabled: true
    # operator:
    #   resources: {}
    # exporter:
    #   resources: {}
    ruler:
      enabled: true
      replicas: 2
    #   resources: {}
  logging:
    enabled: true
    logsidecar:
      enabled: true
      replicas: 2
      # resources: {}
  metrics_server:
    enabled: true
  monitoring:
    storageClass: ""
    node_exporter:
      port: 9100
      # resources: {}
    # kube_rbac_proxy:
    #   resources: {}
    # kube_state_metrics:
    #   resources: {}
    prometheus:
    #   replicas: 1
      volumeSize: 40Gi
    #   resources: {}
    #   operator:
    #     resources: {}
    # alertmanager:
    #   replicas: 1
    #   resources: {}
    # notification_manager:
    #   resources: {}
    #   operator:
    #     resources: {}
    #   proxy:
    #     resources: {}
    gpu:
      nvidia_dcgm_exporter:
        enabled: false
        # resources: {}
  multicluster:
    clusterRole: none
  network:
    networkpolicy:
      enabled: false
    ippool:
      type: calico
    topology:
      type: weave-scope
  openpitrix:
    store:
      enabled: true
  servicemesh:
    enabled: false
    istio:
      components:
        ingressGateways:
        - name: istio-ingressgateway
          enabled: false
        cni:
          enabled: false
  edgeruntime:
    enabled: true
    kubeedge:
      enabled: true
      cloudCore:
        cloudHub:
          advertiseAddress:
            - 192.168.30.203
        service:
          cloudhubNodePort: "30000"
          cloudhubQuicNodePort: "30001"
          cloudhubHttpsNodePort: "30002"
          cloudstreamNodePort: "30003"
          tunnelNodePort: "30004"
        # resources: {}
        # hostNetWork: false
      iptables-manager:
        enabled: true
        mode: "external"
        # resources: {}
      # edgeService:
      #   resources: {}
  gatekeeper:
    enabled: false
    # controller_manager:
    #   resources: {}
    # audit:
    #   resources: {}
  terminal:
    timeout: 600

export KKZONE=cn
./kk add nodes -f config-nodes-add.yaml  # 运行前，先执行第二步骤，确保node上已经有必须要的组件

 

2.配置新节点

 2.1 yum 源

 mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo-history
 wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
 yum install -y wget
 yum clean all
 yum makecache

 2.2  安装依赖组件，看官网文档中，这几个组件是必须的

  yum -y install socat ipset conntrack ebtables ipvsadm

3.配置流水线-netcore环境

项目是由netcore语言开发

 3.1 配置字典 jenkins-casc-config。找到 jenkins_user.yaml

  添加 dotnetcore 容器环境,添加后等待一分钟左右，流水线才会出现环境配置

- name: "dotnetcore"
            namespace: "kubesphere-devops-worker"
            label: "dotnetcore"
            nodeUsageMode: "EXCLUSIVE"
            idleMinutes: 0
            containers:
            - name: "dotnetcore"
              image: "harbor.emergen.cn/library/dotnet-emergen:3-5-6"     # 指定自定义dotnet镜像
              command: "cat"
              args: ""
              ttyEnabled: true
              privileged: true
              resourceRequestCpu: "100m"
              resourceLimitCpu: "4000m"
              resourceRequestMemory: "100Mi"
              resourceLimitMemory: "8192Mi"
              alwaysPullImage: true
            - name: "jnlp"
              #image: "jenkins/jnlp-slave:3.27-1"
              image: "harbor.emergen.cn/kubesphereio/jenkins/inbound-agent:4.10-2"
              #command: "jenkins-slave"
              args: "^${computer.jnlpmac} ^${computer.name}"
              resourceRequestCpu: "50m"
              resourceRequestMemory: "400Mi"
              resourceLimitMemory: "1536Mi"
            imagePullSecrets:           # 指定私有仓库凭证
              - name: harborsecret
            workspaceVolume:
              emptyDirWorkspaceVolume:
                memory: false
            volumes:
            - hostPathVolume:
                hostPath: "/var/run/docker.sock"
                mountPath: "/var/run/docker.sock"
            - hostPathVolume:                            # 将nuget包缓存持久化到hostPath
                hostPath: "/var/data/jenkins_nuget_cache"
                mountPath: "/root/.nuget"
            yaml: |
              spec:
                affinity:
                  nodeAffinity:
                    preferredDuringSchedulingIgnoredDuringExecution:
                    - weight: 1
                      preference:
                        matchExpressions:
                        - key: node-role.kubernetes.io/worker
                          operator: In
                          values:
                          - ci
                tolerations:
                - key: "node.kubernetes.io/ci"
                  operator: "Exists"
                  effect: "NoSchedule"  
                - key: "node.kubernetes.io/ci"
                  operator: "Exists"
                  effect: "PreferNoSchedule"
                containers:
                - name: "dotnetcore"
                  resources:
                    requests:
                      ephemeral-storage: "1Gi"
                    limits:
                      ephemeral-storage: "10Gi"
                  volumeMounts:
                  - name: config-volume
                    mountPath: /root/.nuget/NuGet/NuGet.Config
                    subPath: NuGet.Config
                volumes:
                - name: config-volume
                  configMap:
                    name: ks-devops-agent
                    items:
                    - key: NugetSetting
                      path: NuGet.Config
                securityContext:
                  fsGroup: 1000
          
          - name: "dotnetcore2-emergen" # 自定义 Jenkins Agent 的名称。
            label: "dotnetcore2-emergen" # 自定义 Jenkins Agent 的标签。若要指定多个标签，请用空格来分隔标签。
            inheritFrom: "dotnetcore" # 该自定义 Jenkins Agent 所继承的现有容器组模板的名称。
            containers:
            - name: "dotnetcore" # 该自定义 Jenkins Agent 所继承的现有容器组模板中指定的容器名称。
              image: "harbor.emergen.cn/library/dotnet-emergen:2.2" # 可以使用自己的镜像。
            imagePullSecrets:
              - name: harborsecret
          
          - name: "dotnetcore6-emergen" # 自定义 Jenkins Agent 的名称。
            label: "dotnetcore6-emergen" # 自定义 Jenkins Agent 的标签。若要指定多个标签，请用空格来分隔标签。
            inheritFrom: "dotnetcore" # 该自定义 Jenkins Agent 所继承的现有容器组模板的名称。
            containers:
            - name: "dotnetcore" # 该自定义 Jenkins Agent 所继承的现有容器组模板中指定的容器名称。
              image: "harbor.emergen.cn/library/dotnet-emergen:v6" # 可以使用自己的镜像。
            imagePullSecrets:
              - name: harborsecret

 

4. 配置流水线-pipline

注意: 涉及到 credentialsId，都是在devops 凭据中配置，其他是运行参数

pipeline {
  agent {
    node {
      label 'dotnetcore'
    }
  }
  
  stages {
    stage('拉取代码') {
      steps {
        git(url: "${GIT_REPOSITORY_URL}", credentialsId: 'gitlab-auth', branch: "${BRANCH}", changelog: true, poll: false)
      }
    }
    stage('编译推送') {
      steps {
        container("dotnetcore"){
          sh 'dotnet restore "./demo/demo.csproj" '
          sh 'dotnet build ./demo/demo.csproj -c Release -o dist --force'
          sh 'podman build -t $REGISTRY/$HARBOR_NAMESPACE/$APP_NAME:$BUILD_NUMBER .'
          withCredentials([usernamePassword(credentialsId : 'harbor-auth' ,passwordVariable : 'HARBOR_PASSWORD' ,usernameVariable : 'HARBOR_USERNAME' ,)]) {
            sh '''echo "$HARBOR_PASSWORD" | podman login $REGISTRY -u "$HARBOR_USERNAME" --password-stdin
'''
            sh '''podman push $REGISTRY/$HARBOR_NAMESPACE/$APP_NAME:$BUILD_NUMBER
'''
          }
        }
      }
    }
    
    stage('部署') {
      steps {
         container ('dotnetcore') {
          withCredentials([
            kubeconfigFile(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
              sh 'envsubst < webapi.yaml | kubectl apply -f -'
            }
        }
      }
    }
  }

}

 

 5. netcore环境的dockerfile

 

FROM registry.cn-beijing.aliyuncs.com/kubesphereio/builder-base:v3.2.2-podman

RUN sed -e 's|^mirrorlist=|#mirrorlist=|g' \
         -e 's|^#baseurl=http://mirror.centos.org/centos|baseurl=https://mirrors.tuna.tsinghua.edu.cn/centos|g' \
         -i.bak \
         /etc/yum.repos.d/CentOS-*.repo && \
    rpm -Uvh https://packages.microsoft.com/config/centos/7/packages-microsoft-prod.rpm && \
    yum install -y dotnet-sdk-3.1 && \
    yum install -y dotnet-sdk-5.0 && \
    yum install -y dotnet-sdk-6.0 && \
    yum clean all
#RUN dotnet tool install --global dotnet-sonarscanner --version 5.0.4

#ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/bin/sonar-scanner-3.3.0.1492-linux/bin:/root/.nuget/tools:/root/.dotnet/tools
ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/root/.nuget/tools:/root/.dotnet/tools

CMD ["dotnet", "--list-sdks"]

 6.个人理解

流水线的运行会启动一个pod ，这个pod将使用agnet.node.label中定义的镜像，这个文档开始讲的jenkins_user.yaml是相互关联的

如果没用使用container(){} 包含住脚本的话，他将在base的容器中运行。他们之间的应该是文件共享的一个方式