使用KubeOperator安装k8s集群需要注意的事项

1.检查kubelet和容器运行时所使用的是相同的 cgroup 驱动

事先解决办法：提前修改安装脚本，然后再进行k8s集群的安装

路径：/opt/kubeoperator/data/kobe/project/ko/roles/prepare/containerd/templates/config.toml.j2

官方建议：都使用 systemd 作为默认驱动

若已经安装好k8s集群，则按下面的步骤进行修改

# kubectl get cm -n kube-system
# kubectl edit cm kubelet-config-1.22 -n kube-system

参数：cgroupDriver: systemd

或者使用这个办法
# cat /var/lib/kubelet/config.yaml

参数：cgroupDriver: systemd

容器运行时是：containerd

# cat /etc/containerd/config.toml

参数：systemd_cgroup = true

若容器运行时(containerd)的cgroup 驱动是cgroupfs，则可以直接修改其配置文件，然后重启

更新所有节点的 cgroup 驱动

对于集群中的每一个节点：

• 执行命令 kubectl drain --ignore-daemonsets，以 腾空节点
• 执行命令 systemctl stop kubelet，以停止 kubelet
• 停止容器运行时
• 修改容器运行时 cgroup 驱动为 systemd
• 在文件 /var/lib/kubelet/config.yaml 中添加设置 cgroupDriver: systemd
• 启动容器运行时
• 执行命令 systemctl start kubelet，以启动 kubelet
• 执行命令 kubectl uncordon ，以 取消节点隔离

在节点上依次执行上述步骤，确保工作负载有充足的时间被调度到其他节点。流程完成后，确认所有节点和工作负载均健康如常。

2.使用自定义安装的harbor作为镜像仓库，采用自定义证书的形式，https方式访问的，就算是在kubePi上配置添加并授权了这个镜像仓库后，在k8s上部署服务，从该仓库镜像拉取镜像还是不能拉取，还得手动修改containerd的配置文件，再添加上跳过证书验证的配置才行

事先解决办法：提前修改安装脚本，然后再进行k8s集群的安装

路径：/opt/kubeoperator/data/kobe/project/ko/roles/prepare/containerd/templates/config.toml.j2

    [plugins.cri.registry]
      [plugins.cri.registry.mirrors]
        [plugins.cri.registry.mirrors."registy.myharbor.com"]
          endpoint = [
            "https://registy.myharbor.com"
          ]
      [plugins.cri.registry.configs]
        [plugins.cri.registry.configs."registy.myharbor.com".tls]
          insecure_skip_verify = true

事先修改安装脚本，添加上自定义的harbor镜像信息，跳过ssl验证，然后再进行k8s集群的安装

修改后的配置文件内容：

# cat /etc/containerd/config.toml
root = "/var/lib/containerd"
state = "/run/containerd"
oom_score = 0

[grpc]
  address = "/run/containerd/containerd.sock"
  uid = 0
  gid = 0
  max_recv_message_size = 16777216
  max_send_message_size = 16777216

[debug]
  address = ""
  uid = 0
  gid = 0
  level = ""

[metrics]
  address = ""
  grpc_histogram = false

[cgroup]
  path = ""

[plugins]
  [plugins.cgroups]
    no_prometheus = false
  [plugins.cri]
    stream_server_address = "127.0.0.1"
    stream_server_port = "0"
    enable_selinux = false
    sandbox_image = "registry.kubeoperator.io:8082/kubeoperator/pause:3.5"
    stats_collect_period = 10
    systemd_cgroup = true
    enable_tls_streaming = false
    max_container_log_line_size = 16384
    [plugins.cri.containerd]
      snapshotter = "overlayfs"
      no_pivot = false
      [plugins.cri.containerd.default_runtime]
        runtime_type = "io.containerd.runtime.v1.linux"
        runtime_engine = ""
        runtime_root = ""
      [plugins.cri.containerd.untrusted_workload_runtime]
        runtime_type = ""
        runtime_engine = ""
        runtime_root = ""
    [plugins.cri.cni]
      bin_dir = "/opt/cni/bin"
      conf_dir = "/etc/cni/net.d"
      conf_template = ""
    [plugins.cri.registry]
      [plugins.cri.registry.mirrors]
        [plugins.cri.registry.mirrors."10.16.16.110:8082"]
          endpoint = [
            "http://10.16.16.110:8082"
          ]
        [plugins.cri.registry.mirrors."10.16.16.110:8083"]
          endpoint = [
            "http://10.16.16.110:8083"
          ]
        [plugins.cri.registry.mirrors."registry.kubeoperator.io:8082"]
          endpoint = [
            "http://10.16.16.110:8082"
          ]
        [plugins.cri.registry.mirrors."registry.kubeoperator.io:8083"]
          endpoint = [
            "http://10.16.16.110:8083"
          ]
        [plugins.cri.registry.mirrors."docker.io"]
          endpoint = [
            "https://docker.mirrors.ustc.edu.cn",
            "http://hub-mirror.c.163.com"
          ]
        [plugins.cri.registry.mirrors."registy.myharbor.com"]
          endpoint = [
            "https://registy.myharbor.com"
          ]
        [plugins.cri.registry.mirrors."gcr.io"]
          endpoint = [
            "https://gcr.mirrors.ustc.edu.cn"
          ]
        [plugins.cri.registry.mirrors."k8s.gcr.io"]
          endpoint = [
            "https://gcr.mirrors.ustc.edu.cn/google-containers/"
          ]
        [plugins.cri.registry.mirrors."quay.io"]
          endpoint = [
            "https://quay.mirrors.ustc.edu.cn"
          ]
      [plugins.cri.registry.configs]
        [plugins.cri.registry.configs."registy.myharbor.com".tls]
          insecure_skip_verify = true
    [plugins.cri.x509_key_pair_streaming]
      tls_cert_file = ""
      tls_key_file = ""
  [plugins.diff-service]
    default = ["walking"]
  [plugins.linux]
    shim = "containerd-shim"
    runtime = "runc"
    runtime_root = ""
    no_shim = false
    shim_debug = false
  [plugins.opt]
    path = "/opt/containerd"
  [plugins.restart]
    interval = "10s"
  [plugins.scheduler]
    pause_threshold = 0.02
    deletion_threshold = 0
    mutation_threshold = 100
    schedule_delay = "0s"
    startup_delay = "100ms"