k8s源码分析9-检查k8s集群准入配置和其他准备工作

本节重点总结 :

• k8s集群检查操作
• 新建项目 kube-mutating-webhook-inject-pod，准备工作

k8s集群检查操作

检查k8s集群否启用了准入注册 API：

• 执行kubectl api-versions |grep admission
• 如果有下面的结果说明已启用

 kubectl api-versions |grep admission
admissionregistration.k8s.io/v1
admissionregistration.k8s.io/v1beta1

检查 apiserver 中启用了MutatingAdmissionWebhook和ValidatingAdmissionWebhook两个准入控制插件

• 1.20以上的版本默认已经启用 ，在默认启用的命令行中 enable-admission-plugins

./kube-apiserver -h | grep enable-admission-plugins
      --admission-control strings              Admission is divided into two phases. In the first phase, only mutating admission plugins run. In the second phase, only validating admission plugins run. The names in the below list may represent a validating plugin, a mutating plugin, or both. The order of plugins in which they are passed to this flag does not matter. Comma-delimited list of: AlwaysAdmit, AlwaysDeny, AlwaysPullImages, CertificateApproval, CertificateSigning, CertificateSubjectRestriction, DefaultIngressClass, DefaultStorageClass, DefaultTolerationSeconds, DenyServiceExternalIPs, EventRateLimit, ExtendedResourceToleration, ImagePolicyWebhook, LimitPodHardAntiAffinityTopology, LimitRanger, MutatingAdmissionWebhook, NamespaceAutoProvision, NamespaceExists, NamespaceLifecycle, NodeRestriction, OwnerReferencesPermissionEnforcement, PersistentVolumeClaimResize, PersistentVolumeLabel, PodNodeSelector, PodSecurity, PodSecurityPolicy, PodTolerationRestriction, Priority, ResourceQuota, RuntimeClass, SecurityContextDeny, ServiceAccount, StorageObjectInUseProtection, TaintNodesByCondition, ValidatingAdmissionWebhook. (DEPRECATED: Use --enable-admission-plugins or --disable-admission-plugins instead. Will be removed in a future version.)
      --enable-admission-plugins strings       admission plugins that should be enabled in addition to default enabled ones (NamespaceLifecycle, LimitRanger, ServiceAccount, TaintNodesByCondition, PodSecurity, Priority, DefaultTolerationSeconds, DefaultStorageClass, StorageObjectInUseProtection, PersistentVolumeClaimResize, RuntimeClass, CertificateApproval, CertificateSigning, CertificateSubjectRestriction, DefaultIngressClass, MutatingAdmissionWebhook, ValidatingAdmissionWebhook, ResourceQuota). Comma-delimited list of admission plugins: AlwaysAdmit, AlwaysDeny, AlwaysPullImages, CertificateApproval, CertificateSigning, CertificateSubjectRestriction, DefaultIngressClass, DefaultStorageClass, DefaultTolerationSeconds, DenyServiceExternalIPs, EventRateLimit, ExtendedResourceToleration, ImagePolicyWebhook, LimitPodHardAntiAffinityTopology, LimitRanger, MutatingAdmissionWebhook, NamespaceAutoProvision, NamespaceExists, NamespaceLifecycle, NodeRestriction, OwnerReferencesPermissionEnforcement, PersistentVolumeClaimResize, PersistentVolumeLabel, PodNodeSelector, PodSecurity, PodSecurityPolicy, PodTolerationRestriction, Priority, ResourceQuota, RuntimeClass, SecurityContextDeny, ServiceAccount, StorageObjectInUseProtection, TaintNodesByCondition, ValidatingAdmissionWebhook. The order of plugins in this flag does not matter.

编写 webhook

• 新建项目 kube-mutating-webhook-inject-pod

go mod init kube-mutating-webhook-inject-pod

注入sidecar容器的配置文件设计

• 因为要注入一个容器，需要定义容器的相关配置所以复用k8s pod中container段的yaml
• 同时要挂载注入容器的配置，所以要复用k8s pod 中volumes的yaml
• 新建config.yaml如下

containers:
  - name: sidecar-nginx
    image: nginx:1.12.2
    imagePullPolicy: IfNotPresent
    ports:
      - containerPort: 80
    volumeMounts:
      - name: nginx-conf
        mountPath: /etc/nginx
volumes:
  - name: nginx-conf
    configMap:
      name: nginx-configmap

对应的go代码

• 新建 pkg/webhook.go

package main

import (
	corev1 "k8s.io/api/core/v1"
)
type Config struct {
	Containers []corev1.Container `yaml:"containers"`
	Volumes    []corev1.Volume    `yaml:"volumes"`
}

解析配置文件的函数

func loadConfig(configFile string) (*Config, error) {
	data, err := ioutil.ReadFile(configFile)
	if err != nil {
		return nil, err
	}
	glog.Infof("New configuration: sha256sum %x", sha256.Sum256(data))

	var cfg Config
	if err := yaml.Unmarshal(data, &cfg); err != nil {
		return nil, err
	}

	return &cfg, nil
}

编写webhook server的配置

// Webhook Server options
type webHookSvrOptions struct {
	port           int    // 监听https的端口
	certFile       string // https x509 证书路径
	keyFile        string // https x509 证书私钥路径
	sidecarCfgFile string // 注入sidecar容器的配置文件路径
}

• 在main中通过命令行传入默认值并解析

package main

import (
	"flag"
	"github.com/golang/glog"
)

func main() {
	var runOption webHookSvrOptions

	// get command line parameters
	flag.IntVar(&runOption.port, "port", 8443, "Webhook server port.")
	flag.StringVar(&runOption.certFile, "tlsCertFile", "/etc/webhook/certs/cert.pem", "File containing the x509 Certificate for HTTPS.")
	flag.StringVar(&runOption.keyFile, "tlsKeyFile", "/etc/webhook/certs/key.pem", "File containing the x509 private key to --tlsCertFile.")
	//flag.StringVar(&runOption.sidecarCfgFile, "sidecarCfgFile", "/etc/webhook/config/sidecarconfig.yaml", "File containing the mutation configuration.")
	flag.StringVar(&runOption.sidecarCfgFile, "sidecarCfgFile", "config.yaml", "File containing the mutation configuration.")
	flag.Parse()

	sidecarConfig, err := loadConfig(runOption.sidecarCfgFile)
	if err != nil {
		glog.Errorf("Failed to load configuration: %v", err)
		return
	}
	glog.Infof("[sidecarConfig:%v]", sidecarConfig)

}

加载tls x509证书

	pair, err := tls.LoadX509KeyPair(runOption.certFile, runOption.keyFile)
	if err != nil {
		glog.Errorf("Failed to load key pair: %v", err)
	}

定义webhookhttp server，并构造

• webhook.go中

type webhookServer struct {
	sidecarConfig *Config      // 注入sidecar容器的配置
	server        *http.Server // http serer
}

• main中

	webhooksvr := &webhookServer{
		sidecarConfig: sidecarConfig,
		server: &http.Server{
			Addr:      fmt.Sprintf(":%v", runOption.port),
			TLSConfig: &tls.Config{Certificates: []tls.Certificate{pair}},
		},
	}

webhookServer的mutate handler并关联path

• webhook.go

func (ws *webhookServer) serveMutate(w http.ResponseWriter, r *http.Request) {

}

• main.go

	mux := http.NewServeMux()
	mux.HandleFunc("/mutate", webhooksvr.serveMutate)
	webhooksvr.server.Handler = mux

	// start webhook server in new rountine
	go func() {
		if err := webhooksvr.server.ListenAndServeTLS("", ""); err != nil {
			glog.Errorf("Failed to listen and serve webhook server: %v", err)
		}
	}()

• 意思是请求 /mutate 由webhooksvr.serveMutate处理

main中监听 退出信号

	// listening OS shutdown singal
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
	<-signalChan

	glog.Infof("Got OS shutdown signal, shutting down webhook server gracefully...")
	webhooksvr.server.Shutdown(context.Background())