Kubernetes 资源预留（二）
后续
继Kubernetes 资源预留（一），查阅相关资料及参考阿里云ACK的资源预留配置，深思熟虑后，决定采取以下配置，分为二个部分
阿里云ACK配置
- 具体配置
# 阿里云kubelet详细配置 iZj6c1ubia3yzi7sohsjtpZ:/etc/systemd/system/kubelet.service.d# cat 10-kubeadm.conf [Service] EnvironmentFile=-/etc/kubernetes/kubelet-customized-args.conf Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" Environment="KUBELET_SYSTEM_PODS_ARGS=--max-pods 64 --pod-max-pids 16384 --pod-manifest-path=/etc/kubernetes/manifests" Environment="KUBELET_NETWORK_ARGS=--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --dynamic-config-dir=/etc/kubernetes/kubelet-config --v=3" Environment="KUBELET_DNS_ARGS=--enable-controller-attach-detach=true --cluster-dns=192.168.0.10 --pod-infra-container-image=registry-vpc.cn-hongkong.aliyuncs.com/acs/pause:3.2 --enable-load-reader --cluster-domain=cluster.local --cloud-provider=external --hostname-override=cn-hongkong.172.19.1.134 --provider-id=cn-hongkong.i-j6c1ubia3yzi7sohsjtp" Environment="KUBELET_AUTHZ_ARGS=--authorization-mode=Webhook --authentication-token-webhook=true --anonymous-auth=false --client-ca-file=/etc/kubernetes/pki/ca.crt" Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=systemd" Environment="KUBELET_CERTIFICATE_ARGS=--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --tls-cert-file=/var/lib/kubelet/pki/kubelet.crt --tls-private-key-file=/var/lib/kubelet/pki/kubelet.key --rotate-certificates=true --cert-dir=/var/lib/kubelet/pki" ExecStart= ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_NETWORK_ARGS $KUBELET_DNS_ARGS $KUBELET_AUTHZ_ARGS $KUBELET_CGROUP_ARGS $KUBELET_CERTIFICATE_ARGS $KUBELET_EXTRA_ARGS $KUBELET_CUSTOMIZED_ARGS # 阿里云集群节点资源预留配置 iZj6c1ubia3yzi7sohsjtpZ:/etc/systemd/system/kubelet.service.d# curl -sk https://172.19.1.134:6443/api/v1/nodes/cn-hongkong.172.19.1.134/proxy/configz | jq -r { "kubeletconfig": { "staticPodPath": "/etc/kubernetes/manifests", "syncFrequency": "1m0s", "fileCheckFrequency": "20s", "httpCheckFrequency": "20s", "address": "0.0.0.0", "port": 10250, "readOnlyPort": 10255, "tlsCertFile": "/var/lib/kubelet/pki/kubelet.crt", "tlsPrivateKeyFile": "/var/lib/kubelet/pki/kubelet.key", "tlsCipherSuites": [ "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", "TLS_RSA_WITH_AES_256_GCM_SHA384", "TLS_RSA_WITH_AES_128_GCM_SHA256" ], "rotateCertificates": true, "authentication": { "x509": { "clientCAFile": "/etc/kubernetes/pki/ca.crt" }, "webhook": { "enabled": true, "cacheTTL": "2m0s" }, "anonymous": { "enabled": false } }, "authorization": { "mode": "Webhook", "webhook": { "cacheAuthorizedTTL": "5m0s", "cacheUnauthorizedTTL": "30s" } }, "registryPullQPS": 5, "registryBurst": 10, "eventRecordQPS": 5, "eventBurst": 10, "enableDebuggingHandlers": true, "healthzPort": 10248, "healthzBindAddress": "127.0.0.1", "oomScoreAdj": -999, "clusterDomain": "cluster.local", "clusterDNS": [ "192.168.0.10" ], "streamingConnectionIdleTimeout": "4h0m0s", "nodeStatusUpdateFrequency": "10s", "nodeStatusReportFrequency": "5m0s", "nodeLeaseDurationSeconds": 40, "imageMinimumGCAge": "2m0s", "imageGCHighThresholdPercent": 85, "imageGCLowThresholdPercent": 80, "volumeStatsAggPeriod": "1m0s", "cgroupsPerQOS": true, "cgroupDriver": "systemd", "cpuManagerPolicy": "none", "cpuManagerReconcilePeriod": "10s", "topologyManagerPolicy": "none", "runtimeRequestTimeout": "2m0s", "hairpinMode": "promiscuous-bridge", "maxPods": 64, "podPidsLimit": 16384, "resolvConf": "/etc/resolv.conf", "cpuCFSQuota": true, "cpuCFSQuotaPeriod": "100ms", "maxOpenFiles": 1000000, "contentType": "application/vnd.kubernetes.protobuf", "kubeAPIQPS": 5, "kubeAPIBurst": 10, "serializeImagePulls": true, "evictionHard": { "imagefs.available": "15%", "memory.available": "300Mi", "nodefs.available": "10%", "nodefs.inodesFree": "5%" }, "evictionPressureTransitionPeriod": "5m0s", "enableControllerAttachDetach": true, "makeIPTablesUtilChains": true, "iptablesMasqueradeBit": 14, "iptablesDropBit": 15, "failSwapOn": true, "containerLogMaxSize": "10Mi", "containerLogMaxFiles": 5, "configMapAndSecretChangeDetectionStrategy": "Watch", "systemReserved": { "memory": "300Mi", "pid": "10000" }, "kubeReserved": { "memory": "400Mi", "pid": "10000" }, "enforceNodeAllocatable": [ "pods" ] } }
systemReserved & kubeReserved
关于三种资源控制cpu, memroy, ephemeral-storage，该如何设置，解释如下，如果有不对请指教
- 是否需要开启enforeceNodeAllocatable，我认为是不需要的，对于CPU/MEM大可不必强制开启，如果强制开启那么会有一个结果，就是如果预留值设置的高/低都会受影响，高则浪费；低则压力，CPU作为可压缩的资源，最重要任务就是做计算，任务结束CPU释放；MEM虽然说是一种不可压缩的资源，但是如果预留值过小，势必会触发OOM，凉凉；存储，硬盘值钱吗，合理设置报警即可
- 参考配置如下（但是个人认为还是预留一小部分CPU/MEM即可，以免Kubernetes Pod过度分配，给OS带来负载压力）
systemReserved: cpu: 500m memory: 500Mi kubeReserved: cpu: 500m memory: 500Mi
evictionHard
evictionHard的主要功能是，防止Kubernetes节点的资源被过度消耗，避免触发节点OOM，在这里要理解Node OOM & Pod 驱逐（前者是在资源极度缺乏下由Node触发OOM机制；后者是根据evictionHard的阈值，在触发到阈值将Pod驱逐，具体参考驱逐策略及处理方式，官方引文）
- 具体参考性配置如下
evictionHard: memory.available: "500Mi" nodefs.available: "10%" nodefs.inodesFree: "5%" imagefs.available: "15%" evictionMinimumReclaim: memory.available: "0Mi" nodefs.available: "500Mi" imagefs.available: "2Gi"
