k8s-学习笔记1-部署k8s
规划
docker采用systemd部署
pod网络172.7.0.0/16，docker bip设置为172.7.x.1/24
cluster网络 192.168.0.0/16
k8s版本 1.15.12
cni使用flannel VxLAN
kubeproxy使用ipvs模式
master部署双节点，带一个slb
ectd三节点
# docker配置修改
cat > /etc/docker/daemon.json <<\EOF
{
"registry-mirrors": ["https://70m1xlvx.mirror.aliyuncs.com"],
"exec-opts": ["native.cgroupdriver=systemd"],
"bip": "172.7.18.1/24"
}
EOF
systemctl restart docker
systemctl enable docker
证书签发
证书规划
ca：所有证书的根证书
etcd-peer：etcd节点之间双向通信的双向证书
client：apiserver访问etcd，apiserver访问kubelet，kebelet访问apiserver，flannel访问etcd的客户端证书
apiserver：apiserver提供服务的服务端证书
kubelet：kubelet提供服务给apiserver访问的服务端证书
kube-proxy-cilent：kubeproxy访问apiserver的客户端证书
# 下载cfssl curl -s -L -o /usr/bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 curl -s -L -o /usr/bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 curl -s -L -o /usr/bin/cfssl-certinfo https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 chmod +x /usr/bin/cfssl*
# 创建ca配置文件
mkdir /opt/certs/
cat > /opt/certs/ca-config.json <<\EOF
{
"signing": {
"default": {
"expiry": "175200h"
},
"profiles": {
"server": {
"expiry": "175200h",
"usages": [
"signing",
"key encipherment",
"server auth"
]
},
"client": {
"expiry": "175200h",
"usages": [
"signing",
"key encipherment",
"client auth"
]
},
"peer": {
"expiry": "175200h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
EOF
# 创建CA证书签名请求
cat > /opt/certs/ca-csr.json <<\EOF
{
"CN": "e-dewin",
"hosts": [
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "hangzhou",
"L": "hangzhou",
"O": "e-dewin",
"OU": "ops"
}
],
"ca": {
"expiry": "175200h"
}
}
EOF
# CA证书和私钥
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
# 把etcd集群的ip都放进hosts
cat > /opt/certs/etcd-peer-csr.json <<\EOF
{
"CN": "etcd-peer",
"hosts": [
"192.168.18.24",
"192.168.18.51",
"192.168.18.18"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "hangzhou",
"L": "hangzhou",
"O": "e-dewin",
"OU": "ops"
}
]
}
EOF
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer etcd-peer-csr.json | cfssljson -bare etcd-peer
# 客户端证书
cat > /opt/certs/client-csr.json <<\EOF
{
"CN": "k8s-node",
"hosts": [
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "hangzhou",
"L": "hangzhou",
"O": "e-dewin",
"OU": "ops"
}
]
}
EOF
cd /opt/certs
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client-csr.json | cfssljson -bare client
cat > /opt/certs/apiserver-csr.json <<\EOF
{
"CN": "apiserver",
"hosts": [
"127.0.0.1",
"192.168.0.1",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local",
"192.168.18.18",
"192.168.18.51",
"192.168.18.56"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "hangzhou",
"L": "hangzhou",
"O": "e-dewin",
"OU": "ops"
}
]
}
EOF
cd /opt/certs
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server apiserver-csr.json | cfssljson -bare apiserver
# 需要安装kubelet的ip加进去
cat > /opt/certs/kubelet-csr.json <<\EOF
{
"CN": "kubelet-node",
"hosts": [
"127.0.0.1"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "hangzhou",
"L": "hangzhou",
"O": "e-dewin",
"OU": "ops"
}
]
}
EOF
cd /opt/certs
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server kubelet-csr.json | cfssljson -bare kubelet
# kube-proxy客户端证书
cat > /opt/certs/kube-proxy-csr.json <<\EOF
{
"CN": "system:kube-proxy",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "hangzhou",
"L": "hangzhou",
"O": "e-dewin",
"OU": "ops"
}
]
}
EOF
cd /opt/certs
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client kube-proxy-csr.json | cfssljson -bare kube-proxy-client
# 设置私钥 cd /opt/certs/ chmod 600 *key.pem
所有节点安装supervisor
yum install supervisor -y systemctl start supervisord systemctl enable supervisord
部署etcd三节点集群
每次部署需要修改etcd name，三个url地址，集群初始化地址
# 传输etcd文件
cd /opt/
wget http://repo.e-dewin.com/package/k8s/etcd-v3.1.20-linux-amd64.tar.gz
tar xf etcd-v3.1.20-linux-amd64.tar.gz
ln -s /opt/etcd-v3.1.20-linux-amd64 /opt/etcd
rm -f etcd-v3.1.20-linux-amd64.tar.gz
# 生成配置文件
cat > /opt/etcd/etcd-server-startup.sh <<\EOF
#!/bin/sh
./etcd --name etcd-server-1 \
--data-dir /data/etcd/etcd-server \
--listen-peer-urls https://192.168.18.24:2380 \
--listen-client-urls https://192.168.18.24:2379,http://127.0.0.1:2379 \
--quota-backend-bytes 8000000000 \
--initial-advertise-peer-urls https://192.168.18.24:2380 \
--advertise-client-urls https://192.168.18.24:2379,http://127.0.0.1:2379 \
--initial-cluster etcd-server-1=https://192.168.18.24:2380,etcd-server-2=https://192.168.18.51:2380,etcd-server-3=https://192.168.18.18:2380 \
--ca-file ./certs/ca.pem \
--cert-file ./certs/etcd-peer.pem \
--key-file ./certs/etcd-peer-key.pem \
--client-cert-auth \
--trusted-ca-file ./certs/ca.pem \
--peer-ca-file ./certs/ca.pem \
--peer-cert-file ./certs/etcd-peer.pem \
--peer-key-file ./certs/etcd-peer-key.pem \
--peer-client-cert-auth \
--peer-trusted-ca-file ./certs/ca.pem \
--log-output stdout
EOF
# 传输证书
mkdir -p /opt/etcd/certs
# 上传etcd-peer证书，ca证书
chmod 600 /opt/etcd/certs/*-key.pem
# 创建用户和数据目录
useradd -s /sbin/nologin -M etcd
mkdir -p /data/etcd /data/logs/etcd-server
chown -R etcd.etcd /data/etcd /data/logs/etcd-server/ /opt/etcd/
chmod +x /opt/etcd/etcd-server-startup.sh
# 生成启动文件
cat > /etc/supervisord.d/etcd-server.ini <<\EOF
[program:etcd-server]
command=/opt/etcd/etcd-server-startup.sh ; the program (relative uses PATH, can take args)
numprocs=1 ; number of processes copies to start (def 1)
directory=/opt/etcd ; directory to cwd to before exec (def no cwd)
autostart=true ; start at supervisord start (default: true)
autorestart=true ; retstart at unexpected quit (default: true)
startsecs=22 ; number of secs prog must stay running (def. 1)
startretries=3 ; max # of serial start failures (default 3)
exitcodes=0,2 ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT ; signal used to kill process (default TERM)
stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10)
user=etcd ; setuid to this UNIX account to run the program
redirect_stderr=false ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/etcd-server/etcd.stdout.log ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4 ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false ; emit events on stdout writes (default false)
stderr_logfile=/data/logs/etcd-server/etcd.stderr.log ; stderr log path, NONE for none; default AUTO
stderr_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stderr_logfile_backups=4 ; # of stderr logfile backups (default 10)
stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stderr_events_enabled=false ; emit events on stderr writes (default false)
EOF
# 启动
supervisorctl update
supervisorctl status
# 三个节点部署完成后，检查集群 /opt/etcd/etcdctl cluster-health /opt/etcd/etcdctl member list
api-server双节点
apiserver地址： https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.15.md#server-binaries
需要修改的是etcd地址
所有证书都传上去，下面的服务就不需要上传了
双节点部署完成后加一个负载均衡，提供一个vip作为apiserver的地址
# 安装服务
cd /opt
wget http://repo.e-dewin.com/package/k8s/kubernetes-server-linux-amd64.tar.gz
tar xf kubernetes-server-linux-amd64.tar.gz -C /opt
mv /opt/kubernetes /opt/kubernetes-v1.15.12-linux-amd64
ln -s /opt/kubernetes-v1.15.12-linux-amd64 /opt/kubernetes
mkdir /opt/kubernetes/server/bin/{cert,conf}
cd /opt/kubernetes/server/bin/cert
# 上传证书
chmod 600 *-key.pem
# 生成审计配置
cat > /opt/kubernetes/server/bin/conf/audit.yaml <<\EOF
apiVersion: audit.k8s.io/v1beta1 # This is required.
kind: Policy
# Don't generate audit events for all requests in RequestReceived stage.
omitStages:
- "RequestReceived"
rules:
# Log pod changes at RequestResponse level
- level: RequestResponse
resources:
- group: ""
# Resource "pods" doesn't match requests to any subresource of pods,
# which is consistent with the RBAC policy.
resources: ["pods"]
# Log "pods/log", "pods/status" at Metadata level
- level: Metadata
resources:
- group: ""
resources: ["pods/log", "pods/status"]
# Don't log requests to a configmap called "controller-leader"
- level: None
resources:
- group: ""
resources: ["configmaps"]
resourceNames: ["controller-leader"]
# Don't log watch requests by the "system:kube-proxy" on endpoints or services
- level: None
users: ["system:kube-proxy"]
verbs: ["watch"]
resources:
- group: "" # core API group
resources: ["endpoints", "services"]
# Don't log authenticated requests to certain non-resource URL paths.
- level: None
userGroups: ["system:authenticated"]
nonResourceURLs:
- "/api*" # Wildcard matching.
- "/version"
# Log the request body of configmap changes in kube-system.
- level: Request
resources:
- group: "" # core API group
resources: ["configmaps"]
# This rule only applies to resources in the "kube-system" namespace.
# The empty string "" can be used to select non-namespaced resources.
namespaces: ["kube-system"]
# Log configmap and secret changes in all other namespaces at the Metadata level.
- level: Metadata
resources:
- group: "" # core API group
resources: ["secrets", "configmaps"]
# Log all other resources in core and extensions at the Request level.
- level: Request
resources:
- group: "" # core API group
- group: "extensions" # Version of group should NOT be included.
# A catch-all rule to log all other requests at the Metadata level.
- level: Metadata
# Long-running requests like watches that fall under this rule will not
# generate an audit event in RequestReceived.
omitStages:
- "RequestReceived"
EOF
# 启动脚本
cat > /opt/kubernetes/server/bin/kube-apiserver.sh <<\EOF
#!/bin/bash
./kube-apiserver \
--apiserver-count 2 \
--audit-log-path /data/logs/kubernetes/kube-apiserver/audit-log \
--audit-policy-file ./conf/audit.yaml \
--authorization-mode RBAC \
--client-ca-file ./cert/ca.pem \
--requestheader-client-ca-file ./cert/ca.pem \
--enable-admission-plugins NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota \
--etcd-cafile ./cert/ca.pem \
--etcd-certfile ./cert/client.pem \
--etcd-keyfile ./cert/client-key.pem \
--etcd-servers https://192.168.18.24:2379,https://192.168.18.18:2379,https://192.168.18.51:2379 \
--service-account-key-file ./cert/ca-key.pem \
--service-cluster-ip-range 192.168.0.0/16 \
--service-node-port-range 3000-29999 \
--target-ram-mb=1024 \
--kubelet-client-certificate ./cert/client.pem \
--kubelet-client-key ./cert/client-key.pem \
--log-dir /data/logs/kubernetes/kube-apiserver \
--tls-cert-file ./cert/apiserver.pem \
--tls-private-key-file ./cert/apiserver-key.pem \
--v 2
EOF
chmod +x /opt/kubernetes/server/bin/kube-apiserver.sh
mkdir -p /data/logs/kubernetes/kube-apiserver
cat > /etc/supervisord.d/kube-apiserver.ini <<\EOF
[program:kube-apiserver]
command=/opt/kubernetes/server/bin/kube-apiserver.sh ; the program (relative uses PATH, can take args)
numprocs=1 ; number of processes copies to start (def 1)
directory=/opt/kubernetes/server/bin ; directory to cwd to before exec (def no cwd)
autostart=true ; start at supervisord start (default: true)
autorestart=true ; retstart at unexpected quit (default: true)
startsecs=22 ; number of secs prog must stay running (def. 1)
startretries=3 ; max # of serial start failures (default 3)
exitcodes=0,2 ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT ; signal used to kill process (default TERM)
stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10)
user=root ; setuid to this UNIX account to run the program
redirect_stderr=false ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/kubernetes/kube-apiserver/apiserver.stdout.log ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4 ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false ; emit events on stdout writes (default false)
stderr_logfile=/data/logs/kubernetes/kube-apiserver/apiserver.stderr.log ; stderr log path, NONE for none; default AUTO
stderr_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stderr_logfile_backups=4 ; # of stderr logfile backups (default 10)
stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stderr_events_enabled=false ; emit events on stderr writes (default false)
EOF
supervisorctl update
supervisorctl status
controller-manager双节点
controller-manager和scheduler参数里的master都是本机api-server的http地址，没有交叉使用，所以不需要TLS证书。
不需要修改
cat > /opt/kubernetes/server/bin/kube-controller-manager.sh <<\EOF #!/bin/sh ./kube-controller-manager \ --cluster-cidr 172.7.0.0/16 \ --leader-elect true \ --log-dir /data/logs/kubernetes/kube-controller-manager \ --master http://127.0.0.1:8080 \ --service-account-private-key-file ./cert/ca-key.pem \ --service-cluster-ip-range 192.168.0.0/16 \ --root-ca-file ./cert/ca.pem \ --v 2 EOF chmod +x /opt/kubernetes/server/bin/kube-controller-manager.sh mkdir -p /data/logs/kubernetes/kube-controller-manager cat > /etc/supervisord.d/kube-conntroller-manager.ini <<\EOF [program:kube-controller-manager] command=/opt/kubernetes/server/bin/kube-controller-manager.sh ; the program (relative uses PATH, can take args) numprocs=1 ; number of processes copies to start (def 1) directory=/opt/kubernetes/server/bin ; directory to cwd to before exec (def no cwd) autostart=true ; start at supervisord start (default: true) autorestart=true ; retstart at unexpected quit (default: true) startsecs=22 ; number of secs prog must stay running (def. 1) startretries=3 ; max # of serial start failures (default 3) exitcodes=0,2 ; 'expected' exit codes for process (default 0,2) stopsignal=QUIT ; signal used to kill process (default TERM) stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) user=root ; setuid to this UNIX account to run the program redirect_stderr=false ; redirect proc stderr to stdout (default false) stdout_logfile=/data/logs/kubernetes/kube-controller-manager/controll.stdout.log ; stdout log path, NONE for none; default AUTO stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB) stdout_logfile_backups=4 ; # of stdout logfile backups (default 10) stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) stdout_events_enabled=false ; emit events on stdout writes (default false) stderr_logfile=/data/logs/kubernetes/kube-controller-manager/controll.stderr.log ; stderr log path, NONE for none; default AUTO stderr_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB) stderr_logfile_backups=4 ; # of stderr logfile backups (default 10) stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) stderr_events_enabled=false ; emit events on stderr writes (default false) EOF supervisorctl update supervisorctl status
scheduler双节点
不需要修改
cat > /opt/kubernetes/server/bin/kube-scheduler.sh <<\EOF #!/bin/sh ./kube-scheduler \ --leader-elect \ --log-dir /data/logs/kubernetes/kube-scheduler \ --master http://127.0.0.1:8080 \ --v 2 EOF chmod +x /opt/kubernetes/server/bin/kube-scheduler.sh mkdir -p /data/logs/kubernetes/kube-scheduler cat > /etc/supervisord.d/kube-scheduler.ini <<\EOF [program:kube-scheduler] command=/opt/kubernetes/server/bin/kube-scheduler.sh ; the program (relative uses PATH, can take args) numprocs=1 ; number of processes copies to start (def 1) directory=/opt/kubernetes/server/bin ; directory to cwd to before exec (def no cwd) autostart=true ; start at supervisord start (default: true) autorestart=true ; retstart at unexpected quit (default: true) startsecs=22 ; number of secs prog must stay running (def. 1) startretries=3 ; max # of serial start failures (default 3) exitcodes=0,2 ; 'expected' exit codes for process (default 0,2) stopsignal=QUIT ; signal used to kill process (default TERM) stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) user=root ; setuid to this UNIX account to run the program redirect_stderr=false ; redirect proc stderr to stdout (default false) stdout_logfile=/data/logs/kubernetes/kube-scheduler/scheduler.stdout.log ; stdout log path, NONE for none; default AUTO stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB) stdout_logfile_backups=4 ; # of stdout logfile backups (default 10) stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) stdout_events_enabled=false ; emit events on stdout writes (default false) stderr_logfile=/data/logs/kubernetes/kube-scheduler/scheduler.stderr.log ; stderr log path, NONE for none; default AUTO stderr_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB) stderr_logfile_backups=4 ; # of stderr logfile backups (default 10) stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) stderr_events_enabled=false ; emit events on stderr writes (default false) EOF supervisorctl update supervisorctl status
master健康状态检查
ln -s /opt/kubernetes/server/bin/kubectl /usr/bin/kubectl
# kubectl get cs
NAME STATUS MESSAGE ERROR
controller-manager Healthy ok
scheduler Healthy ok
etcd-1 Healthy {"health": "true"}
etcd-2 Healthy {"health": "true"}
etcd-0 Healthy {"health": "true"}
kubelet
server是apiserver的vip，需要修改
hostname-oveerride改成ip
cluster-dns改成coredns的集群ip
# 创建集群 cd /opt/kubernetes/server/bin/conf kubectl config set-cluster k8s-1 \ --certificate-authority=/opt/kubernetes/server/bin/cert/ca.pem \ --embed-certs=true \ --server=https://192.168.18.56:6443 \ --kubeconfig=kubelet.kubeconfig # 创建用户 kubectl config set-credentials k8s-node --client-certificate=/opt/kubernetes/server/bin/cert/client.pem --client-key=/opt/kubernetes/server/bin/cert/client-key.pem --embed-certs=true --kubeconfig=kubelet.kubeconfig kubectl config set-context k8s-1-context \ --cluster=k8s-1 \ --user=k8s-node \ --kubeconfig=kubelet.kubeconfig kubectl config use-context k8s-1-context --kubeconfig=kubelet.kubeconfig
# 创建认证资源，让k8s-node具有运算节点权限 cat > /opt/kubernetes/server/bin/conf/k8s-node.yaml <<\EOF apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: k8s-node roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:node subjects: - apiGroup: rbac.authorization.k8s.io kind: User name: k8s-node EOF kubectl create -f k8s-node.yaml kubectl get clusterrolebinding k8s-node
# infra_pod基础镜像，准备好放到自己的仓库 # registry-vpc.cn-hangzhou.aliyuncs.com/devvops/rhel7-pod-infrastructure:v3.4 # kubelet启动脚本 cat > /opt/kubernetes/server/bin/kubelet.sh <<\EOF #!/bin/sh ./kubelet \ --anonymous-auth=false \ --cgroup-driver systemd \ --cluster-dns 192.168.0.2 \ --cluster-domain cluster.local \ --runtime-cgroups=/systemd/system.slice --kubelet-cgroups=/systemd/system.slice \ --fail-swap-on="false" \ --client-ca-file ./cert/ca.pem \ --tls-cert-file ./cert/kubelet.pem \ --tls-private-key-file ./cert/kubelet-key.pem \ --hostname-override 192.168.18.18 \ --image-gc-high-threshold 20 \ --image-gc-low-threshold 10 \ --kubeconfig ./conf/kubelet.kubeconfig \ --log-dir /data/logs/kubernetes/kube-kubelet \ --pod-infra-container-image registry-vpc.cn-hangzhou.aliyuncs.com/devvops/rhel7-pod-infrastructure:v3.4 \ --root-dir /data/kubelet EOF chmod +x /opt/kubernetes/server/bin/kubelet.sh mkdir -p /data/logs/kubernetes/kube-kubelet /data/kubelet cat > /etc/supervisord.d/kube-kubelet.ini <<\EOF [program:kube-kubelet] command=/opt/kubernetes/server/bin/kubelet.sh ; the program (relative uses PATH, can take args) numprocs=1 ; number of processes copies to start (def 1) directory=/opt/kubernetes/server/bin ; directory to cwd to before exec (def no cwd) autostart=true ; start at supervisord start (default: true) autorestart=true ; retstart at unexpected quit (default: true) startsecs=22 ; number of secs prog must stay running (def. 1) startretries=3 ; max # of serial start failures (default 3) exitcodes=0,2 ; 'expected' exit codes for process (default 0,2) stopsignal=QUIT ; signal used to kill process (default TERM) stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) user=root ; setuid to this UNIX account to run the program redirect_stderr=false ; redirect proc stderr to stdout (default false) stdout_logfile=/data/logs/kubernetes/kube-kubelet/kubelet.stdout.log ; stdout log path, NONE for none; default AUTO stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB) stdout_logfile_backups=4 ; # of stdout logfile backups (default 10) stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) stdout_events_enabled=false ; emit events on stdout writes (default false) stderr_logfile=/data/logs/kubernetes/kube-kubelet/kubelet.stderr.log ; stderr log path, NONE for none; default AUTO stderr_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB) stderr_logfile_backups=4 ; # of stderr logfile backups (default 10) stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) stderr_events_enabled=false ; emit events on stderr writes (default false) EOF supervisorctl update supervisorctl status
# 检查node kubectl get node
# 设置node角色 kubectl label node 192.168.18.18 node-role.kubernetes.io/master= --overwrite=true kubectl label node 192.168.18.18 node-role.kubernetes.io/node= --overwrite=true kubectl label node 192.168.18.51 node-role.kubernetes.io/master= --overwrite=true kubectl label node 192.168.18.51 node-role.kubernetes.io/node= --overwrite=true
kube-proxy（ipvs）
server是apiserver vip
hostname overide改成自己ip
# 配置生成kube-proxy.kubeconfig
cd /opt/kubernetes/server/bin/conf
kubectl config set-cluster k8s-1 \
--certificate-authority=/opt/kubernetes/server/bin/cert/ca.pem \
--embed-certs=true \
--server=https://192.168.18.56:6443 \
--kubeconfig=kube-proxy.kubeconfig
kubectl config set-credentials kube-proxy \
--client-certificate=/opt/kubernetes/server/bin/cert/kube-proxy-client.pem \
--client-key=/opt/kubernetes/server/bin/cert/kube-proxy-client-key.pem \
--embed-certs=true \
--kubeconfig=kube-proxy.kubeconfig
kubectl config set-context k8s-1-context \
--cluster=k8s-1 \
--user=kube-proxy \
--kubeconfig=kube-proxy.kubeconfig
kubectl config use-context k8s-1-context --kubeconfig=kube-proxy.kubeconfig
# 加载系统ipvs模块
cd /root/
cat > /root/ipvs.sh <<\EOF
#!/bin/bash
ipvs_mods_dir="/usr/lib/modules/$(uname -r)/kernel/net/netfilter/ipvs"
for i in $(ls $ipvs_mods_dir|grep -o "^[^.]*")
do
/sbin/modinfo -F filename $i &>/dev/null
if [ $? -eq 0 ];then
/sbin/modprobe $i
fi
done
EOF
chmod +x /root/ipvs.sh
./ipvs.sh
# kube-proxy启动脚本，使用ipvs的never queue永不排队调度算法
cat > /opt/kubernetes/server/bin/kube-proxy.sh <<\EOF
#!/bin/sh
./kube-proxy \
--cluster-cidr 172.7.0.0/16 \
--hostname-override 192.168.18.51 \
--proxy-mode=ipvs \
--ipvs-scheduler=nq \
--kubeconfig ./conf/kube-proxy.kubeconfig
EOF
chmod +x /opt/kubernetes/server/bin/kube-proxy.sh
mkdir -p /data/logs/kubernetes/kube-proxy
cat > /etc/supervisord.d/kube-proxy.ini <<\EOF
[program:kube-proxy]
command=/opt/kubernetes/server/bin/kube-proxy.sh ; the program (relative uses PATH, can take args)
numprocs=1 ; number of processes copies to start (def 1)
directory=/opt/kubernetes/server/bin ; directory to cwd to before exec (def no cwd)
autostart=true ; start at supervisord start (default: true)
autorestart=true ; retstart at unexpected quit (default: true)
startsecs=22 ; number of secs prog must stay running (def. 1)
startretries=3 ; max # of serial start failures (default 3)
exitcodes=0,2 ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT ; signal used to kill process (default TERM)
stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10)
user=root ; setuid to this UNIX account to run the program
redirect_stderr=false ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/kubernetes/kube-proxy/proxy.stdout.log ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4 ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false ; emit events on stdout writes (default false)
stderr_logfile=/data/logs/kubernetes/kube-proxy/proxy.stderr.log ; stderr log path, NONE for none; default AUTO
stderr_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stderr_logfile_backups=4 ; # of stderr logfile backups (default 10)
stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stderr_events_enabled=false ; emit events on stderr writes (default false)
EOF
supervisorctl update
supervisorctl status
# 验证，这时候应该有集群ip到apiserver的调度
yum install -y ipvsadm
ipvsadm -Ln
flannel
subnet改为docker bip
public ip是自己ip
# etcd主节点增加配置，使用VxLAN模型
./etcdctl set /coreos.com/network/config '{"Network": "172.7.0.0/16", "Backend": {"Type": "VxLAN"}}'
# 下载包
cd /opt
wget http://repo.e-dewin.com/package/k8s/flannel-v0.11.0-linux-amd64.tar.gz
mkdir /opt/flannel-v0.11.0/
tar xf flannel-v0.11.0-linux-amd64.tar.gz -C /opt/flannel-v0.11.0/
rm -f flannel-v0.11.0-linux-amd64.tar.gz
ln -s flannel-v0.11.0/ flannel
# 下载证书
cd /opt/flannel
mkdir cert;cd cert
# 上传
chmod 600 *key.pem
# 配置
cat > /opt/flannel/subnet.env <<\EOF
FLANNEL_NETWORK=172.7.0.0/16
FLANNEL_SUBNET=172.7.18.1/24
FLANNEL_MTU=1500
FLANNEL_IPMASQ=false
EOF
cat > /opt/flannel/flanneld.sh <<\EOF
#!/bin/sh
./flanneld \
--public-ip=192.168.18.18 \
--etcd-endpoints=https://192.168.18.51:2379,https://192.168.18.18:2379,https://192.168.18.24:2379 \
--etcd-keyfile=./cert/client-key.pem \
--etcd-certfile=./cert/client.pem \
--etcd-cafile=./cert/ca.pem \
--iface=eth0 \
--subnet-file=./subnet.env \
--healthz-port=2401
EOF
chmod +x /opt/flannel/flanneld.sh
mkdir -p /data/logs/flanneld
cat > /etc/supervisord.d/flanneld.ini <<\EOF
[program:flanneld]
command=/opt/flannel/flanneld.sh ; the program (relative uses PATH, can take args)
numprocs=1 ; number of processes copies to start (def 1)
directory=/opt/flannel ; directory to cwd to before exec (def no cwd)
autostart=true ; start at supervisord start (default: true)
autorestart=true ; retstart at unexpected quit (default: true)
startsecs=22 ; number of secs prog must stay running (def. 1)
startretries=3 ; max # of serial start failures (default 3)
exitcodes=0,2 ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT ; signal used to kill process (default TERM)
stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10)
user=root ; setuid to this UNIX account to run the program
redirect_stderr=false ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/flanneld/flanneld.stdout.log ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4 ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false ; emit events on stdout writes (default false)
stderr_logfile=/data/logs/flanneld/flanneld.stderr.log ; stderr log path, NONE for none; default AUTO
stderr_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stderr_logfile_backups=4 ; # of stderr logfile backups (default 10)
stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stderr_events_enabled=false ; emit events on stderr writes (default false)
EOF
supervisorctl update
supervisorctl status
coredns
apiVersion: v1
kind: ServiceAccount
metadata:
name: coredns
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
kubernetes.io/bootstrapping: rbac-defaults
addonmanager.kubernetes.io/mode: Reconcile
name: system:coredns
rules:
- apiGroups:
- ""
resources:
- endpoints
- services
- pods
- namespaces
verbs:
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
labels:
kubernetes.io/bootstrapping: rbac-defaults
addonmanager.kubernetes.io/mode: EnsureExists
name: system:coredns
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:coredns
subjects:
- kind: ServiceAccount
name: coredns
namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
name: coredns
namespace: kube-system
data:
Corefile: |
.:53 {
errors
log
health
ready
kubernetes cluster.local 192.168.0.0/16
forward . 100.100.2.136
cache 30
loop
reload
loadbalance
}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: coredns
namespace: kube-system
labels:
k8s-app: coredns
kubernetes.io/name: "CoreDNS"
spec:
replicas: 2
selector:
matchLabels:
k8s-app: coredns
template:
metadata:
labels:
k8s-app: coredns
spec:
priorityClassName: system-cluster-critical
serviceAccountName: coredns
containers:
- name: coredns
image: registry.cn-hangzhou.aliyuncs.com/devvops/coredns:v1.6.1
args:
- -conf
- /etc/coredns/Corefile
volumeMounts:
- name: config-volume
mountPath: /etc/coredns
ports:
- containerPort: 53
name: dns
protocol: UDP
- containerPort: 53
name: dns-tcp
protocol: TCP
- containerPort: 9153
name: metrics
protocol: TCP
livenessProbe:
httpGet:
path: /health
port: 8080
scheme: HTTP
initialDelaySeconds: 60
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
dnsPolicy: Default
volumes:
- name: config-volume
configMap:
name: coredns
items:
- key: Corefile
path: Corefile
---
apiVersion: v1
kind: Service
metadata:
name: coredns
namespace: kube-system
labels:
k8s-app: coredns
kubernetes.io/cluster-service: "true"
kubernetes.io/name: "CoreDNS"
spec:
selector:
k8s-app: coredns
clusterIP: 192.168.0.2
ports:
- name: dns
port: 53
protocol: UDP
- name: dns-tcp
port: 53
- name: metrics
port: 9153
protocol: TCP
检查
# 在宿主机使用coredns的clusterip作为dns解析 dig -t A www.baidu.com @192.168.0.2 # 进入一个容器查看解析规则 cat /etc/resolv.conf nameserver 192.168.0.2 search default.svc.cluster.local svc.cluster.local cluster.local node.consul options ndots:5 # 外网解析 ping baidu.com PING baidu.com (220.181.38.148): 56 data bytes 64 bytes from 220.181.38.148: seq=0 ttl=49 time=33.983 ms # svc域名解析 ping nginx-ds PING nginx-ds (192.168.57.205): 56 data bytes 64 bytes from 192.168.57.205: seq=0 ttl=64 time=0.072 ms 64 bytes from 192.168.57.205: seq=1 ttl=64 time=0.064 ms
kubeadmin
https://blog.csdn.net/wenjianfeng/article/details/93347306
https://www.cnblogs.com/python-cat/p/10831419.html
master部署
前提条件：docker已部署
添加yum源
cat > /etc/yum.repos.d/kubernetes.repo <<\EOF [Kubernetes] name=Kubernetes baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64 enabled=1 gpgcheck=1 repo_gpgcheck=1 gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg EOF
安装
kubelet 运行在 Cluster 所有节点上，负责启动 Pod 和容器。
kubeadm 用于初始化 Cluster。
kubectl 是 Kubernetes 命令行工具。通过 kubectl 可以部署和管理应用，查看各种资源，创建、删除和更新各种组件。
yum install kubelet kubeadm kubectl -y
检查swap和桥接
cat /proc/sys/net/bridge/bridge-nf-call-iptables
1
free -m
total used free shared buff/cache available
Mem: 991 151 365 7 475 674
Swap: 0 0 0
初始化
--image-repository string：这个用于指定从什么位置来拉取镜像
--kubernetes-version string：指定kubenets版本号（看一下之前安装的版本）
--apiserver-advertise-address 指明用 Master 的哪个 interface 与 Cluster 的其他节点通信。如果 Master 有多个 interface，建议明确指定，如果不指定，kubeadm 会自动选择有默认网关的 interface。
--pod-network-cidr指定 Pod 网络的范围。Kubernetes 支持多种网络方案，而且不同网络方案对 --pod-network-cidr有自己的要求，这里设置为10.244.0.0/16 是因为我们将使用 flannel 网络方案，必须设置成这个 CIDR。
kubeadm init --image-repository registry.aliyuncs.com/google_containers --kubernetes-version v1.18.2 --apiserver-advertise-address 10.0.0.80 --pod-network-cidr=10.244.0.0/16
成功的话是以下信息，按照提示，先配置集群信息
Your Kubernetes control-plane has initialized successfully!
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
Then you can join any number of worker nodes by running the following on each as root:
kubeadm join 10.0.0.80:6443 --token 7yasz7.oouxo59ravh4eitz \
--discovery-token-ca-cert-hash sha256:545678e30451193c605e170ba3f8b8e500b9a5c7d0bb54bd00eb7c7113a4308d
配置完，kubectl就能用了
获取组件状态
kubectl get cs
NAME STATUS MESSAGE ERROR
scheduler Healthy ok
etcd-0 Healthy {"health":"true"}
controller-manager Healthy ok
为了使用更便捷，启用 kubectl 命令的自动补全功能。
echo "source <(kubectl completion bash)" >> ~/.bashrc
配置flannel网络
要让 Kubernetes Cluster 能够工作，必须安装 Pod 网络，否则 Pod 之间无法通信。
Kubernetes 支持多种网络方案，这里我们先使用 flannel，后面还会讨论 Canal。
# kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
部署node
cat > /etc/yum.repos.d/kubernetes.repo <<\EOF [Kubernetes] name=Kubernetes baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64 enabled=1 gpgcheck=1 repo_gpgcheck=1 gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg EOF
yum install kubelet kubeadm kubectl -y
echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables
node加入集群使用的token，默认是有有效期的，过期之后node就需要使用新的token
# kubeadm token create W0602 14:40:46.337640 28438 configset.go:202] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io] x56kjm.7e6t3iz2x9y3d93t
# kubeadm token list TOKEN TTL EXPIRES USAGES DESCRIPTION EXTRA GROUPS yjd9de.axl9y80ztcjqevce 23h 2020-06-03T14:22:48+08:00 authentication,signing <none> system:bootstrappers:kubeadm:default-node-token
kubeadm join 10.0.0.80:6443 --token 7yasz7.oouxo59ravh4eitz \
--discovery-token-ca-cert-hash sha256:545678e30451193c605e170ba3f8b8e500b9a5c7d0bb54bd00eb7c7113a4308d
systemctl restart kubelet systemctl enable kubelet
