kubernetes-二进制部署

systemctl restart kube-proxy
systemctl restart kubelet

💗、单点安装文档：
https://k8s.abcdocker.com/

💗、服务器互信，免密钥登陆：
for i in 192.168.2.9;do

ssh-keygen -t rsa -P "" -f /root/.ssh/id_rsa

expect -c "
spawn ssh-copy-id -i /root/.ssh/id_rsa.pub root@192.168.2.9
expect {
"yes/no" {send "yes\r"; exp_continue}
"password" {send "666666\r"; exp_continue}
"Password" {send "666666\r";}
} "
done
💗、关闭分区：
swapoff -a

💗、关闭防火墙：
systemctl stop firewalld
systemctl disable firewalld
setenforce 0
sed -i '/SELINUX/s/enforcing/disabled/' /etc/selinux/config

💗、测试etcd是否可用：
export ETCDCTL_API=3
etcdctl --cacert=/etc/etcd/ssl/etcd-root-ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints=https://192.168.211.7:2379 endpoint health

💗、kubernetes证书目录：
[root@k8s-master-01 ~]# cd /root/kubernets_ssl
[root@k8s-master-01 kubernets_ssl]# ls
admin.csr admin.pem k8s-gencert.json k8s-root-ca-key.pem kube-proxy-csr.json kube-proxy.pem kubernetes-csr.json token.csv
admin-csr.json audit-policy.yaml k8s-root-ca.csr k8s-root-ca.pem kube-proxy-key.pem kubernetes.csr kubernetes-key.pem
admin-key.pem bootstrap.kubeconfig k8s-root-ca-csr.json kube-proxy.csr kube-proxy.kubeconfig kubernetes-csr.jso kubernetes.pem

💗、创建证书：
[root@k8s-master-01 kubernets_ssl]# ls
admin-csr.json k8s-gencert.json k8s-root-ca-csr.json kube-proxy-csr.json kubernetes-csr.json

💗、生成kubernetes证书：
[root@k8s-master-01 kubernets_ssl]# cfssl gencert --initca=true k8s-root-ca-csr.json | cfssljson --bare k8s-root-ca
[root@k8s-master-01 kubernets_ssl]# ls
admin-csr.json k8s-gencert.json k8s-root-ca.csr k8s-root-ca-csr.json k8s-root-ca-key.pem k8s-root-ca.pem kube-proxy-csr.json kubernetes-csr.json
[root@k8s-master-01 kubernets_ssl]# ls
admin.csr admin-key.pem k8s-gencert.json k8s-root-ca-csr.json k8s-root-ca.pem kube-proxy-csr.json kube-proxy.pem kubernetes-csr.json kubernetes.pem
admin-csr.json admin.pem k8s-root-ca.csr k8s-root-ca-key.pem kube-proxy.csr kube-proxy-key.pem kubernetes.csr kubernetes-key.pem

💗、生成boostrap配置：
${BOOTSTRAP_TOKEN},kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF
[root@k8s-master-01 kubernets_ssl]# ls
admin.csr admin-key.pem k8s-gencert.json k8s-root-ca-csr.json k8s-root-ca.pem kube-proxy-csr.json kube-proxy.pem kubernetes-csr.json kubernetes.pem
admin-csr.json admin.pem k8s-root-ca.csr k8s-root-ca-key.pem kube-proxy.csr kube-proxy-key.pem kubernetes.csr kubernetes-key.pem token.csv

💗、log日志：
[root@k8s-master-01 kubernets_ssl]# mkdir -p /var/log/kube-audit /usr/libexec/kubernetes
[root@k8s-master-01 kubernets_ssl]# chown -R kube:kube /var/log/kube-audit /usr/libexec/kubernetes
[root@k8s-master-01 kubernets_ssl]# chmod -R 755 /var/log/kube-audit /usr/libexec/kubernetes

💗、分发证书到此目录：
[root@k8s-master-01 kubernets_ssl]# cd /etc/kubernetes/ssl
[root@k8s-master-01 ssl]# ls
admin-key.pem k8s-root-ca-key.pem kubelet-client-2018-08-14-11-15-40.pem kubelet.crt kube-proxy-key.pem kubernetes-key.pem
admin.pem k8s-root-ca.pem kubelet-client-current.pem kubelet.key kube-proxy.pem kubernetes.pem

💗、启动kubelet and Kube-proxy：
systemctl daemon-reload
systemctl restart kube-proxy
systemctl restart kubelet

💗、查看日志：
journalctl -fu kubelet

💗、查看note节点：
[root@k8s-master-01 kubernetes]# kubectl get csr
NAME AGE REQUESTOR CONDITION
node-csr-rSDdXM7PdhA6iFPhcr4MVqadkQ8VrGQF1ZRnqczfy04 1m kubelet-bootstrap Pending

💗、从Mster节点上将hyperkuber kubelet kubectl kube-proxy 拷贝至node上
for i in hyperkube kubelet kubectl kube-proxy;do
scp ./kubernetes/server/bin/$i 192.168.2.9:/usr/bin/
ssh 192.168.2.8 chmod 755 /usr/bin/$i
done

💗、分发K8s证书：
cd /root/kubernets_ssl/
for IP in 192.168.2.9;do
ssh $IP mkdir -p /etc/kubernetes/ssl
scp *.pem $IP:/etc/kubernetes/ssl
scp *.kubeconfig token.csv audit-policy.yaml $IP:/etc/kubernetes
ssh $IP useradd -s /sbin/nologin/ kube
ssh $IP chown -R kube:kube /etc/kubernetes/ssl
done

💗、分发ETCD证书：
for IP in 192.168.2.8 192.168.2.9;do
cd /root/etcd_ssl
ssh $IP mkdir -p /etc/etcd/ssl
scp .pem $IP:/etc/etcd/ssl
ssh $IP chmod -R 644 /etc/etcd/ssl/
ssh $IP chmod 755 /etc/etcd/ssl
done

💗、note节点创建 nginx 代理：
mkdir -p /etc/nginx
cat > /etc/nginx/nginx.conf <<EOF
error_log stderr notice;

worker_processes auto;
events {
multi_accept on;
use epoll;
worker_connections 1024;
}

stream {
upstream kube_apiserver {
least_conn;
server 192.168.2.7:6443 weight=20 max_fails=1 fail_timeout=10s;
#server中代理master的IP
}

server {
    listen        0.0.0.0:6443;
    proxy_pass    kube_apiserver;
    proxy_timeout 10m;
    proxy_connect_timeout 1s;
}

}
EOF

servcer 中代理的ip应该是master中的apiserver端口

chmod +r /etc/nginx/nginx.conf

docker run -it -d -p 127.0.0.1:6443:6443 -v /etc/nginx:/etc/nginx --name nginx-proxy --net=host --restart=on-failure:5 --memory=512M nginx:1.13.5-alpine

💗、为了保证 nginx 的可靠性，综合便捷性考虑，node 节点上的 nginx 使用 docker 启动，同时 使用 systemd 来守护：
cat >/etc/systemd/system/nginx-proxy.service <<EOF
[Unit]
Description=kubernetes apiserver docker wrapper
Wants=docker.socket
After=docker.service

[Service]
User=root
PermissionsStartOnly=true
ExecStart=/usr/bin/docker start nginx-proxy
Restart=always
RestartSec=15s
TimeoutStartSec=30s

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl start nginx-proxy
systemctl enable nginx-proxy

💗、查看master端口:
sed -i 's#192.168.2.7#127.0.0.1#g' /etc/kubernetes/bootstrap.kubeconfig
查看端口：
netstat -antup |grep 6443

💗、启动kubelet之前最好将kube-proxy重启：
systemctl restart kube-proxy
systemctl enable kubelet

systemctl daemon-reload
systemctl restart kubelet
systemctl enable kubelet

💗、允许证书申请：
查看证书申请：
[root@k8s-master-01 etcd_ssl]# kubectl get csr
NAME AGE REQUESTOR CONDITION
node-csr-epAWxTV6LsmpZZDmYstiCd4r5ZvRtBxWC5vjh6AljKs 57m kubelet-bootstrap Pending
node-csr-ftu-E8KytasffNiZ3sNnxgz7OZDVT2uO8IkdRgKGkJA 57m kubelet-bootstrap Pending
node-csr-rSDdXM7PdhA6iFPhcr4MVqadkQ8VrGQF1ZRnqczfy04 1h kubelet-bootstrap Pending
签发证书：
[root@k8s-master-01 etcd_ssl]# kubectl get csr | grep Pending | awk '{print $1}' | xargs kubectl certificate approve
certificatesigningrequest.certificates.k8s.io/node-csr-epAWxTV6LsmpZZDmYstiCd4r5ZvRtBxWC5vjh6AljKs approved
certificatesigningrequest.certificates.k8s.io/node-csr-ftu-E8KytasffNiZ3sNnxgz7OZDVT2uO8IkdRgKGkJA approved
certificatesigningrequest.certificates.k8s.io/node-csr-rSDdXM7PdhA6iFPhcr4MVqadkQ8VrGQF1ZRnqczfy04 approved
查看note节点：
[root@k8s-master-01 etcd_ssl]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master-01 Ready 2m v1.11.2
k8s-node-02 Ready 2m v1.11.2
k8s-node-03 Ready 2m v1.11.2
master NotReady 2d v1.11.2
node NotReady 1d v1.11.2
💗、认证后自动生成了kubelet kubeconfig 文件和公私钥：
[root@k8s-master-01 etcd_ssl]# ls -l /etc/kubernetes/ssl/kubelet*
-rw-------. 1 root root 1622 8月 16 12:41 /etc/kubernetes/ssl/kubelet-client-2018-08-16-12-41-58.pem
lrwxrwxrwx. 1 root root 58 8月 16 12:41 /etc/kubernetes/ssl/kubelet-client-current.pem -> /etc/kubernetes/ssl/kubelet-client-2018-08-16-12-41-58.pem
-rw-r--r--. 1 root root 2197 8月 16 11:18 /etc/kubernetes/ssl/kubelet.crt
-rw-------. 1 root root 1675 8月 16 11:18 /etc/kubernetes/ssl/kubelet.key

注意：

apiserver如果不启动后续没法操作
kubelet里面配置的IP地址都是本机（master配置node）
Node服务上先启动nginx-proxy在启动kube-proxy。kube-proxy里面地址配置本机127.0.0.1:6443实际上就是master:6443