一键部署高可用RancherServer
环境说明
四台节点
123 nginx
124 node1
125 node2
126 node3
nginx节点实现自动ssh免密登录
建立免密登录连接不能使用root账号 必须在所有节点上创建一个相同名称的普通账号,同时给这个普通账号sudo权限同时设置执行sudo的时候不需要输入密码
在编写脚本的时候用户的家目录最后不要~ 最好指定为绝对路径 这样可以避免一些错误
centos用户建立ssh免密连接
不能直接root用户创建公私钥,必须要创建一个普通用户
普通用户执行docker命令
sudo groupadd docker #添加docker用户组
sudo gpasswd - a $USER docker #将当前用户添加至docker用户组
执行 groupadd docker的时候提示已经存在,所以后面的两条指令都不会执行导致后面在执行docker指令的时候出现异常
普通用户没有sudo权限
通过rke部署k8s集群
#!/usr/bin/bash echo "安装shell模块expect" sudo yum -y install expect echo "安装expect模块成功" echo "安装ntpdate" sudo yum install -y ntpdate echo "开始安装yml配置读取模块" function parse_yaml() { local yaml_file=$1 local prefix=$2 local s local w local fs s='[[:space:]]*' w='[a-zA-Z0-9_.-]*' fs="$(echo @|tr @ '\034')" ( sed -ne '/^--/s|--||g; s|\"|\\\"|g; s/\s*$//g;' \ -e "/#.*[\"\']/!s| #.*||g; /^#/s|#.*||g;" \ -e "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \ -e "s|^\($s\)\($w\)$s[:-]$s\(.*\)$s\$|\1$fs\2$fs\3|p" | awk -F"$fs" '{ indent = length($1)/2; if (length($2) == 0) { conj[indent]="+";} else {conj[indent]="";} vname[indent] = $2; for (i in vname) {if (i > indent) {delete vname[i]}} if (length($3) > 0) { vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")} printf("%s%s%s%s=(\"%s\")\n", "'"$prefix"'",vn, $2, conj[indent-1],$3); } }' | sed -e 's/_=/+=/g' \ -e '/\..*=/s|\.|_|' \ -e '/\-.*=/s|\-|_|' ) < "$yaml_file" } function create_variables() { local yaml_file="$1" eval "$(parse_yaml "$yaml_file")" } create_variables config/system-init.yml echo "yml配置读取模块成功" echo "获取yml的配置信息" echo $fqdn echo $nginxIp echo $rancher1Ip echo $rancher2Ip echo $rancher3Ip echo $userpasswd echo $sshuser #------------------------------------------# # 建立ssh连接互信 #------------------------------------------# echo '开始建立ssh互信连接' if [ ! -f ~/.ssh/id_rsa ];then ssh-keygen -t rsa -P "" -f ~/.ssh/id_rsa else echo "id_rsa has created ..." fi expect <<EOF set timeout 10 spawn ssh-copy-id -i /home/$sshuser/.ssh/id_rsa.pub $sshuser@$rancher1Ip expect { "yes/no" { send "yes\n";exp_continue} "password" { send "$userpasswd\n"} } spawn ssh-copy-id -i /home/$sshuser/.ssh/id_rsa.pub $sshuser@$rancher2Ip expect { "yes/no" { send "yes\n";exp_continue} "password" { send "$userpasswd\n"} } spawn ssh-copy-id -i /home/$sshuser/.ssh/id_rsa.pub $sshuser@$rancher3Ip expect { "yes/no" { send "yes\n";exp_continue} "password" { send "$userpasswd\n"} } expect eof EOF echo '建立ssh互信连接成功' hosts=("$rancher1Ip" "$rancher2Ip" "$rancher3Ip") echo '同步节点时间' sudo ntpdate time1.aliyun.com for ip in ${hosts[@]};do ssh $sshuser@$ip "yum install -y ntpdate" ssh $sshuser@$ip "sudo ntpdate time1.aliyun.com" done echo '同步节点时间成功' echo '配置nameserver' cp -r config/resolv.conf /etc/resolv.conf for ip in ${hosts[@]};do scp -r config/resolv.conf $sshuser@$ip:~/resolv.conf ssh $sshuser@$ip "sudo cp ~/resolv.conf /etc/resolv.conf" done echo '配置nameserver成功' echo "设置开放端口" sudo firewall-cmd --permanent --add-port=80/tcp sudo firewall-cmd --permanent --add-port=443/tcp sudo firewall-cmd --reload for ip in ${hosts[@]};do ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=22/tcp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=80/tcp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=443/tcp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=2376/tcp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=2379/tcp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=2380/tcp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=6443/tcp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=8472/udp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=8472/tcp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=9099/tcp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=10250/tcp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=10254/tcp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=30000-32767/tcp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=30000-32767/udp" ssh $sshuser@$ip "sudo firewall-cmd --reload" done echo "设置端口结束" echo "禁用swap" for ip in ${hosts[@]};do ssh $sshuser@$ip "sudo swapoff -a" done echo "禁用swap结束" echo '开始给集群节点安装docker' for ip in ${hosts[@]};do scp -r pkg/docker/docker-ce.repo $sshuser@$ip:~/ ssh $sshuser@$ip "sudo cp ~/docker-ce.repo /etc/yum.repos.d/docker-ce.repo && sudo yum install -y docker-ce-18.09.3-3.el7" scp -r pkg/docker/daemon.json $sshuser@$ip:~/ ssh $sshuser@$ip "sudo groupadd docker && sudo gpasswd -a $sshuser docker" ssh $sshuser@$ip "sudo mkdir -p /etc/docker/ && sudo cp ~/daemon.json /etc/docker/daemon.json && sudo systemctl start docker" done echo '安装docker结束' echo '开始给集群节点加载镜像' for ip in ${hosts[@]};do scp -r images/ $sshuser@$ip:~/ ssh $sshuser@$ip "sudo docker load -i ~/images/autoscaler.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/calico-node.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/cni.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/coredns.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/defaultbackend.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/etcd.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/flannel.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/flexvol.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/hyperkube.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/ingress-controller.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/metrics.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/metrics-server.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/nginx.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/pause.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/rancher.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/rancher-agent.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/tools.tar" done echo '加载镜像结束' echo "安装kubectl" sudo cp pkg/kubectl/kubernets.repo /etc/yum.repos.d/kubernets.repo sudo yum install -y kubectl echo "kubectl安装结束" echo "安装rke" cp pkg/rke/rke ~/rke && sudo chmod +x ~/rke cp config/rancher-cluster.yml ~/rancher-cluster.yml sed -i "s/node1/$rancher1Ip/" ~/rancher-cluster.yml sed -i "s/node2/$rancher2Ip/" ~/rancher-cluster.yml sed -i "s/node3/$rancher3Ip/" ~/rancher-cluster.yml echo "rke安装结束" echo "安装k8s集群" ~/rke up --config=/home/admin/rancher-cluster.yml mkdir ~/.kube cp ~/kube_config_rancher-cluster.yml /home/admin/.kube/config echo "安装k8s集群成功"
在k8s上部署rancher server
#!/usr/bin/bash echo "开始安装yml配置读取模块" function parse_yaml() { local yaml_file=$1 local prefix=$2 local s local w local fs s='[[:space:]]*' w='[a-zA-Z0-9_.-]*' fs="$(echo @|tr @ '\034')" ( sed -ne '/^--/s|--||g; s|\"|\\\"|g; s/\s*$//g;' \ -e "/#.*[\"\']/!s| #.*||g; /^#/s|#.*||g;" \ -e "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \ -e "s|^\($s\)\($w\)$s[:-]$s\(.*\)$s\$|\1$fs\2$fs\3|p" | awk -F"$fs" '{ indent = length($1)/2; if (length($2) == 0) { conj[indent]="+";} else {conj[indent]="";} vname[indent] = $2; for (i in vname) {if (i > indent) {delete vname[i]}} if (length($3) > 0) { vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")} printf("%s%s%s%s=(\"%s\")\n", "'"$prefix"'",vn, $2, conj[indent-1],$3); } }' | sed -e 's/_=/+=/g' \ -e '/\..*=/s|\.|_|' \ -e '/\-.*=/s|\-|_|' ) < "$yaml_file" } function create_variables() { local yaml_file="$1" eval "$(parse_yaml "$yaml_file")" } create_variables config/system-init.yml echo "yml配置读取模块成功" echo "获取yml的配置信息" echo $fqdn echo $sshuser echo "开始生成自签名证书" mkdir -p /home/$sshuser/certs/ cp pkg/certs/create_self-signed-cert.sh /home/$sshuser/certs/ sudo chmod +x /home/$sshuser/certs/create_self-signed-cert.sh cd /home/$sshuser/certs/ && ./create_self-signed-cert.sh --ssl-domain=$fqdn echo "生成证书完毕" echo "创建secret" kubectl delete secret tls-rancher-ingress -n cattle-system kubectl delete secret tls-ca -n cattle-system kubectl create ns cattle-system kubectl -n cattle-system create secret tls tls-rancher-ingress --cert=/home/$sshuser/certs/tls.crt --key=/home/$sshuser/certs/tls.key kubectl -n cattle-system create secret generic tls-ca --from-file=/home/$sshuser/certs/cacerts.pem echo "创建secret结束" echo "helm安装rancher" sudo cp pkg/helm/helm /home/$sshuser/helm && sudo chmod +x /home/$sshuser/helm /home/$sshuser/helm repo add rancher-stable http://rancher-mirror.oss-cn-beijing.aliyuncs.com/server-charts/stable /home/$sshuser/helm repo update /home/$sshuser/helm install rancher rancher-stable/rancher --namespace cattle-system --set hostname=$fqdn --set ingress.tls.source=secret --set privateCA=true echo "helm安装rancher成功" echo "配置nginx" sudo cp pkg/nginx/nginx.repo /etc/yum.repos.d/nginx.repo sudo yum install -y nginx sudo cp pkg/nginx/nginx.conf /etc/nginx/nginx.conf sudo sed -i "s/node1/$rancher1Ip/" /etc/nginx/nginx.conf sudo sed -i "s/node2/$rancher2Ip/" /etc/nginx/nginx.conf sudo sed -i "s/node3/$rancher3Ip/" /etc/nginx/nginx.conf sudo systemctl start nginx echo "配置nginx结束"
回滚卸载残留文件
cat /proc/mounts | grep "kubelet" | awk '{print $2}' | xargs umount
for mount in $(mount | grep tmpfs | grep '/var/lib/kubelet' | awk '{print $3}') /var/lib/kubelet /var/lib/rancher; do umount $mount; done
kubelet里面的文件必须先进行卸载 然后才能进行删除
#!/bin/bash function parse_yaml() { local yaml_file=$1 local prefix=$2 local s local w local fs s='[[:space:]]*' w='[a-zA-Z0-9_.-]*' fs="$(echo @|tr @ '\034')" ( sed -ne '/^--/s|--||g; s|\"|\\\"|g; s/\s*$//g;' \ -e "/#.*[\"\']/!s| #.*||g; /^#/s|#.*||g;" \ -e "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \ -e "s|^\($s\)\($w\)$s[:-]$s\(.*\)$s\$|\1$fs\2$fs\3|p" | awk -F"$fs" '{ indent = length($1)/2; if (length($2) == 0) { conj[indent]="+";} else {conj[indent]="";} vname[indent] = $2; for (i in vname) {if (i > indent) {delete vname[i]}} if (length($3) > 0) { vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")} printf("%s%s%s%s=(\"%s\")\n", "'"$prefix"'",vn, $2, conj[indent-1],$3); } }' | sed -e 's/_=/+=/g' \ -e '/\..*=/s|\.|_|' \ -e '/\-.*=/s|\-|_|' ) < "$yaml_file" } function create_variables() { local yaml_file="$1" eval "$(parse_yaml "$yaml_file")" } create_variables config/system-init.yml echo "yml配置读取模块成功" echo "获取yml的配置信息" echo $rancher1Ip echo $rancher2Ip echo $rancher3Ip echo $userpasswd echo $sshuser read -p "确定要对此集群进行回滚操作?[y/n] " input case $input in [yY]*) #------------------------------------------# # 建立ssh连接互信 #------------------------------------------# echo 'rke删除k8s集群' /home/$sshuser/rke remove --config=/home/$sshuser/rancher-cluster.yml echo 'rke删除k8s集群结束' echo '开始建立ssh互信连接' if [ ! -f ~/.ssh/id_rsa ];then ssh-keygen -t rsa -P "" -f ~/.ssh/id_rsa else echo "id_rsa has created ..." fi expect <<EOF set timeout 10 spawn ssh-copy-id -i /home/$sshuser/.ssh/id_rsa.pub $sshuser@$rancher1Ip expect { "yes/no" { send "yes\n";exp_continue} "password" { send "$userpasswd\n"} } spawn ssh-copy-id -i /home/$sshuser/.ssh/id_rsa.pub $sshuser@$rancher2Ip expect { "yes/no" { send "yes\n";exp_continue} "password" { send "$userpasswd\n"} } spawn ssh-copy-id -i/home/$sshuser/.ssh/id_rsa.pub $sshuser@$rancher3Ip expect { "yes/no" { send "yes\n";exp_continue} "password" { send "$userpasswd\n"} } expect eof EOF echo '建立ssh互信连接成功' hosts=("$rancher1Ip" "$rancher2Ip" "$rancher3Ip") echo '开始清空集群节点' for ip in ${hosts[@]};do ssh $sshuser@$ip "docker rm -f $(docker ps -qa)" ssh $sshuser@$ip "docker rmi `docker images -q`" ssh $sshuser@$ip "for mount in $(mount | grep tmpfs | grep '/var/lib/kubelet' | awk '{ print $3 }') /var/lib/kubelet /var/lib/rancher; do umount $mount; done" ssh $sshuser@$ip "sudo rm -rf /etc/ceph \ /etc/etcd \ /etc/kubernetes \ /etc/cni \ /opt/cni \ /run/secrets/kubernetes.io \ /run/calico \ /run/flannel \ /var/lib/calico \ /var/lib/cni \ /var/lib/kubelet \ /var/lib/etcd \ /var/log/containers \ /var/log/pods \ /var/run/calico" done echo '清空集群节点成功' echo '卸载docker' # for ip in ${hosts[@]};do # echo $ip # ssh $sshuser@$ip "sudo yum remove -y docker-ce.x86_64" # ssh $sshuser@$ip "sudo yum remove -y containerd.io.x86_64" # ssh $sshuser@$ip "sudo yum remove -y docker-ce-cli.x86_64" # ssh $sshuser@$ip "sudo rm -fr /var/lib/docker" # ssh $sshuser@$ip "sudo rm -fr /etc/docker" # done echo '卸载docker完成' ;; [nN]*) exit ;; *) echo "输入非法,请输入y或者n" exit ;; esac
测试结束
telnet 端口不通并且在防火墙中已经放开
端口必须处于监听状态才能被连接
连接端口成功
docker镜像的导入导出
需要保存原有镜像的标签信息
docker save rancher/calico-cni:v3.13.4 -o cni.tar
docker save rancher/hyperkube:v1.18.3-rancher2 -o hyperkube.tar
批量导入镜像
for i in $(ls /home/admin/images)
do
docker load -i $i
done
本文来自博客园，作者：不懂123，转载请注明原文链接：https://www.cnblogs.com/yxh168/p/13380821.html
