一键部署高可用RancherServer

环境说明

      四台节点 

      123  nginx

      124  node1

      125  node2

      126  node3

nginx节点实现自动ssh免密登录

          建立免密登录连接不能使用root账号 必须在所有节点上创建一个相同名称的普通账号,同时给这个普通账号sudo权限同时设置执行sudo的时候不需要输入密码

          在编写脚本的时候用户的家目录最后不要~  最好指定为绝对路径 这样可以避免一些错误

centos用户建立ssh免密连接

    不能直接root用户创建公私钥,必须要创建一个普通用户

    

普通用户执行docker命令  

       sudo  groupadd  docker                        #添加docker用户组
       sudo  gpasswd -  a   $USER docker     #将当前用户添加至docker用户组

   

   

   执行 groupadd docker的时候提示已经存在,所以后面的两条指令都不会执行导致后面在执行docker指令的时候出现异常

    

普通用户没有sudo权限

    

      

通过rke部署k8s集群

#!/usr/bin/bash

echo "安装shell模块expect"
sudo yum -y install expect
echo "安装expect模块成功"

echo "安装ntpdate"
sudo yum install -y ntpdate

echo "开始安装yml配置读取模块"
function parse_yaml() {
    local yaml_file=$1
    local prefix=$2
    local s
    local w
    local fs
    s='[[:space:]]*'
    w='[a-zA-Z0-9_.-]*'
    fs="$(echo @|tr @ '\034')"
    (
        sed -ne '/^--/s|--||g; s|\"|\\\"|g; s/\s*$//g;' \
            -e "/#.*[\"\']/!s| #.*||g; /^#/s|#.*||g;" \
            -e  "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \
            -e "s|^\($s\)\($w\)$s[:-]$s\(.*\)$s\$|\1$fs\2$fs\3|p" |
        awk -F"$fs" '{
            indent = length($1)/2;
            if (length($2) == 0) { conj[indent]="+";} else {conj[indent]="";}
            vname[indent] = $2;
            for (i in vname) {if (i > indent) {delete vname[i]}}
                if (length($3) > 0) {
                    vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")}
                    printf("%s%s%s%s=(\"%s\")\n", "'"$prefix"'",vn, $2, conj[indent-1],$3);
                }
            }' |
        sed -e 's/_=/+=/g' \
            -e '/\..*=/s|\.|_|' \
            -e '/\-.*=/s|\-|_|'
    ) < "$yaml_file"
}
function create_variables() {
    local yaml_file="$1"
    eval "$(parse_yaml "$yaml_file")"
}

create_variables config/system-init.yml
echo "yml配置读取模块成功"


echo "获取yml的配置信息"
echo $fqdn
echo $nginxIp
echo $rancher1Ip
echo $rancher2Ip
echo $rancher3Ip
echo $userpasswd
echo $sshuser


#------------------------------------------#
# 建立ssh连接互信
#------------------------------------------#

echo '开始建立ssh互信连接'
if [ ! -f ~/.ssh/id_rsa ];then
 ssh-keygen -t rsa -P "" -f ~/.ssh/id_rsa
else
 echo "id_rsa has created ..."
fi

expect <<EOF
 set timeout 10 
 spawn ssh-copy-id -i /home/$sshuser/.ssh/id_rsa.pub $sshuser@$rancher1Ip
 expect {
                "yes/no" { send "yes\n";exp_continue}   
                "password" { send "$userpasswd\n"}
        }

 spawn ssh-copy-id -i /home/$sshuser/.ssh/id_rsa.pub $sshuser@$rancher2Ip
 expect {
                "yes/no" { send "yes\n";exp_continue}   
                "password" { send "$userpasswd\n"}
        }

 spawn ssh-copy-id -i /home/$sshuser/.ssh/id_rsa.pub $sshuser@$rancher3Ip
 expect {
                "yes/no" { send "yes\n";exp_continue}   
                "password" { send "$userpasswd\n"}
        }
 expect eof

EOF
 echo '建立ssh互信连接成功'

hosts=("$rancher1Ip" "$rancher2Ip" "$rancher3Ip")
echo '同步节点时间'
sudo  ntpdate time1.aliyun.com
for ip in ${hosts[@]};do
      ssh  $sshuser@$ip "yum install -y ntpdate"
      ssh  $sshuser@$ip "sudo ntpdate time1.aliyun.com"
done
echo '同步节点时间成功'

echo '配置nameserver'
cp -r config/resolv.conf /etc/resolv.conf
for ip in ${hosts[@]};do
    scp -r config/resolv.conf $sshuser@$ip:~/resolv.conf
    ssh $sshuser@$ip "sudo cp ~/resolv.conf /etc/resolv.conf"
done
echo '配置nameserver成功'


echo "设置开放端口"
    sudo firewall-cmd --permanent --add-port=80/tcp
    sudo firewall-cmd --permanent --add-port=443/tcp
    sudo firewall-cmd --reload
for ip in ${hosts[@]};do
   ssh  $sshuser@$ip "sudo firewall-cmd --permanent --add-port=22/tcp"
   ssh  $sshuser@$ip "sudo firewall-cmd --permanent --add-port=80/tcp"
   ssh  $sshuser@$ip "sudo firewall-cmd --permanent --add-port=443/tcp"
   ssh  $sshuser@$ip "sudo firewall-cmd --permanent --add-port=2376/tcp"
   ssh  $sshuser@$ip "sudo firewall-cmd --permanent --add-port=2379/tcp"
   ssh  $sshuser@$ip "sudo firewall-cmd --permanent --add-port=2380/tcp"
   ssh  $sshuser@$ip "sudo firewall-cmd --permanent --add-port=6443/tcp"
   ssh  $sshuser@$ip "sudo firewall-cmd --permanent --add-port=8472/udp"
   ssh  $sshuser@$ip "sudo firewall-cmd --permanent --add-port=8472/tcp"
   ssh  $sshuser@$ip "sudo firewall-cmd --permanent --add-port=9099/tcp"
   ssh  $sshuser@$ip "sudo firewall-cmd --permanent --add-port=10250/tcp"
   ssh  $sshuser@$ip "sudo firewall-cmd --permanent --add-port=10254/tcp"
   ssh  $sshuser@$ip "sudo firewall-cmd --permanent --add-port=30000-32767/tcp"
   ssh  $sshuser@$ip "sudo firewall-cmd --permanent --add-port=30000-32767/udp"
   ssh  $sshuser@$ip "sudo firewall-cmd --reload"
done
echo "设置端口结束"

echo "禁用swap"
for ip in ${hosts[@]};do
    ssh  $sshuser@$ip "sudo  swapoff -a"
done
echo "禁用swap结束"
   
echo '开始给集群节点安装docker'
for ip in ${hosts[@]};do
   scp -r pkg/docker/docker-ce.repo $sshuser@$ip:~/
   ssh  $sshuser@$ip "sudo cp ~/docker-ce.repo /etc/yum.repos.d/docker-ce.repo && sudo yum install -y docker-ce-18.09.3-3.el7"
   scp -r pkg/docker/daemon.json $sshuser@$ip:~/
   ssh  $sshuser@$ip "sudo groupadd docker && sudo gpasswd -a $sshuser docker"
   ssh  $sshuser@$ip "sudo mkdir -p /etc/docker/ && sudo cp ~/daemon.json /etc/docker/daemon.json && sudo systemctl start docker"
done
echo '安装docker结束'


echo '开始给集群节点加载镜像'
for ip in ${hosts[@]};do
  scp -r images/ $sshuser@$ip:~/
  ssh  $sshuser@$ip "sudo docker load -i ~/images/autoscaler.tar"
  ssh  $sshuser@$ip "sudo docker load -i ~/images/calico-node.tar"
  ssh  $sshuser@$ip "sudo docker load -i ~/images/cni.tar"
  ssh  $sshuser@$ip "sudo docker load -i ~/images/coredns.tar"
  ssh  $sshuser@$ip "sudo docker load -i ~/images/defaultbackend.tar"
  ssh  $sshuser@$ip "sudo docker load -i ~/images/etcd.tar"
  ssh  $sshuser@$ip "sudo docker load -i ~/images/flannel.tar"
  ssh  $sshuser@$ip "sudo docker load -i ~/images/flexvol.tar"
  ssh  $sshuser@$ip "sudo docker load -i ~/images/hyperkube.tar"
  ssh  $sshuser@$ip "sudo docker load -i ~/images/ingress-controller.tar"
  ssh  $sshuser@$ip "sudo docker load -i ~/images/metrics.tar"
  ssh  $sshuser@$ip "sudo docker load -i ~/images/metrics-server.tar"
  ssh  $sshuser@$ip "sudo docker load -i ~/images/nginx.tar"
  ssh  $sshuser@$ip "sudo docker load -i ~/images/pause.tar"
  ssh  $sshuser@$ip "sudo docker load -i ~/images/rancher.tar"
  ssh  $sshuser@$ip "sudo docker load -i ~/images/rancher-agent.tar"
  ssh  $sshuser@$ip "sudo docker load -i ~/images/tools.tar"
done
echo '加载镜像结束'


echo "安装kubectl"
  sudo cp pkg/kubectl/kubernets.repo /etc/yum.repos.d/kubernets.repo
  sudo yum install -y kubectl
echo "kubectl安装结束"

echo "安装rke"
  cp pkg/rke/rke ~/rke && sudo chmod +x ~/rke
  cp config/rancher-cluster.yml ~/rancher-cluster.yml
  sed -i "s/node1/$rancher1Ip/" ~/rancher-cluster.yml
  sed -i "s/node2/$rancher2Ip/" ~/rancher-cluster.yml
  sed -i "s/node3/$rancher3Ip/" ~/rancher-cluster.yml
echo "rke安装结束"

echo "安装k8s集群"
  ~/rke up --config=/home/admin/rancher-cluster.yml
  mkdir ~/.kube
  cp ~/kube_config_rancher-cluster.yml  /home/admin/.kube/config
echo "安装k8s集群成功"

在k8s上部署rancher server

#!/usr/bin/bash
echo "开始安装yml配置读取模块"
function parse_yaml() {
    local yaml_file=$1
    local prefix=$2
    local s
    local w
    local fs
    s='[[:space:]]*'
    w='[a-zA-Z0-9_.-]*'
    fs="$(echo @|tr @ '\034')"
    (
        sed -ne '/^--/s|--||g; s|\"|\\\"|g; s/\s*$//g;' \
            -e "/#.*[\"\']/!s| #.*||g; /^#/s|#.*||g;" \
            -e  "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \
            -e "s|^\($s\)\($w\)$s[:-]$s\(.*\)$s\$|\1$fs\2$fs\3|p" |
        awk -F"$fs" '{
            indent = length($1)/2;
            if (length($2) == 0) { conj[indent]="+";} else {conj[indent]="";}
            vname[indent] = $2;
            for (i in vname) {if (i > indent) {delete vname[i]}}
                if (length($3) > 0) {
                    vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")}
                    printf("%s%s%s%s=(\"%s\")\n", "'"$prefix"'",vn, $2, conj[indent-1],$3);
                }
            }' |
        sed -e 's/_=/+=/g' \
            -e '/\..*=/s|\.|_|' \
            -e '/\-.*=/s|\-|_|'
    ) < "$yaml_file"
}
function create_variables() {
    local yaml_file="$1"
    eval "$(parse_yaml "$yaml_file")"
}

create_variables config/system-init.yml
echo "yml配置读取模块成功"

echo "获取yml的配置信息"
echo $fqdn
echo $sshuser


echo "开始生成自签名证书"
   mkdir -p /home/$sshuser/certs/
   cp pkg/certs/create_self-signed-cert.sh /home/$sshuser/certs/
   sudo chmod +x /home/$sshuser/certs/create_self-signed-cert.sh
   cd /home/$sshuser/certs/ &&   ./create_self-signed-cert.sh --ssl-domain=$fqdn
echo "生成证书完毕"

echo "创建secret"
    kubectl delete secret tls-rancher-ingress -n cattle-system
    kubectl delete secret tls-ca -n cattle-system
    kubectl create ns cattle-system
    kubectl -n cattle-system create secret tls tls-rancher-ingress --cert=/home/$sshuser/certs/tls.crt --key=/home/$sshuser/certs/tls.key
    kubectl -n cattle-system create secret generic tls-ca --from-file=/home/$sshuser/certs/cacerts.pem
echo "创建secret结束"

echo "helm安装rancher"
     sudo cp pkg/helm/helm /home/$sshuser/helm && sudo chmod +x /home/$sshuser/helm
     /home/$sshuser/helm repo add rancher-stable http://rancher-mirror.oss-cn-beijing.aliyuncs.com/server-charts/stable
     /home/$sshuser/helm repo update
     /home/$sshuser/helm install rancher rancher-stable/rancher  --namespace cattle-system  --set hostname=$fqdn --set ingress.tls.source=secret --set privateCA=true
echo "helm安装rancher成功"

echo "配置nginx"
     sudo cp pkg/nginx/nginx.repo /etc/yum.repos.d/nginx.repo
     sudo yum install -y nginx
     sudo cp pkg/nginx/nginx.conf /etc/nginx/nginx.conf
     sudo sed -i "s/node1/$rancher1Ip/" /etc/nginx/nginx.conf
     sudo sed -i "s/node2/$rancher2Ip/" /etc/nginx/nginx.conf
     sudo sed -i "s/node3/$rancher3Ip/" /etc/nginx/nginx.conf
     sudo systemctl start nginx
echo "配置nginx结束"

回滚卸载残留文件

        cat /proc/mounts | grep "kubelet" | awk '{print $2}' | xargs umount

        for mount in $(mount | grep tmpfs | grep '/var/lib/kubelet' | awk '{print $3}') /var/lib/kubelet /var/lib/rancher; do umount $mount; done

        kubelet里面的文件必须先进行卸载 然后才能进行删除

       

#!/bin/bash

function parse_yaml() {
    local yaml_file=$1
    local prefix=$2
    local s
    local w
    local fs
    s='[[:space:]]*'
    w='[a-zA-Z0-9_.-]*'
    fs="$(echo @|tr @ '\034')"
    (
        sed -ne '/^--/s|--||g; s|\"|\\\"|g; s/\s*$//g;' \
            -e "/#.*[\"\']/!s| #.*||g; /^#/s|#.*||g;" \
            -e  "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \
            -e "s|^\($s\)\($w\)$s[:-]$s\(.*\)$s\$|\1$fs\2$fs\3|p" |
        awk -F"$fs" '{
            indent = length($1)/2;
            if (length($2) == 0) { conj[indent]="+";} else {conj[indent]="";}
            vname[indent] = $2;
            for (i in vname) {if (i > indent) {delete vname[i]}}
                if (length($3) > 0) {
                    vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")}
                    printf("%s%s%s%s=(\"%s\")\n", "'"$prefix"'",vn, $2, conj[indent-1],$3);
                }
            }' |
        sed -e 's/_=/+=/g' \
            -e '/\..*=/s|\.|_|' \
            -e '/\-.*=/s|\-|_|'
    ) < "$yaml_file"
}

function create_variables() {
    local yaml_file="$1"
    eval "$(parse_yaml "$yaml_file")"
}

create_variables config/system-init.yml
echo "yml配置读取模块成功"


echo "获取yml的配置信息"
echo $rancher1Ip
echo $rancher2Ip
echo $rancher3Ip
echo $userpasswd
echo $sshuser


read -p "确定要对此集群进行回滚操作?[y/n] " input

case $input in 
    [yY]*)
#------------------------------------------#
# 建立ssh连接互信
#------------------------------------------#

echo 'rke删除k8s集群'
 /home/$sshuser/rke remove --config=/home/$sshuser/rancher-cluster.yml
echo 'rke删除k8s集群结束'

echo '开始建立ssh互信连接'
if [ ! -f ~/.ssh/id_rsa ];then
 ssh-keygen -t rsa -P "" -f ~/.ssh/id_rsa
else
 echo "id_rsa has created ..."
fi

expect <<EOF
 set timeout 10 
 spawn ssh-copy-id -i /home/$sshuser/.ssh/id_rsa.pub $sshuser@$rancher1Ip
 expect {
                "yes/no" { send "yes\n";exp_continue}   
                "password" { send "$userpasswd\n"}
        }

 spawn ssh-copy-id -i /home/$sshuser/.ssh/id_rsa.pub $sshuser@$rancher2Ip
 expect {
                "yes/no" { send "yes\n";exp_continue}   
                "password" { send "$userpasswd\n"}
        }

 spawn ssh-copy-id -i/home/$sshuser/.ssh/id_rsa.pub $sshuser@$rancher3Ip
 expect {
                "yes/no" { send "yes\n";exp_continue}   
                "password" { send "$userpasswd\n"}
        }
 expect eof

EOF
 echo '建立ssh互信连接成功'

hosts=("$rancher1Ip" "$rancher2Ip" "$rancher3Ip")

echo '开始清空集群节点'
for ip in ${hosts[@]};do
    ssh $sshuser@$ip "docker rm -f $(docker ps -qa)"
    ssh $sshuser@$ip "docker rmi `docker images -q`"
    ssh $sshuser@$ip "for mount in $(mount | grep tmpfs | grep '/var/lib/kubelet' | awk '{ print $3 }') /var/lib/kubelet /var/lib/rancher; do umount $mount; done"
    ssh $sshuser@$ip "sudo rm -rf /etc/ceph \
                       /etc/etcd \
                       /etc/kubernetes \
                       /etc/cni \
                       /opt/cni \
                       /run/secrets/kubernetes.io \
                       /run/calico \
                       /run/flannel \
                       /var/lib/calico \
                       /var/lib/cni \
                       /var/lib/kubelet \
                       /var/lib/etcd \
                       /var/log/containers \
                       /var/log/pods \
                       /var/run/calico"
done
echo '清空集群节点成功'

echo '卸载docker'
# for ip in ${hosts[@]};do
#     echo $ip
#     ssh $sshuser@$ip "sudo yum remove -y docker-ce.x86_64"
#     ssh $sshuser@$ip "sudo yum remove -y containerd.io.x86_64"
#     ssh $sshuser@$ip "sudo yum remove -y docker-ce-cli.x86_64"
#     ssh $sshuser@$ip "sudo rm -fr /var/lib/docker"
#     ssh $sshuser@$ip "sudo rm -fr /etc/docker"
# done
echo '卸载docker完成'
;;
[nN]*)
  exit
  ;;
*)
  echo "输入非法,请输入y或者n"
  exit
  ;;
esac

 测试结束

telnet 端口不通并且在防火墙中已经放开

    

端口必须处于监听状态才能被连接

     

     连接端口成功

docker镜像的导入导出

         需要保存原有镜像的标签信息

    

   docker  save  rancher/calico-cni:v3.13.4 -o cni.tar

   docker  save  rancher/hyperkube:v1.18.3-rancher2 -o hyperkube.tar

   批量导入镜像 

   for i in $(ls /home/admin/images)
     do
           docker load -i $i
    done