二进制部署etcd-三个集群方案

etcd的二进制部署

还有什么问题，想咨询的，加群：582337768。 这个群不是我的，但是我在里面，但是还是那句话，我也不懂。

三个节点信息

node01ip=192.168.1.11
node02ip=192.168.1.12
node03ip=192.168.1.13

创建证书

# 下载制作证书的二进制文件
wget --no-check-certificate https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
chmod +x cfssl_linux-amd64
mv cfssl_linux-amd64 /usr/local/bin/cfssl

wget --no-check-certificate https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
chmod +x cfssljson_linux-amd64
mv cfssljson_linux-amd64 /usr/local/bin/cfssljson

wget --no-check-certificate https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
chmod +x cfssl-certinfo_linux-amd64
mv cfssl-certinfo_linux-amd64 /usr/local/bin/cfssl-certinfo

# 创建ca证书
mkdir /tmp/ssl
cd /tmp/ssl
cat > ca-config.json  << EOF
{
  "signing": {
    "default": {
      "expiry": "876000h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "87600h"
      }
    }
  }
}
EOF
# 创建 CA 证书签名请求
cat > ca-csr.json << EOF
{
  "CN": "kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "GuangDong",
      "L": "ShenZhen",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF
/usr/local/bin/cfssl gencert -initca ca-csr.json | cfssljson -bare ca
ls ca*

# 创建etcd证书
cat > etcd-csr.json << EOF
{
  "CN": "etcd",
  "hosts": [
    "127.0.0.1",
    "${node01ip}",
    "${node02ip}",
    "${node03ip}",
    "k8s01.example.com",
    "k8s02.example.com",
    "k8s03.example.com"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "GuangDong",
      "L": "ShenZhen",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

/usr/local/bin/cfssl gencert -ca=ca.pem \
-ca-key=ca-key.pem \
-config=ca-config.json \
-profile=kubernetes etcd-csr.json | cfssljson -bare etcd

mkdir -p /etc/kubernetes/ssl
cp * /etc/kubernetes/ssl
cd ..

下载etcd二进制文件

# 三台机器都下载
ETCD_VER=v3.5.14

# choose either URL
GOOGLE_URL=https://storage.googleapis.com/etcd
GITHUB_URL=https://github.com/etcd-io/etcd/releases/download
DOWNLOAD_URL=${GOOGLE_URL}

rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
rm -rf /tmp/etcd-download-test && mkdir -p /tmp/etcd-download-test

curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1
rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz

mv /tmp/etcd-download-test/etcd* /usr/local/bin/

# 生成配置文件
cat > /lib/systemd/system/etcd.service << \EOF
[Unit]
Description=Etcd Server
After=network.target

[Service]
Type=notify
WorkingDirectory=/var/lib/etcd
EnvironmentFile=-/etc/etcd/etcd.conf
# set GOMAXPROCS to number of processors
ExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /usr/local/bin/etcd"

[Install]
WantedBy=multi-user.target
EOF

cat > etcd.conf << EOF
#[member]
ETCD_NAME="node-name"
ETCD_DATA_DIR="/var/lib/etcd"
#ETCD_SNAPSHOT_COUNTER="10000"
#ETCD_HEARTBEAT_INTERVAL="100"
#ETCD_ELECTION_TIMEOUT="1000"
ETCD_LISTEN_PEER_URLS="https://node-ip:2380"
ETCD_LISTEN_CLIENT_URLS="https://node-ip:2379,https://127.0.0.1:2379,http://127.0.0.1:4001"
#ETCD_MAX_SNAPSHOTS="5"
#ETCD_MAX_WALS="5"
#ETCD_CORS=""
#[cluster]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://node-ip:2380"
# if you use different ETCD_NAME (e.g. test),
# set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..."
ETCD_INITIAL_CLUSTER="etcd-node01=https://${node01ip}:2380,etcd-node02=https://${node02ip}:2380,etcd-node03=https://${node03ip}:2380"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_INITIAL_CLUSTER_TOKEN="k8s-etcd-cluster"
ETCD_ADVERTISE_CLIENT_URLS="https://node-ip:2379"
#[security]
CLIENT_CERT_AUTH="true"
ETCD_CA_FILE="/etc/kubernetes/ssl/ca.pem"
ETCD_CERT_FILE="/etc/kubernetes/ssl/etcd.pem"
ETCD_KEY_FILE="/etc/kubernetes/ssl/etcd-key.pem"
ETCD_TRUSTED_CA_FILE="/etc/kubernetes/ssl/ca.pem"
PEER_CLIENT_CERT_AUTH="true"
ETCD_PEER_CA_FILE="/etc/kubernetes/ssl/ca.pem"
ETCD_PEER_CERT_FILE="/etc/kubernetes/ssl/etcd.pem"
ETCD_PEER_KEY_FILE="/etc/kubernetes/ssl/etcd-key.pem"
ETCD_PEER_TRUSTED_CA_FILE="/etc/kubernetes/ssl/ca.pem"
EOF

name_id=1
for ip in ${node01ip} ${node02ip} ${node03ip}
do
  name=etcd-node0${name_id}
  sed  "s#node-name#${name}#g" etcd.conf > ${name}.conf
  sed -i "s#node-ip#${ip}#g" ${name}.conf
  ((name_id++))
done
mkdir -p /etc/etcd
cp etcd-node01.conf /etc/etcd/etcd.conf
mkdir -p /var/lib/etcd

分发给三台机器，然后

systemctl daemon-reload
systemctl enable etcd
systemctl start etcd
systemctl status etcd

查看健康情况

export ETCDCTL_API=3
/usr/local/bin/etcdctl \
  --endpoints=https://${node01ip}:2379 \
  --cacert=/etc/kubernetes/ssl/ca.pem \
  --cert=/etc/kubernetes/ssl/etcd.pem \
  --key=/etc/kubernetes/ssl/etcd-key.pem \
  endpoint  health

使用域名方式测试部署

node01ip=192.168.1.11
node02ip=192.168.1.12
node03ip=192.168.1.13
domain01=node01.hankbook.cn
domain02=node02.hankbook.cn
domain03=node03.hankbook.cn

/etc/hosts

echo 192.168.1.11 node01.hankbook.cn >> /etc/hosts
echo 192.168.1.12 node02.hankbook.cn >> /etc/hosts
echo 192.168.1.13 node03.hankbook.cn >> /etc/hosts

创建证书

# 下载制作证书的二进制文件
wget --no-check-certificate https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
chmod +x cfssl_linux-amd64
mv cfssl_linux-amd64 /usr/local/bin/cfssl

wget --no-check-certificate https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
chmod +x cfssljson_linux-amd64
mv cfssljson_linux-amd64 /usr/local/bin/cfssljson

wget --no-check-certificate https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
chmod +x cfssl-certinfo_linux-amd64
mv cfssl-certinfo_linux-amd64 /usr/local/bin/cfssl-certinfo

# 创建ca证书
mkdir /tmp/ssl
cd /tmp/ssl
cat >> ca-config.json  << EOF
{
  "signing": {
    "default": {
      "expiry": "876000h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "87600h"
      }
    }
  }
}
EOF
# 创建 CA 证书签名请求
cat >> ca-csr.json << EOF
{
  "CN": "kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "GuangDong",
      "L": "ShenZhen",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF
/usr/local/bin/cfssl gencert -initca ca-csr.json | cfssljson -bare ca
ls ca*

# 创建etcd证书
cat >> etcd-csr.json << EOF
{
  "CN": "etcd",
  "hosts": [
    "127.0.0.1",
    "${domain01}",
    "${domain02}",
    "${domain03}"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "GuangDong",
      "L": "ShenZhen",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

/usr/local/bin/cfssl gencert -ca=ca.pem \
-ca-key=ca-key.pem \
-config=ca-config.json \
-profile=kubernetes etcd-csr.json | cfssljson -bare etcd

mkdir -p /etc/kubernetes/ssl
cp * /etc/kubernetes/ssl
cd ..

下载etcd二进制文件

# 三台机器都下载
ETCD_VER=v3.5.14

# choose either URL
GOOGLE_URL=https://storage.googleapis.com/etcd
GITHUB_URL=https://github.com/etcd-io/etcd/releases/download
DOWNLOAD_URL=${GOOGLE_URL}

rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
rm -rf /tmp/etcd-download-test && mkdir -p /tmp/etcd-download-test

curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1
rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz

mv /tmp/etcd-download-test/etcd* /usr/local/bin/

# 生成配置文件
cat > /lib/systemd/system/etcd.service << \EOF
[Unit]
Description=Etcd Server
After=network.target

[Service]
Type=notify
WorkingDirectory=/var/lib/etcd
EnvironmentFile=-/etc/etcd/etcd.conf
# set GOMAXPROCS to number of processors
ExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /usr/local/bin/etcd"

[Install]
WantedBy=multi-user.target
EOF

cat > etcd.conf << EOF
#[member]
ETCD_NAME="node-name"
ETCD_DATA_DIR="/var/lib/etcd"
#ETCD_SNAPSHOT_COUNTER="10000"
#ETCD_HEARTBEAT_INTERVAL="100"
#ETCD_ELECTION_TIMEOUT="1000"
ETCD_LISTEN_PEER_URLS="https://node-ip:2380"
ETCD_LISTEN_CLIENT_URLS="https://node-ip:2379,https://127.0.0.1:2379,http://127.0.0.1:4001"
#ETCD_MAX_SNAPSHOTS="5"
#ETCD_MAX_WALS="5"
#ETCD_CORS=""
#[cluster]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://node-ip:2380"
# if you use different ETCD_NAME (e.g. test),
# set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..."
ETCD_INITIAL_CLUSTER="etcd-node01=https://${domain01}:2380,etcd-node02=https://${domain02}:2380,etcd-node03=https://${domain03}:2380"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_INITIAL_CLUSTER_TOKEN="k8s-etcd-cluster"
ETCD_ADVERTISE_CLIENT_URLS="https://node-ip:2379"
#[security]
CLIENT_CERT_AUTH="true"
ETCD_CA_FILE="/etc/kubernetes/ssl/ca.pem"
ETCD_CERT_FILE="/etc/kubernetes/ssl/etcd.pem"
ETCD_KEY_FILE="/etc/kubernetes/ssl/etcd-key.pem"
ETCD_TRUSTED_CA_FILE="/etc/kubernetes/ssl/ca.pem"
PEER_CLIENT_CERT_AUTH="true"
ETCD_PEER_CA_FILE="/etc/kubernetes/ssl/ca.pem"
ETCD_PEER_CERT_FILE="/etc/kubernetes/ssl/etcd.pem"
ETCD_PEER_KEY_FILE="/etc/kubernetes/ssl/etcd-key.pem"
ETCD_PEER_TRUSTED_CA_FILE="/etc/kubernetes/ssl/ca.pem"
EOF

name_id=1
for ip in ${node01ip} ${node02ip} ${node03ip}
do
    name=etcd-node0${name_id}
    sed  "s#node-name#${name}#g" etcd.conf > ${name}.conf
    sed -i "s#node-ip#${ip}#g" ${name}.conf  # 这个需要改成绑定IP的操作，不能用域名，也就是listen 2379 和2380
    ((name_id++))
done
mkdir -p /etc/etcd
cp etcd-node01.conf /etc/etcd/etcd.conf

分发给三台机器，然后

systemctl daemon-reload
systemctl enable etcd
systemctl start etcd
systemctl status etcd

查看健康情况

export ETCDCTL_API=3
/usr/local/bin/etcdctl \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/ssl/ca.pem \
  --cert=/etc/kubernetes/ssl/etcd.pem \
  --key=/etc/kubernetes/ssl/etcd-key.pem \
  endpoint  health --cluster=true