centos8 搭建etcd集群

实验环境

hostname	ip
bjcy-62	172.16.0.62
bjcy-81	172.16.0.81
bjcy-82	172.16.0.82

创建证书

证书服务器搭建: 

创建根证书的config配置文件

文件路径:  /opt/certs/ca-config.json  

{
    "signing": {
        "default": {
            "expiry": "175200h"
        },
        "profiles": {
            "server": {
                "expiry": "175200h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth"
                ]
            },
            "client": {
                "expiry": "175200h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "client auth"
                ]
            },
            "peer": {                
                "expiry": "175200h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth",
                    "client auth"
                ]
            }
        }
    }
}

创建etcd申请文件

[root@djcy-200 certs]# cat /opt/certs/etcd-peer-csr.json
{
    "CN": "k8s-etcd",
    "hosts": [
        "172.16.0.62",
        "172.16.0.81",
        "172.16.0.82",
        "172.16.0.83",
        "172.16.0.84"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "Beijing",
            "L": "Beijing",
            "O": "devops",
            "OU": "ops"
        }
    ]
}

授权

[root@djcy-200 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json --profile=peer etcd-peer-csr.json 

[root@djcy-200 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json --profile=peer etcd-peer-csr.json  | cfssl-json -bare etcd-peer

[root@djcy-200 certs]# ll
total 36
-rw-r--r-- 1 root root  840 Aug  4 17:46 ca-config.json
-rw-r--r-- 1 root root  997 Aug  4 17:45 ca.csr
-rw-r--r-- 1 root root  334 Aug  4 17:44 ca-csr.json
-rw------- 1 root root 1675 Aug  4 17:45 ca-key.pem
-rw-r--r-- 1 root root 1350 Aug  4 17:45 ca.pem
-rw-r--r-- 1 root root 1078 Aug  4 17:53 etcd-peer.csr
-rw-r--r-- 1 root root  398 Aug  4 17:52 etcd-peer-csr.json
-rw------- 1 root root 1675 Aug  4 17:53 etcd-peer-key.pem
-rw-r--r-- 1 root root 1444 Aug  4 17:53 etcd-peer.pem

etcd集群搭建

基础环境所有机器一致

下载地址: https://github.com/etcd-io/etcd/releases/

创建用户

[root@djcy-81 ~]# useradd -M -s /sbin/nologin etcd
[root@djcy-81 ~]# id etcd
uid=1000(etcd) gid=1000(etcd) groups=1000(etcd)

下载软件

[root@djcy-81 ~]# wget https://github.com/etcd-io/etcd/releases/download/v3.3.23/etcd-v3.3.23-linux-amd64.tar.gz
tar xf etcd-v3.3.23-linux-amd64.tar.gz -C /opt/
mv /opt/etcd-v3.3.23-linux-amd64/ /opt/etcd-v3.3.23
ln -s /opt/etcd-v3.3.23 /opt/etcd

创建所需目录

 mkdir -p /opt/etcd/certs /data/etcd/etcd-server /data/logs/etcd-server/ -p

拷贝认证key

[root@djcy-81 certs]# ll
total 12
-rw-r--r-- 1 etcd etcd 1350 Aug  4 19:33 ca.pem
-rw------- 1 etcd etcd 1675 Aug  4 19:33 etcd-peer-key.pem
-rw-r--r-- 1 etcd etcd 1444 Aug  4 19:33 etcd-peer.pem

编写启动脚本

启动脚本 

[root@djcy-81 etcd]# cat etcd-server-startup.sh 
#!/bin/sh
./etcd --name etcd-server-81 \                                # 更换实际名称
       --data-dir /data/etcd/etcd-server \
       --listen-peer-urls https://172.16.0.81:2380 \           # 更换实际IP
       --listen-client-urls https://172.16.0.81:2379,http://127.0.0.1:2379 \   # 更换实际IP
       --quota-backend-bytes 8000000000 \
       --initial-advertise-peer-urls https://172.16.0.81:2380 \                 # 更换实际IP
       --advertise-client-urls https://172.16.0.81:2379,http://127.0.0.1:2379 \  # 更换实际IP
       --initial-cluster  etcd-server-81=https://172.16.0.81:2380,etcd-server-82=https://172.16.0.82:2380,etcd-server-62=https://172.16.0.62:2380 \
       --ca-file ./certs/ca.pem \
       --cert-file ./certs/etcd-peer.pem \
       --key-file ./certs/etcd-peer-key.pem \
       --client-cert-auth  \
       --trusted-ca-file ./certs/ca.pem \
       --peer-ca-file ./certs/ca.pem \
       --peer-cert-file ./certs/etcd-peer.pem \
       --peer-key-file ./certs/etcd-peer-key.pem \
       --peer-client-cert-auth \
       --peer-trusted-ca-file ./certs/ca.pem \
       --log-output stdout

添加权限

chown -R etcd.etcd /opt/etcd-v3.3.23/ /data/etcd/ /data/logs/etcd-server/

supervisor管理etcd

安装supervisor

dnf install supervisor -y

启动并设为开机启动

systemctl start supervisord.service 
systemctl enable supervisord.service 

配置etcd启动文件

[root@djcy-81 ~]# cat /etc/supervisord.d/etcd-server.ini
[program:etcd-server]
command=/opt/etcd/etcd-server-startup.sh                        ; the program (relative uses PATH, can take args)	
numprocs=1                                                      ; number of processes copies to start (def 1)
directory=/opt/etcd                                             ; directory to cwd to before exec (def no cwd)
autostart=true                                                  ; start at supervisord start (default: true)
autorestart=true                                                ; retstart at unexpected quit (default: true)
startsecs=30                                                    ; number of secs prog must stay running (def. 1)
startretries=3                                                  ; max # of serial start failures (default 3)
exitcodes=0,2                                                   ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT                                                 ; signal used to kill process (default TERM)
stopwaitsecs=10                                                 ; max num secs to wait b4 SIGKILL (default 10)
user=etcd                                                       ; setuid to this UNIX account to run the program
redirect_stderr=true                                            ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/etcd-server/etcd.stdout.log           ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB                                    ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4                                        ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB                                     ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false                                     ; emit events on stdout writes (default false)

启动

supervisorctl reload
supervisorctl up
supervisorctl status

验证集群

ln -s etcdctl /usr/bin/


[root@bjcy-81 etcd]# pwd
/opt/etcd
[root@bjcy-81 etcd]#  ./etcdctl cluster-health
member 59947c46adf706a9 is healthy: got healthy result from http://127.0.0.1:2379
member 6413ab1ba568b3ed is healthy: got healthy result from http://127.0.0.1:2379
member 742bb1a2c426596e is healthy: got healthy result from http://127.0.0.1:2379
cluster is healthy
[root@bjcy-81 etcd]# ./etcdctl member list
59947c46adf706a9: name=etcd-server-82 peerURLs=https://172.16.0.82:2380 clientURLs=http://127.0.0.1:2379,https://172.16.0.82:2379 isLeader=true
6413ab1ba568b3ed: name=etcd-server-62 peerURLs=https://172.16.0.62:2380 clientURLs=http://127.0.0.1:2379,https://172.16.0.62:2379 isLeader=false
742bb1a2c426596e: name=etcd-server-81 peerURLs=https://172.16.0.81:2380 clientURLs=http://127.0.0.1:2379,https://172.16.0.81:2379 isLeader=false