ETCD集群+ TLS认证
目录
1、下载etcd cfssl相关命令
# 下载etcd cfssl相关命令
[root@1a32vla0168zzzz cfssl]# ll /app/etcd/bin/
cfssl cfssl-certinfo cfssljson etcd etcdctl etcdutl
[root@1a32vla0168zzzz appdeploy]# cd /app/etcd/cfssl/
[root@1a32vla0168zzzz cfssl]# ls
ca.csr ca-csr.json ca-key.pem ca.pem config.json server.csr server.json server-key.pem server.pem
[root@1a32vla0168zzzz cfssl]# cat config.json
{
"signing": {
"default": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "87600h"
}
}
}
[root@1a32vla0168zzzz cfssl]# cat ca-csr.json
{
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"O": "etcd",
"OU": "etcd",
"L": "apisix",
"ST": "apisix",
"C": "china"
}
],
"CN": "etcd"
}
[root@1a32vla0168zzzz cfssl]# cat server.json
{
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"O": "etcd",
"OU": "etcd",
"L": "apisix",
"ST": "apisix",
"C": "china"
}
],
"CN": "etcd",
"hosts": [
"xxx.xxx.xxx.5",
"xxx.xxx.xxx.6",
"xxx.xxx.xxx.7"
]
}
[root@1a32vla0168zzzz cfssl]# cfssl gencert --initca=true ca-csr.json | cfssljson --bare ca
[root@1a32vla0168zzzz cfssl]# cfssl gencert --ca ca.pem --ca-key ca-key.pem --config config.json server.json | cfssljson --bare server
# 复制 .pem文件至其他节点
2、配置文件 /app/etcd/conf/conf.yml
etcd01（etcd02和etcd03配置替换下ip即可），注意https http的配置细节
# vim /app/etcd/conf/conf.yml
name: etcd01
data-dir: /app/etcd/data
initial-advertise-peer-urls: https://xxx.xxx.xxx.5:2380
listen-peer-urls: https://xxx.xxx.xxx.5:2380
listen-client-urls: https://xxx.xxx.xxx.5:2379,http://127.0.0.1:2379
advertise-client-urls: https://xxx.xxx.xxx.5:2379
initial-cluster-token: apisix-etcd-cluster
initial-cluster: etcd01=https://xxx.xxx.xxx.5:2380,etcd02=https://xxx.xxx.xxx.6:2380,etcd03=https://xxx.xxx.xxx.7:2380
initial-cluster-state: new
# [security]
client-transport-security:
client-cert-auth: true
trusted-ca-file: /app/etcd/cfssl/ca.pem
cert-file: /app/etcd/cfssl/server.pem
key-file: /app/etcd/cfssl/server-key.pem
peer-transport-security:
client-cert-auth: true
trusted-ca-file: /app/etcd/cfssl/ca.pem
cert-file: /app/etcd/cfssl/server.pem
key-file: /app/etcd/cfssl/server-key.pem
3、用系统命令启动，创建 /usr/lib/systemd/system/etcd.service
[unix_http_server]
file=//app/etcd/supervisor/upervisor.sock ; the path to the socket file
[supervisord]
logfile=/app/etcd/supervisor/supervisor.log ; main log file; default $CWD/supervisord.log
logfile_maxbytes=50MB
logfile_backups=5
loglevel=info
pidfile=/app/etcd/supervisor/supervisord.pid ; supervisord pidfile; default supervisord.pid
nodaemon=false
silent=false
minfds=1024
minprocs=200
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
[supervisorctl]
serverurl=unix:///app/etcd/supervisor/supervisor.sock
[program:apisix-etcd]
directory = /app/etcd/
command=/app/etcd/bin/etcd --config-file=/app/etcd/conf/conf.yml
autostart=true
startsecs=5
startretries=3
autorestart=true ; 程序崩溃时自动重启，重启次数是有限制的，默认为3次
redirect_stderr=true ; 重定向输出的日志
stdout_syslog=true
stdout_logfile=/app/etcd/supervisor/stdout.log ; stdout log path, NONE for none; default AUTO
stdout_logfile_backups=5
stderr_logfile_backups=5
stderr_syslog=true
启动服务
systemctl daemon-reload && systemctl start etcd.service
注意：新集群中节点启动时，尽量快速同时启动所有节点，单独启动会报错
4、用 supervisor启动
[root@1a32vla0168zzzz etcd]# cat supervisor/supervisor.conf
[unix_http_server]
file=//app/etcd/supervisor/upervisor.sock ; the path to the socket file
[supervisord]
logfile=/app/etcd/supervisor/supervisor.log ; main log file; default $CWD/supervisord.log
logfile_maxbytes=50MB
logfile_backups=5
loglevel=info
pidfile=/app/etcd/supervisor/supervisord.pid ; supervisord pidfile; default supervisord.pid
nodaemon=false
silent=false
minfds=1024
minprocs=200
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
[supervisorctl]
serverurl=unix:///app/etcd/supervisor/supervisor.sock
[program:apisix-etcd]
directory = /app/etcd/
command=/app/etcd/bin/etcd --config-file=/app/etcd/conf/conf.yml
autostart=true
startsecs=5
startretries=3
autorestart=true ; 程序崩溃时自动重启，重启次数是有限制的，默认为3次
redirect_stderr=true ; 重定向输出的日志
stdout_syslog=true
stdout_logfile=/app/etcd/supervisor/stdout.log ; stdout log path, NONE for none; default AUTO
stdout_logfile_backups=5
stderr_logfile_backups=5
stderr_syslog=true
5、TLS验证
注意：https而不是http
[root@1a32vla0168zzzz etcd]# etcdctl endpoint health --endpoints "https://xxx.xxx.xxx.5:2379,https://xxx.xxx.xxx.6:2379,https://xxx.xxx.xxx.7:2379"
{"level":"warn","ts":1659374088.913921,"logger":"client","caller":"v3/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000032380/xxx.xxx.xxx.5:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: authentication handshake failed: x509: certificate signed by unknown authority\""}
{"level":"warn","ts":1659374088.9140902,"logger":"client","caller":"v3/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00032e700/xxx.xxx.xxx.6:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: authentication handshake failed: x509: certificate signed by unknown authority\""}
{"level":"warn","ts":1659374088.9141088,"logger":"client","caller":"v3/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0000c08c0/xxx.xxx.xxx.7:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: authentication handshake failed: x509: certificate signed by unknown authority\""}
https://xxx.xxx.xxx.5:2379 is unhealthy: failed to commit proposal: context deadline exceeded
https://xxx.xxx.xxx.6:2379 is unhealthy: failed to commit proposal: context deadline exceeded
https://xxx.xxx.xxx.7:2379 is unhealthy: failed to commit proposal: context deadline exceeded
Error: unhealthy cluster
# 不带证书验证会发现提示都是unhealthy，必须带上证书
[root@1a32vla0168zzzz etcd]# etcdctl --cacert /app/etcd/cfssl/ca.pem --cert /app/etcd/cfssl/server.pem --key /app/etcd/cfssl/server-key.pem --endpoints "https://xxx.xxx.xxx.5:2379,https://xxx.xxx.xxx.6:2379,https://xxx.xxx.xxx.7:2379" endpoint health
https://xxx.xxx.xxx.5:2379 is healthy: successfully committed proposal: took = 9.169585ms
https://xxx.xxx.xxx.6:2379 is healthy: successfully committed proposal: took = 10.263672ms
https://xxx.xxx.xxx.7:2379 is healthy: successfully committed proposal: took = 13.415932ms
[root@1a32vla0168zzzz etcd]#
6、用户认证
[root@1a32vla0169zzzz etcd]# etcdctl auth status
Authentication Status: false
AuthRevision: 1
[root@1a32vla0169zzzz etcd]# etcdctl user list
[root@1a32vla0169zzzz etcd]# etcdctl role list
[root@1a32vla0169zzzz etcd]# etcdctl role add root
Role root created
[root@1a32vla0169zzzz etcd]# etcdctl user add root:123456
User root created
[root@1a32vla0169zzzz etcd]# etcdctl user grant-role root root
Role root is granted to user root
# 创建读写权限和仅读权限的用户和角色
[root@1a32vla0169zzzz etcd]# etcdctl role add read_write
Role read_write created
[root@1a32vla0169zzzz etcd]# etcdctl role add read_only
Role read_only created
[root@1a32vla0169zzzz etcd]# etcdctl role grant-permission read_write --prefix=true readwrite /
Role read_write updated
[root@1a32vla0169zzzz etcd]# etcdctl role grant-permission read_only --prefix=true read /
Role read_only updated
[root@1a32vla0169zzzz etcd]# etcdctl user add admin:admin123
User admin created
[root@1a32vla0169zzzz etcd]# etcdctl user add monitor:monitor123
User monitor created
[root@1a32vla0169zzzz etcd]# etcdctl user grant-role admin read_write
Role read_write is granted to user admin
[root@1a32vla0169zzzz etcd]# etcdctl user grant-role monitor read_only
Role read_only is granted to user monitor
# 开启用户认证
[root@1a32vla0169zzzz etcd]# etcdctl auth enable
Authentication Enabled
此时，验证需要带上用户密码
# 不带用户密码会报错
# etcdctl --cacert /app/etcd/cfssl/ca.pem --cert /app/etcd/cfssl/server.pem --key /app/etcd/cfssl/server-key.pem --endpoints "https://xxx.xxx.xxx.5:2379,https://xxx.xxx.xxx.6:2379,https://xxx.xxx.xxx.7:2379" endpoint health
{"level":"warn","ts":1659374402.6467996,"logger":"client","caller":"v3/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00040ca80/xxx.xxx.xxx.5:2379","attempt":0,"error":"rpc error: code = PermissionDenied desc = etcdserver: permission denied"}
{"level":"warn","ts":1659374402.6468987,"logger":"client","caller":"v3/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0000b28c0/xxx.xxx.xxx.7:2379","attempt":0,"error":"rpc error: code = PermissionDenied desc = etcdserver: permission denied"}
{"level":"warn","ts":1659374402.6474524,"logger":"client","caller":"v3/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0000321c0/xxx.xxx.xxx.6:2379","attempt":0,"error":"rpc error: code = PermissionDenied desc = etcdserver: permission denied"}
https://xxx.xxx.xxx.7:2379 is healthy: successfully committed proposal: took = 15.426246ms
https://xxx.xxx.xxx.5:2379 is healthy: successfully committed proposal: took = 15.216552ms
https://xxx.xxx.xxx.6:2379 is healthy: successfully committed proposal: took = 15.952318ms
# etcdctl --cacert /app/etcd/cfssl/ca.pem --cert /app/etcd/cfssl/server.pem --key /app/etcd/cfssl/server-key.pem --endpoints "https://xxx.xxx.xxx.5:2379,https://xxx.xxx.xxx.6:2379,https://xxx.xxx.xxx.7:2379" endpoint health --user=root:123456
https://xxx.xxx.xxx.6:2379 is healthy: successfully committed proposal: took = 933.418µs
https://xxx.xxx.xxx.5:2379 is healthy: successfully committed proposal: took = 915.707µs
https://xxx.xxx.xxx.7:2379 is healthy: successfully committed proposal: took = 1.209063ms
