现象:
执行 ETCDCTL_API=3 /opt/etcd/bin/etcdctl --endpoints="http://apisix-etcd-0.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379,http://apisix-etcd-1.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379,http://apisix-etcd-2.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379" endpoint status -w table 查看存储情况,告警信息提示NOSPACE
分析原因:etcd存储满了,这边存储空间并非磁盘空间，而是etcd默认设置了空间配额，默认为8G
处理方法:
#先重新选主【适用3个etcd包括主节点都挂掉的情况】
kubectl exec -it sts/apisix-etcd -n ingress-apisix -- /opt/bitnami/etcd/bin/etcdctl --endpoints="http://apisix-etcd-0.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379,http://apisix-etcd-1.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379,http://apisix-etcd-2.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379" member update 62c3c5516bb89b91【member ID】 --peer-urls=http://apisix-etcd-1.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2380
#之前的主节点降为:follower节点
kubectl exec -it sts/apisix-etcd -n ingress-apisix -- /opt/bitnami/etcd/bin/etcdctl --endpoints="http://apisix-etcd-0.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379,http://apisix-etcd-1.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379,http://apisix-etcd-2.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379" member update 62c3c5516bb89xv5 【member ID】--peer-urls=http://apisix-etcd-0.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2380
#这行命令的作用是获取Etcd集群中的当前修订版本(revision)号
rev=$(ETCDCTL_API=3 /opt/etcd/bin/etcdctl --endpoints="http://apisix-etcd-0.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379,http://apisix-etcd-1.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379,http://apisix-etcd-2.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379" endpoint status -w json | egrep -o '"revision":[0-9]*' | egrep -o '[0-9]*'| awk 'NR==1{print $1}')
#用于在Etcd中进行修订版本的压缩操作
ETCDCTL_API=3 /opt/etcd/bin/etcdctl --endpoints="http://apisix-etcd-0.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379,http://apisix-etcd-1.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379,http://apisix-etcd-2.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379" compact $rev
#用于在 Etcd 存储空间中进行碎片整理操作，注意:碎片整理会阻塞对etcd的读写操作，大量数据的defrag建议逐台进行，以免影响集群稳定性，一般在业务低峰期执行此操作
ETCDCTL_API=3 /opt/etcd/bin/etcdctl --endpoints="http://apisix-etcd-0.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379,http://apisix-etcd-1.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379,http://apisix-etcd-2.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379" defrag
#查看Etcd 集群的节点状态信息
ETCDCTL_API=3 /opt/etcd/bin/etcdctl --endpoints="http://apisix-etcd-0.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379,http://apisix-etcd-1.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379,http://apisix-etcd-2.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379" endpoint status -w table
#列出告警状态
ETCDCTL_API=3 /opt/etcd/bin/etcdctl --endpoints="http://apisix-etcd-0.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379,http://apisix-etcd-1.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379,http://apisix-etcd-2.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379" alarm list
#解除 Etcd 集群中的警报状态
ETCDCTL_API=3 /opt/etcd/bin/etcdctl --endpoints="http://apisix-etcd-0.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379,http://apisix-etcd-1.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379,http://apisix-etcd-2.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379" alarm disarm
