k8s部署redis集群（cluster模式）

参考：https://www.cnblogs.com/LiuChang-blog/p/15898005.html

一、redis集群部署(三主三从)

1.1yaml文件准备

（1） 配置文件说明
# 部署所在命名空间: redis-cluster

[root@k8s-master1 ~]# cd /root/redis-cluster/
[root@k8s-master1 redis-cluster]# ls -l
total 8
# redis 配置文件使用 configmap 方式进行挂载
# fix-ip.sh 脚本的作用用于当 redis 集群某 pod 重建后 Pod IP 发生变化，在 /data/nodes.conf 中将新的 Pod IP 替换原 Pod IP。不
# 然集群会出问题。
# redis.conf 文件
-rw-r--r-- 1 root root 2389 Feb 14 20:25 redis-cluster-configmap.yml
# StatefulSet、Service 配置文件
-rw-r--r-- 1 root root 1527 Feb 14 20:25 redis-cluster.yml

（2） redis-cluster-configmap.yml
[root@k8s-master1 redis-cluster]# cat redis-cluster-configmap.yml
apiVersion: v1
kind: ConfigMap
metadata:
  name: redis-cluster
  namespace: uat-app
data:
  fix-ip.sh: |
    #!/bin/sh
    CLUSTER_CONFIG="/data/nodes.conf"
    if [ -f ${CLUSTER_CONFIG} ]; then
      if [ -z "${POD_IP}" ]; then
        echo "Unable to determine Pod IP address!"
        exit 1
      fi
      echo "Updating my IP to ${POD_IP} in ${CLUSTER_CONFIG}"
      sed -i.bak -e '/myself/ s/[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}/'${POD_IP}'/' ${CLUSTER_CONFIG}
    fi
    exec "$@"
  redis.conf: |
    bind 0.0.0.0
    protected-mode yes
    port 6379
    tcp-backlog 2048
    timeout 0
    tcp-keepalive 300
    daemonize no
    supervised no
    pidfile /var/run/redis.pid
    loglevel notice
    logfile /data/redis.log
    databases 16
    always-show-logo yes
    stop-writes-on-bgsave-error yes
    rdbcompression yes
    rdbchecksum yes
    dbfilename dump.rdb
    dir /data
    masterauth redis@2022   #设置密码，自行修改
    replica-serve-stale-data yes
    replica-read-only no
    repl-diskless-sync no
    repl-diskless-sync-delay 5
    repl-disable-tcp-nodelay no
    replica-priority 100
    requirepass redis@2022   #设置密码，自行修改
    maxclients 32768
    #maxmemory 6g
    maxmemory-policy allkeys-lru
    lazyfree-lazy-eviction no
    lazyfree-lazy-expire no
    lazyfree-lazy-server-del no
    replica-lazy-flush no
    appendonly yes
    appendfilename "appendonly.aof"
    appendfsync everysec
    no-appendfsync-on-rewrite no
    auto-aof-rewrite-percentage 100
    auto-aof-rewrite-min-size 64mb
    aof-load-truncated yes
    aof-use-rdb-preamble yes
    lua-time-limit 5000
    cluster-enabled yes
    cluster-config-file /data/nodes.conf
    cluster-node-timeout 15000
    slowlog-log-slower-than 10000
    slowlog-max-len 128
    latency-monitor-threshold 0
    notify-keyspace-events ""
    hash-max-ziplist-entries 512
    hash-max-ziplist-value 64
    list-max-ziplist-size -2
    list-compress-depth 0
    set-max-intset-entries 512
    zset-max-ziplist-entries 128
    zset-max-ziplist-value 64
    hll-sparse-max-bytes 3000
    stream-node-max-bytes 4096
    stream-node-max-entries 100
    activerehashing yes
    client-output-buffer-limit normal 0 0 0
    client-output-buffer-limit replica 256mb 64mb 60
    client-output-buffer-limit pubsub 32mb 8mb 60
    hz 10
    dynamic-hz yes
    aof-rewrite-incremental-fsync yes
    rdb-save-incremental-fsync yes

（3） redis-cluster.yml
[root@k8s-master1 redis-cluster]# cat redis-cluster.yml
---
apiVersion: v1
kind: Service
metadata:
  namespace: uat-app
  name: redis-cluster
spec:
  clusterIP: None
  ports:
  - port: 6379
    targetPort: 6379
    name: client
  - port: 16379
    targetPort: 16379
    name: gossip
  selector:
    app: redis-cluster
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  namespace: uat-app
  name: redis-cluster
spec:
  serviceName: redis-cluster
  replicas: 6
  selector:
    matchLabels:
      app: redis-cluster
  template:
    metadata:
      labels:
        app: redis-cluster
    spec:
      containers:
      - name: redis
        image: 172.16.43.156/app/redis:6.2.6
        ports:
        - containerPort: 6379
          name: client
        - containerPort: 16379
          name: gossip
        command: ["/etc/redis/fix-ip.sh", "redis-server", "/etc/redis/redis.conf"]
        env:
        - name: POD_IP
          valueFrom:
            fieldRef:
              fieldPath: status.podIP
        volumeMounts:
        - name: conf
          mountPath: /etc/redis/
          readOnly: false
        - name: data
          mountPath: /data
          readOnly: false
      volumes:
      - name: conf
        configMap:
          name: redis-cluster
          defaultMode: 0755
  volumeClaimTemplates:
  - metadata:
      name: data
      #annotations:
      #  volume.beta.kubernetes.io/storage-class: "redis-nfs-storage"
    spec:
      storageClassName: "uat-nfs-storage"
      accessModes:
        - ReadWriteMany
      resources:
        requests:
          storage: 10Gi

1.2部署

（1） 创建命名空间
[root@k8s-master1 redis-cluster]# kubectl create namespace uat-app
​
（2） 应用yml文件
[root@k8s-master1 redis-cluster]# ls | xargs -i kubectl apply -f {}
configmap/redis-cluster created
service/redis-cluster created
statefulset.apps/redis-cluster created

(3) 查看pod
[root@k8s-master1 redis-cluster]# kubectl get pod -n uat-app |grep  redis

(4 )查看pvc
[root@k8s-master1 redis-cluster]# kubectl get pvc -n uat-app

(5)查看pv
[root@k8s01-zongshuai redis-cluster]# kubectl get pv -n uat-app |grep uat-app/data-redis-cluster

(6) 查看svc
[root@k8s-master1 redis-cluster]# kubectl get svc,ep -n uat-app

(8 )初始化 Redis Cluster 集群
1、注意: 必须使用 ip 进行初始化 redis 集群，使用域名会报如下错误
Node redis-cluster-1.redis-cluster.uat-app.svc.cluster.local:6379 replied with error:
ERR Invalid node address specified: redis-cluster-0.redis-cluster.uat-app.svc.cluster.local:6379
​
2、 获取 Redis 集群 6 个节点 Pod 的 ip 地址
1) 方式一
[root@k8s-master1 redis-cluster]# kubectl get pod -n  ua-app -o wide
​
2) 方式二
# kubectl run -i --tty --image busybox:1.28.4 dns-test --restart=Never --rm /bin/sh
/ # nslookup redis-cluster.uat-app.svc.cluster.local # 应用连接 redis 集群时使用下面 pod 的域名

(3) 创建集群
[root@k8s01-zongshuai redis-cluster]# kubectl exec -it pod/redis-cluster-0 -n uat-app -- bash
root@redis-cluster-0:/data# redis-cli -a redis@2022 --cluster create \
> 10.244.1.222:6379  \
> 10.244.2.118:6379 \
> 10.244.0.159:6379 \
> 10.244.0.158:6379 \
> 10.244.1.223:6379 \
> 10.244.2.119:6379 \
> --cluster-replicas 1

 

9 验证 Redis Cluster 集群

[root@k8s-master1 redis-cluster]# kubectl exec -it pod/redis-cluster-0 -n uat-app -- bash
# redis-cli -h redis-cluster-1.redis-cluster.uat-app.svc.cluster.local -c -a 'liuchang@2022'
redis-cluster-1.redis-cluster.uat-app.svc.cluster.local:6379> cluster info
redis-cluster-1.redis-cluster.uat-app.svc.cluster.local:6379> cluster nodes

2.3 测试

1 查看 redis 集群 pod
[root@k8s-master1 redis-cluster]# kubectl get pod -n redis-cluster -o wide
NAME             READY   STATUS   RESTARTS   AGE   IP               NODE    
redis-cluster-0   1/1     Running   0         3h1m   172.27.36.87     k8s-node1
redis-cluster-1   1/1     Running   0         3h1m   172.27.169.155   k8s-node2
redis-cluster-2   1/1     Running   0         3h     172.27.36.88     k8s-node1
redis-cluster-3   1/1     Running   0         3h     172.27.169.156   k8s-node2
redis-cluster-4   1/1     Running   0         3h     172.27.36.89     k8s-node1
redis-cluster-5   1/1     Running   0         3h     172.27.169.157   k8s-node2
​
2 删除任意一个 pod(删除名称为 redis-cluster-3 的 pod)
[root@k8s-master1 redis-cluster]# kubectl delete pod/redis-cluster-3 -n redis-cluster
pod "redis-cluster-3" deleted
​
3 pod 被重新拉起(还占用原来的pvc 和 pv)
[root@k8s-master1 redis-cluster]# kubectl get pod -n redis-cluster -o wide
NAME             READY   STATUS   RESTARTS   AGE   IP               NODE    
redis-cluster-0   1/1     Running   0         3h5m   172.27.36.87     k8s-node1
redis-cluster-1   1/1     Running   0         3h4m   172.27.169.155   k8s-node2
redis-cluster-2   1/1     Running   0         3h4m   172.27.36.88     k8s-node1
redis-cluster-3   1/1     Running   0         3s     172.27.169.158   k8s-node2
redis-cluster-4   1/1     Running   0         3h4m   172.27.36.89     k8s-node1
redis-cluster-5   1/1     Running   0         3h4m   172.27.169.157   k8s-node2
​
注:
可以看到名称为 redis-cluster-3 的 pod 启动时长 AGE 为 3s，IP 由原来的 172.27.169.156 变为 172.27.169.158，
/data/nodes.conf 文件中 "myself" 对应的 ip 被 fix-ip.sh 脚本修改，redis 集群修复时会将该 ip 同步到其它 pod
节的 /data/nodes.conf 文件中，从而保证整个 redis 集群的可用性。
​
4 验证集群状态
[root@k8s-master1 redis-cluster]# kubectl exec -it pod/redis-cluster-4 -n redis-cluster -- bash
# redis-cli -h redis-cluster-5.redis-cluster.redis-cluster.svc.cluster.local -c -a 'liuchang@2022'
redis-cluster-5.redis-cluster.redis-cluster.svc.cluster.local:6379> cluster info
redis-cluster-5.redis-cluster.redis-cluster.svc.cluster.local:6379> cluster nodes
​
注: 由下图可以看到集群状态又恢复了正常

 

 

2.4 补充

1 如果整个 redis 集群的 pod 全部都挂掉了，pod自动拉起后，集群不可用，需要重建集群。
​
2 重建集群的方法1: 删除 redis 集群所有的资源，然后重新创建 redis 集群
(1) 删除 redis 集群中所有的 pod
[root@k8s-master1 redis-cluster]# kubectl delete -f redis-cluster.yml
​
(2) 删除 redis 集群中所有的 pvc(pv)
[root@k8s-master1 redis-cluster]# kubectl delete pvc/data-redis-cluster-0 -n redis-cluster
[root@k8s-master1 redis-cluster]# kubectl delete pvc/data-redis-cluster-1 -n redis-cluster
[root@k8s-master1 redis-cluster]# kubectl delete pvc/data-redis-cluster-2 -n redis-cluster
[root@k8s-master1 redis-cluster]# kubectl delete pvc/data-redis-cluster-3 -n redis-cluster
[root@k8s-master1 redis-cluster]# kubectl delete pvc/data-redis-cluster-4 -n redis-cluster
[root@k8s-master1 redis-cluster]# kubectl delete pvc/data-redis-cluster-5 -n redis-cluster
​
(3) 删除 redis 集群中 pod 对应的 nfs 持久化存储目录
[root@k8s_nfs ~]# rm -rf /ifs/kubernetes/archived-redis-cluster-data-redis-cluster-*/
​
(4) 重新创建 redis 集群
[root@k8s-master1 redis-cluster]# kubectl apply -f redis-cluster.yml
​
3 重建集群的方法2: 在原有 redis 集群的基础上进行修复
(1) 删除 redis 集群中所有的 pod
[root@k8s-master1 redis-cluster]# kubectl delete -f redis-cluster.yml
​
(2) 找到 redis 集群中 pod 对应的 nfs 持久化存储目录后删除 nodes.conf
[root@k8s_nfs ~]# rm -f /ifs/kubernetes/redis-cluster-data-redis-cluster-0*/nodes.conf*
[root@k8s_nfs ~]# rm -f /ifs/kubernetes/redis-cluster-data-redis-cluster-1*/nodes.conf*
[root@k8s_nfs ~]# rm -f /ifs/kubernetes/redis-cluster-data-redis-cluster-2*/nodes.conf*
[root@k8s_nfs ~]# rm -f /ifs/kubernetes/redis-cluster-data-redis-cluster-3*/nodes.conf*
[root@k8s_nfs ~]# rm -f /ifs/kubernetes/redis-cluster-data-redis-cluster-4*/nodes.conf*
[root@k8s_nfs ~]# rm -f /ifs/kubernetes/redis-cluster-data-redis-cluster-5*/nodes.conf*
​
(3) 重新创建 redis 集群
[root@k8s-master1 redis-cluster]# kubectl apply -f redis-cluster.yml