基于StatefulSet控制器运行Redis Cluster

redis集群规划

redis0——>master

redis1——>master

redis2——>master

redis3——>slave

redis4——>slave

redis5——>slave

构建镜像

创建redis.conf

新增集群配置和aof持久化，注意要将集群的配置文件nodes.conf放在nfs共享存储上，可以使用脚本动态修改pod ip

root@deploy:/dockerfile/project/redis-cluster# cat redis.conf 
bind 0.0.0.0
port 6379
daemonize no
pidfile /var/run/redis_6379.pid
loglevel warning
logfile "/usr/local/redis/logs/redis.log"
dbfilename dump.rdb
dir /usr/local/redis/data
save 900 1
save 5 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error no
requirepass 123456
cluster-enabled yes
cluster-config-file /usr/local/redis/data/nodes.conf
cluster-node-timeout 5000
appendonly yes
appendfilename "appendonly.aof"

nfs server配置

创建nfs共享目录

root@harbor:/data/k8sdata# mkdir redis-cluster/redis{0..5} -p

配置nfs server

root@harbor:/data/k8sdata# cat /etc/exports
/data/k8sdata  *(rw,sync,no_root_squash)
root@harbor:/data/k8sdata# exportfs -r

创建构建镜像脚本

root@master1:/dockerfile/project/redis-cluster# vim build-command.sh
#!/bin/bash
TAG=$1
nerdctl build -t harbor.cncf.net/project/redis:${TAG} .

nerdctl push harbor.cncf.net/project/redis:${TAG}

创建Dockerfile

root@deploy:/dockerfile/project/redis-cluster# cat Dockerfile 
FROM harbor.cncf.net/os/ubuntu:20.04

ADD redis-6.2.6.tar.gz /usr/local/src

RUN cd /usr/local/src/redis-6.2.6 && \
make MALLOC=jemalloc && \
make PREFIX=/usr/local/redis install

RUN cd /usr/local/redis && \
mkdir /usr/local/redis/conf && \
mkdir /usr/local/redis/logs && \
mkdir /usr/local/redis/data

ENV PATH=$PATH:/usr/local/redis/bin

ADD redis.conf /usr/local/redis/conf/redis.conf

EXPOSE 6379

RUN rm -f /usr/local/src/redis-6.2.6.tar.gz

创建k8s资源

使用statefulset控制器创建redis集群

创建pv

root@master1:/dockerfile/project/redis-cluster/pv# cat redis-cluster-pv.yaml 
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-cluster-pv0
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.100.15
    path: /data/k8sdata/redis-cluster/redis0 

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-cluster-pv1
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.100.15
    path: /data/k8sdata/redis-cluster/redis1

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-cluster-pv2
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.100.15
    path: /data/k8sdata/redis-cluster/redis2

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-cluster-pv3
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.100.15
    path: /data/k8sdata/redis-cluster/redis3 

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-cluster-pv4
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.100.15
    path: /data/k8sdata/redis-cluster/redis4

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-cluster-pv5
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.100.15
    path: /data/k8sdata/redis-cluster/redis5
root@master1:/dockerfile/project/redis-cluster/pv# kubectl apply -f redis-cluster-pv.yaml 
persistentvolume/redis-cluster-pv0 created
persistentvolume/redis-cluster-pv1 created
persistentvolume/redis-cluster-pv2 created
persistentvolume/redis-cluster-pv3 created
persistentvolume/redis-cluster-pv4 created
persistentvolume/redis-cluster-pv5 created

查看pv

创建redis statefulset控制器和svc

root@deploy:/dockerfile/project/redis-cluster# cat redis.yaml 
apiVersion: v1
kind: Service
metadata:
  name: redis
  namespace: redis-test
  labels:
    app: redis
spec:
  selector:
    app: redis
    appCluster: redis-cluster
  ports:
  - name: client
    port: 6379
    targetPort: 6379
  - name: cluster
    port: 16379
    targetPort: 16379
  clusterIP: None
  
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis
  namespace: redis-test
spec:
  serviceName: redis
  replicas: 6
  selector:
    matchLabels:
      app: redis
      appCluster: redis-cluster
  template:
    metadata:
      labels:
        app: redis
        appCluster: redis-cluster
    spec:
      terminationGracePeriodSeconds: 20
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - redis
              topologyKey: kubernetes.io/hostname
      containers:
      - name: redis
        image: harbor.cncf.net/project/redis:6.2.6
        imagePullPolicy: Always
        command: ["redis-server", "/usr/local/redis/conf/redis.conf"]
        env:
        - name: POD_IP
          valueFrom:
            fieldRef:
              fieldPath: status.podIP
        resources:
          requests:
            cpu: "500m"
            memory: "500Mi"
        ports:
        - containerPort: 6379
          name: redis
          protocol: TCP
        - containerPort: 16379
          name: cluster
          protocol: TCP
        volumeMounts:
        - name: data
          mountPath: /usr/local/redis/data
  volumeClaimTemplates:
  - metadata:
      name: data
      namespace: redis-test
    spec:
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 5Gi

查看pods

查看pvc

初始化redis创建集群

进入data-redis-0 pod

root@master1:/dockerfile/project/redis-cluster# kubectl exec -it redis-0 -- bash

安装dig

root@redis-0:/# apt install dnsutils -y

初始化命令

redis-cli -a 123456 --cluster create --cluster-replicas 1 \
`dig +short redis-0.redis.redis-test.svc.cluster.local`:6379 \
`dig +short redis-1.redis.redis-test.svc.cluster.local`:6379 \
`dig +short redis-2.redis.redis-test.svc.cluster.local`:6379 \
`dig +short redis-3.redis.redis-test.svc.cluster.local`:6379 \
`dig +short redis-4.redis.redis-test.svc.cluster.local`:6379 \
`dig +short redis-5.redis.redis-test.svc.cluster.local`:6379

创建redis集群