redis单节点与集群模式
23. redis实战
- Dockerfile
root@k8s-master1:~/k8s-data/dockerfile/web/magedu/redis# cat Dockerfile
#Redis Image
FROM harbor.nbrhce.com/baseimages/centos:7.9.2009
MAINTAINER zhangshijie "zhangshijie@magedu.net"
ADD redis-4.0.14.tar.gz /usr/local/src
RUN ln -sv /usr/local/src/redis-4.0.14 /usr/local/redis \
&& yum install -y make gcc \
&& cd /usr/local/redis \
&& make \
&& cp src/redis-cli /usr/local/bin/ \
&& cp src/redis-server /usr/local/bin/ \
&& mkdir -pv /data/redis-data
ADD redis.conf /usr/local/redis/redis.conf
ADD run_redis.sh /usr/local/redis/run_redis.sh
RUN chmod +x /usr/local/redis/run_redis.sh
EXPOSE 6379
CMD ["/usr/local/redis/run_redis.sh"]
- redis.conf 参考 也可以不用写太多
root@k8s-master1:~/k8s-data/dockerfile/web/magedu/redis# egrep -v '^#|^$' redis.conf
#使用默认路由都可以访问
bind 0.0.0.0
protected-mode yes
port 6379
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize yes
supervised no
pidfile /var/run/redis_6379.pid
loglevel notice
logfile ""
databases 16
always-show-logo yes
#设置快照
save 900 1
save 5 1
save 300 10
save 60 10000
#如果宕机了可以继续往存储目录写
stop-writes-on-bgsave-error no
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
dir /data/redis-data
slave-serve-stale-data yes
slave-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
slave-priority 100
requirepass 123456
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
slave-lazy-flush no
appendonly no
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-use-rdb-preamble no
lua-time-limit 5000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
aof-rewrite-incremental-fsync yes
- 运行执行的命令
root@k8s-master1:~/k8s-data/dockerfile/web/magedu/redis# cat run_redis.sh
#!/bin/bash
chmod 777 /usr/sbin/redis-server
/usr/local/bin/redis-server /usr/local/redis/redis.conf
tail -f /etc/hosts
- 构建的命令
root@k8s-master1:~/k8s-data/dockerfile/web/magedu/redis# cat build-command.sh
#!/bin/bash
TAG=$1
nerdctl build -t harbor.nbrhce.com/demo/redis:${TAG} .
sleep 3
nerdctl push harbor.nbrhce.com/demo/redis:${TAG}
23.1 单节点YAML
- Deployment
root@k8s-master1:~/k8s-data/yaml/magedu/redis# cat redis.yaml
kind: Deployment
#apiVersion: extensions/v1beta1
apiVersion: apps/v1
metadata:
labels:
app: devops-redis
name: deploy-devops-redis
namespace: redis
spec:
replicas: 1
selector:
matchLabels:
app: devops-redis
template:
metadata:
labels:
app: devops-redis
spec:
containers:
- name: redis-container
image: harbor.nbrhce.com/demo/redis:v4.0.14
imagePullPolicy: Always
volumeMounts:
- mountPath: "/data/redis-data/"
name: redis-datadir
volumes:
- name: redis-datadir
persistentVolumeClaim:
claimName: redis-datadir-pvc-1
---
kind: Service
apiVersion: v1
metadata:
labels:
app: devops-redis
name: srv-devops-redis
namespace: redis
spec:
type: NodePort
ports:
- name: http
port: 6379
targetPort: 6379
nodePort: 36379
selector:
app: devops-redis
sessionAffinity: ClientIP
sessionAffinityConfig:
clientIP:
timeoutSeconds: 10800
- PV
#在nfs服务器上创建这个目录/data/k8sdata/redis-datadir-1
root@k8s-master1:~/k8s-data/yaml/magedu/redis/pv# cat redis-persistentvolume.yaml
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: redis-datadir-pv-1
spec:
capacity:
storage: 10Gi
accessModes:
- ReadWriteOnce
nfs:
path: /data/k8sdata/redis-datadir-1
server: 10.0.0.109
- PVC
root@k8s-master1:~/k8s-data/yaml/magedu/redis/pv# cat redis-persistentvolumeclaim.yaml
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: redis-datadir-pvc-1
namespace: redis
spec:
volumeName: redis-datadir-pv-1
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
23.2 集群YAML
root@k8s-master1:~/k8s-data/yaml/magedu/redis-cluster# cat redis.yaml
apiVersion: v1
kind: Service
metadata:
name: redis
namespace: redis-cluster
labels:
app: redis
spec:
selector:
app: redis
appCluster: redis-cluster
ports:
- name: redis
port: 6379
clusterIP: None
---
apiVersion: v1
kind: Service
metadata:
name: redis-access
namespace: redis-cluster
labels:
app: redis
spec:
selector:
app: redis
appCluster: redis-cluster
ports:
- name: redis-access
protocol: TCP
port: 6379
targetPort: 6379
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: redis
namespace: redis-cluster
spec:
serviceName: redis
replicas: 3
selector:
matchLabels:
app: redis
appCluster: redis-cluster
template:
metadata:
labels:
app: redis
appCluster: redis-cluster
spec:
terminationGracePeriodSeconds: 20
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: app
operator: In
values:
- redis
topologyKey: kubernetes.io/hostname
containers:
- name: redis
image: redis:4.0.14
command:
- "redis-server"
args:
- "/etc/redis/redis.conf"
- "--protected-mode"
- "no"
resources:
requests:
cpu: "500m"
memory: "500Mi"
ports:
- containerPort: 6379
name: redis
protocol: TCP
- containerPort: 16379
name: cluster
protocol: TCP
volumeMounts:
- name: conf
mountPath: /etc/redis
- name: data
mountPath: /var/lib/redis
volumes:
- name: conf
configMap:
name: redis-conf
items:
- key: redis.conf
path: redis.conf
volumeClaimTemplates:
- metadata:
name: data
namespace: redis-cluster
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 5Gi
- redis.conf配置文件
root@k8s-master1:~/k8s-data/yaml/magedu/redis-cluster# cat redis.conf
appendonly yes
cluster-enabled yes
cluster-config-file /var/lib/redis/nodes.conf
cluster-node-timeout 5000
dir /var/lib/redis
port 6379
- configmap
kubectl create configmap redis-conf --from-file=redis.conf -n redis-cluster
kubectl get configmaps -n redis-cluster redis-conf -o yaml
- PV
#提前创建好存储目录 在NFS服务器上
root@k8s-master1:~#mkdir /data/k8sdata/{redis0,redis1,redis2,redis3,redis4,redis5}
root@k8s-master1:~/k8s-data/yaml/magedu/redis-cluster/pv# cat redis-cluster-pv.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: redis-cluster-pv0
spec:
capacity:
storage: 5Gi
accessModes:
- ReadWriteOnce
nfs:
server: 10.0.0.109
path: /data/k8sdata/redis0
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: redis-cluster-pv1
spec:
capacity:
storage: 5Gi
accessModes:
- ReadWriteOnce
nfs:
server: 10.0.0.109
path: /data/k8sdata/redis1
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: redis-cluster-pv2
spec:
capacity:
storage: 5Gi
accessModes:
- ReadWriteOnce
nfs:
server: 10.0.0.109
path: /data/k8sdata/redis2
- 运行一个检测容器 redis4的初始化方式
kubectl run -it ubuntu1804 --image=ubuntu:18.04 --restart=Never -n redis-cluster bash
apt update
apt install python2.7 python-pip redis-tools dnsutils iputils-ping net-tools
pip install --upgrade pip
pip install redis-trib==0.5.1
#初始化redis集群 一主两从 对应三个PV
#创建master节点
redis-trib.py create \
`dig +short redis-0.redis.redis-cluster.svc.cluster.local`:6379
#设置副本把slave节点加入集群中
redis-trib.py replicate \
--master-addr `dig +short redis-0.redis.redis-cluster.svc.cluster.local`:6379 \
--slave-addr `dig +short redis-1.redis.redis-cluster.svc.cluster.local`:6379
#设置副本把slave节点加入集群中
redis-trib.py replicate \
--master-addr `dig +short redis-0.redis.redis-cluster.svc.cluster.local`:6379 \
--slave-addr `dig +short redis-1.redis.redis-cluster.svc.cluster.local`:6379
#要是初始化三主三从（一个master对应一个slave）需要对应6个PV
#创建三个主
redis-trib.py create \
`dig +short redis-0.redis.redis-cluster.cluster.local`:6379 \
`dig +short redis-1.redis.redis-cluster.cluster.local`:6379 \
`dig +short redis-2.redis.redis-cluster.svc.cluster.local`:6379
#一个从加入一个主
redis-trib.py replicate \
--master-addr `dig +short redis-0.redis.redis-cluster.svc.cluster.local`:6379 \
--slave-addr `dig +short redis-3.redis.redis-cluster.svc.cluster.local`:6379
#一个从加入一个主
redis-trib.py replicate \
--master-addr `dig +short redis-1.redis.redis-cluster.svc.cluster.local`:6379 \
--slave-addr `dig +short redis-4.redis.redis-cluster.svc.cluster.local`:6379
#一个从加入一个主
redis-trib.py replicate \
--master-addr `dig +short redis-2.redis.redis-cluster.svc.cluster.local`:6379 \
--slave-addr `dig +short redis-5.redis.redis-cluster.svc.cluster.local`:6379
- 查看集群状态
127.0.0.1:6379> CLUSTER INFO
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:3
cluster_size:1
cluster_current_epoch:2
cluster_my_epoch:0
cluster_stats_messages_ping_sent:1337
cluster_stats_messages_pong_sent:1275
cluster_stats_messages_sent:2612
cluster_stats_messages_ping_received:1275
cluster_stats_messages_pong_received:1337
cluster_stats_messages_received:2612
#查看集群是什么角色
127.0.0.1:6379> info replication
role:master
connected_slaves:2
slave0:ip=10.200.169.160,port=6379,state=online,offset=42,lag=1
slave1:ip=10.200.36.104,port=6379,state=online,offset=42,lag=0
master_replid:b1a3f5758b0741d1591ca58fb1452a3fbf6f7910
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:42
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:42
127.0.0.1:6379>
# 查看集群有几个节点
127.0.0.1:6379> CLUSTER NODES
a2841545b149a760c6820fd7ce4ed1dcd353dedc 10.200.169.160:6379@16379 slave f915e9f561c8922246cda5263f6d0f11ac808bfb 0 1670735335364 1 connected
f915e9f561c8922246cda5263f6d0f11ac808bfb 10.200.107.195:6379@16379 myself,master - 0 1670735333000 0 connected 0-16383
aeccf2ee66fc6ee9d59d86f37b27193b539a1851 10.200.36.104:6379@16379 slave f915e9f561c8922246cda5263f6d0f11ac808bfb 0 1670735335364 2 connected
- 效果图
