redis 哨兵部署
redis 生产哨兵的测试
1、helm release的版本
helm repo add bitnami https://charts.bitnami.com/bitnami
helm pull bitnami/redis --version 18.2.2
sentinel-redis]# ls
redis redis-18.2.2.tgz
values.yaml 配置文件
注意配置文件的cm部分，先使用默认的configmap配置然后再单独修改cm,这样防止部署的时候第一次启动就报错。
这里把官方的注释和空行都过滤了，详细的配置可以参考官方的注释说明。
jgj-sentinel-redis]# grep -Ev '#|^$' redis/values.yaml
global:
imageRegistry: "swr.cn-south-1.myhuaweicloud.com/aaa"
imagePullSecrets: ["registry-huawei"]
storageClass: "csi-disk"
redis:
password: "AdVji0a"
kubeVersion: ""
nameOverride: ""
fullnameOverride: ""
commonLabels: {}
commonAnnotations: {}
secretAnnotations: {}
clusterDomain: cluster.local
extraDeploy: []
useHostnames: true
nameResolutionThreshold: 5
nameResolutionTimeout: 5
diagnosticMode:
enabled: false
command:
- sleep
args:
- infinity
image:
registry: swr.cn-south-1.myhuaweicloud.com/aaa
repository: redis
tag: 7.2.3-debian-11-r0
digest: ""
pullPolicy: IfNotPresent
pullSecrets: ["registry-huawei"]
debug: false
architecture: replication
auth:
enabled: true
sentinel: true
password: "AdVji0a"
existingSecret: ""
existingSecretPasswordKey: ""
usePasswordFiles: false
commonConfiguration: |-
appendonly no
save ""
existingConfigmap: ""
master:
count: 1
configuration: ""
disableCommands:
- FLUSHDB
- FLUSHALL
command: []
args: []
enableServiceLinks: true
preExecCmds: []
extraFlags: []
extraEnvVars: []
extraEnvVarsCM: ""
extraEnvVarsSecret: ""
containerPorts:
redis: 6379
startupProbe:
enabled: false
initialDelaySeconds: 20
periodSeconds: 5
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
livenessProbe:
enabled: true
initialDelaySeconds: 20
periodSeconds: 5
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
readinessProbe:
enabled: true
initialDelaySeconds: 20
periodSeconds: 5
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 5
customStartupProbe: {}
customLivenessProbe: {}
customReadinessProbe: {}
resources:
limits: {cpu: 2, memory: 4G}
requests: {cpu: 1, memory: 4G}
podSecurityContext:
enabled: true
fsGroup: 1001
containerSecurityContext:
enabled: true
runAsUser: 1001
runAsGroup: 0
runAsNonRoot: true
allowPrivilegeEscalation: false
seccompProfile:
type: RuntimeDefault
capabilities:
drop:
- ALL
kind: StatefulSet
schedulerName: ""
updateStrategy:
type: RollingUpdate
minReadySeconds: 0
priorityClassName: ""
hostAliases: []
podLabels: {}
podAnnotations: {}
shareProcessNamespace: false
podAffinityPreset: ""
podAntiAffinityPreset: soft
nodeAffinityPreset:
type: ""
key: ""
values: []
affinity: {}
nodeSelector: {}
tolerations: []
topologySpreadConstraints: []
dnsPolicy: ""
dnsConfig: {}
lifecycleHooks: {}
extraVolumes: []
extraVolumeMounts: []
sidecars: []
initContainers: []
persistence:
enabled: true
medium: ""
sizeLimit: ""
path: /data
subPath: ""
subPathExpr: ""
storageClass: ""
accessModes:
- ReadWriteOnce
size: 2Gi
annotations: {}
labels: {}
selector: {}
dataSource: {}
existingClaim: ""
persistentVolumeClaimRetentionPolicy:
enabled: false
whenScaled: Retain
whenDeleted: Retain
service:
type: ClusterIP
ports:
redis: 6379
nodePorts:
redis: ""
externalTrafficPolicy: Cluster
extraPorts: []
internalTrafficPolicy: Cluster
clusterIP: ""
loadBalancerIP: ""
loadBalancerSourceRanges: []
externalIPs: []
annotations: {}
sessionAffinity: None
sessionAffinityConfig: {}
terminationGracePeriodSeconds: 30
serviceAccount:
create: false
name: ""
automountServiceAccountToken: true
annotations: {}
replica:
replicaCount: 3
configuration: ""
disableCommands:
- FLUSHDB
- FLUSHALL
command: []
args: []
enableServiceLinks: true
preExecCmds: []
extraFlags: []
extraEnvVars: []
extraEnvVarsCM: ""
extraEnvVarsSecret: ""
externalMaster:
enabled: false
host: ""
port: 6379
containerPorts:
redis: 6379
startupProbe:
enabled: true
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 22
livenessProbe:
enabled: true
initialDelaySeconds: 20
periodSeconds: 5
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
readinessProbe:
enabled: true
initialDelaySeconds: 20
periodSeconds: 5
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 5
customStartupProbe: {}
customLivenessProbe: {}
customReadinessProbe: {}
resources:
limits: {cpu: 2, memory: 2G}
requests: {cpu: 1, memory: 2G}
podSecurityContext:
enabled: true
fsGroup: 1001
containerSecurityContext:
enabled: true
runAsUser: 1001
runAsGroup: 0
runAsNonRoot: true
allowPrivilegeEscalation: false
seccompProfile:
type: RuntimeDefault
capabilities:
drop:
- ALL
schedulerName: ""
updateStrategy:
type: RollingUpdate
minReadySeconds: 0
priorityClassName: ""
podManagementPolicy: ""
hostAliases: []
podLabels: {}
podAnnotations: {}
shareProcessNamespace: false
podAffinityPreset: ""
podAntiAffinityPreset: soft
nodeAffinityPreset:
type: ""
key: ""
values: []
affinity: {}
nodeSelector: {}
tolerations: []
topologySpreadConstraints: []
dnsPolicy: ""
dnsConfig: {}
lifecycleHooks: {}
extraVolumes: []
extraVolumeMounts: []
sidecars: []
initContainers: []
persistence:
enabled: true
medium: ""
sizeLimit: ""
path: /data
subPath: ""
subPathExpr: ""
storageClass: ""
accessModes:
- ReadWriteOnce
size: 2Gi
annotations: {}
labels: {}
selector: {}
dataSource: {}
existingClaim: ""
persistentVolumeClaimRetentionPolicy:
enabled: false
whenScaled: Retain
whenDeleted: Retain
service:
type: ClusterIP
ports:
redis: 6379
nodePorts:
redis: ""
externalTrafficPolicy: Cluster
internalTrafficPolicy: Cluster
extraPorts: []
clusterIP: ""
loadBalancerIP: ""
loadBalancerSourceRanges: []
annotations: {}
sessionAffinity: None
sessionAffinityConfig: {}
terminationGracePeriodSeconds: 30
autoscaling:
enabled: false
minReplicas: 1
maxReplicas: 11
targetCPU: ""
targetMemory: ""
serviceAccount:
create: false
name: ""
automountServiceAccountToken: true
annotations: {}
sentinel:
enabled: true
image:
registry: docker.io
repository: redis-sentinel
tag: 7.2.3-debian-11-r0
digest: ""
pullPolicy: IfNotPresent
pullSecrets: []
debug: false
annotations: {}
masterSet: mymaster
quorum: 3
getMasterTimeout: 99
automateClusterRecovery: false
redisShutdownWaitFailover: true
downAfterMilliseconds: 60000
failoverTimeout: 180000
parallelSyncs: 1
configuration: ""
command: []
args: []
enableServiceLinks: true
preExecCmds: []
extraEnvVars: []
extraEnvVarsCM: ""
extraEnvVarsSecret: ""
externalMaster:
enabled: false
host: ""
port: 6379
containerPorts:
sentinel: 26379
startupProbe:
enabled: true
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 22
livenessProbe:
enabled: true
initialDelaySeconds: 20
periodSeconds: 10
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 6
readinessProbe:
enabled: true
initialDelaySeconds: 20
periodSeconds: 5
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 6
customStartupProbe: {}
customLivenessProbe: {}
customReadinessProbe: {}
persistence:
enabled: false
storageClass: ""
accessModes:
- ReadWriteOnce
size: 100Mi
annotations: {}
labels: {}
selector: {}
dataSource: {}
medium: ""
sizeLimit: ""
persistentVolumeClaimRetentionPolicy:
enabled: false
whenScaled: Retain
whenDeleted: Retain
resources:
limits: {cpu: 2, memory: 2G}
requests: {cpu: 1, memory: 2G}
containerSecurityContext:
enabled: true
runAsUser: 1001
runAsGroup: 0
runAsNonRoot: true
allowPrivilegeEscalation: false
seccompProfile:
type: RuntimeDefault
capabilities:
drop:
- ALL
lifecycleHooks: {}
extraVolumes: []
extraVolumeMounts: []
service:
type: ClusterIP
ports:
redis: 6379
sentinel: 26379
nodePorts:
redis: ""
sentinel: ""
externalTrafficPolicy: Cluster
extraPorts: []
clusterIP: ""
loadBalancerIP: ""
loadBalancerSourceRanges: []
annotations: {}
sessionAffinity: None
sessionAffinityConfig: {}
headless:
annotations: {}
terminationGracePeriodSeconds: 30
serviceBindings:
enabled: false
networkPolicy:
enabled: false
allowExternal: true
extraIngress: []
extraEgress: []
ingressNSMatchLabels: {}
ingressNSPodMatchLabels: {}
metrics:
allowExternal: true
ingressNSMatchLabels: {}
ingressNSPodMatchLabels: {}
podSecurityPolicy:
create: false
enabled: false
rbac:
create: false
rules: []
serviceAccount:
create: true
name: ""
automountServiceAccountToken: true
annotations: {}
pdb:
create: false
minAvailable: 1
maxUnavailable: ""
tls:
enabled: false
authClients: true
autoGenerated: false
existingSecret: ""
certificatesSecret: ""
certFilename: ""
certKeyFilename: ""
certCAFilename: ""
dhParamsFilename: ""
metrics:
enabled: false
image:
registry: docker.io
repository: bitnami/redis-exporter
tag: 1.55.0-debian-11-r0
digest: ""
pullPolicy: IfNotPresent
pullSecrets: []
startupProbe:
enabled: false
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
livenessProbe:
enabled: true
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
readinessProbe:
enabled: true
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 3
customStartupProbe: {}
customLivenessProbe: {}
customReadinessProbe: {}
command: []
redisTargetHost: "localhost"
extraArgs: {}
extraEnvVars: []
containerSecurityContext:
enabled: true
runAsUser: 1001
runAsGroup: 0
runAsNonRoot: true
allowPrivilegeEscalation: false
seccompProfile:
type: RuntimeDefault
capabilities:
drop:
- ALL
extraVolumes: []
extraVolumeMounts: []
resources:
limits: {}
requests: {}
podLabels: {}
podAnnotations:
prometheus.io/scrape: "true"
prometheus.io/port: "9121"
service:
type: ClusterIP
port: 9121
externalTrafficPolicy: Cluster
extraPorts: []
loadBalancerIP: ""
loadBalancerSourceRanges: []
annotations: {}
clusterIP: ""
serviceMonitor:
enabled: false
namespace: ""
interval: 30s
scrapeTimeout: ""
relabellings: []
metricRelabelings: []
honorLabels: false
additionalLabels: {}
podTargetLabels: []
sampleLimit: false
targetLimit: false
podMonitor:
enabled: false
namespace: ""
interval: 30s
scrapeTimeout: ""
relabellings: []
metricRelabelings: []
honorLabels: false
additionalLabels: {}
podTargetLabels: []
sampleLimit: false
targetLimit: false
prometheusRule:
enabled: false
namespace: ""
additionalLabels: {}
rules: []
volumePermissions:
enabled: false
image:
registry: docker.io
repository: bitnami/os-shell
tag: 11-debian-11-r90
digest: ""
pullPolicy: IfNotPresent
pullSecrets: []
resources:
limits: {}
requests: {}
containerSecurityContext:
runAsUser: 0
sysctl:
enabled: false
image:
registry: docker.io
repository: bitnami/os-shell
tag: 11-debian-11-r90
digest: ""
pullPolicy: IfNotPresent
pullSecrets: []
command: []
mountHostSys: false
resources:
limits: {}
requests: {}
useExternalDNS:
enabled: false
suffix: ""
annotationKey: external-dns.alpha.kubernetes.io/
additionalAnnotations: {}
部署 redis
helm -n jian-butler install redis redis/
修改configmap
ckubectl -n jian-butler get cm redis-configuration -o yaml
apiVersion: v1
data:
master.conf: |-
dir /tmp
# User-supplied master configuration:
rename-command FLUSHDB ""
rename-command FLUSHALL ""
appendonly no
save ""
maxclients 65535
maxmemory 4G
slowlog-log-slower-than 10000
slowlog-max-len 128
timeout 10
tcp-keepalive 30
# End of master configuration
redis.conf: |-
appendonly no
save ""
dir /tmp
maxclients 65535
maxmemory 4G
slowlog-log-slower-than 10000
slowlog-max-len 128
timeout 10
tcp-keepalive 30
replica.conf: |-
dir /data
appendonly yes
rename-command FLUSHDB ""
rename-command FLUSHALL ""
# End of replica configuration
sentinel.conf: |-
dir "/tmp"
port 26379
sentinel monitor mymaster redis-node-0.redis-headless.jian-butler.svc.cluster.local 6379 3
sentinel down-after-milliseconds mymaster 60000
sentinel failover-timeout mymaster 180000
sentinel parallel-syncs mymaster 1
# User-supplied sentinel configuration:
# End of sentinel configuration
kind: ConfigMap
metadata:
annotations:
meta.helm.sh/release-name: redis
meta.helm.sh/release-namespace: jian-butler
labels:
app.kubernetes.io/instance: redis
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/name: redis
app.kubernetes.io/version: 7.2.3
helm.sh/chart: redis-18.2.2
name: redis-configuration
namespace: jian-butler
查看修改之后的redis 哨兵状态。
jgj-sentinel-redis]# ckubectl -n jian-butler get pod
NAME READY STATUS RESTARTS AGE
redis-node-0 2/2 Running 0 15m
redis-node-1 2/2 Running 0 16m
redis-node-2 2/2 Running 0 16m
修改内核参数,优化配置
修改之后会重启redis的每个pod
initContainers:
- image: busybox
command:
- sh
- -c
- |
sysctl -w net.core.somaxconn=65535
sysctl -w fs.file-max=1048576
imagePullPolicy: Always
name: setsysctl
securityContext:
privileged: true
