安装rocketmq DLedger 高可用集群

架构说明:

RocketMQ Dledger 集群架构

 RocketMQ 4.5 以前的版本大多都是采用 Master-Slave 架构来部署，能在一定程度上保证数据的不丢失，也能保证一定的可用性。

但是那种方式 的缺陷很明显，最大的问题就是当 Master Broker 挂了之后 ，没办法让 Slave Broker 自动 切换为新的 Master Broker，需要手动更改配置将 Slave Broker 设置为 Master Broker，以及重启机器，这个非常麻烦。

在手式运维的期间，可能会导致系统的不可用。

使用 Dledger 技术要求至少由三个 Broker 组成 ，一个 Master 和两个 Slave，这样三个 Broker 就可以组成一个 Group ，也就是三个 Broker 可以分组来运行。一但 Master 宕机，Dledger 就可以从剩下的两个 Broker 中选举一个 Master 继续对外提供服务。

高可用

三个 NameServer 极端情况下，确保集群的可用性，任何两个 NameServer 挂掉也不会影响信息的整体使用。
在上图中每个 Master Broker 都有两个 Slave Broker，这样可以保证可用性，如在同一个 Dledger Group 中 Master Broker 宕机后，Dledger 会去行投票将剩下的节点晋升为 Master Broker。

高并发

假设某个Topic的每秒十万消息的写入， 可以增加 Master Broker 然后十万消息的写入会分别分配到不同的 Master Broker ，如有5台 Master Broker 那每个 Broker 就会承载2万的消息写入。

可伸缩

如果消息数量增大，需要存储更多的数量和最高的并发，完全可以增加 Broker ，这样可以线性扩展集群。

海量消息

数据都是分布式存储的，每个Topic的数据都会分布在不同的 Broker 中，如果需要存储更多的数据，只需要增加 Master Broker 就可以了。

安装rocketmq DLedger 高可用集群

系统环境：

centos 7.x
rocketmq 4.8.0 (至少要4.5+)
docker 18.09.9
docker-compose 1.29.2

服务器

10.200.26.26 部署组件：nameserver, ${brokerName}-n0, ${brokerName}-n1, ${brokerName}-n2,rocketmq-console
10.200.26.28 部署组件：nameserver, ${brokerName}-n0, ${brokerName}-n1, ${brokerName}-n2

  • 最低高可用是3台（1组broker)

  • 生产可以采用6台，分别 部署2 组broker

目录 （所有服务器）

mkdir -p /data/docker_app/rocketmq/data/broker-n{0..2}/store
mkdir -p /data/docker_app/rocketmq/data/broker-n{0..2}/conf
mkdir -p /data/docker_app/rocketmq/data/broker-n{0..2}/logs
mkdir -p /data/docker_app/rocketmq/data/broker-n{0..2}/commitlog

配置文件-20.200.26.26

vim /data/docker_app/rocketmq/data/broker-n0/conf/broker.conf

brokerName=RaftNode00
listenPort=30911
brokerClusterName = DefaultCluster
brokerId = 0
deleteWhen = 04
fileReservedTime = 48
brokerRole = ASYNC_MASTER
flushDiskType = ASYNC_FLUSH
brokerIP1=10.200.26.26
brokerIP2=10.200.26.26
namesrvAddr=10.200.26.26:9876;10.200.26.28:9876
storePathRootDir=/home/rocketmq/store
storePathCommitLog=/home/rocketmq/commitlog
autoCreateTopicEnable=true
autoCreateSubscriptionGroup=true
# 与 dledger 相关的属性
enableDLegerCommitLog=true
dLegerGroup=RaftNode00
dLegerPeers=n0-10.200.26.26:40911;n1-10.200.26.26:40912;n2-10.200.26.26:40913
dLegerSelfId=n0

vim /data/docker_app/rocketmq/data/broker-n1/conf/broker.conf

brokerName=RaftNode00
listenPort=30912
brokerClusterName = DefaultCluster
brokerId = 1
deleteWhen = 04
fileReservedTime = 48
brokerRole = ASYNC_MASTER
flushDiskType = ASYNC_FLUSH
brokerIP1=10.200.26.26
brokerIP2=10.200.26.26
namesrvAddr=10.200.26.26:9876;10.200.26.28:9876
storePathRootDir=/home/rocketmq/store
storePathCommitLog=/home/rocketmq/commitlog
autoCreateTopicEnable=true
autoCreateSubscriptionGroup=true
# 与 dledger 相关的属性
enableDLegerCommitLog=true
dLegerGroup=RaftNode00
dLegerPeers=n0-10.200.26.26:40911;n1-10.200.26.26:40912;n2-10.200.26.26:40913
dLegerSelfId=n1

vim /data/docker_app/rocketmq/data/broker-n2/conf/broker.conf

brokerName=RaftNode00
listenPort=30913
brokerClusterName = DefaultCluster
brokerId = 2
deleteWhen = 04
fileReservedTime = 48
brokerRole = ASYNC_MASTER
flushDiskType = ASYNC_FLUSH
brokerIP1=10.200.26.26
brokerIP2=10.200.26.26
namesrvAddr=10.200.26.26:9876;10.200.26.28:9876
storePathRootDir=/home/rocketmq/store
storePathCommitLog=/home/rocketmq/commitlog
autoCreateTopicEnable=true
autoCreateSubscriptionGroup=true
# 与 dledger 相关的属性
enableDLegerCommitLog=true
dLegerGroup=RaftNode00
dLegerPeers=n0-10.200.26.26:40911;n1-10.200.26.26:40912;n2-10.200.26.26:40913
dLegerSelfId=n2

编辑 docker-compose文件-20.200.26.26

version: '2'
services:
  #Service for nameserver
  rmnamesrv:
    restart: always
    image: foxiswho/rocketmq:4.8.0
    container_name: rmnamesrv 
    ports:
      - 9876:9876
    environment:
      - JAVA_OPT_EXT=-Xms512M -Xmx512M -Xmn128m
      - TZ=Asia/Shanghai
    volumes:
      - ./data/namesrv/logs:/home/rocketmq/logs/rocketmqlogs
    command: sh mqnamesrv

  #Service for broker
  rmqbroker-n0:
    restart: always
    image: foxiswho/rocketmq:4.8.0
    container_name: rmqbroker-n0
    ports:
      - 30911:30911
      - 40911:40911
    environment:
      - NAMESRV_ADDR=10.200.26.26:9876;10.200.26.28:9876
      - JAVA_OPT_EXT=-Xms512M -Xmx512M -Xmn128m
      - TZ=Asia/Shanghai
  
    volumes:
      - ./data/broker-n0/logs:/home/rocketmq/logs
      - ./data/broker-n0/store:/home/rocketmq/store
      - ./data/broker-n0/commitlog:/home/rocketmq/commitlog
      - ./data/broker-n0/conf/broker.conf:/home/rocketmq/rocketmq-4.8.0/conf/broker.conf
    command: sh mqbroker -c /home/rocketmq/rocketmq-4.8.0/conf/broker.conf
  rmqbroker-n1:
    restart: always
    image: foxiswho/rocketmq:4.8.0
    container_name: rmqbroker-n1
    ports:
      - 30912:30912
      - 40912:40912
    environment:
      - NAMESRV_ADDR=10.200.26.26:9876;10.200.26.28:9876
      - JAVA_OPT_EXT=-Xms512M -Xmx512M -Xmn128m
      - TZ=Asia/Shanghai

    volumes:
      - ./data/broker-n1/logs:/home/rocketmq/logs
      - ./data/broker-n1/store:/home/rocketmq/store
      - ./data/broker-n1/commitlog:/home/rocketmq/commitlog
      - ./data/broker-n1/conf/broker.conf:/home/rocketmq/rocketmq-4.8.0/conf/broker.conf
    command: sh mqbroker -c /home/rocketmq/rocketmq-4.8.0/conf/broker.conf

  rmqbroker-n2:
    restart: always
    image: foxiswho/rocketmq:4.8.0
    container_name: rmqbroker-n2
    ports: 
      - 30913:30913
      - 40913:40913
    environment:
      - NAMESRV_ADDR=10.200.26.26:9876;10.200.26.28:9876
      - JAVA_OPT_EXT=-Xms512M -Xmx512M -Xmn128m
      - TZ=Asia/Shanghai

    volumes:
      - ./data/broker-n2/logs:/home/rocketmq/logs
      - ./data/broker-n2/store:/home/rocketmq/store
      - ./data/broker-n2/commitlog:/home/rocketmq/commitlog
      - ./data/broker-n2/conf/broker.conf:/home/rocketmq/rocketmq-4.8.0/conf/broker.conf
    command: sh mqbroker -c /home/rocketmq/rocketmq-4.8.0/conf/broker.conf

  rmqconsole:
    restart: always
    image: styletang/rocketmq-console-ng
    container_name: rmqconsole
    ports:
      - 8081:8080
    environment:
        TZ: "Asia/Shanghai"
        JAVA_OPTS: "-Drocketmq.namesrv.addr=10.200.26.26:9876;10.200.26.28:9876 -Dcom.rocketmq.sendMessageWithVIPChannel=false"
    depends_on:
      - rmnamesrv

配置文件-20.200.26.28

vim /data/docker_app/rocketmq/data/broker-n0/conf/broker.conf

brokerName=RaftNode01
listenPort=30911
brokerClusterName = DefaultCluster
brokerId = 0
deleteWhen = 04
fileReservedTime = 48
brokerRole = ASYNC_MASTER
flushDiskType = ASYNC_FLUSH
brokerIP1=10.200.26.28
brokerIP2=10.200.26.28
namesrvAddr=10.200.26.26:9876;10.200.26.28:9876
storePathRootDir=/home/rocketmq/store
storePathCommitLog=/home/rocketmq/commitlog
autoCreateTopicEnable=true
autoCreateSubscriptionGroup=true
# 与 dledger 相关的属性
enableDLegerCommitLog=true
dLegerGroup=RaftNode01
dLegerPeers=n0-10.200.26.28:40911;n1-10.200.26.28:40912;n2-10.200.26.28:40913
dLegerSelfId=n0

vim /data/docker_app/rocketmq/data/broker-n1/conf/broker.conf


brokerName=RaftNode01
listenPort=30912
brokerClusterName = DefaultCluster
brokerId = 1
deleteWhen = 04
fileReservedTime = 48
brokerRole = ASYNC_MASTER
flushDiskType = ASYNC_FLUSH
brokerIP1=10.200.26.28
brokerIP2=10.200.26.28
namesrvAddr=10.200.26.26:9876;10.200.26.28:9876
storePathRootDir=/home/rocketmq/store
storePathCommitLog=/home/rocketmq/commitlog
autoCreateTopicEnable=true
autoCreateSubscriptionGroup=true
# 与 dledger 相关的属性
enableDLegerCommitLog=true
dLegerGroup=RaftNode01
dLegerPeers=n0-10.200.26.28:40911;n1-10.200.26.28:40912;n2-10.200.26.28:40913
dLegerSelfId=n1

vim /data/docker_app/rocketmq/data/broker-n2/conf/broker.conf

brokerName=RaftNode01
listenPort=30913
brokerClusterName = DefaultCluster
brokerId = 3
deleteWhen = 04
fileReservedTime = 48
brokerRole = ASYNC_MASTER
flushDiskType = ASYNC_FLUSH
brokerIP1=10.200.26.28
brokerIP2=10.200.26.28
namesrvAddr=10.200.26.26:9876;10.200.26.28:9876;10.200.26.29:9876
storePathRootDir=/home/rocketmq/store
storePathCommitLog=/home/rocketmq/store/commitlog
autoCreateTopicEnable=true
autoCreateSubscriptionGroup=true
# 与 dledger 相关的属性
enableDLegerCommitLog=true
dLegerGroup=RaftNode01
dLegerPeers=n0-10.200.26.28:40911;n1-10.200.26.28:40912;n2-10.200.26.28:40913
dLegerSelfId=n2

编辑 docker-compose文件-20.200.26.28

vim /data/docker_app/rocketmq/docker-compose.yaml

version: '2'
services:
  #Service for nameserver
  rmnamesrv:
    restart: always
    image: foxiswho/rocketmq:4.8.0
    container_name: rmnamesrv 
    ports:
      - 9876:9876
    environment:
      - JAVA_OPT_EXT=-Xms512M -Xmx512M -Xmn128m
      - TZ=Asia/Shanghai
    volumes:
      - ./data/namesrv/logs:/home/rocketmq/logs/rocketmqlogs
    command: sh mqnamesrv

  #Service for broker
  rmqbroker-n0:
    restart: always
    image: foxiswho/rocketmq:4.8.0
    container_name: rmqbroker-n0
    ports:
      - 30911:30911
      - 40911:40911
    environment:
      - NAMESRV_ADDR=10.200.26.26:9876;10.200.26.28:9876
      - JAVA_OPT_EXT=-Xms512M -Xmx512M -Xmn128m
      - TZ=Asia/Shanghai
  
    volumes:
      - ./data/broker-n0/logs:/home/rocketmq/logs
      - ./data/broker-n0/store:/home/rocketmq/store
      - ./data/broker-n0/commitlog:/home/rocketmq/commitlog
      - ./data/broker-n0/conf/broker.conf:/home/rocketmq/rocketmq-4.8.0/conf/broker.conf
    command: sh mqbroker -c /home/rocketmq/rocketmq-4.8.0/conf/broker.conf
  rmqbroker-n1:
    restart: always
    image: foxiswho/rocketmq:4.8.0
    container_name: rmqbroker-n1
    ports:
      - 30912:30912
      - 40912:40912
    environment:
      - NAMESRV_ADDR=10.200.26.26:9876;10.200.26.28:9876
      - JAVA_OPT_EXT=-Xms512M -Xmx512M -Xmn128m
      - TZ=Asia/Shanghai

    volumes:
      - ./data/broker-n1/logs:/home/rocketmq/logs
      - ./data/broker-n1/store:/home/rocketmq/store
      - ./data/broker-n1/commitlog:/home/rocketmq/commitlog
      - ./data/broker-n1/conf/broker.conf:/home/rocketmq/rocketmq-4.8.0/conf/broker.conf
    command: sh mqbroker -c /home/rocketmq/rocketmq-4.8.0/conf/broker.conf

  rmqbroker-n2:
    restart: always
    image: foxiswho/rocketmq:4.8.0
    container_name: rmqbroker-n2
    ports: 
      - 30913:30913
      - 40913:40913
    environment:
      - NAMESRV_ADDR=10.200.26.26:9876;10.200.26.28:9876
      - JAVA_OPT_EXT=-Xms512M -Xmx512M -Xmn128m
      - TZ=Asia/Shanghai

    volumes:
      - ./data/broker-n2/logs:/home/rocketmq/logs
      - ./data/broker-n2/store:/home/rocketmq/store
      - ./data/broker-n2/commitlog:/home/rocketmq/commitlog
      - ./data/broker-n2/conf/broker.conf:/home/rocketmq/rocketmq-4.8.0/conf/broker.conf
    command: sh mqbroker -c /home/rocketmq/rocketmq-4.8.0/conf/broker.conf

启动服务

  docker-compose up -d

访问 mq 控制 台

http://10.200.26.26:8081/#/cluster

提示： 如果 走监控需要开通映像端口：broker: 30909 (rocketmq-exporter这个需要）

安装rockemq-exporter插件

https://github.com/apache/rocketmq-exporter

提示： 如果 走监控需要开通映像端口：broker:  30909  (rocketmq-exporter这个需要）

vim docker-compose.yaml

version: '3'
services:
  rocketmq-exporter:
    image: harbor.xxx.com/rocketmq-exporter
    restart: always
    command:
    - '--rocketmq.config.namesrvAddr=10.x.x.x:9876'
    - '--rocketmq.config.rocketmqVersion=V4_8_0'
    ports:
    - '5557:5557'

配置prometheus


- targets: ['10.x.x.x:5557']
  labels:
    project: hcloud
    instance: 10.x.x.x.x:9876
    app: rocketmq
    env: pro
-------

rules/xxx.yaml

groups:
- name: rocketmq
  rules:
  - alert: RocketMQ Exporter is Down
    expr: up{job="rocketmq"} == 0
    for: 20s
    labels:
      severity: '灾难'
    annotations:
      summary: RocketMQ {{ $labels.instance }} is down
  - alert: RocketMQ 存在消息积压
    expr: (sum(irate(rocketmq_producer_offset{project='hcloud'}[1m])) by (topic)  - on(topic) group_right sum(irate(rocketmq_consumer_offset{project='hcloud'}[1m])) by (group,topic)) > 5
    for: 5m
    labels:
      severity: '警告'
    annotations:
      summary: RocketMQ (group={{ $labels.group }} topic={{ $labels.topic }})积压数 = {{ .Value }}
  - alert: GroupGetLatencyByStoretime 消费组的消费延时时间过高
    expr: rocketmq_group_get_latency_by_storetime{project='hcloud',group !="storetransferauto"}/1000  > 5 and rate(rocketmq_group_get_latency_by_storetime{project='hcloud',group !="storetransferauto"}[5m]) >0
    for: 3m
    labels:
      severity: 警告
    annotations:
      description: 'consumer {{$labels.group}} on {{$labels.broker}}, {{$labels.topic}} consume time lag behind message store time
        and (behind value is {{$value}}).'
      summary: 消费组的消费延时时间过高
  - alert: RocketMQClusterProduceHigh 集群TPS > 20
    expr: sum(rocketmq_producer_tps{project='hcloud'}) by (cluster) >= 20
    for: 3m
    labels:
      severity: 警告
    annotations:
      description: '{{$labels.cluster}} Sending tps too high. now TPS = {{ .Value }}'
      summary: cluster send tps too high

配置补充说明（硬盘延迟高，发送消息失败，可以加)

#broker配置
#broker检测队列中的消息等待时间（默认是200毫秒，自行配置）
waitTimeMillsInSendQueue=6000
#系统页面缓存繁忙超时时间(翻译),默认值 1000
osPageCacheBusyTimeOutMills=5000

配置性优化

文档:https://segmentfault.com/a/1190000038318572

posted @ 2022-09-26 14:53  清风木  阅读(881)  评论(0)  编辑  收藏  举报