k8s部署rocketmq 双主

由于apache 官网的 docker image 是单点，要实现集群方式部署。

 

rocketmq 分为 nameserver 和 broker ， 对于之间调用频繁的服务，会增加网络压力， 所以 考虑又网络通信 变成 进程间通信， 那么1个pod 中包含两个container。

 

关于rocketmq 下载编译， https://www.cnblogs.com/fengjian2016/p/10150394.html

拿到 编译后的  apache-rocketmq , 需要修改一些配置， 例如：

1. rm -rf  删除 rocketmq/bin/*.cmd

2 .把runserver.sh  HEAP_OPTS 变成环境变量， 把runbroker.sh  HEAP_OPTS 变成环境变量

  

3. 指定消息存储路径

[root@rocketmq1 conf]#mkdir -p /data/rocketmq/store
[root@rocketmq1 conf]#mkdir -p /data/rocketmq/store/commitlog
[root@rocketmq1 conf]#mkdir -p /data/rocketmq/logs 

4. 指定日志输出路径

/data/rocketmq/conf/ 中的xml文件
sed -i 's#${user.home}#/data/rocketmq/logs#g' *.xml

5. 删除多余配置文件 2m-2s 2master 等等， 启动的时候k8s command 直接指向文件
/data/rocketmq/conf/ 目录下新建 broker-a.conf  broker-b.conf 2个文件

6. 配置修改

　　(1) rocketmqHome 修改

　　(2) namesrvAddr 修改，强调  通一个pod 必须写localhost 或者 127.0.0.1 要不然调不通

 

[root@harbor conf]# cat broker-a.conf 
rocketmqHome=/data/rocketmq
namesrvAddr=localhost:9876;rocketmq2:9876
brokerName=broker-a
brokerClusterName=DefaultCluster
brokerId=0
brokerPermission=6
#在发送消息时，自动创建服务器不存在的topic，默认创建的队列数
defaultTopicQueueNums=4
#是否允许 Broker 自动创建Topic，建议线下开启，线上关闭
autoCreateTopicEnable=true
clusterTopicEnable=true
brokerTopicEnable=true
#是否允许 Broker 自动创建订阅组，建议线下开启，线上关闭
autoCreateSubscriptionGroup=true
sendMessageThreadPoolNums=1
pullMessageThreadPoolNums=20
queryMessageThreadPoolNums=10
adminBrokerThreadPoolNums=16
clientManageThreadPoolNums=32
consumerManageThreadPoolNums=32
heartbeatThreadPoolNums=2
endTransactionThreadPoolNums=12
flushConsumerOffsetInterval=5000
flushConsumerOffsetHistoryInterval=60000
rejectTransactionMessage=false
fetchNamesrvAddrByAddressServer=false
sendThreadPoolQueueCapacity=10000
pullThreadPoolQueueCapacity=100000
queryThreadPoolQueueCapacity=20000
clientManagerThreadPoolQueueCapacity=1000000
consumerManagerThreadPoolQueueCapacity=1000000
heartbeatThreadPoolQueueCapacity=50000
endTransactionPoolQueueCapacity=100000
filterServerNums=0
longPollingEnable=true
shortPollingTimeMills=1000
notifyConsumerIdsChangedEnable=true
highSpeedMode=false
commercialEnable=true
commercialTimerCount=1
commercialTransCount=1
commercialBigCount=1
commercialBaseCount=1
transferMsgByHeap=true
maxDelayTime=40
regionId=DefaultRegion
registerBrokerTimeoutMills=6000
slaveReadEnable=false
disableConsumeIfConsumerReadSlowly=false
consumerFallbehindThreshold=17179869184
brokerFastFailureEnable=true
waitTimeMillsInSendQueue=200
waitTimeMillsInPullQueue=5000
waitTimeMillsInHeartbeatQueue=31000
waitTimeMillsInTransactionQueue=3000
startAcceptSendRequestTimeStamp=0
traceOn=true
enableCalcFilterBitMap=false
expectConsumerNumUseFilter=32
maxErrorRateOfBloomFilter=20
filterDataCleanTimeSpan=86400000
filterSupportRetry=false
enablePropertyFilter=false
compressedRegister=false
forceRegister=true
registerNameServerPeriod=30000
transactionTimeOut=6000
transactionCheckMax=15
transactionCheckInterval=60000
#Broker 对外服务的监听端口
listenPort=10911
serverWorkerThreads=8
serverCallbackExecutorThreads=0
serverSelectorThreads=3
serverOnewaySemaphoreValue=256
serverAsyncSemaphoreValue=64
serverChannelMaxIdleTimeSeconds=120
serverSocketSndBufSize=131072
serverSocketRcvBufSize=131072
serverPooledByteBufAllocatorEnable=true
useEpollNativeSelector=false
clientWorkerThreads=4
clientCallbackExecutorThreads=2
clientOnewaySemaphoreValue=65535
clientAsyncSemaphoreValue=65535
connectTimeoutMillis=3000
channelNotActiveInterval=60000
clientChannelMaxIdleTimeSeconds=120
clientSocketSndBufSize=131072
clientSocketRcvBufSize=131072
clientPooledByteBufAllocatorEnable=false
clientCloseSocketIfTimeout=false
useTLS=false
storePathRootDir=/data/rocketmq/store
storePathCommitLog=/root/rocketmq/store/commitlog
#commitLog每个文件的大小默认1G
mapedFileSizeCommitLog=1073741824
#ConsumeQueue每个文件默认存60W条，根据业务情况调整,
mapedFileSizeConsumeQueue=3000000
enableConsumeQueueExt=false
mappedFileSizeConsumeQueueExt=50331648
bitMapLengthConsumeQueueExt=64
flushIntervalCommitLog=500
commitIntervalCommitLog=200
useReentrantLockWhenPutMessage=false
flushCommitLogTimed=false
flushIntervalConsumeQueue=1000
cleanResourceInterval=10000
deleteCommitLogFilesInterval=100
deleteConsumeQueueFilesInterval=100
destroyMapedFileIntervalForcibly=120000
redeleteHangedFileInterval=120000
#删除文件时间点，默认凌晨 2点
deleteWhen=02
#检测物理文件磁盘空间
diskMaxUsedSpaceRatio=75
#文件保留时间，默认 72 小时
fileReservedTime=128
putMsgIndexHightWater=600000
#限制的消息大小
maxMessageSize=65536
checkCRCOnRecover=true
flushCommitLogLeastPages=4
commitCommitLogLeastPages=4
flushLeastPagesWhenWarmMapedFile=4096
flushConsumeQueueLeastPages=2
flushCommitLogThoroughInterval=10000
commitCommitLogThoroughInterval=200
flushConsumeQueueThoroughInterval=60000
maxTransferBytesOnMessageInMemory=262144
maxTransferCountOnMessageInMemory=32
maxTransferBytesOnMessageInDisk=65536
maxTransferCountOnMessageInDisk=8
accessMessageInMemoryMaxRatio=40
messageIndexEnable=true
maxHashSlotNum=5000000
maxIndexNum=20000000
maxMsgsNumBatch=64
messageIndexSafe=false
haListenPort=10912
haSendHeartbeatInterval=5000
haHousekeepingInterval=20000
haTransferBatchSize=32768
haMasterAddress=
haSlaveFallbehindMax=268435456
brokerRole=ASYNC_MASTER
flushDiskType=ASYNC_FLUSH
syncFlushTimeout=5000
messageDelayLevel=1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h
flushDelayOffsetInterval=10000
cleanFileForciblyEnable=true
warmMapedFileEnable=false
offsetCheckInSlave=false
debugLockEnable=false
duplicationEnable=false
diskFallRecorded=true
osPageCacheBusyTimeOutMills=1000
defaultQueryMaxNum=32
transientStorePoolEnable=false
transientStorePoolSize=5
fastFailIfNoBufferInStorePool=false

 

[root@harbor conf]# cat broker-b.conf 
rocketmqHome=/data/rocketmq
namesrvAddr=rocketmq1:9876;localhost:9876
brokerName=broker-b
brokerClusterName=DefaultCluster
brokerId=0
brokerPermission=6
#在发送消息时，自动创建服务器不存在的topic，默认创建的队列数
defaultTopicQueueNums=4
#是否允许 Broker 自动创建Topic，建议线下开启，线上关闭
autoCreateTopicEnable=true
clusterTopicEnable=true
brokerTopicEnable=true
#是否允许 Broker 自动创建订阅组，建议线下开启，线上关闭
autoCreateSubscriptionGroup=true
sendMessageThreadPoolNums=1
pullMessageThreadPoolNums=20
queryMessageThreadPoolNums=10
adminBrokerThreadPoolNums=16
clientManageThreadPoolNums=32
consumerManageThreadPoolNums=32
heartbeatThreadPoolNums=2
endTransactionThreadPoolNums=12
flushConsumerOffsetInterval=5000
flushConsumerOffsetHistoryInterval=60000
rejectTransactionMessage=false
fetchNamesrvAddrByAddressServer=false
sendThreadPoolQueueCapacity=10000
pullThreadPoolQueueCapacity=100000
queryThreadPoolQueueCapacity=20000
clientManagerThreadPoolQueueCapacity=1000000
consumerManagerThreadPoolQueueCapacity=1000000
heartbeatThreadPoolQueueCapacity=50000
endTransactionPoolQueueCapacity=100000
filterServerNums=0
longPollingEnable=true
shortPollingTimeMills=1000
notifyConsumerIdsChangedEnable=true
highSpeedMode=false
commercialEnable=true
commercialTimerCount=1
commercialTransCount=1
commercialBigCount=1
commercialBaseCount=1
transferMsgByHeap=true
maxDelayTime=40
regionId=DefaultRegion
registerBrokerTimeoutMills=6000
slaveReadEnable=false
disableConsumeIfConsumerReadSlowly=false
consumerFallbehindThreshold=17179869184
brokerFastFailureEnable=true
waitTimeMillsInSendQueue=200
waitTimeMillsInPullQueue=5000
waitTimeMillsInHeartbeatQueue=31000
waitTimeMillsInTransactionQueue=3000
startAcceptSendRequestTimeStamp=0
traceOn=true
enableCalcFilterBitMap=false
expectConsumerNumUseFilter=32
maxErrorRateOfBloomFilter=20
filterDataCleanTimeSpan=86400000
filterSupportRetry=false
enablePropertyFilter=false
compressedRegister=false
forceRegister=true
registerNameServerPeriod=30000
transactionTimeOut=6000
transactionCheckMax=15
transactionCheckInterval=60000
#Broker 对外服务的监听端口
listenPort=10911
serverWorkerThreads=8
serverCallbackExecutorThreads=0
serverSelectorThreads=3
serverOnewaySemaphoreValue=256
serverAsyncSemaphoreValue=64
serverChannelMaxIdleTimeSeconds=120
serverSocketSndBufSize=131072
serverSocketRcvBufSize=131072
serverPooledByteBufAllocatorEnable=true
useEpollNativeSelector=false
clientWorkerThreads=4
clientCallbackExecutorThreads=2
clientOnewaySemaphoreValue=65535
clientAsyncSemaphoreValue=65535
connectTimeoutMillis=3000
channelNotActiveInterval=60000
clientChannelMaxIdleTimeSeconds=120
clientSocketSndBufSize=131072
clientSocketRcvBufSize=131072
clientPooledByteBufAllocatorEnable=false
clientCloseSocketIfTimeout=false
useTLS=false
storePathRootDir=/data/rocketmq/store
storePathCommitLog=/root/rocketmq/store/commitlog
#commitLog每个文件的大小默认1G
mapedFileSizeCommitLog=1073741824
#ConsumeQueue每个文件默认存60W条，根据业务情况调整,
mapedFileSizeConsumeQueue=3000000
enableConsumeQueueExt=false
mappedFileSizeConsumeQueueExt=50331648
bitMapLengthConsumeQueueExt=64
flushIntervalCommitLog=500
commitIntervalCommitLog=200
useReentrantLockWhenPutMessage=false
flushCommitLogTimed=false
flushIntervalConsumeQueue=1000
cleanResourceInterval=10000
deleteCommitLogFilesInterval=100
deleteConsumeQueueFilesInterval=100
destroyMapedFileIntervalForcibly=120000
redeleteHangedFileInterval=120000
#删除文件时间点，默认凌晨 2点
deleteWhen=02
#检测物理文件磁盘空间
diskMaxUsedSpaceRatio=75
#文件保留时间，默认 72 小时
fileReservedTime=128
putMsgIndexHightWater=600000
#限制的消息大小
maxMessageSize=65536
checkCRCOnRecover=true
flushCommitLogLeastPages=4
commitCommitLogLeastPages=4
flushLeastPagesWhenWarmMapedFile=4096
flushConsumeQueueLeastPages=2
flushCommitLogThoroughInterval=10000
commitCommitLogThoroughInterval=200
flushConsumeQueueThoroughInterval=60000
maxTransferBytesOnMessageInMemory=262144
maxTransferCountOnMessageInMemory=32
maxTransferBytesOnMessageInDisk=65536
maxTransferCountOnMessageInDisk=8
accessMessageInMemoryMaxRatio=40
messageIndexEnable=true
maxHashSlotNum=5000000
maxIndexNum=20000000
maxMsgsNumBatch=64
messageIndexSafe=false
haListenPort=10912
haSendHeartbeatInterval=5000
haHousekeepingInterval=20000
haTransferBatchSize=32768
haMasterAddress=
haSlaveFallbehindMax=268435456
brokerRole=ASYNC_MASTER
flushDiskType=ASYNC_FLUSH
syncFlushTimeout=5000
messageDelayLevel=1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h
flushDelayOffsetInterval=10000
cleanFileForciblyEnable=true
warmMapedFileEnable=false
offsetCheckInSlave=false
debugLockEnable=false
duplicationEnable=false
diskFallRecorded=true
osPageCacheBusyTimeOutMills=1000
defaultQueryMaxNum=32
transientStorePoolEnable=false
transientStorePoolSize=5
fastFailIfNoBufferInStorePool=false

 

broker Dockerfile

[root@harbor rocketmq-broker]# cat Dockerfile 
FROM centos

MAINTAINER fengjian <fengjian@senyint.com>

ENV TZ "Asia/Shanghai"
ENV TERM xterm
ENV MALLOC_ARENA_MAX=1
ENV JAVA_HOME /data/jdk
ENV CLASSPATH $JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
ENV CATALINA_HOME /data/tomcat
ENV CATALINA_BASE /data/tomcat
ENV PATH $PATH:$JAVA_HOME/bin:$CATALINA_HOME/lib:$CATALINA_HOME/bin
ENV ROCKETMQ_HOME /data/rocketmq

RUN rm /etc/yum.repos.d/* -rf
ADD Centos-7.repo /etc/yum.repos.d/
ADD epel.repo /etc/yum.repos.d/
RUN yum clean all
RUN yum -y install net-tools bind-utils sysstat

ADD localtime  /etc/
ADD jdk.tar.gz /data
ADD profile /etc
ADD sysctl.conf /etc
ADD 20-nproc.conf /etc/security/limits.d/
ADD limits.conf /etc/security/

ADD rocketmq.tar.gz /data/
WORKDIR ${ROCKETMQ_HOME}/bin

EXPOSE 10909 10911

 

 

nameserver  Dockerfile

FROM centos

MAINTAINER fengjian <fengjian@senyint.com>

ENV TZ "Asia/Shanghai"
ENV TERM xterm
ENV MALLOC_ARENA_MAX=1
ENV JAVA_HOME /data/jdk
ENV CLASSPATH $JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
ENV CATALINA_HOME /data/tomcat
ENV CATALINA_BASE /data/tomcat
ENV PATH $PATH:$JAVA_HOME/bin:$CATALINA_HOME/lib:$CATALINA_HOME/bin
ENV ROCKETMQ_HOME /data/rocketmq

RUN rm /etc/yum.repos.d/* -rf
ADD Centos-7.repo /etc/yum.repos.d/
ADD epel.repo /etc/yum.repos.d/
RUN yum clean all
RUN yum -y install net-tools bind-utils sysstat

ADD localtime  /etc/
ADD jdk.tar.gz /data
ADD profile /etc
ADD sysctl.conf /etc
ADD 20-nproc.conf /etc/security/limits.d/
ADD limits.conf /etc/security/

ADD rocketmq.tar.gz /data/
WORKDIR ${ROCKETMQ_HOME}/bin

EXPOSE 9876

ENTRYPOINT ["sh","mqnamesrv"]

 

cat ceph-rocketmq1-pvc.yaml

kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: rocketmq1-brokerstore-pvc
  annotations:
    volume.beta.kubernetes.io/storage-class: ceph-rbd-rocketmq
spec:
  accessModes:
  - ReadWriteOnce  #必须写ReadWriteOnce,否则报错。
  resources:
    requests:
      storage: 20Gi
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: rocketmq1-brokerlog-pvc
  annotations:
    volume.beta.kubernetes.io/storage-class: ceph-rbd-rocketmq
spec:
  accessModes:
  - ReadWriteOnce  #必须写ReadWriteOnce,否则报错。
  resources:
    requests:
      storage: 20Gi
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: rocketmq1-namesrvlog-pvc
  annotations:
    volume.beta.kubernetes.io/storage-class: ceph-rbd-rocketmq
spec:
  accessModes:
  - ReadWriteOnce  #必须写ReadWriteOnce,否则报错。
  resources:
    requests:
      storage: 20Gi

 

rocketmq1 yaml文件

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: rocketmq1
spec:
  replicas: 1
  template:
    metadata:
     labels:
       app: rocketmq1
    spec:
      containers:
      - name: namesrv
        image: 192.168.200.10/fengjian/rocketmq-nameserver:4.3
        imagePullPolicy: Always
        ports:
          - containerPort: 9876
        env:
        - name: ROCKETMQ_HEAP_OPTS
          value : "-Xms1g -Xmx1g -Xmn512m"
        volumeMounts:
        - name: rocketmq1namesrvlog
          mountPath: /data/rocketmq/logs
      - name: broker
        image: 192.168.200.10/fengjian/rocketmq-broker:4.3
        imagePullPolicy: Always
        ports:
          - name: brockerserver
            containerPort: 10909
          - name: brockerslave
            containerPort: 10911
        command: ["/data/rocketmq/bin/mqbroker"]
        args: ["-c", "/data/rocketmq/conf/broker-a.conf"]
        env:
        - name: ROCKETMQ_HEAP_OPTS
          value : "-Xms2g -Xmx2g -Xmn1g"
        volumeMounts:
          - name: brokerlogs
            mountPath: /data/rocketmq/logs
          - name: brokerstore
            mountPath: /data/rocketmq/store
      volumes:
      - name: rocketmq1namesrvlog
        persistentVolumeClaim:
          claimName: rocketmq1-namesrvlog-pvc
      - name: brokerlogs
        persistentVolumeClaim:
          claimName: rocketmq1-brokerlog-pvc
      - name: brokerstore
        persistentVolumeClaim:
          claimName: rocketmq1-brokerstore-pvc


---
apiVersion: v1
kind: Service
metadata:
  name: rocketmq1
spec:
  ports:
  - name: brokerslave
    port: 10909
    protocol: TCP
    targetPort: 10909
  - name: brokerserver
    port: 10911
    protocol: TCP
    targetPort: 10911
  - name: namesrvport
    port: 9876
    protocol: TCP
    targetPort: 9876
  selector:
    app: rocketmq1

 

 

还要提前定义pvc 卷

vim  rocketmq2.yaml

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: rocketmq1
spec:
  replicas: 1
  template:
    metadata:
     labels:
       app: rocketmq1
    spec:
      containers:
      - name: namesrv
        image: 192.168.200.10/fengjian/rocketmq-nameserver:4.3
        imagePullPolicy: Always
        ports:
          - containerPort: 9876
        env:
        - name: ROCKETMQ_HEAP_OPTS
          value : "-Xms1g -Xmx1g -Xmn512m"
        volumeMounts:
        - name: rocketmq1namesrvlog
          mountPath: /data/rocketmq/logs
      - name: broker
        image: 192.168.200.10/fengjian/rocketmq-broker:4.3
        imagePullPolicy: Always
        ports:
          - name: brockerserver
            containerPort: 10909
          - name: brockerslave
            containerPort: 10911
        command: ["/data/rocketmq/bin/mqbroker"]
        args: ["-c", "/data/rocketmq/conf/broker-b.conf"]
        env:
        - name: ROCKETMQ_HEAP_OPTS
          value : "-Xms2g -Xmx2g -Xmn1g"
        volumeMounts:
          - name: brokerlogs
            mountPath: /data/rocketmq/logs
          - name: brokerstore
            mountPath: /data/rocketmq/store
      volumes:
      - name: rocketmq1namesrvlog
        persistentVolumeClaim:
          claimName: rocketmq1-namesrvlog-pvc
      - name: brokerlogs
        persistentVolumeClaim:
          claimName: rocketmq1-brokerlog-pvc
      - name: brokerstore
        persistentVolumeClaim:
          claimName: rocketmq1-brokerstore-pvc


---
apiVersion: v1
kind: Service
metadata:
  name: rocketmq1
spec:
  ports:
  - name: brokerslave
    port: 10909
    protocol: TCP
    targetPort: 10909
  - name: brokerserver
    port: 10911
    protocol: TCP
    targetPort: 10911
  - name: namesrvport
    port: 9876
    protocol: TCP
    targetPort: 9876
  selector:
    app: rocketmq1