RocketMQ简单使用(一)简单消息、顺序消息、延迟消息、消费者
研究下其简单使用。
0. pom
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId>
<version>4.8.0</version>
</dependency>
1. 简单消息
这里使用三种消息的发送方式: 同步发送、异步发送、单向发送，以及消息的消费。
- 同步发送
package com.zd.bx.rocketmq.simple;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.remoting.common.RemotingHelper;
/**
* 同步发送消息: 可靠同步传输应用场景广泛，如重要通知消息、短信通知、短信营销系统等
*/
public class SyncProducer {
public static void main(String[] args) throws Exception {
//Instantiate with a producer group name.
DefaultMQProducer producer = new
DefaultMQProducer("syncProducer");
// Specify name server addresses.
producer.setNamesrvAddr("192.168.13.111:9876");
//Launch the instance.
producer.start();
for (int i = 0; i < 100; i++) {
//Create a message instance, specifying topic, tag and message body.
Message msg = new Message("syncTopic" /* Topic */,
"TagA" /* Tag */,
"keys" + i, /* Keys */
("Hello RocketMQ " +
i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */
);
//Call send message to deliver message to one of brokers.
SendResult sendResult = producer.send(msg);
System.out.printf("%s%n", sendResult);
}
//Shut down once the producer instance is not longer in use.
producer.shutdown();
}
}
结果:
SendResult [sendStatus=SEND_OK, msgId=7F0000017CA018B4AAC25BF224D30000, offsetMsgId=C0A80D6F00002A9F000000000010B9F4, messageQueue=MessageQueue [topic=syncTopic, brokerName=DEFAULT_BROKER, queueId=0], queueOffset=50]
SendResult [sendStatus=SEND_OK, msgId=7F0000017CA018B4AAC25BF224E00001, offsetMsgId=C0A80D6F00002A9F000000000010BABD, messageQueue=MessageQueue [topic=syncTopic, brokerName=DEFAULT_BROKER, queueId=1], queueOffset=50]
SendResult [sendStatus=SEND_OK, msgId=7F0000017CA018B4AAC25BF224E40002, offsetMsgId=C0A80D6F00002A9F000000000010BB86, messageQueue=MessageQueue [topic=syncTopic, brokerName=DEFAULT_BROKER, queueId=2], queueOffset=50]
SendResult [sendStatus=SEND_OK, msgId=7F0000017CA018B4AAC25BF224E80003, offsetMsgId=C0A80D6F00002A9F000000000010BC4F, messageQueue=MessageQueue [topic=syncTopic, brokerName=DEFAULT_BROKER, queueId=3], queueOffset=50]
...
- 异步发送消息
package com.zd.bx.rocketmq.simple;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.SendCallback;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.remoting.common.RemotingHelper;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
/**
* 异步发送消息: 一般用于响应时间敏感的业务场景。
*/
public class AsyncProducer {
public static void main(String[] args) throws Exception {
//Instantiate with a producer group name.
DefaultMQProducer producer = new DefaultMQProducer("asyncProducer");
// Specify name server addresses.
producer.setNamesrvAddr("192.168.13.111:9876");
//Launch the instance.
producer.start();
producer.setRetryTimesWhenSendAsyncFailed(0);
int messageCount = 100;
final CountDownLatch countDownLatch = new CountDownLatch(messageCount);
for (int i = 0; i < messageCount; i++) {
try {
final int index = i;
Message msg = new Message("asyncTopic",
"TagA",
"OrderID188",
"Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET));
producer.send(msg, new SendCallback() {
@Override
public void onSuccess(SendResult sendResult) {
countDownLatch.countDown();
System.out.printf("%-10d OK %s %n", index, sendResult.getMsgId());
}
@Override
public void onException(Throwable e) {
countDownLatch.countDown();
System.out.printf("%-10d Exception %s %n", index, e);
e.printStackTrace();
}
});
} catch (Exception e) {
e.printStackTrace();
}
}
countDownLatch.await(5, TimeUnit.SECONDS);
producer.shutdown();
}
}
结果:
0 OK 7F000001631C18B4AAC25BF3A67E0000
6 OK 7F000001631C18B4AAC25BF3A67E0002
3 OK 7F000001631C18B4AAC25BF3A67E0005
1 OK 7F000001631C18B4AAC25BF3A67E0007
4 OK 7F000001631C18B4AAC25BF3A67E0004
7 OK 7F000001631C18B4AAC25BF3A67E0001
...
- 单向发送
package com.zd.bx.rocketmq.simple;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.remoting.common.RemotingHelper;
/**
* 单向发送消息： 单向传输用于需要中等可靠性的情况，例如日志收集
*/
public class OnewayProducer {
public static void main(String[] args) throws Exception {
//Instantiate with a producer group name.
DefaultMQProducer producer = new DefaultMQProducer("oneWayProducer");
// Specify name server addresses.
producer.setNamesrvAddr("192.168.13.111:9876");
//Launch the instance.
producer.start();
for (int i = 0; i < 100; i++) {
//Create a message instance, specifying topic, tag and message body.
Message msg = new Message("oneWayTopic" /* Topic */,
"TagB" /* Tag */,
("Hello RocketMQ " +
i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */
);
//Call send message to deliver message to one of brokers.
producer.sendOneway(msg);
}
//Wait for sending to complete
Thread.sleep(5000);
producer.shutdown();
}
}
- 消费者
(1)推送模式的消息消费者
package com.zd.bx.rocketmq.simple;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.message.MessageExt;
import java.util.List;
public class Consumer {
public static void main(String[] args) throws InterruptedException, MQClientException {
// 定义一个push消费者
DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("myTestConsumerGroup");
// 指定从第一条消息开始消费
// consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
// 指定nameserver
consumer.setNamesrvAddr("192.168.13.111:9876");
// 指定消费的topic与tag
consumer.subscribe("syncTopic", "*");
// 指定使用 广播模式进行消费，默认为集群模式
// consumer.setMessageModel(MessageModel.BROADCASTING);
consumer.registerMessageListener(new MessageListenerConcurrently() {
@Override
public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
ConsumeConcurrentlyContext context) {
for (MessageExt msg : msgs) {
System.out.printf("%s Receive New Messages: %s body: %s %n", Thread.currentThread().getName(), msgs, new String(msg.getBody()));
}
return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
}
});
consumer.start();
System.out.printf("Consumer Started.%n");
}
}
(2)拉取模式的消费者代码
package com.zd.bx.rocketmq.simple;
import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.message.MessageExt;
import java.util.List;
public class PullConsumer {
public static void main(String[] args) throws InterruptedException, MQClientException {
// 1、创建DefaultLitePullConsumer对象
DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer("LitePullConsumer");
// 2、设置namesrv地址
litePullConsumer.setNamesrvAddr("192.168.13.111:9876");
// 3、订阅消费主题
litePullConsumer.subscribe("syncTopic", "*");
// 4、启动消费对象
litePullConsumer.start();
// 可以修改默认的阻塞时间
// litePullConsumer.setPollTimeoutMillis(0);
try {
// 5、循环开始消费消息
while (true) {
// 拉取消息，无消息时会阻塞 (默认会阻塞5s, 没有消息则返回一个空集合)
List<MessageExt> messageExts = litePullConsumer.poll();
System.out.printf("%s messageExts.size(): %s %n",System.currentTimeMillis(), messageExts.size());
messageExts.stream().forEach(msg -> {
// 业务逻辑
System.out.printf("%s Receive New Messages, body: %s %n", Thread.currentThread().getName(), new String(msg.getBody()));
});
// 同步消费位置。不执行该方法，应用重启会存在重复消费。
if (messageExts.size() > 0) {
litePullConsumer.commitSync();
}
}
} finally {
litePullConsumer.shutdown();
}
}
}
(3)关于消息消费模式广播模式与集群模式的区别
广播消费模式下：相同ConsumerGroup 的每个Consumer实例都接收同一个Topic 的全量消息，即每条消息都会被发送到ConsumerGroup 中的每个每个Consumer。消费进度保存在Consumer端，因为广播模式每个Consumer 都会消费，但是其进度不同。
集群消费模式下(默认，又被称为负载均衡模式)，相同ConsumerGroup 的每个Consumer 实例平摊同一个Topic 的消息，即消息只会被发送到ConsumerGroup的某个Consumer。消费进度保存在broker 中。因为同一条消息只被消费一次，所以需要服务端broker 进行记录下次进行合理的投递。
简单理解。对于多实例的Consumer(同时启动多个相同的Consumer)，集群模式会平摊Topic的消息；广播模式，每个实例都会消费所有的消息的消息，出现重复消费。 由于上述的特性，消费进度保存也略有不同，集群模式下消费进度保存在broker中；广播模式保存在consumer实例中。服务端可以查看集群模式的消费进度：
[root@redisnode1 config]# pwd
/root/store/config
[root@redisnode1 config]# cat consumerOffset.json
{
"offsetTable":{
"syncTopic@myTestConsumerGroup":{0:124,1:128,2:124,3:124
},
"%RETRY%myTestConsumerGroup@myTestConsumerGroup":{0:0
},
"syncTopic@LitePullConsumer":{0:115,1:118,2:117,3:115
}
}
}
2. 顺序消息
1. 什么是顺序消息
顺序消息是指，严格按照消息的发送顺序进行消费的消息。
默认情况下，生产者会把以 RoundRobin 轮询方式发送到不同的Queue 分区队列； 而消费消息时会从多个Queue 上拉取消息，这种情况下的发送和消费是不能保证顺序的。 如果将消息仅发送到同一个Queue 中，消费时也就从这个Queue 上拉取消息，就保证了消息的顺序性。
举个例子：订单状态队列(ORDER_STATUS), 其下有四个Queue 队列。我们发送一个订单的状态时：001未支付-》001已支付-》001发货中-》001发货成功； 这四个状态必须严格按照顺序进行消费，所以就引入了顺序消息。
2. 有序性分类
全局有序\分区有序。 全局有序是指该topic只有一个队列；分区有序是指在有多个Queue 的情况下，在定义Producer时我们指定消息队列选择器，将相关的消息发送到相同的队列，来保证在同一个队列。
3. 代码演示
全局有序很简单，只需要将生产者队列数量设置为1即可。
分区有序可以在调send 的时候传递MessageQueueSelector 进行队列的负载均衡， 负载均衡肯定会选择一个key 作为路由的值：可以是msg的ke， 也可以是调用send 的时候传递第三个参数。
package com.zd.bx.rocketmq.order;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.MessageQueueSelector;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.remoting.common.RemotingHelper;
import java.util.List;
public class Producer {
public static void main(String[] args) throws Exception {
DefaultMQProducer producer = new DefaultMQProducer("orderProducer");
producer.setNamesrvAddr("192.168.13.111:9876");
// 设置queue 的数量为1， 则为全局有序
// producer.setDefaultTopicQueueNums(1);
producer.start();
String[] tags = new String[]{"TagA", "TagB", "TagC", "TagD", "TagE"};
for (int i = 0; i < 100; i++) {
int orderId = i % 10;
Message msg = new Message("TopicTest", tags[i % tags.length], "KEY" + i,
("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET));
msg.setKeys(i + "");
SendResult sendResult = producer.send(msg, new MessageQueueSelector() {
@Override
public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
// 第一种方法是根据key 来进行路由
// Integer id = Integer.valueOf(msg.getKeys());
// 第二种就是传递参数来进行路由，send 方法的第三个参数会传递到arg参数上面
Integer id = (Integer) arg;
int index = id % mqs.size();
return mqs.get(index);
}
}, orderId);
System.out.printf("%s%n", sendResult);
}
producer.shutdown();
}
}
3. 延迟消息
如果记得没错，rabbitMQ的延迟消息需要借助于消息的生存时间和死信队列实现延迟。
1. 延迟消息的等级
延迟消息的等级定义在服务端MessageStoreConfig.java 中，如下：
private String messageDelayLevel = "1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h";
比如，如果指定等级为3， 则延迟为10s， 延迟等级从1开始。如果想增加其他的配置，可以在rocketMQ安装目录的conf 目录中进行配置。
2. 延迟消息原理
如下图：
具体实现方案是：
Producer 将消息发送到Broker 之后，Broker 先将消息发送到commitLog 文件，然后将其分发到相应的consumerqueue。 不过，在分发之前，系统会判断消息中是否带有延迟等级，没有则直接正常分发；若有：
(1) 修改消息的topic 为SCHEDULE_TOPIC_XXXX 目录。
(2) 根据延时等级，在consumequeue 目录中SCHEDULE_TOPIC_XXXX 主题下创建出相应的queueId 与consumequeue 文件(如果没有这些目录与文件)。这里需要注意：延迟等级与queueId 的对应关系为 queueId = delayLevel -1
(3) 修改消息索引单元内容。索引单元中的MessagetagHashCode 部分原本存放的是消息的Tag的Hash 值。现在修改为消息的投递时间。投递时间=消息存储时间+延迟时间， 也就是投递时间存的是其真正需要被分发回原queue 的时间。
(4) 投递延迟消息：Broker 内部有一个延迟消息服务类ScheduleMessageService，负责将消息投递到目标Topic(内部用Timer 定时器实现)
(5) 将消息重新写入commitlog，形成新的索引条目分发到相应的queue 中
3. 代码演示
package com.zd.bx.rocketmq.delay;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.common.message.Message;
public class Producer {
public static void main(String[] args) throws Exception {
// Instantiate a producer to send scheduled messages
DefaultMQProducer producer = new DefaultMQProducer("delayProducer");
producer.setNamesrvAddr("192.168.13.111:9876");
// Launch producer
producer.start();
int totalMessagesToSend = 100;
for (int i = 0; i < totalMessagesToSend; i++) {
Message message = new Message("delayTopicTest", ("Hello delay message " + i).getBytes());
// This message will be delivered to consumer 10 seconds later.
message.setDelayTimeLevel(3);
producer.send(message);
}
producer.shutdown();
}
}
如果想测试效果，可以先启动消费者，然后启动生产者查看其效果。
启动后到服务器rocketMQ 存储目录查看主题目录其队列信息如下：(默认在当前用户的store 目录)
[root@redisnode1 consumequeue]# pwd
/root/store/consumequeue
[root@redisnode1 consumequeue]# ls
asyncTopic delayTopicTest oneWayTopic SCHEDULE_TOPIC_XXXX syncTopic TopicTest
[root@redisnode1 consumequeue]# ls -R ./delayTopicTest/
./delayTopicTest/:
0 1 2 3
./delayTopicTest/0:
00000000000000000000
./delayTopicTest/1:
00000000000000000000
./delayTopicTest/2:
00000000000000000000
./delayTopicTest/3:
00000000000000000000
[root@redisnode1 consumequeue]# ls -R ./SCHEDULE_TOPIC_XXXX/
./SCHEDULE_TOPIC_XXXX/:
2
./SCHEDULE_TOPIC_XXXX/2:
00000000000000000000
可以看出第一次发出延迟消息后会多出一个SCHEDULE_TOPIC_XXXX目录，内部会根据消息的延迟等级创建相应的queue 目录(queueId = delayLevel -1 , 所以queue 的目录ID为2)。
补充：关于rocketMQ消息发送者与消费者
- rocketmq底层使用了netty 相关的东西，这里只简单的看下。通过线程模型或者debug 跟代码可以看到，比如：
(1)debug查看：
生产者在org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl#sendDefaultImpl 对消息进行包装、选择队列、重试调用
private SendResult sendDefaultImpl(
Message msg,
final CommunicationMode communicationMode,
final SendCallback sendCallback,
final long timeout
) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
this.makeSureStateOK();
Validators.checkMessage(msg, this.defaultMQProducer);
final long invokeID = random.nextLong();
long beginTimestampFirst = System.currentTimeMillis();
long beginTimestampPrev = beginTimestampFirst;
long endTimestamp = beginTimestampFirst;
TopicPublishInfo topicPublishInfo = this.tryToFindTopicPublishInfo(msg.getTopic());
if (topicPublishInfo != null && topicPublishInfo.ok()) {
boolean callTimeout = false;
MessageQueue mq = null;
Exception exception = null;
SendResult sendResult = null;
int timesTotal = communicationMode == CommunicationMode.SYNC ? 1 + this.defaultMQProducer.getRetryTimesWhenSendFailed() : 1;
int times = 0;
String[] brokersSent = new String[timesTotal];
for (; times < timesTotal; times++) {
String lastBrokerName = null == mq ? null : mq.getBrokerName();
MessageQueue mqSelected = this.selectOneMessageQueue(topicPublishInfo, lastBrokerName);
if (mqSelected != null) {
mq = mqSelected;
brokersSent[times] = mq.getBrokerName();
try {
beginTimestampPrev = System.currentTimeMillis();
if (times > 0) {
//Reset topic with namespace during resend.
msg.setTopic(this.defaultMQProducer.withNamespace(msg.getTopic()));
}
long costTime = beginTimestampPrev - beginTimestampFirst;
if (timeout < costTime) {
callTimeout = true;
break;
}
sendResult = this.sendKernelImpl(msg, mq, communicationMode, sendCallback, topicPublishInfo, timeout - costTime);
endTimestamp = System.currentTimeMillis();
this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false);
switch (communicationMode) {
case ASYNC:
return null;
case ONEWAY:
return null;
case SYNC:
if (sendResult.getSendStatus() != SendStatus.SEND_OK) {
if (this.defaultMQProducer.isRetryAnotherBrokerWhenNotStoreOK()) {
continue;
}
}
return sendResult;
default:
break;
}
} catch (RemotingException e) {
endTimestamp = System.currentTimeMillis();
this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true);
log.warn(String.format("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e);
log.warn(msg.toString());
exception = e;
continue;
} catch (MQClientException e) {
endTimestamp = System.currentTimeMillis();
this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true);
log.warn(String.format("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e);
log.warn(msg.toString());
exception = e;
continue;
} catch (MQBrokerException e) {
endTimestamp = System.currentTimeMillis();
this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true);
log.warn(String.format("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e);
log.warn(msg.toString());
exception = e;
if (this.defaultMQProducer.getRetryResponseCodes().contains(e.getResponseCode())) {
continue;
} else {
if (sendResult != null) {
return sendResult;
}
throw e;
}
} catch (InterruptedException e) {
endTimestamp = System.currentTimeMillis();
this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false);
log.warn(String.format("sendKernelImpl exception, throw exception, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e);
log.warn(msg.toString());
log.warn("sendKernelImpl exception", e);
log.warn(msg.toString());
throw e;
}
} else {
break;
}
}
if (sendResult != null) {
return sendResult;
}
String info = String.format("Send [%d] times, still failed, cost [%d]ms, Topic: %s, BrokersSent: %s",
times,
System.currentTimeMillis() - beginTimestampFirst,
msg.getTopic(),
Arrays.toString(brokersSent));
info += FAQUrl.suggestTodo(FAQUrl.SEND_MSG_FAILED);
MQClientException mqClientException = new MQClientException(info, exception);
if (callTimeout) {
throw new RemotingTooMuchRequestException("sendDefaultImpl call timeout");
}
if (exception instanceof MQBrokerException) {
mqClientException.setResponseCode(((MQBrokerException) exception).getResponseCode());
} else if (exception instanceof RemotingConnectException) {
mqClientException.setResponseCode(ClientErrorCode.CONNECT_BROKER_EXCEPTION);
} else if (exception instanceof RemotingTimeoutException) {
mqClientException.setResponseCode(ClientErrorCode.ACCESS_BROKER_TIMEOUT);
} else if (exception instanceof MQClientException) {
mqClientException.setResponseCode(ClientErrorCode.BROKER_NOT_EXIST_EXCEPTION);
}
throw mqClientException;
}
validateNameServerSetting();
throw new MQClientException("No route info of this topic: " + msg.getTopic() + FAQUrl.suggestTodo(FAQUrl.NO_TOPIC_ROUTE_INFO),
null).setResponseCode(ClientErrorCode.NOT_FOUND_TOPIC_EXCEPTION);
}
然后调用到org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl#sendKernelImpl 进行消息转换，msg 转换为SendMessageRequestHeader 对象
private SendResult sendKernelImpl(final Message msg,
final MessageQueue mq,
final CommunicationMode communicationMode,
final SendCallback sendCallback,
final TopicPublishInfo topicPublishInfo,
final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
long beginStartTime = System.currentTimeMillis();
String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName());
if (null == brokerAddr) {
tryToFindTopicPublishInfo(mq.getTopic());
brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName());
}
SendMessageContext context = null;
if (brokerAddr != null) {
brokerAddr = MixAll.brokerVIPChannel(this.defaultMQProducer.isSendMessageWithVIPChannel(), brokerAddr);
byte[] prevBody = msg.getBody();
try {
//for MessageBatch,ID has been set in the generating process
if (!(msg instanceof MessageBatch)) {
MessageClientIDSetter.setUniqID(msg);
}
boolean topicWithNamespace = false;
if (null != this.mQClientFactory.getClientConfig().getNamespace()) {
msg.setInstanceId(this.mQClientFactory.getClientConfig().getNamespace());
topicWithNamespace = true;
}
int sysFlag = 0;
boolean msgBodyCompressed = false;
if (this.tryToCompressMessage(msg)) {
sysFlag |= MessageSysFlag.COMPRESSED_FLAG;
msgBodyCompressed = true;
}
final String tranMsg = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED);
if (Boolean.parseBoolean(tranMsg)) {
sysFlag |= MessageSysFlag.TRANSACTION_PREPARED_TYPE;
}
if (hasCheckForbiddenHook()) {
CheckForbiddenContext checkForbiddenContext = new CheckForbiddenContext();
checkForbiddenContext.setNameSrvAddr(this.defaultMQProducer.getNamesrvAddr());
checkForbiddenContext.setGroup(this.defaultMQProducer.getProducerGroup());
checkForbiddenContext.setCommunicationMode(communicationMode);
checkForbiddenContext.setBrokerAddr(brokerAddr);
checkForbiddenContext.setMessage(msg);
checkForbiddenContext.setMq(mq);
checkForbiddenContext.setUnitMode(this.isUnitMode());
this.executeCheckForbiddenHook(checkForbiddenContext);
}
if (this.hasSendMessageHook()) {
context = new SendMessageContext();
context.setProducer(this);
context.setProducerGroup(this.defaultMQProducer.getProducerGroup());
context.setCommunicationMode(communicationMode);
context.setBornHost(this.defaultMQProducer.getClientIP());
context.setBrokerAddr(brokerAddr);
context.setMessage(msg);
context.setMq(mq);
context.setNamespace(this.defaultMQProducer.getNamespace());
String isTrans = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED);
if (isTrans != null && isTrans.equals("true")) {
context.setMsgType(MessageType.Trans_Msg_Half);
}
if (msg.getProperty("__STARTDELIVERTIME") != null || msg.getProperty(MessageConst.PROPERTY_DELAY_TIME_LEVEL) != null) {
context.setMsgType(MessageType.Delay_Msg);
}
this.executeSendMessageHookBefore(context);
}
SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();
requestHeader.setProducerGroup(this.defaultMQProducer.getProducerGroup());
requestHeader.setTopic(msg.getTopic());
requestHeader.setDefaultTopic(this.defaultMQProducer.getCreateTopicKey());
requestHeader.setDefaultTopicQueueNums(this.defaultMQProducer.getDefaultTopicQueueNums());
requestHeader.setQueueId(mq.getQueueId());
requestHeader.setSysFlag(sysFlag);
requestHeader.setBornTimestamp(System.currentTimeMillis());
requestHeader.setFlag(msg.getFlag());
requestHeader.setProperties(MessageDecoder.messageProperties2String(msg.getProperties()));
requestHeader.setReconsumeTimes(0);
requestHeader.setUnitMode(this.isUnitMode());
requestHeader.setBatch(msg instanceof MessageBatch);
if (requestHeader.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
String reconsumeTimes = MessageAccessor.getReconsumeTime(msg);
if (reconsumeTimes != null) {
requestHeader.setReconsumeTimes(Integer.valueOf(reconsumeTimes));
MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_RECONSUME_TIME);
}
String maxReconsumeTimes = MessageAccessor.getMaxReconsumeTimes(msg);
if (maxReconsumeTimes != null) {
requestHeader.setMaxReconsumeTimes(Integer.valueOf(maxReconsumeTimes));
MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_MAX_RECONSUME_TIMES);
}
}
SendResult sendResult = null;
switch (communicationMode) {
case ASYNC:
Message tmpMessage = msg;
boolean messageCloned = false;
if (msgBodyCompressed) {
//If msg body was compressed, msgbody should be reset using prevBody.
//Clone new message using commpressed message body and recover origin massage.
//Fix bug:https://github.com/apache/rocketmq-externals/issues/66
tmpMessage = MessageAccessor.cloneMessage(msg);
messageCloned = true;
msg.setBody(prevBody);
}
if (topicWithNamespace) {
if (!messageCloned) {
tmpMessage = MessageAccessor.cloneMessage(msg);
messageCloned = true;
}
msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultMQProducer.getNamespace()));
}
long costTimeAsync = System.currentTimeMillis() - beginStartTime;
if (timeout < costTimeAsync) {
throw new RemotingTooMuchRequestException("sendKernelImpl call timeout");
}
sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage(
brokerAddr,
mq.getBrokerName(),
tmpMessage,
requestHeader,
timeout - costTimeAsync,
communicationMode,
sendCallback,
topicPublishInfo,
this.mQClientFactory,
this.defaultMQProducer.getRetryTimesWhenSendAsyncFailed(),
context,
this);
break;
case ONEWAY:
case SYNC:
long costTimeSync = System.currentTimeMillis() - beginStartTime;
if (timeout < costTimeSync) {
throw new RemotingTooMuchRequestException("sendKernelImpl call timeout");
}
sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage(
brokerAddr,
mq.getBrokerName(),
msg,
requestHeader,
timeout - costTimeSync,
communicationMode,
context,
this);
break;
default:
assert false;
break;
}
if (this.hasSendMessageHook()) {
context.setSendResult(sendResult);
this.executeSendMessageHookAfter(context);
}
return sendResult;
} catch (RemotingException e) {
if (this.hasSendMessageHook()) {
context.setException(e);
this.executeSendMessageHookAfter(context);
}
throw e;
} catch (MQBrokerException e) {
if (this.hasSendMessageHook()) {
context.setException(e);
this.executeSendMessageHookAfter(context);
}
throw e;
} catch (InterruptedException e) {
if (this.hasSendMessageHook()) {
context.setException(e);
this.executeSendMessageHookAfter(context);
}
throw e;
} finally {
msg.setBody(prevBody);
msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultMQProducer.getNamespace()));
}
}
throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null);
}
然后调用到org.apache.rocketmq.client.impl.MQClientAPIImpl#sendMessageSync 发送消息并且处理回调：
private SendResult sendMessageSync(
final String addr,
final String brokerName,
final Message msg,
final long timeoutMillis,
final RemotingCommand request
) throws RemotingException, MQBrokerException, InterruptedException {
RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis);
assert response != null;
return this.processSendResponse(brokerName, msg, response, addr);
}
然后调用到 org.apache.rocketmq.remoting.netty.NettyRemotingClient#invokeSync 走netty的channel 进行发送消息，最终会调用到：org.apache.rocketmq.remoting.netty.NettyRemotingAbstract#invokeSyncImpl
public RemotingCommand invokeSyncImpl(Channel channel, RemotingCommand request, long timeoutMillis) throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException {
final int opaque = request.getOpaque();
RemotingCommand var9;
try {
final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, timeoutMillis, (InvokeCallback)null, (SemaphoreReleaseOnlyOnce)null);
this.responseTable.put(opaque, responseFuture);
final SocketAddress addr = channel.remoteAddress();
channel.writeAndFlush(request).addListener(new ChannelFutureListener() {
public void operationComplete(ChannelFuture f) throws Exception {
if (f.isSuccess()) {
responseFuture.setSendRequestOK(true);
} else {
responseFuture.setSendRequestOK(false);
NettyRemotingAbstract.this.responseTable.remove(opaque);
responseFuture.setCause(f.cause());
responseFuture.putResponse((RemotingCommand)null);
NettyRemotingAbstract.log.warn("send a request command to channel <" + addr + "> failed.");
}
}
});
RemotingCommand responseCommand = responseFuture.waitResponse(timeoutMillis);
if (null == responseCommand) {
if (responseFuture.isSendRequestOK()) {
throw new RemotingTimeoutException(RemotingHelper.parseSocketAddressAddr(addr), timeoutMillis, responseFuture.getCause());
}
throw new RemotingSendRequestException(RemotingHelper.parseSocketAddressAddr(addr), responseFuture.getCause());
}
var9 = responseCommand;
} finally {
this.responseTable.remove(opaque);
}
return var9;
}
调用链如下：
可以看到类似于dubbo的RPC调用，封装一个统一的request 对象用于交互，然后回调封装成统一的ResponseFuture 对象处理回调。
(2)线程模型:
2. 关于消息消费者和消费者组
消息的投递是以消费者组为单位进行的：如果两个不同名称的消费者组监听同一个Topic，那么两个消费者组接收到的消息是一样的，都是整个Topic的全量消息。消费者组内的消费者(可以理解为一个实例就是一个消费者，一个消费者内部又是有多个线程进行消息的消费，线程数量可以修改)受消费模式的影响：广播模式下每个实例都是全量消息，集群模式(默认)相当于所有实例平分整个Topic的消息。
测试代码:消费者修改线程数量以及消费模式然后查看结果
消费者组myTestConsumerGroup:(启动两个实例，充当两个消费者)
package com.zd.bx.rocketmq.simple;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.message.MessageExt;
import java.util.List;
public class PushConsumer {
public static void main(String[] args) throws InterruptedException, MQClientException {
// 定义一个push消费者
DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("myTestConsumerGroup");
// 设置线程数量
consumer.setConsumeThreadMax(4);
consumer.setConsumeThreadMin(2);
// 指定nameserver
consumer.setNamesrvAddr("192.168.13.111:9876");
// 指定消费的topic与tag
consumer.subscribe("syncTopic", "*");
// 指定使用 广播模式进行消费，默认为集群模式
// consumer.setMessageModel(MessageModel.BROADCASTING);
consumer.registerMessageListener(new MessageListenerConcurrently() {
@Override
public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
ConsumeConcurrentlyContext context) {
for (MessageExt msg : msgs) {
System.out.printf("%s Receive New Messages, body: %s %n", Thread.currentThread().getName(), new String(msg.getBody()));
}
return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
}
});
consumer.start();
System.out.printf("Consumer Started.%n");
}
}
消费者组myTestConsumerGroup2(启动两个实例):
package com.zd.bx.rocketmq.simple;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.message.MessageExt;
import java.util.List;
public class PushConsumer2 {
public static void main(String[] args) throws InterruptedException, MQClientException {
// 定义一个push消费者
DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("myTestConsumerGroup2");
// 设置线程数量
consumer.setConsumeThreadMax(4);
consumer.setConsumeThreadMin(2);
// 指定从第一条消息开始消费
// consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
// 指定nameserver
consumer.setNamesrvAddr("192.168.13.111:9876");
// 指定消费的topic与tag
consumer.subscribe("syncTopic", "*");
// 指定使用 广播模式进行消费，默认为集群模式
// consumer.setMessageModel(MessageModel.BROADCASTING);
consumer.registerMessageListener(new MessageListenerConcurrently() {
@Override
public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
ConsumeConcurrentlyContext context) {
for (MessageExt msg : msgs) {
System.out.printf("%s Receive New Messages, body: %s %n", Thread.currentThread().getName(), new String(msg.getBody()));
}
return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
}
});
consumer.start();
System.out.printf("Consumer Started.%n");
}
}
生产者:
package com.zd.bx.rocketmq.simple;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.remoting.common.RemotingHelper;
/**
* 同步发送消息: 可靠同步传输应用场景广泛，如重要通知消息、短信通知、短信营销系统等
*/
public class SyncProducer {
public static void main(String[] args) throws Exception {
DefaultMQProducer producer = new
DefaultMQProducer("syncProducer");
// Specify name server addresses.
producer.setNamesrvAddr("192.168.13.111:9876");
//Launch the instance.
producer.start();
for (int i = 0; i < 5; i++) {
Message msg = new Message("syncTopic" /* Topic */,
"TagA" /* Tag */,
"keys" + i, /* Keys */
("Hello RocketMQ " +
i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */
);
SendResult sendResult = producer.send(msg);
System.out.printf("%s%n", sendResult);
}
producer.shutdown();
}
}
(1)默认消费模式集群模式下结果如下:(可以看到两个组接收到的是全量的消息，组内的两个实例平分消息)
(2)广播模式下结果如下:(可以看到两个实例收到的都是全量的消息，组内的两个实例也都是全量消息)
【当你用心写完每一篇博客之后,你会发现它比你用代码实现功能更有成就感!】
