rocketmq客户端消费过程
RocketMQ的消息发送方式主要含syncSend()同步发送、asyncSend()异步发送、sendOneWay()三种方式，sendOneWay()也是异步发送，区别在于不需等待Broker返回确认，
所以可能会存在信息丢失的状况，但吞吐量更高，具体需根据业务情况选用。
一个队列只会被消费组内的一个消费者消费，即如果topic相同，但是有多个consumerGroup，可能有A、B两个,那么此时一个队列会被A、B共同消费。只不过只会被A和B中各自的一个服务器消费。
详情见 https://codetd.com/article/12138580#3_54 中的第3点。
当实例化RocketMQTemplate过程中因为实现了InitializingBean接口，会执行afterPropertiesSet()方法
this.producer.start();
之后执行
public void start() throws MQClientException { this.defaultMQProducerImpl.start(); 。。。。。。。。。。。。 }
之后执行
public void start(boolean startFactory) throws MQClientException { ..................... this.mQClientFactory.start(); ..................... }
之后执行
MQClientInstance public void start() throws MQClientException { //各种线程的start
//NettyRemotingClient实现Netty客户器端功能，接受数据包，在客户器端处理后发送给服务端。
this.mQClientAPIImpl.start();
//获取nameserver的地址 默认每3秒更新Topic规则 清除下线的borker并且发送心跳到broker 持久化消费者进度 适配执行消费请求的线程池的核心线程书大小 this.startScheduledTask();
//启动消息拉取服务，循环拉取阻塞队列pullRequestQueue this.pullMessageService.start();
//负载均衡服务开始 this.rebalanceService.start(); this.defaultMQProducer.getDefaultMQProducerImpl().start(false); this.log.info("the client factory [{}] start OK", this.clientId); this.serviceState = ServiceState.RUNNING; }
this.pullMessageService.start()解析
1 public void run() { 2 this.log.info(this.getServiceName() + " service started"); 3 4 while(!this.isStopped()) { 5 try {
//PullRequest是一个阻塞队列 6 PullRequest pullRequest = (PullRequest)this.pullRequestQueue.take(); 7 this.pullMessage(pullRequest); 8 } catch (InterruptedException var2) { 9 } catch (Exception var3) { 10 this.log.error("Pull Message Service Run Method exception", var3); 11 } 12 } 13 14 this.log.info(this.getServiceName() + " service end"); 15 }
之后会执行拉取消息
1 public void pullMessage(final PullRequest pullRequest) {
//校验队列是否被丢弃 2 final ProcessQueue processQueue = pullRequest.getProcessQueue(); 3 if (processQueue.isDropped()) { 4 log.info("the pull request[{}] is dropped.", pullRequest.toString()); 5 return; 6 } 7 8 pullRequest.getProcessQueue().setLastPullTimestamp(System.currentTimeMillis()); 9 //校验消费端的情况，状态是否正常，是否暂停 10 try { 11 this.makeSureStateOK(); 12 } catch (MQClientException e) { 13 log.warn("pullMessage exception, consumer state not ok", e); 14 this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException); 15 return; 16 } 17 18 if (this.isPause()) { 19 log.warn("consumer was paused, execute pull request later. instanceName={}, group={}", this.defaultMQPushConsumer.getInstanceName(), this.defaultMQPushConsumer.getConsumerGroup()); 20 this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_SUSPEND); 21 return; 22 } 23 //获取该队列中的消息数量 24 long cachedMessageCount = processQueue.getMsgCount().get();
//获取该队列中的消息大小（M） 25 long cachedMessageSizeInMiB = processQueue.getMsgSize().get() / (1024 * 1024); 26 //如果该队列中的消息数量>每个队列中最大的消息数量(默认1000) 这里做这些判断主要是进行流控，即rocketmq的削峰原理就在这里 27 if (cachedMessageCount > this.defaultMQPushConsumer.getPullThresholdForQueue()) {
//等待50ms再次去拉取，从这里可以看出实际上拉取消息是实时的，没有时间间隔的， 28 this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL);
//该队列的流控次数每达到1000次则打印日志 29 if ((queueFlowControlTimes++ % 1000) == 0) { 30 log.warn( 31 "the cached message count exceeds the threshold {}, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, pullRequest={}, flowControlTimes={}", 32 this.defaultMQPushConsumer.getPullThresholdForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, pullRequest, queueFlowControlTimes); 33 } 34 return; 35 } 36 //如果该队列中的消息大小（M）超过100M(默认)，则执行流控，实现同上 37 if (cachedMessageSizeInMiB > this.defaultMQPushConsumer.getPullThresholdSizeForQueue()) { 38 this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL); 39 if ((queueFlowControlTimes++ % 1000) == 0) { 40 log.warn( 41 "the cached message size exceeds the threshold {} MiB, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, pullRequest={}, flowControlTimes={}", 42 this.defaultMQPushConsumer.getPullThresholdSizeForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, pullRequest, queueFlowControlTimes); 43 } 44 return; 45 } 46 //如果不是顺序消费,即并发消费 47 if (!this.consumeOrderly) {
//获取单队列并行消费允许的最大跨度 关键代码：这里用了读写锁保证并发 this.msgTreeMap.lastKey() - this.msgTreeMap.firstKey();
/*
RocketMQ是以consumer group+queue为单位是管理消费进度的，以一个consumer offset标记这个这个消费组在这条queue上的消费进度。
如果某已存在的消费组出现了新消费实例的时候，依靠这个组的消费进度，就可以判断第一次是从哪里开始拉取的。
每次消息成功后，本地的消费进度会被更新，然后由定时器定时同步到broker，以此持久化消费进度。
mq消费端做幂等原因：
由于消费进度只是记录了一个下标，就可能出现拉取了100条消息如 2101-2200的消息，后面99条都消费结束了，只有2101消费一直没有结束的情况。
在这种情况下，RocketMQ为了保证消息肯定被消费成功，消费进度职能维持在2101，直到2101也消费结束了，本地的消费进度才会一下子更新到2200。
在这种设计下，就有消费大量重复的风险。如2101在还没有消费完成的时候消费实例突然退出（机器断电，或者被kill）。
这条queue的消费进度还是维持在2101，当queue重新分配给新的实例的时候，新的实例从broker上拿到的消费进度还是维持在2101，
这时候就会又从2101开始消费，2102-2200这批消息实际上已经被消费过还是会投递一次。
*/
//单队列并行消费允许的最大跨度>2000(默认) 比如有1000条消息，offset=2这条消息一直没有消费完，但是其他都消费完了，但是会实时拉取消息，当多次拉取条消息 // 直到offset=2003了，此时offset=2的这条还没有消费完成，mq就会认为有阻塞，会实行流控
48 if (processQueue.getMaxSpan() > this.defaultMQPushConsumer.getConsumeConcurrentlyMaxSpan()) { 49 this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL); 50 if ((queueMaxSpanFlowControlTimes++ % 1000) == 0) { 51 log.warn( 52 "the queue's messages, span too long, so do flow control, minOffset={}, maxOffset={}, maxSpan={}, pullRequest={}, flowControlTimes={}",
//本次拉取的总消息数量中最小的offset 和最大的offset 53 processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), processQueue.getMaxSpan(), 54 pullRequest, queueMaxSpanFlowControlTimes); 55 } 56 return; 57 } 58 } else {
//顺序消费 59 if (processQueue.isLocked()) { 60 if (!pullRequest.isLockedFirst()) { 61 final long offset = this.rebalanceImpl.computePullFromWhere(pullRequest.getMessageQueue()); 62 boolean brokerBusy = offset < pullRequest.getNextOffset(); 63 log.info("the first time to pull message, so fix offset from broker. pullRequest: {} NewOffset: {} brokerBusy: {}", 64 pullRequest, offset, brokerBusy); 65 if (brokerBusy) { 66 log.info("[NOTIFYME]the first time to pull message, but pull request offset larger than broker consume offset. pullRequest: {} NewOffset: {}", 67 pullRequest, offset); 68 } 69 70 pullRequest.setLockedFirst(true); 71 pullRequest.setNextOffset(offset); 72 } 73 } else { 74 this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException); 75 log.info("pull message later because not locked in broker, {}", pullRequest); 76 return; 77 } 78 } 79 //获取该队列的topic、consumerGroup、tag信息 80 final SubscriptionData subscriptionData = this.rebalanceImpl.getSubscriptionInner().get(pullRequest.getMessageQueue().getTopic()); 81 if (null == subscriptionData) { 82 this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException); 83 log.warn("find the consumer's subscription failed, {}", pullRequest); 84 return; 85 } 86 87 final long beginTimestamp = System.currentTimeMillis(); 88 //回调函数， 在这个方法被调用，里面的逻辑执行到onSuccess()时才会真正执行到这里 this.pullAPIWrapper.pullKernelImpl(。。。，pullCallback) //真正调用的地方：private void pullMessageAsync(....) -> pullCallback.onSuccess(pullResult); pullCallback.onException(e);
//下面会对这个回调函数进行具体解析
89 PullCallback pullCallback = new PullCallback() { 90 。。。。。。。。。。。。。。。。。。。。。191 }; 192 193 boolean commitOffsetEnable = false; 194 long commitOffsetValue = 0L; //如果是集群模式，此处读取方式是ReadOffsetType.READ_FROM_MEMORY 表示从内存中获取消费进度 也有从broker中获取进度
195 if (MessageModel.CLUSTERING == this.defaultMQPushConsumer.getMessageModel()) { 196 commitOffsetValue = this.offsetStore.readOffset(pullRequest.getMessageQueue(), ReadOffsetType.READ_FROM_MEMORY); 197 if (commitOffsetValue > 0) { 198 commitOffsetEnable = true; 199 } 200 } 201 202 String subExpression = null; 203 boolean classFilter = false; //再次获取该队列的topic、consumerGroup、tag信息，之前已经获取了，不明白这里为啥要再次获取一次？？
204 SubscriptionData sd = this.rebalanceImpl.getSubscriptionInner().get(pullRequest.getMessageQueue().getTopic()); 205 if (sd != null) { 206 if (this.defaultMQPushConsumer.isPostSubscriptionWhenPull() && !sd.isClassFilterMode()) { //就是tag的值
207 subExpression = sd.getSubString(); 208 } 209 210 classFilter = sd.isClassFilterMode(); 211 } 212 213 int sysFlag = PullSysFlag.buildSysFlag( 214 commitOffsetEnable, // commitOffset 215 true, // suspend 216 subExpression != null, // subscription 217 classFilter // class filter 218 ); //拉取服务的真正实现入口：kernel：内核
219 try { 220 this.pullAPIWrapper.pullKernelImpl( 221 pullRequest.getMessageQueue(), 222 subExpression, 223 subscriptionData.getExpressionType(), 224 subscriptionData.getSubVersion(), 225 pullRequest.getNextOffset(), 226 this.defaultMQPushConsumer.getPullBatchSize(), 227 sysFlag, 228 commitOffsetValue, 229 BROKER_SUSPEND_MAX_TIME_MILLIS, 230 CONSUMER_TIMEOUT_MILLIS_WHEN_SUSPEND, 231 CommunicationMode.ASYNC, 232 pullCallback 233 ); 234 } catch (Exception e) { 235 log.error("pullKernelImpl exception", e); 236 this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException); 237 } 238 }
之后拉取消息的核心实现：
1 public PullResult pullKernelImpl( 2 final MessageQueue mq, 3 final String subExpression, 4 final String expressionType, 5 final long subVersion, 6 final long offset, 7 final int maxNums, 8 final int sysFlag, 9 final long commitOffset, 10 final long brokerSuspendMaxTimeMillis, 11 final long timeoutMillis, 12 final CommunicationMode communicationMode, 13 final PullCallback pullCallback 14 ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { //获取broker的地址
15 FindBrokerResult findBrokerResult = 16 this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(), 17 this.recalculatePullFromWhichNode(mq), false); 18 if (null == findBrokerResult) { 19 this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); 20 findBrokerResult = 21 this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(), 22 this.recalculatePullFromWhichNode(mq), false); 23 } 24 25 if (findBrokerResult != null) { 26 { 27 // check version 28 if (!ExpressionType.isTagType(expressionType) 29 && findBrokerResult.getBrokerVersion() < MQVersion.Version.V4_1_0_SNAPSHOT.ordinal()) { 30 throw new MQClientException("The broker[" + mq.getBrokerName() + ", " 31 + findBrokerResult.getBrokerVersion() + "] does not upgrade to support for filter message by " + expressionType, null); 32 } 33 } 34 int sysFlagInner = sysFlag; 35 //如果是备用机器？？ 36 if (findBrokerResult.isSlave()) { 37 sysFlagInner = PullSysFlag.clearCommitOffsetFlag(sysFlagInner); 38 } 39 //通过netty拉取信息，这里组装请求信息 40 PullMessageRequestHeader requestHeader = new PullMessageRequestHeader(); 41 requestHeader.setConsumerGroup(this.consumerGroup); 42 requestHeader.setTopic(mq.getTopic()); 43 requestHeader.setQueueId(mq.getQueueId()); 44 requestHeader.setQueueOffset(offset); 45 requestHeader.setMaxMsgNums(maxNums);//一次拉取的最大数据条数 46 requestHeader.setSysFlag(sysFlagInner); 47 requestHeader.setCommitOffset(commitOffset); 48 requestHeader.setSuspendTimeoutMillis(brokerSuspendMaxTimeMillis); 49 requestHeader.setSubscription(subExpression); 50 requestHeader.setSubVersion(subVersion); 51 requestHeader.setExpressionType(expressionType); 52 53 String brokerAddr = findBrokerResult.getBrokerAddr(); 54 if (PullSysFlag.hasClassFilterFlag(sysFlagInner)) { 55 brokerAddr = computPullFromWhichFilterServer(mq.getTopic(), brokerAddr); 56 } 57 58 PullResult pullResult = this.mQClientFactory.getMQClientAPIImpl().pullMessage( 59 brokerAddr, 60 requestHeader, 61 timeoutMillis, 62 communicationMode, 63 pullCallback); 64 65 return pullResult; 66 } 67 68 throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); 69 }
异步拉取：
1 private void pullMessageAsync( 2 final String addr, 3 final RemotingCommand request, 4 final long timeoutMillis, 5 final PullCallback pullCallback 6 ) throws RemotingException, InterruptedException { //异步通过netty进行异步调用
7 this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { //拉取消息回调后会执行operationComplete()
8 @Override 9 public void operationComplete(ResponseFuture responseFuture) { 10 RemotingCommand response = responseFuture.getResponseCommand(); 11 if (response != null) { 12 try {
//获取拉取的原始结果，这个pullResult里面就有拉取的数据 13 PullResult pullResult = MQClientAPIImpl.this.processPullResponse(response); 14 assert pullResult != null;
//这里才会真正执行到回调函数，具体解析详见下面 15 pullCallback.onSuccess(pullResult); 16 } catch (Exception e) { 17 pullCallback.onException(e); 18 } 19 } else { 20 if (!responseFuture.isSendRequestOK()) { 21 pullCallback.onException(new MQClientException("send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); 22 } else if (responseFuture.isTimeout()) { 23 pullCallback.onException(new MQClientException("wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, 24 responseFuture.getCause())); 25 } else { 26 pullCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause())); 27 } 28 } 29 } 30 }); 31 }
拉取完成后执行回调函数：
1 PullCallback pullCallback = new PullCallback() { 2 @Override 3 public void onSuccess(PullResult pullResult) { 4 if (pullResult != null) {
//详细解析见下面 5 pullResult = DefaultMQPushConsumerImpl.this.pullAPIWrapper.processPullResult(pullRequest.getMessageQueue(), pullResult, 6 subscriptionData); 7 8 switch (pullResult.getPullStatus()) { 9 case FOUND: 10 long prevRequestOffset = pullRequest.getNextOffset(); 11 pullRequest.setNextOffset(pullResult.getNextBeginOffset()); 12 long pullRT = System.currentTimeMillis() - beginTimestamp; 13 DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullRT(pullRequest.getConsumerGroup(), 14 pullRequest.getMessageQueue().getTopic(), pullRT); 15 16 long firstMsgOffset = Long.MAX_VALUE; 17 if (pullResult.getMsgFoundList() == null || pullResult.getMsgFoundList().isEmpty()) { 18 DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest); 19 } else { 20 firstMsgOffset = pullResult.getMsgFoundList().get(0).getQueueOffset(); 21 22 DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullTPS(pullRequest.getConsumerGroup(), 23 pullRequest.getMessageQueue().getTopic(), pullResult.getMsgFoundList().size()); 24 //判断是否能发送给消费者，并发没有用到这个参数 25 boolean dispatchToConsume = processQueue.putMessage(pullResult.getMsgFoundList()); 26 DefaultMQPushConsumerImpl.this.consumeMessageService.submitConsumeRequest( 27 pullResult.getMsgFoundList(), 28 processQueue, 29 pullRequest.getMessageQueue(), 30 dispatchToConsume); 31 32 if (DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval() > 0) { 33 DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, 34 DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval()); 35 } else { 36 DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest); 37 } 38 } 39 40 if (pullResult.getNextBeginOffset() < prevRequestOffset 41 || firstMsgOffset < prevRequestOffset) { 42 log.warn( 43 "[BUG] pull message result maybe data wrong, nextBeginOffset: {} firstMsgOffset: {} prevRequestOffset: {}", 44 pullResult.getNextBeginOffset(), 45 firstMsgOffset, 46 prevRequestOffset); 47 } 48 49 break; 50 case NO_NEW_MSG: 51 pullRequest.setNextOffset(pullResult.getNextBeginOffset()); 52 53 DefaultMQPushConsumerImpl.this.correctTagsOffset(pullRequest); 54 55 DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest); 56 break; 57 case NO_MATCHED_MSG: 58 pullRequest.setNextOffset(pullResult.getNextBeginOffset()); 59 60 DefaultMQPushConsumerImpl.this.correctTagsOffset(pullRequest); 61 62 DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest); 63 break; 64 case OFFSET_ILLEGAL: 65 log.warn("the pull request offset illegal, {} {}", 66 pullRequest.toString(), pullResult.toString()); 67 pullRequest.setNextOffset(pullResult.getNextBeginOffset()); 68 69 pullRequest.getProcessQueue().setDropped(true); 70 DefaultMQPushConsumerImpl.this.executeTaskLater(new Runnable() { 71 72 @Override 73 public void run() { 74 try { 75 DefaultMQPushConsumerImpl.this.offsetStore.updateOffset(pullRequest.getMessageQueue(), 76 pullRequest.getNextOffset(), false); 77 78 DefaultMQPushConsumerImpl.this.offsetStore.persist(pullRequest.getMessageQueue()); 79 80 DefaultMQPushConsumerImpl.this.rebalanceImpl.removeProcessQueue(pullRequest.getMessageQueue()); 81 82 log.warn("fix the pull request offset, {}", pullRequest); 83 } catch (Throwable e) { 84 log.error("executeTaskLater Exception", e); 85 } 86 } 87 }, 10000); 88 break; 89 default: 90 break; 91 } 92 } 93 } 94 95 @Override 96 public void onException(Throwable e) { 97 if (!pullRequest.getMessageQueue().getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { 98 log.warn("execute the pull request exception", e); 99 } 100 101 DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException); 102 } 103 };
解析：public PullResult processPullResult(。。。。)主要是讲原始消息解析成 MessageExt
1 public PullResult processPullResult(final MessageQueue mq, final PullResult pullResult, 2 final SubscriptionData subscriptionData) { 3 PullResultExt pullResultExt = (PullResultExt) pullResult; 4 //更新mq(topic、队列id、broker)和brokerId的对应关系 5 this.updatePullFromWhichNode(mq, pullResultExt.getSuggestWhichBrokerId());
//如果拉取到了数据 6 if (PullStatus.FOUND == pullResult.getPullStatus()) {
//解析拉取的数据 7 ByteBuffer byteBuffer = ByteBuffer.wrap(pullResultExt.getMessageBinary()); 8 List<MessageExt> msgList = MessageDecoder.decodes(byteBuffer); 9 10 List<MessageExt> msgListFilterAgain = msgList; 11 if (!subscriptionData.getTagsSet().isEmpty() && !subscriptionData.isClassFilterMode()) { 12 msgListFilterAgain = new ArrayList<MessageExt>(msgList.size()); 13 for (MessageExt msg : msgList) { 14 if (msg.getTags() != null) {
//再次过滤 tagsSet是什么？？ 15 if (subscriptionData.getTagsSet().contains(msg.getTags())) { 16 msgListFilterAgain.add(msg); 17 } 18 } 19 } 20 } 21 //钩子，目前没有实现，可扩展 22 if (this.hasHook()) { 23 FilterMessageContext filterMessageContext = new FilterMessageContext(); 24 filterMessageContext.setUnitMode(unitMode); 25 filterMessageContext.setMsgList(msgListFilterAgain); 26 this.executeHook(filterMessageContext); 27 } 28 //MessageExt中属性塞入一些值 29 for (MessageExt msg : msgListFilterAgain) { 30 String traFlag = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED); 31 if (Boolean.parseBoolean(traFlag)) { 32 msg.setTransactionId(msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX)); 33 } 34 MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MIN_OFFSET, 35 Long.toString(pullResult.getMinOffset())); 36 MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MAX_OFFSET, 37 Long.toString(pullResult.getMaxOffset())); 38 msg.setBrokerName(mq.getBrokerName()); 39 } 40 41 pullResultExt.setMsgFoundList(msgListFilterAgain); 42 } 43 44 pullResultExt.setMessageBinary(null); 45 46 return pullResult; 47 }
解析：DefaultMQPushConsumerImpl.this.consumeMessageService.submitConsumeRequest(.....)
1 public void submitConsumeRequest( 2 final List<MessageExt> msgs, 3 final ProcessQueue processQueue, 4 final MessageQueue messageQueue, 5 final boolean dispatchToConsume) {
//消费者一次可以消费的最大数量 6 final int consumeBatchSize = this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize(); 7 if (msgs.size() <= consumeBatchSize) { 8 ConsumeRequest consumeRequest = new ConsumeRequest(msgs, processQueue, messageQueue); 9 try { 10 this.consumeExecutor.submit(consumeRequest); 11 } catch (RejectedExecutionException e) { 12 this.submitConsumeRequestLater(consumeRequest); 13 } 14 } else {
//如果拉取的总的消息数量msgs.size有8笔，consumeBatchSize=2,则分4次去消费 15 for (int total = 0; total < msgs.size(); ) { 16 List<MessageExt> msgThis = new ArrayList<MessageExt>(consumeBatchSize); 17 for (int i = 0; i < consumeBatchSize; i++, total++) { 18 if (total < msgs.size()) { 19 msgThis.add(msgs.get(total)); 20 } else { 21 break; 22 } 23 } 24 //通过线程池执行ConsumeRequest的run() 25 ConsumeRequest consumeRequest = new ConsumeRequest(msgThis, processQueue, messageQueue); 26 try { 27 this.consumeExecutor.submit(consumeRequest); 28 } catch (RejectedExecutionException e) { 29 for (; total < msgs.size(); total++) { 30 msgThis.add(msgs.get(total)); 31 } 32 33 this.submitConsumeRequestLater(consumeRequest); 34 } 35 } 36 } 37 }
解析：ConsumeRequest的run() 这里面会调用listen.consumeMessage(....) 这个方法就是实际上我们业务代码中拿到消费信息的方法
1 public void run() {
//如果该队列被丢弃，直接返回 2 if (this.processQueue.isDropped()) { 3 ConsumeMessageConcurrentlyService.log.info("the message queue not be able to consume, because it's dropped. group={} {}", ConsumeMessageConcurrentlyService.this.consumerGroup, this.messageQueue); 4 } else { 5 int exeSize = this.msgs.size(); 6 MessageListenerConcurrently listener = ConsumeMessageConcurrentlyService.this.messageListener; 7 ConsumeConcurrentlyContext context = new ConsumeConcurrentlyContext(this.messageQueue); 8 ConsumeConcurrentlyStatus status = null; 9 ConsumeMessageContext consumeMessageContext = null;
//作用？？ 10 if (ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) { 11 consumeMessageContext = new ConsumeMessageContext(); 12 consumeMessageContext.setConsumerGroup(ConsumeMessageConcurrentlyService.this.defaultMQPushConsumer.getConsumerGroup()); 13 consumeMessageContext.setProps(new HashMap()); 14 consumeMessageContext.setMq(this.messageQueue); 15 consumeMessageContext.setMsgList(this.msgs); 16 consumeMessageContext.setSuccess(false); 17 ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.executeHookBefore(consumeMessageContext); 18 } 19 20 long beginTimestamp = System.currentTimeMillis(); 21 boolean hasException = false; 22 ConsumeReturnType returnType = ConsumeReturnType.SUCCESS; 23 24 try { 25 ConsumeMessageConcurrentlyService.this.resetRetryTopic(this.msgs); 26 if (this.msgs != null && !this.msgs.isEmpty()) { 27 Iterator var10 = this.msgs.iterator(); 28 29 while(var10.hasNext()) { 30 MessageExt msg = (MessageExt)var10.next(); 31 MessageAccessor.setConsumeStartTimeStamp(msg, String.valueOf(System.currentTimeMillis())); 32 } 33 } 34 35 if (exeSize == 1) { 36 MQContextHelper.readChainContextFrom((Message)this.msgs.get(0), true); 37 } 38 //这个listener的实际对象由DefaultMQPushConsumer传入，所以这里会调用到DefaultMQPushConsumer中的接口实现方法中 39 status = listener.consumeMessage(Collections.unmodifiableList(tis.msgs), context); 40 } catch (Throwable var12) { 41 ConsumeMessageConcurrentlyService.log.warn("consumeMessage exception: {} Group: {} Msgs: {} MQ: {}", new Object[]{RemotingHelper.exceptionSimpleDesc(var12), ConsumeMessageConcurrentlyService.this.consumerGroup, this.msgs, this.messageQueue}); 42 hasException = true; 43 } 44 45 long consumeRT = System.currentTimeMillis() - beginTimestamp; 46 if (null == status) { 47 if (hasException) { 48 returnType = ConsumeReturnType.EXCEPTION; 49 } else { 50 returnType = ConsumeReturnType.RETURNNULL; 51 } 52 } else if (consumeRT >= ConsumeMessageConcurrentlyService.this.defaultMQPushConsumer.getConsumeTimeout() * 60L * 1000L) { 53 returnType = ConsumeReturnType.TIME_OUT; 54 } else if (ConsumeConcurrentlyStatus.RECONSUME_LATER == status) { 55 returnType = ConsumeReturnType.FAILED; 56 } else if (ConsumeConcurrentlyStatus.CONSUME_SUCCESS == status) { 57 returnType = ConsumeReturnType.SUCCESS; 58 } 59 60 if (ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) { 61 consumeMessageContext.getProps().put("ConsumeContextType", returnType.name()); 62 } 63 64 if (null == status) { 65 ConsumeMessageConcurrentlyService.log.warn("consumeMessage return null, Group: {} Msgs: {} MQ: {}", new Object[]{ConsumeMessageConcurrentlyService.this.consumerGroup, this.msgs, this.messageQueue}); 66 status = ConsumeConcurrentlyStatus.RECONSUME_LATER; 67 } 68 69 if (ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) { 70 consumeMessageContext.setStatus(status.toString()); 71 consumeMessageContext.setSuccess(ConsumeConcurrentlyStatus.CONSUME_SUCCESS == status); 72 ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.executeHookAfter(consumeMessageContext); 73 } 74 75 ConsumeMessageConcurrentlyService.this.getConsumerStatsManager().incConsumeRT(ConsumeMessageConcurrentlyService.this.consumerGroup, this.messageQueue.getTopic(), consumeRT); 76 if (!this.processQueue.isDropped()) {
//处理消费结果 详见下面解析 77 ConsumeMessageConcurrentlyService.this.processConsumeResult(status, context, this); 78 } else { 79 ConsumeMessageConcurrentlyService.log.warn("processQueue is dropped without process consume result. messageQueue={}, msgs={}", this.messageQueue, this.msgs); 80 } 81 82 } 83 84 }
解析：processConsumeResult(....) 处理消费结果 这个方法里面会对消费失败的进行消息重试 （生产发送失败也可以重试，但只限于同步发送的情况下）
在确定是否需要重试的时候，进一步处理哪些消息需要重试，也就是哪些消息会发送回broker
1 public void processConsumeResult( 2 final ConsumeConcurrentlyStatus status, 3 final ConsumeConcurrentlyContext context, 4 final ConsumeRequest consumeRequest 5 ) {
//默认是int类型能存储的最大值 6 int ackIndex = context.getAckIndex(); 7 //判断拉取的消息数量是否为空 8 if (consumeRequest.getMsgs().isEmpty()) 9 return; 10 11 switch (status) { 12 case CONSUME_SUCCESS: 13 if (ackIndex >= consumeRequest.getMsgs().size()) { 14 ackIndex = consumeRequest.getMsgs().size() - 1; 15 }
16 int ok = ackIndex + 1;
17 int failed = consumeRequest.getMsgs().size() - ok;
//记录消费成功或者失败的吞吐量？？ TPS 18 this.getConsumerStatsManager().incConsumeOKTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), ok); 19 this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), failed); 20 break; 21 case RECONSUME_LATER: 22 ackIndex = -1; 23 this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), 24 consumeRequest.getMsgs().size()); 25 break; 26 default: 27 break; 28 } 29 30 switch (this.defaultMQPushConsumer.getMessageModel()) { 31 case BROADCASTING: 32 for (int i = ackIndex + 1; i < consumeRequest.getMsgs().size(); i++) { 33 MessageExt msg = consumeRequest.getMsgs().get(i); 34 log.warn("BROADCASTING, the message consume failed, drop it, {}", msg.toString()); 35 } 36 break;
//一般使用集群模式 37 case CLUSTERING: 38 List<MessageExt> msgBackFailed = new ArrayList<MessageExt>(consumeRequest.getMsgs().size());
//正常情况下ackIndex会在上面的代码中被赋值为consumeRequest.getMsgs().size() //所以这里的for循环就可以=for(int i=3;i<3;i++)这种情况了，根本就不会进行循环，也就不会进行消息重试了
39 for (int i = ackIndex + 1; i < consumeRequest.getMsgs().size(); i++) { 40 MessageExt msg = consumeRequest.getMsgs().get(i);
//直接重新发送消息到broker 这个方法的解析流程具体可以看：https://www.cnblogs.com/sunshine-2015/p/9011446.html
//如果msgs有一条消息要重试，则msgs中的所有消息都要重试 41 boolean result = this.sendMessageBack(msg, context); 42 if (!result) { 43 msg.setReconsumeTimes(msg.getReconsumeTimes() + 1); 44 msgBackFailed.add(msg); 45 } 46 } 47 48 if (!msgBackFailed.isEmpty()) { 49 consumeRequest.getMsgs().removeAll(msgBackFailed); 50 //如果发送消息失败，则直接再次消费 51 this.submitConsumeRequestLater(msgBackFailed, consumeRequest.getProcessQueue(), consumeRequest.getMessageQueue()); 52 } 53 break; 54 default: 55 break; 56 }
//https://blog.csdn.net/guolong1983811/article/details/78821913 这篇文章讲了很多下面代码的实现 //removeMessage解析见如下 58 long offset = consumeRequest.getProcessQueue().removeMessage(consumeRequest.getMsgs());
//更新offset,之后就从此offset开始消费 ，这里也是会导致重复消费的地方，详见上面链接中的文章 59 if (offset >= 0 && !consumeRequest.getProcessQueue().isDropped()) { 60 this.defaultMQPushConsumerImpl.getOffsetStore().updateOffset(consumeRequest.getMessageQueue(), offset, true); 61 } 62 }
解析removeMessage
1 public long removeMessage(final List<MessageExt> msgs) { 2 long result = -1; 3 final long now = System.currentTimeMillis(); 4 try { 5 this.lockTreeMap.writeLock().lockInterruptibly(); 6 this.lastConsumeTimestamp = now; 7 try {
//本次从一个队列中拉取的消息总数，不只是这一次消费的数量 8 if (!msgTreeMap.isEmpty()) {
//默认下次从该队列中最大的一个offset中再+1进行 比如本次拉取的队列中的消息的offset是1000，则下次默认从1001开始 9 result = this.queueOffsetMax + 1; 10 int removedCnt = 0; 11 for (MessageExt msg : msgs) {
//已经消费完成的就从msgTreeMap中删除掉 12 MessageExt prev = msgTreeMap.remove(msg.getQueueOffset()); 13 if (prev != null) { 14 removedCnt--; 15 msgSize.addAndGet(0 - msg.getBody().length); 16 } 17 } 18 msgCount.addAndGet(removedCnt); 19 //如果本次从队列中一共拉取了10条，但是最大消费数量为2 ，那么此时的msgTreeMap=8 20 if (!msgTreeMap.isEmpty()) {
//获取剩余未消费的消息集合中的最小的offset,可能拉取了100条 201-300都消费完成了，但是200还没有，
//那么此时的result=200，如果此时断电了，那么消费的进度就维持在了200，就会造成重复消费。 21 result = msgTreeMap.firstKey(); 22 } 23 } 24 } finally { 25 this.lockTreeMap.writeLock().unlock(); 26 } 27 } catch (Throwable t) { 28 log.error("removeMessage exception", t); 29 } 30 31 return result; 32 }
msgTreeMap
