记录一下怎么保证MQ消费消息去重，消息重试

先说 背景，有消息生产，有很多SQL表名称，对应去统计不同表的数据，更新数量，但是这些消息会重复，可能有很多逻辑都要重复执行，可能会速度慢

生产：

这是SQL解析，重要的是这段 ，

tableName是枚举里面固定的，图片中有显示

  RabbitMQSender.sendMessage(MQConfig.FIRST_PAGE_SQL_ROUTINGKEY, tableName, MessageType.COMMON, uuid);

///伪代码 　　　　
　　StatementHandler statementHandler = PluginUtils.realTarget(invocation.getTarget());
            MetaObject metaObject = SystemMetaObject.forObject(statementHandler);
            MappedStatement mappedStatement = (MappedStatement)metaObject.getValue("delegate.mappedStatement");
            if(!SqlCommandType.SELECT.equals(mappedStatement.getSqlCommandType())){
                Object proceed = invocation.proceed();
                firstPageSqlParse(metaObject);
                return proceed;
　　　　　　}

private void firstPageSqlParse(MetaObject metaObject) {
    BoundSql boundSql = (BoundSql) metaObject.getValue("delegate.boundSql");
    String sql = boundSql.getSql();
    CompletableFuture.runAsync(() -> {
        // 语句提取表名：
        String lowSql = sql.toLowerCase();
        String tableName = extractTableName(lowSql);
        FirstPageCountEnum[] values = FirstPageCountEnum.values();
        Set<String> collect = Arrays.stream(values).map(FirstPageCountEnum::getTableName).collect(Collectors.toSet());
        if (collect.contains(tableName)) {
            String uuid = IdUtil.randomUUID();
            RabbitMQSender.sendMessage(MQConfig.FIRST_PAGE_SQL_ROUTINGKEY, tableName, MessageType.COMMON, uuid);
        }
    });
}

private static String extractTableName(String sql) {
    String regex = "(insert\\s+into\\s+|update\\s+|delete\\s+from\\s+)(\\w+)";
    Pattern pattern = Pattern.compile(regex);
    Matcher matcher = pattern.matcher(sql);
    if (matcher.find()) {
        String tableName = matcher.group(2);
        System.out.println(tableName);
        return tableName;
    }
    return "";
}

 

 

消费端：

这就是普通接受部分

 /**
     * 监听单个队列
     * concurrency:并发处理消息数
     */
    @RabbitListener(queues = MQConfig.FIRST_PAGE_SQL)
    @RabbitHandler
    public void notificationQueueReceiver(Message message, Channel channel) throws IOException {
        messageHandler(message, channel);
    }

    private void messageHandler(Message message, Channel channel) throws IOException {
        long deliveryTag = message.getMessageProperties().getDeliveryTag();
        Action action = Action.ACCEPT;
        try {
            MessageBody messageBody = MessageBody.getMessageBody(message);
          
            String sql = MessageBody.getMessageBody(message).getData().toString();
           　// 锁，　
            Object o = CACHE_MQ_MAP_LOCK.get(sql);
            if (o != null) {
                if (CACHE_MQ_MAP.getOrDefault(sql, 0) == 0) {
                    synchronized (o) {
                        if (CACHE_MQ_MAP.getOrDefault(sql, 0) == 0) {
                            CACHE_MQ_MAP.put(sql, 1);
                        }
                        o.notifyAll();
                    }
                }
            }
        } catch (Exception e) {
            log.error("MQ处理消息出错", e);
        } finally {
            // 通过 finally 块来保证 Ack/Nack 会且只会执行一次
            if (action == Action.ACCEPT) {
                // false 只确认当前 consumer 一个消息收到，true 确认所有 consumer 获得的消息。
                channel.basicAck(deliveryTag, false);
            } else {
                // 第二个 boolean 为 false 表示不会重试，为 true 会重新放回队列
                channel.basicReject(deliveryTag, false);
            }
        }
    }

初始化两个MAP 锁，以及去除重复的数据Map

 @PostConstruct
    public void init() {
        CompletableFuture.runAsync(() -> {
            for (FirstPageCountEnum value : FirstPageCountEnum.values()) {
                CACHE_MQ_MAP_LOCK.put(value.getTableName(), new Object());
                try {
//　　为了初始化可以数据更新
                    String uuid = IdUtil.randomUUID();
                    RabbitMQSender.sendMessage(MQConfig.FIRST_PAGE_SQL_ROUTINGKEY, value.getTableName(), MessageType.COMMON, uuid);
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
            // 创建消费者线程，处理逻辑在下面，这里有点问题，创建的线程多了，因为表名称重复了。
            for (FirstPageCountEnum value : FirstPageCountEnum.values()) {
                Thread consumerThread = new Thread(new Consumer(CACHE_MQ_MAP, value.getTableName(), userFirstPageCountService));
                consumerThread.start();
            }
        });
    }
　　// 保存消息的去重复的MAP
    private final ConcurrentHashMap<String, Integer> CACHE_MQ_MAP = new ConcurrentHashMap<>(32);
　　// 保存锁的
    private static final ConcurrentHashMap<String, Object> CACHE_MQ_MAP_LOCK = new ConcurrentHashMap<>(32);

 

创建的消费线程处理的逻辑

 static class Consumer implements Runnable {
        private final ConcurrentHashMap<String, Integer> dataMap;
        private final String tableName;
        private final UserFirstPageCountService userFirstPageCountService;

        public Consumer(ConcurrentHashMap<String, Integer> dataMap, String tableName, UserFirstPageCountService userFirstPageCountService) {
            this.userFirstPageCountService = userFirstPageCountService;
            this.tableName = tableName;
            this.dataMap = dataMap;
        }

        @Override
        public void run() {
            // 消费数据
            while (true) {
                for (String key : dataMap.keySet()) {
                    if (!key.equals(tableName)) {
                        continue;
                    }

                    Object lock = CACHE_MQ_MAP_LOCK.get(key);
                    // 获取数据
                    try {
                        Thread.sleep(2000);
                        int value = dataMap.getOrDefault(key, 0);
                        log.info("Consumed: {},  value:{}", key, value);
                        if (value == 1) {
                            dataMap.put(key, 0);
　　　　　　　　　　　　　　　　//这里业务处理逻辑，里面加了重试。
                            userFirstPageCountService.sqlParse(tableName);
                        }
                        synchronized (lock) {
                            if (dataMap.getOrDefault(key, 0) == 0) {
                                // 设置对应的key为0
                                log.info("wait {},{}", key, 0);
                                lock.wait();
                                log.info("notify {},{}", key, 0);
                            }
                        }
                    } catch (Exception e) {
                        log.error("Consumed error ", e);
                    }

                }
            }
        }
    }

 

这是重试，初始化逻辑可以保证不丢失。 以及这里出错后调用不成功可以不断重试