对RocketMq Client 封装 可以通过配置文件创建生产者 消费者

背景

当我们使用rocketmq 时 需要需要在Java代码里创建producer，consume 当我们生产者与消费者很多时 就会出现很多重复代码 不便于管理维护
为了更加便于管理和维护 所以对rocketmq client 进行封装 可以通过配置文件和注解来创建生产者和消费者 同时可以对多个MQ集群进行操作

定义好yaml文件配置格式

  rocketmq:
    enable: true #是否开启
    address: 192.168.01.01:9876 #mq默认地址
    scanAllPackage: #非必填 用于扫描consume 注解
    producers:
      - address: 192.168.01.01:9876 # 非必填 没有填写就取rocketmq.address的配置
        group: test_topic # 必填生产组名称
        beanName: producer2 #必填 生产者在spring容器的名称 注入的时候要用到
      - address: 192.168.01.02:9876
        group: test_topic1
        beanName: producer1
    consumers:
      - address: 192.168.01.01:9876 # 非必填 没有填写就取rocketmq.address的配置
        group: test_group # 必填消费组名称 
        topic: test_topic #消费的主题
        model: 1 # 1 集群消费 2 广播消费
        tag: '*' # tag
        # listener 消息消费监听器全类名 处理业务的
        listener: org.dome.common.module.rocketmq.test.TestListener

根据我们定义好的rocket mq 配置 开始定义一个配置类 用来将配置转换成对象

@Data
public class RocketMqProperties {
    /**
     * namesev 地址 多个用逗号隔开
     */
    private String address;
    /**
     * 是否开启
     */
    private Boolean enable = false;
    /**
     * 扫描路径 用于注解
     */
    private String scanAllPackage;
    /**
     * 生产者
     */
    private List<ProducerProperties> producers;
    /**
     * 消费者
     */
    private List<ConsumerProperties> consumers;
}

@Data
class ProducerProperties {

    /**
     * 地址
     */
    private String address;
    /**
     * 生产组
     */
    private String group;
    /**
     * bean 名称
     */
    private String beanName;
}

@Data
class ConsumerProperties {
    /**
     * 地址
     */
    private String address;
    /**
     * 消费组
     */
    private String group;
    /**
     * 集群还是广播 1：集群 2：广播
     */
    private Integer model = 1;
    /**
     * 主题
     */
    private String topic;
    /**
     * tag
     */
    private String tag;
    /**
     * 消息监听器  全类名
     */
    private String listener;
    /**
     * 消费方式
     * last：消费者客户端从之前停止的地方开始。 如果是新启动的消费者客户端，根据消费者群体的老化情况，
     * 有两种情况：如果consumer group是最近创建的，最早订阅的消息还没有过期，说明consumer group代表最近上线的业务，
     * 从头开始消费；
     * <p>
     * first：消费者客户端将从最早可用的消息开始，也就是从头开始消费
     * <p>
     * timestamp：Consumer客户端将从指定的时间戳开始，这意味着在consumeTimestamp之前产生的消息将被忽略
     */
    private String consumeFromWhere;
    /**
     * 当consumeFromWhere=timestamp 时这个字段必填写
     * 格式：TODO
     */
    private String consumeTimestamp;
    /**
     * 最小消费线程数量
     */
    private Integer consumeThreadMin = 20;

    /**
     * 最大线程消费数量
     */
    private Integer consumeThreadMax = 20;

    /**
     * MQ消费者每次去拉取消息拉取多少条
     */
    private Integer pullBatchSize;
}

到这一步我们所有的准备工作已经完成 开始书写配置类

我们定义一个RocketMqDynamicRegistryConfig 并实现EnvironmentAware 与BeanDefinitionRegistryPostProcessor 这两个接口 EnvironmentAware的优先级在spring中是高于BeanDefinitionRegistryPostProcessor的 所以我们需要在spring 执行EnvironmentAware时 将配置文件转换成 我们的配置类 然后在BeanDefinitionRegistryPostProcessor的回调中进行生产者与消费者的BeanDefinition创建，并且扫描指定路径下所有通过注解创建的consume 根据注解信息创建BeanDefinition 具体看下面的代码注释

@Slf4j
@ConditionalOnProperty(name = "rocketmq.enable", havingValue = "true")
@Configuration
public class RocketMqDynamicRegistryConfig implements BeanDefinitionRegistryPostProcessor, EnvironmentAware {
    private Environment environment;
    private RocketProperties properties;
    private static final AtomicInteger COUNT = new AtomicInteger(0);
    public static final String APP_NAME_KEY = "spring.application.name";

    public static final String MQ_CONFIG_PREFIX = "rocketmq";

    /**
     * 默认生产者BEAN
     *
     * @return
     */
    @Bean(initMethod = "start", destroyMethod = "shutdown")
    @Primary
    public MqProducer defaultEdgeProducer() {
        String applicationName = environment.getProperty(APP_NAME_KEY);
        MqProducer producer = new MqProducer(applicationName, properties.getAddress(), new ProducerProperties());
        return MqProducer;
    }


    @Override
    public void postProcessBeanDefinitionRegistry(BeanDefinitionRegistry beanDefinitionRegistry) throws BeansException {
        // 从配置文件中获取当前应用的应用名称
        String applicationName = environment.getProperty(APP_NAME_KEY);

        log.info("postProcessBeanDefinitionRegistry");
        if (CollUtil.isNotEmpty(properties.getProducers())) {
            //根据配置类去创建生产者的beanDefinition
            startProducer(beanDefinitionRegistry, properties);
        }
        if (CollUtil.isNotEmpty(properties.getConsumers())) {
            //根据配置去创建consume
            startConsumer(beanDefinitionRegistry, applicationName, properties);
        }
        //判断是否有配置扫描
        if (StrUtil.isNotEmpty(properties.getScanAllPackage())) {
            //获取所有带有MqConsumer的类
            Set<Class<?>> classSet = ClassScanner.scanAllPackageByAnnotation(properties.getScanAllPackage(), MqConsumer.class);
            for (Class<?> aClass : classSet) {
                //类获取注解
                MqConsumer annotation = AnnotationUtils.findAnnotation(aClass, MqConsumer.class);
                //获取注解的topice配置
                String topic = annotation.topic();
                //替换${变量内容} 在注解上可以通过${xxx} 获取配置 有这种写法 参考@Values配置
                topic = environment.resolvePlaceholders(topic);
                //消费模式
                MessageModel model = annotation.MODEL();
                String expression = annotation.subExpression();
                expression = environment.resolvePlaceholders(expression);
                String group = annotation.group();
                group = environment.resolvePlaceholders(group);
                //将注解上的配置转换成ConsumerProperties
                ConsumerProperties consumerProperties = new ConsumerProperties();
                consumerProperties.setAddress(properties.getAddress());
                consumerProperties.setTopic(topic);
                consumerProperties.setGroup(group);
                consumerProperties.setTag(expression);
                consumerProperties.setModel(model == MessageModel.CLUSTERING ? 1 : 2);
                consumerProperties.setListener(aClass.getName());
                // 注册 消费者beanDefinition
                registryConsumer(beanDefinitionRegistry, applicationName, properties, consumerProperties);
            }
        }
    }

    /**
     * 启动生产者
     *
     * @param beanDefinitionRegistry
     * @param properties
     */
    private void startProducer(BeanDefinitionRegistry beanDefinitionRegistry, RocketProperties properties) {
        for (ProducerProperties producerProperties : properties.getProducers()) {
            //构建生产者的 BeanDefinition
            BeanDefinitionBuilder builder = BeanDefinitionBuilder.genericBeanDefinition(MqProducer.class);
            //添加构造方法参数
            builder.addConstructorArgValue(environment.getProperty(APP_NAME_KEY));
            builder.addConstructorArgValue(properties.getAddress());
            builder.addConstructorArgValue(producerProperties);
            //设置bean的初始化方法 也就是启动生产者的方法
            builder.setInitMethodName("start");
            //设置销毁方法
            builder.setDestroyMethodName("shutdown");
            Assert.hasText(producerProperties.getBeanName(), "producer bean name is not null");
            //注册beanDefinition
            beanDefinitionRegistry.registerBeanDefinition(producerProperties.getBeanName(), builder.getBeanDefinition());
        }
    }

    /**
     * 启动消费者
     *
     * @param beanDefinitionRegistry
     * @param applicationName
     * @param properties
     */
    private void startConsumer(BeanDefinitionRegistry beanDefinitionRegistry, String applicationName, RocketProperties properties) {
        for (ConsumerProperties consumerProperties : properties.getConsumers()) {
            registryConsumer(beanDefinitionRegistry, applicationName, properties, consumerProperties);
        }
    }

    private void registryConsumer(BeanDefinitionRegistry beanDefinitionRegistry, String applicationName, RocketProperties properties, ConsumerProperties consumerProperties) {
        BeanDefinitionBuilder builder = BeanDefinitionBuilder.genericBeanDefinition(ConsumerDefinition.class);
        builder.addConstructorArgValue(applicationName);
        builder.addConstructorArgValue(properties.getAddress());
        builder.addConstructorArgValue(consumerProperties);
       //设置消费者初始化方法
        builder.setInitMethodName("start");
      //销毁
        builder.setDestroyMethodName("shutdown");
        Assert.hasText(consumerProperties.getListener(), "consumer listener not null");
        beanDefinitionRegistry.registerBeanDefinition("EdgeConsumerDefinition-" + COUNT.incrementAndGet(), builder.getBeanDefinition());
    }

    @Override
    public void postProcessBeanFactory(ConfigurableListableBeanFactory configurableListableBeanFactory) throws BeansException {
    }

    @Override
    public void setEnvironment(Environment environment) {
        //设置Environment
        this.environment = environment;
        //通过MQ_CONFIG_PREFIX 前缀将配置文件转换成RocketProperties配置类
        BindResult<RocketProperties> bind = Binder.get(environment).bind(MQ_CONFIG_PREFIX, RocketProperties.class);
        properties = bind.get();
    }

    /**
    * 定义一个切面 用于消费消息时 trace_id 生成
    */
    @ConditionalOnClass(Tracer.class)
    @Bean
    public TraceRocketMqConsumerAspect consumerAspect(Tracer tracer) {
        TraceRocketMqConsumerAspect traceRocketMqConsumerAspect = new TraceRocketMqConsumerAspect(tracer);
        return traceRocketMqConsumerAspect;
    }

}

通过上面这个类发现我们实际上不需要自己去控制生产者消费者的启动和关闭 这一步利用spring bean的生命周期来处理 我们只需要定义好BeanDefinition 中的内容即可

生产者BeanDefinition

@Slf4j
public class MqProducer {


    private final String defaultAddress;
    /**
     * 应用程序名称
     */
    private final String applicationName;
    private final ProducerProperties properties;

    private DefaultMQProducer mqProducer;

    public MqProducer(String applicationName, String defaultAddress, ProducerProperties properties) {
        this.applicationName = applicationName;
        this.defaultAddress = defaultAddress;
        this.properties = properties;
    }
    //bean 初始化时 调用该方法进行生产者的启动
    protected void start() throws IllegalAccessException, MQClientException {
        log.info("MqProducer start group:{}", properties.toString());
        if (StringUtils.isNotEmpty(properties.getGroup())) {
            mqProducer = new DefaultMQProducer(properties.getGroup());
        } else {
            mqProducer = new DefaultMQProducer(applicationName);
        }
        if (StringUtils.isNotEmpty(properties.getAddress())) {
            mqProducer.setNamesrvAddr(properties.getAddress());
        } else if (StringUtils.isNotEmpty(defaultAddress)) {
            mqProducer.setNamesrvAddr(defaultAddress);
        } else {
            throw new IllegalAccessException("mq参数地址配置异常");
        }
        mqProducer.start();
    }

    protected void shutdown() {
        if (mqProducer != null) {
            mqProducer.shutdown();
        }
    }

    /**
     * 发送普通消息
     *
     * @param message
     * @return
     * @throws MQBrokerException
     * @throws RemotingException
     * @throws InterruptedException
     * @throws MQClientException
     */
    public Boolean send(Message message) throws MQBrokerException, RemotingException, InterruptedException, MQClientException {
        SendResult result = this.mqProducer.send(message);
        return SendStatus.SEND_OK.equals(result.getSendStatus());
    }

    /**
     * 发送普通消息带超时
     *
     * @param message
     * @param timeout
     * @return
     * @throws MQBrokerException
     * @throws RemotingException
     * @throws InterruptedException
     * @throws MQClientException
     */
    public Boolean send(Message message, Long timeout) throws MQBrokerException, RemotingException, InterruptedException, MQClientException {
        SendResult result = this.mqProducer.send(message, timeout);
        return SendStatus.SEND_OK.equals(result.getSendStatus());
    }

    /**
     * 发送异步消息
     *
     * @param message
     * @param callback
     * @throws MQBrokerException
     * @throws RemotingException
     * @throws InterruptedException
     * @throws MQClientException
     */
    public void sendAsync(Message message, SendCallback callback) throws MQBrokerException, RemotingException, InterruptedException, MQClientException {
        this.mqProducer.send(message, callback);
    }

    /**
     * 发送顺序消息
     * 通过key的hash 确定在那一条队列
     *
     * @param message
     * @param key
     * @return
     * @throws MQBrokerException
     * @throws RemotingException
     * @throws InterruptedException
     * @throws MQClientException
     */
    public Boolean sendOrderMsg(Message message, String key) throws MQBrokerException, RemotingException, InterruptedException, MQClientException {
        SendResult result = this.mqProducer.send(message, new MessageQueueSelector() {
            @Override
            public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
                String key = (String) arg;
                int index = (key.hashCode()& Integer.MAX_VALUE) % mqs.size();
                return mqs.get(index);
            }
        }, key);
        return SendStatus.SEND_OK.equals(result.getSendStatus());
    }

    /**
     * 异步发送顺序消息
     *
     * @param message
     * @param key
     * @param sendCallback
     * @throws MQBrokerException
     * @throws RemotingException
     * @throws InterruptedException
     * @throws MQClientException
     */
    public void sendOrderMsgAsync(Message message, String key, SendCallback sendCallback) throws MQBrokerException, RemotingException, InterruptedException, MQClientException {
        this.mqProducer.send(message, new MessageQueueSelector() {
            @Override
            public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
                String key = (String) arg;
                int index = (key.hashCode()& Integer.MAX_VALUE) % mqs.size();
                return mqs.get(index);
            }
        }, key, sendCallback);
    }
}

Consume BeanDefinition 定义

@Slf4j
public class ConsumerDefinition {
    /**
     * 默认地址
     */
    private final String defaultAddress;
    /**
     * 应用程序名称
     */
    private final String applicationName;
    /**
     * 配置信息
     */
    private final ConsumerProperties consumerProperties;
    private DefaultMQPushConsumer consumer;


    private static final String LAST_CONSUME_FROM_WHERE="last";
    private static final String FIRST_CONSUME_FROM_WHERE="first";
    private static final String TIMESTAMP_CONSUME_FROM_WHERE="timestamp";
    public ConsumerDefinition(String applicationName, String defaultAddress, ConsumerProperties consumerProperties) {
        this.defaultAddress = defaultAddress;
        this.applicationName = applicationName;
        this.consumerProperties = consumerProperties;
    }
    //启动消费者
    protected void start() {
        log.info("EdgeConsumerDefinition start");
        try {
            String address = StringUtils.isNotEmpty(consumerProperties.getAddress()) ? consumerProperties.getAddress() : defaultAddress;
            String group = StringUtils.isNotEmpty(consumerProperties.getGroup()) ? consumerProperties.getGroup() : applicationName;
            consumer = new DefaultMQPushConsumer(group);
            consumer.setNamesrvAddr(address);
            consumer.setMessageModel(consumerProperties.getModel() == 1 ? MessageModel.CLUSTERING : MessageModel.BROADCASTING);
            consumer.subscribe(consumerProperties.getTopic(), consumerProperties.getTag());
            if (StrUtil.isNotEmpty(consumerProperties.getConsumeFromWhere())) {
                if (LAST_CONSUME_FROM_WHERE.equals(consumerProperties.getConsumeFromWhere())){
                    consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);
                }else if (FIRST_CONSUME_FROM_WHERE.equals(consumerProperties.getConsumeFromWhere())){
                    consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
                }else if (TIMESTAMP_CONSUME_FROM_WHERE.equals(consumerProperties.getConsumeFromWhere())){
                    if (StrUtil.isNotEmpty(consumerProperties.getConsumeTimestamp())) {
                        consumer.setConsumeTimestamp(consumerProperties.getConsumeTimestamp());
                    }else {
                        throw new RuntimeException("consumeTimestamp 不能为空。。。");
                    }
                }
            }

            if (consumerProperties.getPullBatchSize() != null) {
                consumer.setPullBatchSize(consumerProperties.getPullBatchSize());
            }
            consumer.setConsumeThreadMax(consumerProperties.getConsumeThreadMax());
            consumer.setConsumeThreadMin(consumerProperties.getConsumeThreadMin());
            String listener = consumerProperties.getListener();
            //从spring 容器中获取消息监听器
            Object bean = SpringUtil.getBean(Class.forName(listener));
            consumer.setMessageListener((MessageListener) bean);
            consumer.start();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }


    protected void shutdown() {
        if (consumer != null) {
            consumer.shutdown();
        }
    }
}