kafka消费者配置

最近的业务接触了一下消息插件kafka ,其他客户端负责发送,我方负责接收消费。功能很简单,但是!!!!我们要求把消息入库,而且数据量每天达到了千万级别,这就日了苟啊!

 

废话不多说,上代码!

 

由于消息需要入库,我使用的是手动提交消息,如果不入库,不要求准确定,使用自动提交就ok

 

消费者配置类

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
package com.asiainfo.hsop.server.kafka;
 
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.config.KafkaListenerContainerFactory;
import org.springframework.kafka.core.ConsumerFactory;
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
import org.springframework.kafka.listener.AbstractMessageListenerContainer;
import org.springframework.kafka.listener.ConcurrentMessageListenerContainer;
import org.springframework.kafka.listener.config.ContainerProperties;
 
import java.util.HashMap;
import java.util.Map;
/**
 * kafka消息队列消费者 配置类
 *
 * @author cyc
 * @version 1.0.0
 * @date 2021-07-02
 *
 * */
@Configuration
@EnableKafka
public class KafkaConsumerConfig {
    /*
    *
    *启动项目之前 需要修改 etc/hosts 文件,把 kafka实例名称和主机ip对应关系填好,否则会报错 ,
    *
    *报错信息如下:
    *           Discovered group coordinator......
    *           (Re-)joining group......
    *           Marking the coordinator......
    *尝试升级kafka-clients ,但是没有啥卵用.......最好的解决方案 就是修改系统hosts文件
    *
    *   主机ip 和 实例名称 对应关系
    *   11.251.10.111 sss-nn-01
    *  
    *
    *  */
    public Map<String, Object> consumerConfig(String consumerGroupId) {
        Map<String, Object> props = new HashMap<>();
        //kafka服务地址
        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "10.253.6.194:6667,10.253.6.195:6667,10.253.6.196:6667,10.253.6.197:6667,10.253.6.198:6667,10.253.6.199:6667,10.253.6.200:6667,10.253.6.201:6667");
        //消费后是否自动提交 当前为false
        props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
        //获取消息后提交偏移量的最大时间
        props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "100");
        //超时时间,服务端没有收到心跳就会认为当前消费者失效
        props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "15000");
        //默认消费组
        props.put(ConsumerConfig.GROUP_ID_CONFIG, consumerGroupId);
        //earliest  从头开始消费、latest获取最新消息 、none 没理解
        props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest");
        //序列化
        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        return props;
    }
 
    public ConsumerFactory<String, String> consumerFactory(String consumerGroupId) {
        return new DefaultKafkaConsumerFactory<>(consumerConfig(consumerGroupId));
    }
 
    /*有多个消费组监听,直接复制改方法,
    * 修改 @Bean(name = "kafkaListenerContainerFactory") name重新命名
    *
    * factory.setConsumerFactory(consumerFactory("etctu_tradeinfolist_test")); 改下消费组 名称
    *
    * 在调用的时候containerFactory="kafkaListenerContainerFactory" 改一下就ok
    *
    * */
    @Bean(name = "kafkaListenerContainerFactory")
    public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
        ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(consumerFactory("etctu_tradeinfolist_test"));<br>     //监听的线程数量,多个线程入库,数据库的id是自增的话,可能导致死锁,建议使用UUID
        factory.setConcurrency(1);
        /*消费者有两种消费模式,一种是kafka实例主动推送push模式,推送速度由kafka决定,很有可能导致消费端阻塞,
        *另一种就是 消费者主动拉取,poll模式
        * */
        factory.getContainerProperties().setPollTimeout(3000);
        //当使用手动提交时必须设置ackMode为MANUAL,否则会报错No Acknowledgment available as an argument, the listener container must have a MANUAL AckMode to populate the Acknowledgment.
        //
        factory.getContainerProperties().setAckMode(AbstractMessageListenerContainer.AckMode.MANUAL);
        // 下面两个条件哪个先满足,就会先使用那个
        factory.getContainerProperties().setAckCount(10);//达到10条数据的时候提交一次
        factory.getContainerProperties().setAckTime(10000);//10s提交一次
        return factory;
    }
 
 
 
     
    @Bean(name = "kafkaListenerContainerFactory2")
    public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory2() { ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(consumerFactory("exit_toll_host_test"));
        factory.setConcurrency(1);
        factory.getContainerProperties().setPollTimeout(3000);
        factory.getContainerProperties().setAckMode(AbstractMessageListenerContainer.AckMode.MANUAL);
        factory.getContainerProperties().setAckCount(10);
        factory.getContainerProperties().setAckTime(10000);
        return factory;
    }
 
   
    @Bean(name = "kafkaListenerContainerFactory3")
    public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory3() { ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(consumerFactory("entryinfo_test"));
        factory.setConcurrency(1);
        factory.getContainerProperties().setPollTimeout(3000);
        //当使用手动提交时必须设置ackMode为MANUAL,否则会报错No Acknowledgment available as an argument, the listener container must have a MANUAL AckMode to populate the Acknowledgment.
        factory.getContainerProperties().setAckMode(AbstractMessageListenerContainer.AckMode.MANUAL);
        factory.getContainerProperties().setAckCount(10);
        factory.getContainerProperties().setAckTime(10000);
        return factory;
    }
 
 
}

 

 

消费者类,里面包含了Hbase 引入的包,没啥用,删掉就行

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
package com.asiainfo.hsop.server.kafka;
 
import com.alibaba.fastjson.JSON;
import com.asiainfo.hsop.base.po.data.kafka.HospKafkaDetail;
 
import com.asiainfo.hsop.common.utils.HBaseUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.kafka.support.Acknowledgment;
import org.springframework.stereotype.Component;
 
import java.io.IOException;
import java.math.BigDecimal;
import java.text.ParseException;
import java.util.*;
 
/**
 *
 * @author cyc
 *
 * @date 2021-07-02
 *
 *
 *
 *
 * */
@Component
public class Consumer {
 
    private Logger logger= LoggerFactory.getLogger(Consumer.class);
 
 
 
 
        @KafkaListener(topics = "highwayetctu",containerFactory="kafkaListenerContainerFactory")
        public void consumer1(ConsumerRecord<?, ?> record, Acknowledgment ack) throws InterruptedException, ParseException, IOException {
 
            //具体业务
<br>        //手动提交
            ack.acknowledge();
        }
 
 
 
 
 
 
         
 
 
 
 
        
        @KafkaListener(topics = "exitList_0706",containerFactory="kafkaListenerContainerFactory2")
        public void consumer2(ConsumerRecord<?, ?> record,Acknowledgment ack ) throws IOException, ParseException {
        //具体业务<br>        //手动提交
            ack.acknowledge();
        }
 
 
 
 
 
        @KafkaListener(topics = "entryinfo",containerFactory="kafkaListenerContainerFactory3")
        public void consumer3(ConsumerRecord<?, ?> record,Acknowledgment ack ) throws IOException, ParseException {
 
             //具体业务<br>      <br>        //手动提交
            ack.acknowledge();
        }
 
   
 
 
 
 
}

 

 

 

 

 

posted @   幽灵中的野孩子  阅读(2632)  评论(0编辑  收藏  举报
编辑推荐:
· AI与.NET技术实操系列:基于图像分类模型对图像进行分类
· go语言实现终端里的倒计时
· 如何编写易于单元测试的代码
· 10年+ .NET Coder 心语,封装的思维:从隐藏、稳定开始理解其本质意义
· .NET Core 中如何实现缓存的预热?
阅读排行:
· 25岁的心里话
· 闲置电脑爆改个人服务器(超详细) #公网映射 #Vmware虚拟网络编辑器
· 零经验选手,Compose 一天开发一款小游戏!
· 因为Apifox不支持离线,我果断选择了Apipost!
· 通过 API 将Deepseek响应流式内容输出到前端
点击右上角即可分享
微信分享提示