SpringBoot集成kafka
环境springboot2.7 +kafka3.0。kafka安装请自行百度，话不多说直接上代码。
1、添加maven依赖
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
</dependency>
2、application.yml 添加kafka配置
spring: logging: level: org.apache.kafka: error kafka: bootstrap-servers: 127.0.0.1:9092 producer: key-serializer: org.apache.kafka.common.serialization.StringSerializer value-serializer: org.apache.kafka.common.serialization.StringSerializer acks: 1 retries: 3 batch-size: 16384 linger-ms: 1 buffer-memory: 33554432 max-request-size: 10485760 consumer: key-deserializer: org.apache.kafka.common.serialization.StringDeserializer value-deserializer: org.apache.kafka.common.serialization.StringDeserializer group-id: test-consumer-group enable-auto-commit: false auto-commit-interval-ms: 1000 session-timeout-ms: 30000 auto-offset-reset: earliest max-poll-records: 100
3、kafka工具类 KafkaUtils.java
import com.alibaba.fastjson.JSONObject; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.kafka.clients.admin.*; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.errors.TopicExistsException; import org.apache.kafka.common.errors.UnknownTopicOrPartitionException; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import java.time.Duration; import java.util.*; import java.util.concurrent.ExecutionException; /** * @Description: kafka工具类 * @Date: 2023/6/14 16:29 * @Version: 1.0 */ @Component @Slf4j public class KafkaUtils { @Value("${spring.kafka.bootstrap-servers}") private String kafkaBootstrapServers; //生产者配置 @Value("${spring.kafka.producer.key-serializer}") private String kafkaProducerKeySerializer; @Value("${spring.kafka.producer.value-serializer}") private String kafkaProducerValueSerializer; @Value("${spring.kafka.producer.acks}") private String kafkaProducerAcks; @Value("${spring.kafka.producer.retries}") private Integer kafkaProducerRetries; @Value("${spring.kafka.producer.batch-size}") private Integer kafkaProducerBatchSize; @Value("${spring.kafka.producer.linger-ms}") private Integer kafkaProducerLingerMs; @Value("${spring.kafka.producer.buffer-memory}") private Long kafkaProducerBufferMemory; @Value("${spring.kafka.producer.max-request-size}") private Integer kafkaProducerMaxRequestSize; //消费者配置 @Value("${spring.kafka.consumer.key-deserializer}") private String kafkaConsumerKeyDeserializer; @Value("${spring.kafka.consumer.value-deserializer}") private String kafkaConsumerValueDeserializer; @Value("${spring.kafka.consumer.group-id}") private String kafkaConsumerGroupId; @Value("${spring.kafka.consumer.enable-auto-commit}") private boolean kafkaConsumerEnableAutoCommit; @Value("${spring.kafka.consumer.auto-commit-interval-ms}") private Integer kafkaConsumerAutoCommitIntervalMs; @Value("${spring.kafka.consumer.session-timeout-ms}") private Integer kafkaConsumerSessionTimeoutMs; @Value("${spring.kafka.consumer.auto-offset-reset}") private String kafkaConsumerAutoOffsetReset; @Value("${spring.kafka.consumer.max-poll-records}") private Integer kafkaConsumerMaxPollRecords; private AdminClient adminClient; /** * 私有静态方法，创建Kafka生产者 * @author o * @return KafkaProducer */ private KafkaProducer<String, String> createProducer() { Properties props = new Properties(); //声明kafka的地址 props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,kafkaBootstrapServers); //0、1 和 -1：0表示只要把消息发送出去就返回成功；1表示只要Leader收到消息就返回成功；-1表示所有副本都写入数据成功才算成功 props.put("acks", kafkaProducerAcks); //重试次数 props.put("retries", kafkaProducerRetries); //批处理的字节数 props.put("batch.size", kafkaProducerBatchSize); //批处理的延迟时间，当批次数据未满之时等待的时间 props.put("linger.ms", kafkaProducerLingerMs); //用来约束KafkaProducer能够使用的内存缓冲的大小的，默认值32MB props.put("buffer.memory", kafkaProducerBufferMemory); //修改最大向kafka推送消息大小 props.put("max.request.size",kafkaProducerMaxRequestSize); props.put("value.serializer", kafkaProducerKeySerializer); props.put("key.serializer", kafkaProducerValueSerializer); return new KafkaProducer<String, String>(props); } /** * 私有静态方法，创建Kafka消费者 * @author o * @return KafkaConsumer */ private KafkaConsumer<String, String> createConsumer() { Properties props = new Properties(); //声明kafka的地址 props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,kafkaBootstrapServers); //每个消费者分配独立的消费者组编号 props.put("group.id", kafkaConsumerGroupId); //如果value合法，则自动提交偏移量 props.put("enable.auto.commit", kafkaConsumerEnableAutoCommit); //设置多久一次更新被消费消息的偏移量 props.put("auto.commit.interval.ms", kafkaConsumerAutoCommitIntervalMs); //设置会话响应的时间，超过这个时间kafka可以选择放弃消费或者消费下一条消息 props.put("session.timeout.ms", kafkaConsumerSessionTimeoutMs); //自动重置offset props.put("auto.offset.reset",kafkaConsumerAutoOffsetReset); //设置每次批量拉取消息数量 props.put("max.poll.records",kafkaConsumerMaxPollRecords); props.put("key.deserializer", kafkaConsumerKeyDeserializer); props.put("value.deserializer", kafkaConsumerValueDeserializer); return new KafkaConsumer<String, String>(props); } /** * 私有静态方法，创建Kafka集群管理员对象 * @author o */ public void createAdmin(String servers){ Properties props = new Properties(); props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG,servers); adminClient = AdminClient.create(props); } /** * 私有静态方法，创建Kafka集群管理员对象 * @author o * @return AdminClient */ private void createAdmin(){ createAdmin(kafkaBootstrapServers); } /** * 传入kafka约定的topic,json格式字符串，发送给kafka集群 * @author o * @param topic * @param jsonMessage */ public void sendMessage(String topic, String jsonMessage) { KafkaProducer<String, String> producer = createProducer(); producer.send(new ProducerRecord<String, String>(topic, jsonMessage), new Callback() { @Override public void onCompletion(RecordMetadata recordMetadata, Exception e) { if(e == null){ log.info("消息发送成功！"); }else { log.info("消息发送失败！"); } } }); producer.close(); } /** * 获取kafka消费者对象 * @author o * @param topic */ public KafkaConsumer<String, String> getConsumer(String topic){ KafkaConsumer<String, String> consumer = createConsumer(); consumer.subscribe(Arrays.asList(topic)); return consumer; } /** * 传入kafka约定的topic消费数据，用于测试，数据最终会输出到控制台上 * @author o * @param topic */ public List<String> consume(String topic) { KafkaConsumer<String, String> consumer = createConsumer(); consumer.subscribe(Arrays.asList(topic)); //while (true) { // ConsumerRecords<String, String> records = consumer.poll(Duration.ofSeconds(100)); // for (ConsumerRecord<String, String> record : records){ // System.out.printf("offset = %d, key = %s, value = %s",record.offset(), record.key(), record.value()); // //System.out.println(); // } //} ConsumerRecords<String, String> records = consumer.poll(Duration.ofSeconds(3)); List<String> msgList = new ArrayList<>(); for (ConsumerRecord<String, String> record : records){ log.info("offset(偏移量) ={}, key(键) = {}, value(值) ={}",record.offset(),record.key(),record.value()); msgList.add(record.value()); } // 同步提交 // kafkaConsumer.commitSync(); // 异步提交 //consumer.commitAsync(); consumer.close(); return msgList; } /** * 传入kafka约定的topic数组，消费数据 * @author o * @param topics */ public void consume(String ... topics) { KafkaConsumer<String, String> consumer = createConsumer(); consumer.subscribe(Arrays.asList(topics)); while (true) { ConsumerRecords<String, String> records = consumer.poll(Duration.ofSeconds(100)); for (ConsumerRecord<String, String> record : records){ System.out.printf("offset = %d, key = %s, value = %s",record.offset(), record.key(), record.value()); System.out.println(); } } } /** * 传入kafka约定的topic,json格式字符串数组，发送给kafka集群 * 用于批量发送消息，性能较高。 * @author o * @param topic * @param jsonMessages * @throws InterruptedException */ public void sendMessage(String topic, String... jsonMessages) throws InterruptedException { KafkaProducer<String, String> producer = createProducer(); for (String jsonMessage : jsonMessages) { producer.send(new ProducerRecord<String, String>(topic, jsonMessage)); } producer.close(); } /** * 传入kafka约定的topic,Map集合，内部转为json发送给kafka集群 <br> * 用于批量发送消息，性能较高。 * @author o * @param topic * @param mapMessageToJSONForArray */ public void sendMessage(String topic, List<Map<Object, Object>> mapMessageToJSONForArray) { KafkaProducer<String, String> producer = createProducer(); for (Map<Object, Object> mapMessageToJSON : mapMessageToJSONForArray) { String array = JSONObject.toJSON(mapMessageToJSON).toString(); producer.send(new ProducerRecord<String, String>(topic, array)); } producer.close(); } /** * 传入kafka约定的topic,Map，内部转为json发送给kafka集群 * @author o * @param topic * @param mapMessageToJSON */ public void sendMessage(String topic, Map<Object, Object> mapMessageToJSON) { KafkaProducer<String, String> producer = createProducer(); String array = JSONObject.toJSON(mapMessageToJSON).toString(); producer.send(new ProducerRecord<String, String>(topic, array)); producer.close(); } /** * 创建主题 * @author o * @param name 主题的名称 * @param numPartitions 主题的分区数 * @param replicationFactor 主题的每个分区的副本因子 */ public void createTopic(String name,int numPartitions,int replicationFactor){ if(adminClient == null) { createAdmin(); } Map<String, String> configs = new HashMap<>(); CreateTopicsResult result = adminClient.createTopics(Arrays.asList(new NewTopic(name, numPartitions, (short) replicationFactor).configs(configs))); //以下内容用于判断创建主题的结果 for (Map.Entry<String, KafkaFuture<Void>> entry : result.values().entrySet()) { try { entry.getValue().get(); System.out.println("主题: "+entry.getKey()+" 创建成功"); } catch (InterruptedException | ExecutionException e) { if (ExceptionUtils.getRootCause(e) instanceof TopicExistsException) { System.out.println("主题: "+entry.getKey()+" 已存在"); } } } } /** * 删除主题 * @author o * @param name 主题的名称 */ public void deleteTopic(String name){ if(adminClient == null) { createAdmin(); } Map<String, String> configs = new HashMap<>(); Collection<String> topics = new ArrayList<>(); topics.add(name); DeleteTopicsResult result = adminClient.deleteTopics(topics); //以下内容用于判断删除主题的结果 for (Map.Entry<String, KafkaFuture<Void>> entry : result.values().entrySet()) { try { entry.getValue().get(); System.out.println("主题: "+entry.getKey()+" 已删除"); } catch (InterruptedException | ExecutionException e) { if (ExceptionUtils.getRootCause(e) instanceof UnknownTopicOrPartitionException) { System.out.println("主题: "+entry.getKey()+" 不存在"); } } } } /** * 查看主题详情 * @author o * @param name 主题的名称 */ public void describeTopic(String name){ if(adminClient == null) { createAdmin(); } Map<String, String> configs = new HashMap<>(); Collection<String> topics = new ArrayList<>(); topics.add(name); DescribeTopicsResult result = adminClient.describeTopics(topics); //以下内容用于显示主题详情的结果 for (Map.Entry<String, KafkaFuture<TopicDescription>> entry : result.values().entrySet()) { try { entry.getValue().get(); System.out.println("topic "+entry.getKey()+" describe"); System.out.println("\t name: "+entry.getValue().get().name()); System.out.println("\t partitions: "); entry.getValue().get().partitions().stream().forEach(p-> { System.out.println("\t\t index: "+p.partition()); System.out.println("\t\t\t leader: "+p.leader()); System.out.println("\t\t\t replicas: "+p.replicas()); System.out.println("\t\t\t isr: "+p.isr()); }); System.out.println("\t internal: "+entry.getValue().get().isInternal()); } catch (InterruptedException | ExecutionException e) { if (ExceptionUtils.getRootCause(e) instanceof UnknownTopicOrPartitionException) { System.out.println("topic "+entry.getKey()+" not exist"); } } } } /** * 查看主题列表 * @author o * @return Set<String> TopicList */ public Set<String> listTopic(){ if(adminClient == null) { createAdmin(); } ListTopicsResult result = adminClient.listTopics(); try { //result.names().get().stream().map(x->x+"\t").forEach(System.out::print); return result.names().get(); } catch (InterruptedException | ExecutionException e) { e.printStackTrace(); return null; } } }
4、自定义监听类KafkaConsumerAutoRun.java
import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.ApplicationArguments; import org.springframework.boot.ApplicationRunner; import org.springframework.stereotype.Component; import java.time.Duration; /** * @Description: kafka自定义监听处理 * @Date: 2023/6/15 14:43 * @Version: 1.0 */ @Component @Slf4j public class KafkaConsumerAutoRun implements ApplicationRunner,DisposableEnabelBean { @Autowired KafkaUtils kafkaUtils; private String topic = "q-10.1.52.30"; private int lock; /** * @description: 消息处理 * @date: 2023/6/15 15:37 * @param * @return boolean */ private boolean deal(){ log.info("消息监听启动"); try { KafkaConsumer<String, String> consumer = kafkaUtils.getConsumer(topic); ConsumerRecords<String, String> records = consumer.poll(Duration.ofSeconds(3)); if(records.isEmpty()){ lock = 1; }else{ for (ConsumerRecord<String, String> record : records) { String value = record.value(); log.info("收到消息：{}",value); //此处处理你的业务逻辑 //此处处理你的业务逻辑 } //异步提交 consumer.commitAsync(); lock = 0; } consumer.close(); } catch (Exception e) { log.info("消息处理deal()失败：{}",e); } return true; } @Override public void start() throws Exception { while (deal()) { if (lock == 1) { log.info("没有消息休息一下"); Thread.sleep(10000L); } } } @Override public void destroy() throws Exception {} @Override public void run(ApplicationArguments args) throws Exception { start(); } }
5、重写监听接口DisposableEnabelBean.java
import org.springframework.beans.factory.DisposableBean; public interface DisposableEnabelBean extends DisposableBean { void start() throws Exception; }
