springboot2.0 快速集成kafka
一、kafka搭建
二、版本
springboot版本
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.0.5.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
kafka版本
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
<version>2.1.2.RELEASE</version>
</dependency>
三、基本的配置application.yml
实际上只有bootstrap-servers是必须配置的。
kafka: producer: key-serializer: org.apache.kafka.common.serialization.StringSerializer value-serializer: org.apache.kafka.common.serialization.StringSerializer bootstrap-servers: 192.168.31.10:9092,192.168.31.10:9093,192.168.31.10:9094 bootstrap-servers: 192.168.31.10:9092,192.168.31.10:9093,192.168.31.10:9094 topic: Name: home.bus.log #自定义topic名称 numPartitions: 2 #自定义分区 replicationFactor: 2 #自定义副本 consumer: group-id: home.bus.log.group.1 auto-offset-reset: latest enable-auto-commit: true auto-commit-interval: 20000 key-deserializer: org.apache.kafka.common.serialization.StringDeserializer value-deserializer: org.apache.kafka.common.serialization.StringDeserializer
四、自定义topic
如果不配置topic属性，系统会用缺省的，当然名字得需要配置，也可以在生产者中直接使用常量
@Configuration @EnableKafka public class KafkaTopicConfig { @Value("${spring.kafka.bootstrap-servers}") private String bootstrapServer; @Value("${spring.kafka.topic.Name}") private String topicName; @Value("${spring.kafka.topic.numPartitions}") private int numPartitions; @Value("${spring.kafka.topic.replicationFactor}") private int replicationFactor; @Bean public KafkaAdmin kafkaAdmin() { Map<String, Object> configs = new HashMap<>(); configs.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServer); return new KafkaAdmin(configs); } @Bean public NewTopic myTopic() { //第三个参数是副本数量，确保集群中配置的数目大于等于副本数量 return new NewTopic(topicName, numPartitions, (short) replicationFactor); } }
通过配置，如果kafka服务器上没有创建topic，则会按照自定义属性来创建，如果topic名称已经创建了，那么NewTopic将不会创建新的topic，无论topic其他分区和副本属性是否相同
五、自定义的producer和consumer
简单的使用，不需要自定义，yml文件也进行了基本的配置，如果需要自定义在参照如下额配置：
//@Configuration //@EnableKafka public class KafkaProducerConfig { @Value("${spring.kafka.producer.bootstrap-servers}") private String bootstrapServer; /* --------------producer configuration-----------------**/ @Bean public Map<String, Object> producerConfigs() { Map<String, Object> props = new HashMap<>(); props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServer); props.put(ProducerConfig.RETRIES_CONFIG, 0); props.put(ProducerConfig.BATCH_SIZE_CONFIG, 16384); props.put(ProducerConfig.LINGER_MS_CONFIG, 1); props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, 33554432); props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class); return props; } @Bean public ProducerFactory<String, String> producerFactory() { return new DefaultKafkaProducerFactory<>(producerConfigs()); } /* --------------kafka template configuration-----------------**/ @Bean public KafkaTemplate<String, String> kafkaTemplate() { KafkaTemplate<String, String> kafkaTemplate = new KafkaTemplate<>(producerFactory()); return kafkaTemplate; } }
//@Configuration //@EnableKafka public class KafkaConsumerConfig { @Value("${spring.kafka.bootstrap-servers}") private String bootstrapServer; @Bean public Map<String, Object> consumerConfigs() { Map<String, Object> props = new HashMap<>(); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServer); props.put(ConsumerConfig.GROUP_ID_CONFIG, "0"); props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true); props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, 100); props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "15000"); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); return props; } @Bean ConcurrentKafkaListenerContainerFactory<String, String> kafkaListenerContainerFactory() { ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>(); factory.setConsumerFactory(consumerFactory()); return factory; } @Bean public ConsumerFactory<String, String> consumerFactory() { return new DefaultKafkaConsumerFactory<>(consumerConfigs()); } }
六、创建生产者producer
根据需要直接调用就可以。
@Service public class KafkaServiceImpl implements MQService { @Value("${spring.mqconfig.mq-enable}") private boolean mqEnable; @Value("${spring.kafka.topic.Name}") private String topicName; private Logger logger = LoggerFactory.getLogger(KafkaServiceImpl.class); @Resource private KafkaTemplate<String, String> kafkaTemplate; public boolean isMqEnable() { return mqEnable; } public void setMqEnable(boolean mqEnable) { this.mqEnable = mqEnable; } @Override @Async("logThread") //异步，可以注销，除非网络出问题，否则发送消息到kafka服务器非常的快 public void sendMessage(String msg) { if(!isMqEnable()) return; long start = System.currentTimeMillis(); kafkaTemplate.send(topicName,msg); long end = System.currentTimeMillis(); logger.info("写入kafka，耗时："+(end-start)+"毫秒"); } }
七、创建消费者consumer
因为生产消息的时候进行了json封装，获取消息的时候对应进行反序列化
@Service public class MQConsumerServiceImpl implements MQConsumerService { private ObjectMapper objectMapper; private List<MQMessage> mqMessageList; private long maxMessageCount=100; @Override public List<MQMessage> getMessage() { return mqMessageList; } @KafkaListener(topics = "${spring.kafka.topic.Name}") private void consumer(ConsumerRecord<?, ?> record) { if(objectMapper==null) objectMapper = new ObjectMapper(); if(mqMessageList==null) mqMessageList = new ArrayList<>(); Optional<?> mqMessage = Optional.ofNullable(record.value()); if (mqMessage.isPresent()) { Object message = mqMessage.get(); try { if(mqMessageList.size()>maxMessageCount) { mqMessageList.remove(0); } MQMessage mq = objectMapper.readValue((String)message, MQMessage.class);//反序列化 mqMessageList.add(mq); }catch (Exception e) { e.printStackTrace(); } } } }
八、显示到页面上
大部分场景，我们并不需要把消息取出来显示，这里为了顺便测试一下消费者，也可以在kafka服务器上用命令查看
bin/kafka-console-consumer.sh --bootstrap-server 192.168.31.10:9092 --topic home.bus.log --from-beginning
@Controller @RequestMapping(value = "/mq") public class mqController { @Resource(name = "MQConsumerServiceImpl") private MQConsumerService mqConsumerService; @RequestMapping(value = "/list") public String list() { return "/mq/mqList"; } @RequestMapping(value = "/getMsgList") @ResponseBody public Object getMsgList(HttpServletRequest request) { int pageSize = 50; try { pageSize = Integer.parseInt(request.getParameter("pageSize")); } catch (Exception e) { e.printStackTrace(); } int pageNumber = 0; try { pageNumber = Integer.parseInt(request.getParameter("pageNumber")) - 1; } catch (Exception e) { e.printStackTrace(); } Map<String, Object> map = new HashMap<>(); String sortName = request.getParameter("sortName") == null ? "roleId" : request.getParameter("sortName"); String sortOrder = request.getParameter("sortOrder") == null ? "asc" : request.getParameter("sortOrder"); Sort sortLocal = new Sort(sortOrder.equalsIgnoreCase("asc") ? Sort.Direction.ASC : Sort.Direction.DESC, sortName); Pageable pageable = PageRequest.of(pageNumber, pageSize, sortLocal); Page<MQMessage> mqMessagePage = new PageImpl<MQMessage>(mqConsumerService.getMessage(),pageable,this.mqConsumerService.getMessage().size()); map.put("total", mqMessagePage.getTotalElements()); map.put("rows", mqMessagePage.getContent()); return map; } }
