kafka入门及使用（不用集成spring）
1、引入坐标
<dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka_2.12</artifactId> <version>2.0.0</version> </dependency>
2、生产者
package com.search.kafka; import java.util.ArrayList; import java.util.List; import java.util.Properties; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; import com.lvmama.searchclient.bean.HotelTimePriceBean; /** * Kafka生产者 * 先启动生产者，发送消息到broker，这里简单发送了10条从0-9的消息，再启动消费者，控制台输出如下： */ public class SimpleKafkaProducer { public static void main(String[] args) { // TODO Auto-generated method stub Properties props = new Properties(); //broker地址 props.put("bootstrap.servers", "localhost:9092"); // "localhost:9092" //请求时候需要验证 props.put("acks", "all"); //请求失败时候需要重试 props.put("retries", 0); //内存缓存区大小 props.put("buffer.memory", 33554432); //指定消息key序列化方式 props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); //指定消息本身的序列化方式 props.put("value.serializer", "com.search.kafka.DataSerializer"); Producer<String, List<HotelTimePriceBean>> producer = new KafkaProducer<>(props); List<HotelTimePriceBean> list = new ArrayList<HotelTimePriceBean>(); for (int i = 0; i < 10; i++) { //i < 10 // 生产一条消息的时间有点长 HotelTimePriceBean bean = new HotelTimePriceBean(); bean.setProductId(Long.valueOf(i)); list.add(bean); } producer.send(new ProducerRecord<>("HOTEL_PRICE_HOTSEARCH_TOPIC", list)); System.out.println("Message sent successfully"); producer.close(); } }
3、消费者package com.search.kafka;
package com.vst.search.kafka; import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Properties; import java.util.Set; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.time.DateFormatUtils; import org.apache.commons.lang3.time.DateUtils; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.searchclient.bean.HotelTimePriceBean; import com.vst.search.beans.RealTimePriceBean; import com.vst.search.common.mapping.PropertiesDataMapping; import com.vst.search.common.util.CommonSearchUtils; import com.vst.search.common.util.RealTimePriceUtils; public enum SimpleKafkaConsumer implements Runnable { INSTANCE; private static final Logger logger = LoggerFactory.getLogger(SimpleKafkaConsumer.class); public static String getJdbcValue(String key) { return PropertiesDataMapping.INSTANCE.getPropsMap("kafka.properties").get(key); } private static final String BOOTSTRAP_SERVERS = getJdbcValue("kafka.bootstrap.servers"); private static final String TOPIC = getJdbcValue("kafka.hotsearch.topic"); private static final String GROUP_ID = getJdbcValue("kafka.hotsearch.group.id"); @Override public void run() { Properties props = new Properties(); props.put("bootstrap.servers", BOOTSTRAP_SERVERS); // "localhost:9092" // 每个消费者分配独立的组号 props.put("group.id", GROUP_ID); // 如果value合法，则自动提交偏移量 props.put("enable.auto.commit", "true"); // 设置多久一次更新被消费消息的偏移量 props.put("auto.commit.interval.ms", "1000"); // 设置会话响应的时间，超过这个时间kafka可以选择放弃消费或者消费下一条消息 props.put("session.timeout.ms", "30000"); // props.put("auto.offset.reset", "earliest"); props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); props.put("value.deserializer", "com.vst.search.kafka.DataDeserializer"); KafkaConsumer<String, List<HotelTimePriceBean>> consumer = new KafkaConsumer<>(props); consumer.subscribe(Collections.singletonList(TOPIC)); // 核心函数1：订阅topic while (true) { if (!"true".equals(CommonSearchUtils.getConstValue("kafka.enable"))) { logger.info("kafka is disable..."); try { Thread.sleep(1000); } catch (InterruptedException e) { logger.error("kafka thread is interrupted..."); } } else { // 核心函数2：long poll,一次拉取回来多个消息,读取数据，读取超时时间为100ms ConsumerRecords<String, List<HotelTimePriceBean>> records = consumer.poll(Duration.ofMillis(100)); for (ConsumerRecord<String, List<HotelTimePriceBean>> record : records) { List<HotelTimePriceBean> value = record.value(); if (CollectionUtils.isNotEmpty(value)) { logger.info("HotSearchConsumer:{}", value); List<RealTimePriceBean> realTimePriceBeans = buildRealTimePriceBeans(value);// 转化 RealTimePriceUtils.buildAndInsertBatch(realTimePriceBeans); // 批量插入 } } } } } }
4、对象序列化
package com.search.kafka; import java.util.List; import java.util.Map; import org.apache.kafka.common.serialization.Serializer; import com.alibaba.fastjson.JSON; import com.searchclient.bean.HotelTimePriceBean; public class DataSerializer implements Serializer<List<HotelTimePriceBean>> { @Override public void configure(Map<String, ?> configs, boolean isKey) {} @Override public byte[] serialize(String topic, List<HotelTimePriceBean> data) { return JSON.toJSONBytes(data); } @Override public void close() {} }
5、对象反序列化
package com.search.kafka; import java.util.List; import java.util.Map; import org.apache.kafka.common.serialization.Deserializer; import com.alibaba.fastjson.JSONArray; import com.searchclient.bean.HotelTimePriceBean; public class DataDeserializer implements Deserializer<List<HotelTimePriceBean>> { @Override public void configure(Map<String, ?> configs, boolean isKey) {} @Override public List<HotelTimePriceBean> deserialize(String topic, byte[] data) { return JSONArray.parseArray(new String(data), HotelTimePriceBean.class); } @Override public void close() {} }
6、启动
package com.search.web.listener; import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.web.context.ContextLoaderListener; import com.search.common.mapping.PropertiesDataMapping; import com.search.elasticsearch.processer.StoreCommissionsProcessor; import com.search.kafka.SimpleKafkaConsumer; public class SearchDataLoader extends ContextLoaderListener implements ServletContextListener { private static final Logger logger = LoggerFactory.getLogger(SearchDataLoader.class); @Override public void contextInitialized(ServletContextEvent event) { // Kafka消费启动监听线程 Thread simpleKafkaConsumerThread = new Thread(SimpleKafkaConsumer.INSTANCE); simpleKafkaConsumerThread.setDaemon(true); simpleKafkaConsumerThread.start(); } }
7、配置监听器
<listener> <description>Used to initialize Search Data.</description> <listener-class>com.search.web.listener.SearchDataLoader</listener-class> </listener>
注：web项目下依赖监听器启动线程
