Kafka之常用API代码示例
其实Kafka作为一款消息队列,本身的api并不多,也不复杂,本文列举出来生产者和消费者常见的api
public class KafkaTestProduer { public static final Logger logger = LoggerFactory.getLogger(KafkaTestProduer.class); public final static String kafkaServer = "10.37.136.172:9092"; public final static String topic = "mxz.test.202010311544"; public static Properties initConfig() { Properties props = new Properties(); props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaServer); props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); props.put(ProducerConfig.CLIENT_ID_CONFIG, "mxz.client"); props.put(ProducerConfig.RETRIES_CONFIG, 3); return props; } public static Properties initConsumerConfig() { Properties props = new Properties(); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaServer); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); props.put(ConsumerConfig.CLIENT_ID_CONFIG, "mxz.client"); props.put(ConsumerConfig.GROUP_ID_CONFIG, "202101051648"); props.put("auto.offset.reset", "earliest"); props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); props.put(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, ConsumerinterceptorTTL.class.getName()); return props; } // public static void main(String[] args) { // // Properties props = initConfig(); // KafkaProducer<String, String> producer = new KafkaProducer<>(props); // // ProducerRecord<String, String> record = new ProducerRecord<>(topic, "batman"); // ProducerRecord<String, String> record2 = new ProducerRecord<>(topic, "superman"); // ProducerRecord<String, String> record3 = new ProducerRecord<>(topic, "aquaman"); // ProducerRecord<String, String> record4 = new ProducerRecord<>(topic, "flash"); // ProducerRecord<String, String> record5 = new ProducerRecord<>(topic, "wonderwoman"); // List<ProducerRecord<String, String>> list = new ArrayList<>(); // list.add(record); // list.add(record2); // list.add(record3); // list.add(record4); // list.add(record5); // try { // for (ProducerRecord<String, String> recordItem : list) { // producer.send(recordItem, new CallbackImpl()); // } // // }finally { // producer.close(); // } // // } public static class CallbackImpl implements Callback { @Override public void onCompletion(RecordMetadata metadata, Exception exception) { if (exception != null) { } else { System.out.print(metadata.topic() + "-" + metadata.partition() + "-" + metadata.offset()); } } } public static void main(String[] args) throws InterruptedException { logger.info("kafka consumer main begins {} "); Properties props = initConsumerConfig(); KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props); consumer.subscribe(Collections.singletonList(topic), new ConsumerRebalanceListener() { @Override public void onPartitionsRevoked(Collection<TopicPartition> partitions) { } @Override public void onPartitionsAssigned(Collection<TopicPartition> partitions) { } }); // ConsumerRecords<String, String> records = consumer.poll(1000); // TopicPartition tp = new TopicPartition(topic, 0); // List<ConsumerRecord<String, String>> list = records.records(tp); // if (list.isEmpty()) { // return; // } // long offset = list.get(0).offset(); Set<TopicPartition> assignment = consumer.assignment(); while (assignment.size() == 0) { consumer.poll(100L); assignment = consumer.assignment(); } for (TopicPartition tpItem : assignment) { consumer.seek(tpItem, 19); } // Set<TopicPartition> assignment = new HashSet<>(); // while (assignment.size() == 0) { // consumer.poll(100L); // assignment = consumer.assignment(); // } // Map<TopicPartition, Long> timestampToSearch = new HashMap<>(); // for (TopicPartition tp : assignment) { // timestampToSearch.put(tp, System.currentTimeMillis() - 1000 * 1000); // } // // Map<TopicPartition, OffsetAndTimestamp> offsets = consumer.offsetsForTimes(timestampToSearch); // for (TopicPartition tp : assignment) { // OffsetAndTimestamp offsetAndTimestamp = offsets.get(tp); // if (offsetAndTimestamp != null) { // consumer.seek(tp, offsetAndTimestamp.offset()); //// consumer.poll(1000); // } // } // for (TopicPartition tp : assignment) { // consumer.seek(tp,10); // } ConsumerRecords<String, String> records = consumer.poll(1000); // // for (TopicPartition tp : records.partitions()) { // List<ConsumerRecord<String, String>> list = records.records(tp); // long lastConsumedOffset = list.get(list.size() - 1).offset() ; // System.out.println("lastConsumedOffset: " + lastConsumedOffset); // } // for (ConsumerRecord<String, String> record : records) { System.out.println(record.value()); } consumer.commitAsync(new OffsetCommitCallback() { @Override public void onComplete(Map<TopicPartition, OffsetAndMetadata> offsets, Exception exception) { if (exception == null) { logger.info("offset {}", offsets); logger.info("time {} ", System.currentTimeMillis()); System.out.println("TIME IS END"); } else { System.out.println(exception); logger.info("offset {}", offsets); logger.info("time {} ", System.currentTimeMillis()); } } }); Thread.sleep(30000); consumer.close(); } }
再加一个生产者指定partition的写法
public ProducerRecord(String topic, Integer partition, Long timestamp, K key, V value, Iterable<Header> headers) { if (topic == null) throw new IllegalArgumentException("Topic cannot be null."); if (timestamp != null && timestamp < 0) throw new IllegalArgumentException( String.format("Invalid timestamp: %d. Timestamp should always be non-negative or null.", timestamp)); if (partition != null && partition < 0) throw new IllegalArgumentException( String.format("Invalid partition: %d. Partition number should always be non-negative or null.", partition)); this.topic = topic; this.partition = partition; this.key = key; this.value = value; this.timestamp = timestamp; this.headers = new RecordHeaders(headers); }