Kafka学习笔记-Java简单操作

转自：http://www.cnblogs.com/edison2012/p/5759223.html

 

Maven依赖包：

 

<dependency>
        <groupId>org.apache.kafka</groupId>
        <artifactId>kafka-clients</artifactId>
        <version>0.8.2.1</version>
</dependency>

<dependency>
    <groupId>org.apache.kafka</groupId>
    <artifactId>kafka_2.11</artifactId>
    <version>0.8.2.1</version>
</dependency>

代码如下：

 

 

import java.util.Properties;

import org.apache.kafka.clients.producer.Callback;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class KafkaProducerTest {

    private static final Logger LOG = LoggerFactory.getLogger(KafkaProducerTest.class);

    private static Properties properties = null;

    static {
        properties = new Properties();
        properties.put("bootstrap.servers", "centos.master:9092,centos.slave1:9092,centos.slave2:9092");
        properties.put("producer.type", "sync");
        properties.put("request.required.acks", "1");
        properties.put("serializer.class", "kafka.serializer.DefaultEncoder");
        properties.put("partitioner.class", "kafka.producer.DefaultPartitioner");
        properties.put("key.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");
//      properties.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        properties.put("value.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");
//      properties.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    }

    public void produce() {
        KafkaProducer<byte[], byte[]> kafkaProducer = new KafkaProducer<byte[],byte[]>(properties);
        ProducerRecord<byte[],byte[]> kafkaRecord = new ProducerRecord<byte[],byte[]>(
                "test", "kkk".getBytes(), "vvv".getBytes());
        kafkaProducer.send(kafkaRecord, new Callback() {
            public void onCompletion(RecordMetadata metadata, Exception e) {
                if(null != e) {
                    LOG.info("the offset of the send record is {}", metadata.offset());
                    LOG.error(e.getMessage(), e);
                }
                LOG.info("complete!");
            }
        });
        kafkaProducer.close();
    }

    public static void main(String[] args) {
        KafkaProducerTest kafkaProducerTest = new KafkaProducerTest();
        for (int i = 0; i < 10; i++) {
            kafkaProducerTest.produce();
        }
    }
}

 

import java.util.List;
import java.util.Map;
import java.util.Properties;

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class KafkaConsumerTest {

    private static final Logger LOG = LoggerFactory.getLogger(KafkaConsumerTest.class);

    public static void main(String[] args) {
        Properties properties = new Properties();
        properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,
                "centos.master:9092,centos.slave1:9092,centos.slave2:9092");
        properties.put(ConsumerConfig.GROUP_ID_CONFIG, "test-consumer-group");
        properties.put(ConsumerConfig.SESSION_TIMEOUT_MS, "1000");
        properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
        properties.put(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY, "range");
//      properties.put(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY, "roundrobin");
        properties.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "10000");
        properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,
                "org.apache.kafka.common.serialization.ByteArrayDeserializer");
        properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
                "org.apache.kafka.common.serialization.ByteArrayDeserializer");

        KafkaConsumer<byte[], byte[]> kafkaConsumer = new KafkaConsumer<byte[], byte[]>(properties);
        kafkaConsumer.subscribe("test");
//      kafkaConsumer.subscribe("*");
        boolean isRunning = true;
        while(isRunning) {
            Map<String, ConsumerRecords<byte[], byte[]>> results = kafkaConsumer.poll(100);
            if (null != results) {
                for (Map.Entry<String, ConsumerRecords<byte[], byte[]>> entry : results.entrySet()) {
                    LOG.info("topic {}", entry.getKey());
                    ConsumerRecords<byte[], byte[]> consumerRecords = entry.getValue();
                    List<ConsumerRecord<byte[], byte[]>> records = consumerRecords.records();
                    for (int i = 0, len = records.size(); i < len; i++) {
                        ConsumerRecord<byte[], byte[]> consumerRecord = records.get(i);
                        LOG.info("topic {} partition {}", consumerRecord.topic(), consumerRecord.partition());
                        try {
                            LOG.info("offset {} value {}", consumerRecord.offset(), new String(consumerRecord.value()));
                        } catch (Exception e) {
                            LOG.error(e.getMessage(), e);
                        }
                    }
                }
            }
        }

        kafkaConsumer.close();

    }

}

 

发现KafkaConsumer的poll方法未实现

 

@Override
public Map<String, ConsumerRecords<K,V>> poll(long timeout) {
     // TODO Auto-generated method stub
     return null;
}

后改为kafka.javaapi.consumer.SimpleConsumer实现，正常运行