[Kafka] - Kafka Java Producer代码实现

根据业务需要可以使用Kafka提供的Java Producer API进行产生数据，并将产生的数据发送到Kafka对应Topic的对应分区中，入口类为：Producer

Kafka的Producer API主要提供下列三个方法：

　　public void send(KeyedMessage<K,V> message) 发送单条数据到Kafka集群

　　public void send(List<KeyedMessage<K,V>> messages) 发送多条数据(数据集)到Kafka集群

　　public void close() 关闭Kafka连接资源

使用Java语言实现Kafka的Consumer详见博客: Java 实现 High Level Consumer API 以及 Java实现LowerLevelConsumerAPI

======================================================================

一、JavaKafkaProducerPartitioner：自定义的数据分区器，功能是：决定输入的key/value键值对的message发送到Topic的那个分区中，返回分区id，范围:[0,分区数量); 这里的实现比较简单，根据key中的数字决定分区的值。具体代码如下：

import kafka.producer.Partitioner;
import kafka.utils.VerifiableProperties;

/**
 * Created by gerry on 12/21.
 */
public class JavaKafkaProducerPartitioner implements Partitioner {

    /**
     * 无参构造函数
     */
    public JavaKafkaProducerPartitioner() {
        this(new VerifiableProperties());
    }

    /**
     * 构造函数，必须给定
     *
     * @param properties 上下文
     */
    public JavaKafkaProducerPartitioner(VerifiableProperties properties) {
        // nothings
    }

    @Override
    public int partition(Object key, int numPartitions) {
        int num = Integer.valueOf(((String) key).replaceAll("key_", "").trim());
        return num % numPartitions;
    }
}

 

二、 JavaKafkaProducer：通过Kafka提供的API进行数据产生操作的测试类；具体代码如下：

import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;
import org.apache.log4j.Logger;

import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.ThreadLocalRandom;

/**
 * Created by gerry on 12/21.
 */
public class JavaKafkaProducer {
    private Logger logger = Logger.getLogger(JavaKafkaProducer.class);
    public static final String TOPIC_NAME = "test";
    public static final char[] charts = "qazwsxedcrfvtgbyhnujmikolp1234567890".toCharArray();
    public static final int chartsLength = charts.length;


    public static void main(String[] args) {
        String brokerList = "192.168.187.149:9092";
        brokerList = "192.168.187.149:9092,192.168.187.149:9093,192.168.187.149:9094,192.168.187.149:9095";
        brokerList = "192.168.187.146:9092";
        Properties props = new Properties();
        props.put("metadata.broker.list", brokerList);
        /**
         * 0表示不等待结果返回<br/>
         * 1表示等待至少有一个服务器返回数据接收标识<br/>
         * -1表示必须接收到所有的服务器返回标识，及同步写入<br/>
         * */
        props.put("request.required.acks", "0");
        /**
         * 内部发送数据是异步还是同步
         * sync：同步, 默认
         * async：异步
         */
        props.put("producer.type", "async");
        /**
         * 设置序列化的类
         * 可选：kafka.serializer.StringEncoder
         * 默认：kafka.serializer.DefaultEncoder
         */
        props.put("serializer.class", "kafka.serializer.StringEncoder");
        /**
         * 设置分区类
         * 根据key进行数据分区
         * 默认是：kafka.producer.DefaultPartitioner ==> 安装key的hash进行分区
         * 可选:kafka.serializer.ByteArrayPartitioner ==> 转换为字节数组后进行hash分区
         */
        props.put("partitioner.class", "JavaKafkaProducerPartitioner");

        // 重试次数
        props.put("message.send.max.retries", "3");

        // 异步提交的时候(async)，并发提交的记录数
        props.put("batch.num.messages", "200");

        // 设置缓冲区大小，默认10KB
        props.put("send.buffer.bytes", "102400");

        // 2. 构建Kafka Producer Configuration上下文
        ProducerConfig config = new ProducerConfig(props);

        // 3. 构建Producer对象
        final Producer<String, String> producer = new Producer<String, String>(config);

        // 4. 发送数据到服务器，并发线程发送
        final AtomicBoolean flag = new AtomicBoolean(true);
        int numThreads = 50;
        ExecutorService pool = Executors.newFixedThreadPool(numThreads);
        for (int i = 0; i < 5; i++) {
            pool.submit(new Thread(new Runnable() {
                @Override
                public void run() {
                    while (flag.get()) {
                        // 发送数据
                        KeyedMessage message = generateKeyedMessage();
                        producer.send(message);
                        System.out.println("发送数据:" + message);

                        // 休眠一下
                        try {
                            int least = 10;
                            int bound = 100;
                            Thread.sleep(ThreadLocalRandom.current().nextInt(least, bound));
                        } catch (InterruptedException e) {
                            e.printStackTrace();
                        }
                    }

                    System.out.println(Thread.currentThread().getName() + " shutdown....");
                }
            }, "Thread-" + i));

        }

        // 5. 等待执行完成
        long sleepMillis = 600000;
        try {
            Thread.sleep(sleepMillis);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        flag.set(false);

        // 6. 关闭资源

        pool.shutdown();
        try {
            pool.awaitTermination(6, TimeUnit.SECONDS);
        } catch (InterruptedException e) {
        } finally {
            producer.close(); // 最后之后调用
        }
    }

    /**
     * 产生一个消息
     *
     * @return
     */
    private static KeyedMessage<String, String> generateKeyedMessage() {
        String key = "key_" + ThreadLocalRandom.current().nextInt(10, 99);
        StringBuilder sb = new StringBuilder();
        int num = ThreadLocalRandom.current().nextInt(1, 5);
        for (int i = 0; i < num; i++) {
            sb.append(generateStringMessage(ThreadLocalRandom.current().nextInt(3, 20))).append(" ");
        }
        String message = sb.toString().trim();
        return new KeyedMessage(TOPIC_NAME, key, message);
    }

    /**
     * 产生一个给定长度的字符串
     *
     * @param numItems
     * @return
     */
    private static String generateStringMessage(int numItems) {
        StringBuilder sb = new StringBuilder();
        for (int i = 0; i < numItems; i++) {
            sb.append(charts[ThreadLocalRandom.current().nextInt(chartsLength)]);
        }
        return sb.toString();
    }
}

 

三、Pom.xml依赖配置如下

<properties>
    <kafka.version>0.8.2.1</kafka.version>
</properties>

<dependencies>
    <dependency>
        <groupId>org.apache.kafka</groupId>
        <artifactId>kafka_2.10</artifactId>
        <version>${kafka.version}</version>
    </dependency>
</dependencies>