Kafka Streams Example
kakfa官网上的几个例子:
环境: kafka集群, eclipse(win10)
pom.xml
<dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka_2.12</artifactId> <version>1.0.0</version> </dependency> <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka-streams</artifactId> <version>1.0.0</version> </dependency> <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka-clients</artifactId> <version>1.0.0</version> </dependency>
一. Pipe
这个例子的功能就是从streams-plaintext-input输入文本, 然后我们写的代码将输入的文本转到streams-pipe-output.
streams-plaintext-input和streams-pipe-output这两个topic,kafka会自动创建.
可以查看topic:
# bin/kafka-topics.sh --describe --zookeeper node1,node2,node3
这将列出所有topic.
数据流是酱婶儿地:
图中蓝色部分要在shell上手中输入。具体命令:(开两个putty)
这个putty产生数据:
# bin/kafka-console-producer.sh --broker-list node1:9092,node2:9092,node3:9092 --topic streams-plaintext-input
这个putty显示producer输入的数据
# bin/kafka-console-consumer.sh --bootstrap-server node3:9092 --from-beginning --topic streams-pipe-output
java代码:(运行在win10的eclipse里)
package ex.pipe; import java.util.Properties; import java.util.concurrent.CountDownLatch; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.Topology; import org.apache.kafka.streams.kstream.KStream; public class Pipe { public static void main(String[] args) throws Exception { Properties config = new Properties(); config.put(StreamsConfig.APPLICATION_ID_CONFIG, "stream-pipe"); // specify a list of host/port pairs to connect to the kakfa cluster config.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.1.121:9092, 192.168.1.122:9092,192.168.1.123:9092"); config.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass()); config.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass()); final StreamsBuilder builder = new StreamsBuilder(); // a source stream from a kakfa topic named streams-plaintext-inpput KStream<String,String> source = builder.stream("streams-plaintext-input"); // write to streams-pipe-output topic source.to("streams-pipe-output"); // concatenate the above two lines into a single line: // builder.stream("streams-plaintext-input").to("streams-pipe-output"); // inpsect what kind of topology is created from this builder final Topology topology = builder.build(); System.out.println(topology.describe()); final KafkaStreams streams = new KafkaStreams(topology, config); final CountDownLatch latch = new CountDownLatch(1); Runtime.getRuntime().addShutdownHook(new Thread("streams-shutdonw-hook") { @Override public void run() { streams.close(); latch.countDown(); } }); try { streams.start(); latch.await(); }catch(Throwable e) { System.exit(1); } } }