storm整合kafka storm-kafka-client
pom.xml-注意jar-log4j
---------------------
<dependencies> <dependency> <groupId>org.apache.storm</groupId> <artifactId>storm-kafka-client</artifactId> <version>1.1.3</version> </dependency> <dependency> <groupId>org.apache.storm</groupId> <artifactId>storm-core</artifactId> <version>1.1.3</version> <scope>provided</scope> </dependency> <dependency> <groupId>com.alibaba</groupId> <artifactId>fastjson</artifactId> <version>1.2.44</version> </dependency> <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka_2.11</artifactId> <version>1.0.0</version> <exclusions> <exclusion> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> </exclusion> <exclusion> <groupId>log4j</groupId> <artifactId>log4j</artifactId> </exclusion> </exclusions> </dependency> <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <version>3.8.1</version> <scope>test</scope> </dependency> </dependencies>
TopologyBuilder builder = new TopologyBuilder();
ByTopicRecordTranslator<String,String> brt = new ByTopicRecordTranslator<>( (r) -> new Values(r.value(),r.topic()),new Fields("values","msg"));
KafkaSpoutConfig<String,String> ksc = KafkaSpoutConfig //bootstrapServers 以及topic(mycall_in)
.builder("192.168.1.3:9092", "mycall_in") //设置group.id .setProp(ConsumerConfig.GROUP_ID_CONFIG, "skc-test") //设置开始消费的气势位置 .setFirstPollOffsetStrategy(FirstPollOffsetStrategy.LATEST) //设置提交消费边界的时长间隔 .setOffsetCommitPeriodMs(10_000) //Translator .setRecordTranslator(brt) .build();
builder.setSpout("kafkaspout", new KafkaSpout<>(ksc), 4); builder.setBolt("mybolt1", new MyBolt1(), 2).shuffleGrouping("kafkaspout");
//set producer properties. Properties props = new Properties(); props.put("bootstrap.servers", "192.168.1.3:9092"); props.put("acks", "1"); props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); @SuppressWarnings({ "unchecked", "rawtypes" }) KafkaBolt bolt = new KafkaBolt() .withProducerProperties(props) .withTopicSelector(new DefaultTopicSelector("mycall_out")) .withTupleToKafkaMapper(new FieldNameBasedTupleToKafkaMapper("", "call")); builder.setBolt("KafkaBolt", bolt, 4).fieldsGrouping("mybolt3", new Fields("call")); Config config = new Config(); config.setNumWorkers(2); config.setNumAckers(0); config.setDebug(false); LocalCluster cluster = new LocalCluster(); cluster.submitTopology("storm-kafka-clients", config, builder.createTopology()); try { //集群运行 //StormSubmitter.submitTopology("storm-kafka-clients", config, builder.createTopology()); Thread.sleep(1000*60*30);//30m cluster.shutdown(); } catch (Exception e) { e.printStackTrace(); }
---
感谢阅读,需完整代码的请联系博主!
<dependency><groupId>org.apache.storm</groupId><artifactId>storm-kafka-client</artifactId><version>1.1.3</version></dependency>