CentOS7搭建Storm集群及基础操作
前提
安装Kafka前需要先安装zookeeper集群，集体安装方法请参照我的另一篇文档
Storm安装
下载
wget https://mirrors.tuna.tsinghua.edu.cn/apache/storm/apache-storm-1.1.0/apache-storm-1.1.0.tar.gz解压
tar -zxvf apache-storm-1.1.0.tar.gz移动文件夹
mv apache-storm-1.1.0 /usr/local/hadoop/vim storm.yamlstorm.zookeeper.servers:
- "192.168.174.200"
- "192.168.174.201"
nimbus.seeds: ["192.168.174.200"]
storm.local.dir: "/usr/local/hadoop/apache-storm-1.1.0/data"- storm.zookeeper.servers:表示zookeeper的集群地址，如果Zookeeper集群使用的不是默认端口，那么还需要配置storm.zookeeper.port
- storm.zookeeper.port: Zookeeper集群的端口号
- storm.local.dir: 用于配置Storm存储少量文件的路径
- nimbus.seeds: 用于配置主控节点的地址，可以配置多个
拷贝文件到其余工作节点
scp apache-storm-1.1.0 salver1:/usr/local/hadoop/Storm操作
启动主控节点
./storm nimbus 1>/dev/null 2>&1 &启动主控节点管理界面
./storm ui 1>/dev/null 2>&1 & 启动工作节点
./storm supervisor 1>/dev/null 2>&1 &访问地址
http://127.0.0.1:8080
运行拓扑
./storm jar storm-book.jar com.TopologyMain /usr/words.txt删除拓扑
./storm kill Getting-Started-Toplogie完整示例
package com;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.AlreadyAliveException;
import org.apache.storm.generated.AuthorizationException;
import org.apache.storm.generated.InvalidTopologyException;
import org.apache.storm.kafka.KafkaSpout;
import org.apache.storm.kafka.SpoutConfig;
import org.apache.storm.kafka.StringScheme;
import org.apache.storm.kafka.ZkHosts;
import org.apache.storm.redis.bolt.RedisStoreBolt;
import org.apache.storm.redis.common.config.JedisPoolConfig;
import org.apache.storm.redis.common.mapper.RedisStoreMapper;
import org.apache.storm.spout.SchemeAsMultiScheme;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.tuple.Fields;
public class MykafkaSpout {
/**
* @param args
* @throws AuthorizationException
*/
public static void main(String[] args) throws AuthorizationException {
// TODO Auto-generated method stub
String host = "127.0.0.1";
int port = 6385;
String topic = "test" ;
ZkHosts zkHosts = new ZkHosts("192.168.174.200:2181,192.168.174.201:2181");
SpoutConfig spoutConfig = new SpoutConfig(zkHosts, topic,
"",
UUID.randomUUID().toString()) ;
List<String> zkServers = new ArrayList<String>() ;
zkServers.add("192.168.174.200");
zkServers.add("192.168.174.201");
spoutConfig.zkServers = zkServers;
spoutConfig.zkPort = 2181;
spoutConfig.socketTimeoutMs = 60 * 1000 ;
spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme()) ;
spoutConfig.startOffsetTime = kafka.api.OffsetRequest.LatestTime();
TopologyBuilder builder = new TopologyBuilder() ;
builder.setSpout("spout", new KafkaSpout(spoutConfig) ,1) ;
builder.setBolt("bolt1", new MyKafkaBolt(), 2).shuffleGrouping("spout") ;
builder.setBolt("MyCountBolt", new MyCountBolt(), 2).fieldsGrouping("bolt1", new Fields("type"));
// 将所有单词及其次数进行汇总输出
builder.setBolt("MyReportBolt", new MyReportBolt(), 2).globalGrouping("MyCountBolt");
JedisPoolConfig poolConfig = new JedisPoolConfig.Builder().setHost(host).setPort(port).setPassword("Apple05101314").build();
RedisStoreMapper storeMapper = new MyCountStoreMapper();
RedisStoreBolt storeBolt = new RedisStoreBolt(poolConfig, storeMapper);
//向redis保存数据
builder.setBolt("redis-store-bolt", storeBolt).globalGrouping("MyReportBolt");
Config conf = new Config ();
conf.setDebug(false) ;
if (args.length > 0) {
try {
StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
} catch (AlreadyAliveException e) {
e.printStackTrace();
} catch (InvalidTopologyException e) {
e.printStackTrace();
}
}else {
LocalCluster localCluster = new LocalCluster();
localCluster.submitTopology("mytopology", conf, builder.createTopology());
}
}
}
package com;
import java.util.HashMap;
import java.util.Map;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.BasicOutputCollector;
import org.apache.storm.topology.IBasicBolt;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
public class MyKafkaBolt extends BaseRichBolt {
private OutputCollector outputCollector;
// key:messageId,Data
private HashMap<String, String> waitAck = new HashMap<String, String>();
public void prepare(Map map, TopologyContext context,
OutputCollector collector) {
// TODO Auto-generated method stub
this.outputCollector = collector;
}
public void execute(Tuple input) {
// TODO Auto-generated method stub
String kafkaMsg = input.getString(0);
if(kafkaMsg!=null){
this.outputCollector.emit(new Values(kafkaMsg));
this.outputCollector.ack(input);
}
}
public void declareOutputFields(OutputFieldsDeclarer declarer) {
// TODO Auto-generated method stub
declarer.declare(new Fields("type"));
}
}package com;
import java.util.HashMap;
import java.util.Map;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.BasicOutputCollector;
import org.apache.storm.topology.IBasicBolt;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
public class MyCountBolt extends BaseRichBolt {
private OutputCollector outputCollector;
private HashMap<String, Integer> count;
public void prepare(Map stormConf, TopologyContext context,
OutputCollector collector) {
// TODO Auto-generated method stub
this.outputCollector = collector;
this.count = new HashMap<String, Integer>();
}
public void execute(Tuple input) {
// TODO Auto-generated method stub
String type = input.getStringByField("type");
int cnt = 1;
if(count.containsKey(type)){
cnt = count.get(type) + 1;
}
count.put(type, cnt);
this.outputCollector.emit(new Values(type, cnt));
this.outputCollector.ack(input);
}
public void declareOutputFields(OutputFieldsDeclarer declarer) {
// TODO Auto-generated method stub
declarer.declare(new Fields("type", "cnt"));
}
}package com;
import org.apache.storm.redis.common.mapper.RedisDataTypeDescription;
import org.apache.storm.redis.common.mapper.RedisStoreMapper;
import org.apache.storm.tuple.ITuple;
public class MyCountStoreMapper implements RedisStoreMapper {
private RedisDataTypeDescription description;
private final String hashKey = "myCount";
public MyCountStoreMapper() {
description = new RedisDataTypeDescription(
RedisDataTypeDescription.RedisDataType.HASH, hashKey);
}
public RedisDataTypeDescription getDataTypeDescription() {
return description;
}
public String getKeyFromTuple(ITuple tuple) {
return tuple.getStringByField("zs");
}
public String getValueFromTuple(ITuple tuple) {
return tuple.getIntegerByField("cnt")+"";
}
}
package com;
import org.apache.storm.redis.bolt.RedisStoreBolt;
import org.apache.storm.redis.common.config.JedisPoolConfig;
import org.apache.storm.redis.common.mapper.RedisStoreMapper;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
import java.util.HashMap;
import java.util.Map;
import org.apache.log4j.Logger;
/**
* Created by gzx on 17-2-6.
*/
public class MyReportBolt extends BaseRichBolt {
private static Logger logger = Logger.getLogger(MyReportBolt.class);
private OutputCollector outputCollector;
private HashMap<String, Integer> count;
public void prepare(Map map, TopologyContext topologyContext,
OutputCollector collector) {
this.count = new HashMap<String, Integer>();
this.outputCollector = collector;
}
/**
* 打印单词及其出现次数
*
* @param tuple
*/
public void execute(Tuple tuple) {
String type = tuple.getStringByField("type");
int cnt = tuple.getIntegerByField("cnt");
count.put(type, cnt);
if (count.containsKey("join") && count.containsKey("out")) {
int join = count.get("join");
int out = count.get("out");
int sy = join-out;
System.out.println("join=" + join);
System.out.println("out=" + out);
//System.out.printf("===当前剩余总数==="+sy+"\r\n");
logger.debug("===当前剩余总数==="+sy);
this.outputCollector.emit(new Values("zs", sy));
this.outputCollector.ack(tuple);
}
}
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("zs", "cnt"));
}
}
