Storm计算网站PV
1.统计PV的流程图
2.Storm代码
模拟数据源：
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import org.apache.commons.io.FileUtils;
import com.yun.redis.PropertyReader;
import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichSpout;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;
/**
* 实时读取PV用户行为日志数据 可以从数据库读取数据，可以从kafka读取数据，可以从文件系统读取数据
*/
public class PVSpout extends BaseRichSpout {
/**
*
*/
private static final long serialVersionUID = 1L;
private SpoutOutputCollector collector;
private Map stormConf;
/**
* 当PVSpout初始化时候调用一次
*
* @param conf
* The Storm configuration for this spout.
* @param context
* 可以获取每个任务的TaskID
* @param collector
* The collector is used to emit tuples from this spout.
*/
@Override
public void open(Map stormConf, TopologyContext context, SpoutOutputCollector collector) {
this.collector = collector;
this.stormConf = stormConf;
}
/**
* 死循环，一直会调用
*/
@Override
public void nextTuple() {
// 获取数据源
try {
String dataDir = PropertyReader.getProperty("pv.properties", "data.dir");
File file = new File(dataDir);
//获取文件列表
Collection<File> listFiles = FileUtils.listFiles(file, new String[]{"log"},true);
for (File f : listFiles) {
//处理文件
List<String> readLines = FileUtils.readLines(f);
for (String line : readLines) {
this.collector.emit(new Values(line));
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* Declare the output schema for all the streams of this topology.
*
* @param declarer
* this is used to declare output stream ids, output fields, and
* whether or not each output stream is a direct stream
*/
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("line"));
}
}
一级bolt：
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
/**
* 获取PVSpout发送的数据，PVTopology开启多线程。 给出每个线程处理的PV数
*
* 在多线程情况下，对PV数据只能局部汇总，不能整体汇总，可以把局部汇总的结果给一个单线程的BOLT进行整体汇总（PVSumBolt）
*/
public class PVBolt extends BaseRichBolt {
/**
*
*/
private static final long serialVersionUID = 1L;
private OutputCollector collector;
private TopologyContext context;
/**
* 实例初始化的时候调用一次
*
* @param stormConf
* The Storm configuration for this bolt.
* @param context
* This object can be used to get information about this task's
* place within the topology, including the task id and component
* id of this task, input and output information, etc.
* @param collector
* The collector is used to emit tuples from this bolt
*/
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector ) {
this.context = context ;
this.collector = collector ;
}
private long pv = 0;
/**
* Process a single tuple of input.
*
* @param input
* The input tuple to be processed.
*/
@Override
public void execute(Tuple input ) {
try {
String line = input.getStringByField("line" );
if (StringUtils.isNotBlank( line)) {
pv++;
}
//System.out.println(Thread.currentThread().getName() + "[" + Thread.currentThread().getId() + "]" +context.getThisTaskId()+ "->" + pv);
//this.collector.emit(new Values(Thread.currentThread().getId(),pv));//仅适合一个线程和一个task情况
this.collector .emit(new Values(context.getThisTaskId(),pv ));// 一个线程和1或多个task的情况,TaskId唯一
this.collector .ack(input );
} catch(Exception e ){
e.printStackTrace();
this.collector .fail(input );
}
}
/**
* Declare the output schema for all the streams of this topology.
*
* @param declarer
* this is used to declare output stream ids, output fields, and
* whether or not each output stream is a direct stream
*/
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer ) {
declarer.declare( new Fields("taskId","pv" ));
}
}
二级bolt：
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.commons.collections.MapUtils;
import org.apache.hadoop.hbase.client.Result;
import com.yun.hbase.HBaseUtils;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Tuple;
/**
* 汇总PVBolt多个线程的结果（必须是单线程，多线程是无法做全局汇总的）
*
*/
public class PVSumBolt extends BaseRichBolt{
/**
*
*/
private static final long serialVersionUID = 1L;
private OutputCollector collector;
private Map<Integer,Long> map = new HashMap<Integer,Long>();
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
this.collector = collector;
}
@Override
public void execute(Tuple input) {
try {
Integer taskId = input.getIntegerByField("taskId");
Long pv = input.getLongByField("pv");
map.put(taskId, pv);//map个数为task实例数
long sum = 0;//获取总数，遍历map 的values，进行sum
for (Entry<Integer, Long> e : map.entrySet()) {
sum += e.getValue();
}
System.out.println("当前时间："+System.currentTimeMillis()+"pv汇总结果：" + "->" + sum);
this.collector.ack(input);
}catch(Exception e){
e.printStackTrace();
this.collector.fail(input);
}
}
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
}
}
PVTopology主程序
import java.util.HashMap;
import java.util.Map;
import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.topology.TopologyBuilder;
/**
* 实时统计PV拓扑
*/
public class PVTopology {
public final static String SPOUT_ID = PVSpout.class.getSimpleName();
public final static String PVBOLT_ID = PVBolt.class.getSimpleName();
public final static String PVTOPOLOGY_ID = PVTopology.class.getSimpleName();
public final static String PVSUMBOLT_ID = PVSumBolt.class.getSimpleName();
public static void main(String[] args) {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout( SPOUT_ID, new PVSpout(),1);
builder.setBolt( PVBOLT_ID, new PVBolt(), 4).setNumTasks(8).shuffleGrouping(SPOUT_ID );//2个线程，4个task(并发度)
builder.setBolt( PVSUMBOLT_ID, new PVSumBolt(), 1).shuffleGrouping(PVBOLT_ID );//单线程汇总
Map<String,Object> conf = new HashMap<String,Object>();
conf.put(Config. TOPOLOGY_RECEIVER_BUFFER_SIZE , 8);
conf.put(Config. TOPOLOGY_TRANSFER_BUFFER_SIZE , 32);
conf.put(Config. TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE , 16384);
conf.put(Config. TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE , 16384);
/* try {
StormSubmitter.submitTopology(PVTOPOLOGY_ID, conf, builder.createTopology());
} catch (Exception e) {
e.printStackTrace();
} */
LocalCluster cluster = new LocalCluster();
cluster.submitTopology( PVTOPOLOGY_ID, conf ,builder.createTopology());
}
}
