Strom的trident单词计数代码

 

/**
 * 单词计数
 */
public class LocalTridentCount {
    
    public static class MyBatchSpout implements IBatchSpout {

        Fields fields;
        HashMap<Long, List<List<Object>>> batches = new HashMap<Long, List<List<Object>>>();
        
        public MyBatchSpout(Fields fields) {
            this.fields = fields;
        }
        @Override
        public void open(Map conf, TopologyContext context) {
        }

        @Override
        public void emitBatch(long batchId, TridentCollector collector) {
            List<List<Object>> batch = this.batches.get(batchId);
            if(batch == null){
                batch = new ArrayList<List<Object>>();
                Collection<File> listFiles = FileUtils.listFiles(new File("d:\\stormtest"), new String[]{"txt"}, true);
                for (File file : listFiles) {
                    List<String> readLines;
                    try {
                        readLines = FileUtils.readLines(file);
                        for (String line : readLines) {
                            batch.add(new Values(line));
                        }
                        FileUtils.moveFile(file, new File(file.getAbsolutePath()+System.currentTimeMillis()));
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                    
                }
                if(batch.size()>0){
                    this.batches.put(batchId, batch);
                }
            }
            for(List<Object> list : batch){
                collector.emit(list);
            }
        }

        @Override
        public void ack(long batchId) {
            this.batches.remove(batchId);
        }

        @Override
        public void close() {
        }

        @Override
        public Map getComponentConfiguration() {
            Config conf = new Config();
            conf.setMaxTaskParallelism(1);
            return conf;
        }

        @Override
        public Fields getOutputFields() {
            return fields;
        }
        
    }
    
    /**
     * 对一行行的数据进行切割成一个个单词
     */
    public static class MySplit extends BaseFunction{

        @Override
        public void execute(TridentTuple tuple, TridentCollector collector) {
            String line = tuple.getStringByField("lines");
            String[] words = line.split("\t");
            for (String word : words) {
                collector.emit(new Values(word));
            }
        }
        
    }
    
    public static class MyWordAgge extends BaseAggregator<Map<String, Integer>>{

        @Override
        public Map<String, Integer> init(Object batchId,
                TridentCollector collector) {
            return new HashMap<String, Integer>();
        }

        @Override
        public void aggregate(Map<String, Integer> val, TridentTuple tuple,
                TridentCollector collector) {
            String key = tuple.getString(0);
            /*Integer integer = val.get(key);
            if(integer==null){
                integer=0;
            }
            integer++;
            val.put(key, integer);*/
            val.put(key, MapUtils.getInteger(val, key, 0)+1);
        }

        @Override
        public void complete(Map<String, Integer> val,
                TridentCollector collector) {
            collector.emit(new Values(val));
        }
        
    }
    
    /**
     * 汇总局部的map，并且打印结果
     *
     */
    public static class MyCountPrint extends BaseFunction{

        HashMap<String, Integer> hashMap = new HashMap<String, Integer>();
        @Override
        public void execute(TridentTuple tuple, TridentCollector collector) {
            Map<String, Integer> map = (Map<String, Integer>)tuple.get(0);
            for (Entry<String, Integer> entry : map.entrySet()) {
                String key = entry.getKey();
                Integer value = entry.getValue();
                Integer integer = hashMap.get(key);
                if(integer==null){
                    integer=0;
                }
                hashMap.put(key, integer+value);
            }
            
            Utils.sleep(1000);
            System.out.println("==================================");
            for (Entry<String, Integer> entry : hashMap.entrySet()) {
                System.out.println(entry);
            }
        }
        
    }
    
    
    public static void main(String[] args) {
        //大体流程:首先设置一个数据源MyBatchSpout,会监控指定目录下文件的变化,当发现有新文件的时候把文件中的数据取出来,
        //然后封装到一个batch中发射出来.就会对tuple中的数据进行处理,把每个tuple中的数据都取出来,然后切割..切割成一个个的单词.
        //单词发射出来之后,会对单词进行分组,会对一批假设有10个tuple,会对这10个tuple分完词之后的单词进行分组, 相同的单词分一块  
        //分完之后聚合 把相同的单词使用同一个聚合器聚合  然后出结果  每个单词出现多少次...
        //进行汇总  先每一批数据局部汇总  最后全局汇总....
        //这个代码也不是很简单...挺多....就是使用批处理的方式.
        
        TridentTopology tridentTopology = new TridentTopology();
        
        tridentTopology.newStream("spoutid", new MyBatchSpout(new Fields("lines")))
            .each(new Fields("lines"), new MySplit(), new Fields("word"))
            .groupBy(new Fields("word"))//用到了分组 对一批tuple中的单词进行分组..
            .aggregate(new Fields("word"), new MyWordAgge(), new Fields("wwwww"))//用到了聚合
            .each(new Fields("wwwww"), new MyCountPrint(), new Fields(""));
        
        LocalCluster localCluster = new LocalCluster();
        String simpleName = TridentMeger.class.getSimpleName();
        localCluster.submitTopology(simpleName, new Config(), tridentTopology.build());
    }
}

指定路径下文件中的内容:

程序运行结果: