Cascading(三)之计数器

同样是官方源码,这个源码稍微复杂一点,坚持看完了.

package com.wyf.cascade;

import java.util.Map;
import java.util.Properties;

import cascading.cascade.Cascade;
import cascading.cascade.CascadeConnector;
import cascading.cascade.Cascades;
import cascading.flow.Flow;
import cascading.flow.FlowConnector;
import cascading.operation.Identity;
import cascading.operation.aggregator.Count;
import cascading.operation.regex.RegexFilter;
import cascading.operation.regex.RegexGenerator;
import cascading.operation.regex.RegexReplace;
import cascading.operation.regex.RegexSplitter;
import cascading.operation.xml.TagSoupParser;
import cascading.operation.xml.XPathGenerator;
import cascading.operation.xml.XPathOperation;
import cascading.pipe.Each;
import cascading.pipe.Every;
import cascading.pipe.GroupBy;
import cascading.pipe.Pipe;
import cascading.pipe.SubAssembly;
import cascading.scheme.SequenceFile;
import cascading.scheme.TextLine;
import cascading.tap.Hfs;
import cascading.tap.Lfs;
import cascading.tap.Tap;
import cascading.tuple.Fields;

/**
 * 
 * 
 * @author: wyf
 * @version: Jul 15, 2013 11:33:04 AM
 */
public class WordCount {

    /**
     * 导入爬虫数据装配
     * 
     * @author: wyf
     * @version: Jul 15, 2013 11:38:02 AM
     */
    @SuppressWarnings("serial")
    private static class ImportCrawlDataAssembly extends SubAssembly {
        public ImportCrawlDataAssembly(String name) {
            //拆分文本行到url和raw
            RegexSplitter regexSplitter = new RegexSplitter(new Fields("url", "raw"));
            Pipe importPipe = new Each(name, new Fields("line"), regexSplitter);
            //删除所有pdf文档
            importPipe = new Each(importPipe, new Fields("url"), new RegexFilter(".*\\.pdf$", true));
            //把":n1"替换为"\n"
            RegexReplace regexReplace = new RegexReplace(new Fields("page"), ":nl:", "\n");
            importPipe = new Each(importPipe, new Fields("raw"), regexReplace, new Fields("url", "page"));

            //此句强制调用
            setTails(importPipe);
        }
    }

    /**
     * 计数器拆分
     * 
     * @author: wyf
     * @version: Jul 15, 2013 11:38:35 AM
     */
    @SuppressWarnings("serial")
    private static class WordCountSplitAssembly extends SubAssembly {
        public WordCountSplitAssembly(String sourceName, String sinkUrlName, String sinkWordName) {
            //创建一个新的组件,计算所有页面中字数,和一个页面中的字数
            Pipe pipe = new Pipe(sourceName);
            
            //利用TagSoup将HTML转成XHTML，只保留"url"和"xml"去掉其它多余的
            pipe = new Each(pipe, new Fields("page"), new TagSoupParser(new Fields("xml")), new Fields("url", "xml"));
            
            //对"xml"字段运用XPath(XML Path Language)表达式,提取"body"元素
            XPathGenerator bodyExtractor = new XPathGenerator(new Fields("body"), XPathOperation.NAMESPACE_XHTML, "//xhtml:body");
            pipe = new Each(pipe, new Fields("xml"), bodyExtractor, new Fields("url", "body"));
            
            //运用另一个XPath表达式删除所有元素,只保留文本节点,删除在"script"元素中的文本节点
            String elementXPath = "//text()[ name(parent::node()) != 'script']";
            XPathGenerator elementRemover = new XPathGenerator(new Fields("words"), XPathOperation.NAMESPACE_XHTML, elementXPath);
            pipe = new Each(pipe, new Fields("body"), elementRemover, new Fields("url", "words"));
            
            //用正则表达式将文档打乱成一个个独立的单词,和填充每个单词(新元组)到当前流使用"url"和"word"字段
            RegexGenerator wordGenerator = new RegexGenerator(new Fields("word"), "(?<!\\pL)(?=\\pL)[^ ]*(?<=\\pL)(?!\\pL)");
            pipe = new Each(pipe, new Fields("words"), wordGenerator, new Fields("url", "word"));

            //按"url"分组
            Pipe urlCountPipe = new GroupBy(sinkUrlName, pipe, new Fields("url", "word"));
            urlCountPipe = new Every(urlCountPipe, new Fields("url", "word"), new Count(), new Fields("url", "word", "count"));

            //按"word"分组
            Pipe wordCountPipe = new GroupBy(sinkWordName, pipe, new Fields("word"));
            wordCountPipe = new Every(wordCountPipe, new Fields("word"), new Count(), new Fields("word", "count"));
            
            setTails(urlCountPipe, wordCountPipe);
        }
    }

    
    /**
     * 运行主函数
     * 
     * @author: wyf
     * @version: Jul 15, 2013 11:41:30 AM
     */
    public static void main(String[] args) {
        //设置当前工作jar
        Properties properties = new Properties();
        FlowConnector.setApplicationJarClass(properties, WordCount.class);
        FlowConnector flowConnector = new FlowConnector(properties);

        String inputPath = "/home/wyf/workspace/HadoopCascading/data/url+page.200.txt";
        String tempPath = "/home/wyf/workspace/HadoopCascading/data";
        String tempPath2 = "/home/wyf/workspace/HadoopCascading/data/local";
        String pagesPath = tempPath + "/pages/";
        String urlsPath = tempPath + "/urls/";
        String wordsPath = tempPath + "/words/";
        String localUrlsPath = tempPath2 + "/urls/";
        String localWordsPath = tempPath2 + "/words/";

        //初始化Pipe管道处理爬虫数据装配
        Pipe importPipe = new ImportCrawlDataAssembly("import pipe");

        //创建tap实例
        Tap localPagesSource = new Lfs(new TextLine(), inputPath);
        Tap importedPages = new Hfs(new SequenceFile(new Fields("url", "page")), pagesPath);

        //链接pipe装配到tap实例
        Flow importPagesFlow = flowConnector.connect("import pages", localPagesSource, importedPages, importPipe);

        //拆分定义的管道到两个管道
        SubAssembly wordCountPipe = new WordCountSplitAssembly("wordcount pipe", "url pipe", "word pipe");

        //创建hadoop SequenceFile 存储计数后的结果
        Tap sinkUrl = new Hfs(new SequenceFile(new Fields("url", "word", "count")), urlsPath);
        Tap sinkWord = new Hfs(new SequenceFile(new Fields("word", "count")), wordsPath);

        //绑定多个pipe和tap,此处指定的是pipe名称
        Map<String, Tap> sinks = Cascades.tapsMap(new String[] { "url pipe", "word pipe" }, Tap.taps(sinkUrl, sinkWord));
        Flow count = flowConnector.connect(importedPages, sinks, wordCountPipe);

        //创建一个装配,导出hadoop sequenceFile 到本地文本文件
        Pipe exportPipe = new Each("export pipe", new Identity());
        Tap localSinkUrl = new Lfs(new TextLine(), localUrlsPath);
        Tap localSinkWord = new Lfs(new TextLine(), localWordsPath);

        Flow exportFromUrl = flowConnector.connect("export url", sinkUrl, localSinkUrl, exportPipe);
        Flow exportFromWord = flowConnector.connect("export word", sinkWord, localSinkWord, exportPipe);

        //装载flow,并执行
        Cascade cascade = new CascadeConnector().connect(importPagesFlow, count, exportFromUrl, exportFromWord);
        cascade.complete();
    }
}