hadoop示例程序Grep分析
该程序实现的是对指定文档中指定单词的词频进行计算
1 /**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18 package org.apache.hadoop.examples;
19
20 import java.util.Random;
21
22 import org.apache.hadoop.conf.Configuration;
23 import org.apache.hadoop.conf.Configured;
24 import org.apache.hadoop.fs.FileSystem;
25 import org.apache.hadoop.fs.Path;
26 import org.apache.hadoop.io.LongWritable;
27 import org.apache.hadoop.io.Text;
28 import org.apache.hadoop.mapred.*;
29 import org.apache.hadoop.mapred.lib.*;
30 import org.apache.hadoop.util.Tool;
31 import org.apache.hadoop.util.ToolRunner;
32
33 /* Extracts matching regexs from input files and counts them. */
34 /* 从input中提取与表达式相符的单词并计算词频*/
35 publicclass Grep extends Configured implements Tool {
36 private Grep() {} // singleton
37 /** 继承自配置基类Configured,并扩展接口Tool
38 *Configured类中有一个变量conf用于存储配置文件
39 *Tool中只有一个方法需要实现
40 *int run(String [] args)用于运行输入参数
41 */
42 publicint run(String[] args) throws Exception {
43 if (args.length <3) {
44 System.out.println("Grep <inDir> <outDir> <regex> [<group>]");
45 ToolRunner.printGenericCommandUsage(System.out);
46 return-1;
47 }
48
49 Path tempDir =
50 new Path("grep-temp-"+
51 Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
52
53 JobConf grepJob =new JobConf(getConf(), Grep.class);//创建job
54
55 try {
56
57 grepJob.setJobName("grep-search"); //job命名
58
59 FileInputFormat.setInputPaths(grepJob, args[0]); //设置job的输入路径
60
61 grepJob.setMapperClass(RegexMapper.class); //设置Mapper类
62 grepJob.set("mapred.mapper.regex", args[2]);
63 if (args.length ==4)
64 grepJob.set("mapred.mapper.regex.group", args[3]);
65
66 grepJob.setCombinerClass(LongSumReducer.class); //设置Combiner类
67 grepJob.setReducerClass(LongSumReducer.class); //设置Reducer类
68
69 FileOutputFormat.setOutputPath(grepJob, tempDir); //设置输出路径
70 grepJob.setOutputFormat(SequenceFileOutputFormat.class); //设置输出格式
71 grepJob.setOutputKeyClass(Text.class); //设置输出键的类
72 grepJob.setOutputValueClass(LongWritable.class); //设置输出值的类
73
74 JobClient.runJob(grepJob); //运行
75
76 JobConf sortJob =new JobConf(Grep.class); //排序,内容同上
77 sortJob.setJobName("grep-sort");
78
79 FileInputFormat.setInputPaths(sortJob, tempDir);
80 sortJob.setInputFormat(SequenceFileInputFormat.class);
81
82 sortJob.setMapperClass(InverseMapper.class);
83
84 sortJob.setNumReduceTasks(1); // write a single file
85 FileOutputFormat.setOutputPath(sortJob, new Path(args[1]));
86 sortJob.setOutputKeyComparatorClass // sort by decreasing freq
87 (LongWritable.DecreasingComparator.class);
88
89 JobClient.runJob(sortJob);
90 }
91 finally {
92 FileSystem.get(grepJob).delete(tempDir, true);
93 }
94 return0;
95 }
96
97 publicstaticvoid main(String[] args) throws Exception {
98 int res = ToolRunner.run(new Configuration(), new Grep(), args);
99 System.exit(res);
100 }
101
102 }
