1 import java.io.IOException;
2 import java.util.Iterator;
3
4 import org.apache.hadoop.conf.Configuration;
5 import org.apache.hadoop.conf.Configured;
6 import org.apache.hadoop.fs.Path;
7 import org.apache.hadoop.io.IntWritable;
8 import org.apache.hadoop.io.Text;
9 import org.apache.hadoop.mapred.FileInputFormat;
10 import org.apache.hadoop.mapred.FileOutputFormat;
11 import org.apache.hadoop.mapred.JobClient;
12 import org.apache.hadoop.mapred.JobConf;
13 import org.apache.hadoop.mapred.KeyValueTextInputFormat;
14 import org.apache.hadoop.mapred.MapReduceBase;
15 import org.apache.hadoop.mapred.Mapper;
16 import org.apache.hadoop.mapred.OutputCollector;
17 import org.apache.hadoop.mapred.Reducer;
18 import org.apache.hadoop.mapred.Reporter;
19 import org.apache.hadoop.mapred.TextOutputFormat;
20 import org.apache.hadoop.util.Tool;
21 import org.apache.hadoop.util.ToolRunner;
22
23 public class Slist_1 extends Configured implements Tool {
24
25 public static class MapClass extends MapReduceBase
26 implements Mapper<Text, Text, IntWritable, IntWritable> {
27
28 private final static IntWritable uno = new IntWritable(1);
29 private IntWritable citationCount = new IntWritable();
30
31 public void map(Text key, Text value,
32 OutputCollector<IntWritable, IntWritable> output,
33 Reporter reporter) throws IOException {
34
35 citationCount.set(Integer.parseInt(value.toString()));
36 output.collect(citationCount, uno);
37 }
38 }
39
40 public static class Reduce extends MapReduceBase
41 implements Reducer<IntWritable,IntWritable,IntWritable,IntWritable>
42 {
43
44 public void reduce(IntWritable key, Iterator<IntWritable> values,
45 OutputCollector<IntWritable, IntWritable>output,
46 Reporter reporter) throws IOException {
47
48 int count = 0;
49 while (values.hasNext()) {
50 count += values.next().get();
51 }
52 output.collect(key, new IntWritable(count));
53 }
54 }
55
56 public int run(String[] args) throws Exception {
57 Configuration conf = getConf();
58
59 JobConf job = new JobConf(conf, Slist_1.class);
60
61 Path in = new Path(args[0]);
62 Path out = new Path(args[1]);
63 FileInputFormat.setInputPaths(job, in);
64 FileOutputFormat.setOutputPath(job, out);
65
66 job.setJobName("Slist_1");
67 job.setMapperClass(MapClass.class);
68 job.setReducerClass(Reduce.class);
69
70 job.setInputFormat(KeyValueTextInputFormat.class);
71 job.setOutputFormat(TextOutputFormat.class);
72 job.setOutputKeyClass(IntWritable.class);
73 job.setOutputValueClass(IntWritable.class);
74 job.set("key.value.separator.in.input.line", ",");
75
76 JobClient.runJob(job);
77
78 return 0;
79 }
80
81 public static void main(String[] args) throws Exception {
82 int res = ToolRunner.run(new Configuration(),
83 new Slist_1(),
84 args);
85
86 System.exit(res);
87 }
88 }
View Code
