一个可以跑的Hadoop的WordCount程序

搭个新环境时总要折腾一下，于是干脆记下来。

程序：

package  com.my;  
  
import  java.io.IOException;  
import  java.util.Iterator;  
import  java.util.StringTokenizer;  
  
import  org.apache.hadoop.fs.Path;  
import  org.apache.hadoop.io.IntWritable;  
import  org.apache.hadoop.io.LongWritable;  
import  org.apache.hadoop.io.Text;  
import  org.apache.hadoop.mapred.FileInputFormat;  
import  org.apache.hadoop.mapred.FileOutputFormat;  
import  org.apache.hadoop.mapred.JobClient;  
import  org.apache.hadoop.mapred.JobConf;  
import  org.apache.hadoop.mapred.MapReduceBase;  
import  org.apache.hadoop.mapred.Mapper;  
import  org.apache.hadoop.mapred.OutputCollector;  
import  org.apache.hadoop.mapred.Reducer;  
import  org.apache.hadoop.mapred.Reporter;  
import  org.apache.hadoop.mapred.TextInputFormat;  
import  org.apache.hadoop.mapred.TextOutputFormat;  
public   class  WordCount  
{  
  
    public   static   class  Map  extends  MapReduceBase  implements   
            Mapper<LongWritable, Text, Text, IntWritable>  
    {  
        private   final   static  IntWritable one =  new  IntWritable( 1 );  
        private  Text word =  new  Text();  
          
        public   void  map(LongWritable key, Text value,  
                OutputCollector<Text, IntWritable> output, Reporter reporter)  
                throws  IOException  
        {  
            String line = value.toString();  
            StringTokenizer tokenizer = new  StringTokenizer(line);  
            while  (tokenizer.hasMoreTokens())  
            {  
                word.set(tokenizer.nextToken());  
                output.collect(word, one);  
            }  
        }  
    }  
  
    public   static   class  Reduce  extends  MapReduceBase  implements   
            Reducer<Text, IntWritable, Text, IntWritable>  
    {  
        public   void  reduce(Text key, Iterator<IntWritable> values,  
                OutputCollector<Text, IntWritable> output, Reporter reporter)  
                throws  IOException  
        {  
            int  sum =  0 ;  
            while  (values.hasNext())  
            {  
                sum += values.next().get();  
            }  
            output.collect(key, new  IntWritable(sum));  
        }  
    }  
  
    public   static   void  main(String[] args)  throws  Exception  
    {  
        JobConf conf = new  JobConf(WordCount. class );  
        conf.setJobName("wordcount" ); 
  
        conf.setOutputKeyClass(Text.class );
        conf.setOutputValueClass(IntWritable.class );
  
        conf.setMapperClass(Map.class );
        conf.setCombinerClass(Reduce.class );
        conf.setReducerClass(Reduce.class ); 
  
        conf.setInputFormat(TextInputFormat.class );
        conf.setOutputFormat(TextOutputFormat.class );
  
        FileInputFormat.setInputPaths(conf, new  Path(args[ 0 ]));  
        FileOutputFormat.setOutputPath(conf, new  Path(args[ 1 ]));  
  
        JobClient.runJob(conf); 
    }  
}

编译命令：

mkdir Myjava

javac -classpath hadoop-core-1.1.2.jar -d Myjava WordCount.java

jar -cvf WordCount.jar -C Myjava .

运行命令：

bin/hadoop jar WordCount.jar com.my.WordCount /src/test.txt /output

 

这一次的是基于hadoop 1.1.2程序。