Hadoop 学习笔记 （十一） MapReduce 求平均成绩

china：
张三 78
李四 89
王五 96
赵六 67
english
张三 80
李四 82
王五    84
赵六 86
math
张三 88
李四 99
王五 66
赵六 77




import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.reduce.IntSumReducer;


public class MyAverage {

    public static class FormatMapper extends Mapper<Object, Text, Text, IntWritable>{
        
        private IntWritable val = new IntWritable();
        
        public void map(Object key, Text value, Context context) throws IOException, InterruptedException{
            String line[] = value.toString().split("\\s");
            val.set(Integer.parseInt(line[1]));
            context.write(new Text(line[0]), val);
        }
    }
    
    public static class AverageReducer extends Reducer<Text, IntWritable, Text, FloatWritable>{
        public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException{
            int count = 0;
            int sum = 0;
            for (IntWritable val : values) {
                sum += val.get();
                count ++;
            }
            float ave = (float)sum / count;
            context.write(key, new FloatWritable(ave));
        }
    }
    
    
    
    public static void main(String[] args) throws Exception {
        String dir_in = "hdfs://localhost:9000/in_average";
        String dir_out = "hdfs://localhost:9000/out_average";

        Path in = new Path(dir_in);
        Path out = new Path(dir_out);
        
        Configuration conf = new Configuration();
        Job averageJob = new Job(conf, "my_average");

        averageJob.setJarByClass(MyAverage.class);

        averageJob.setInputFormatClass(TextInputFormat.class);
        averageJob.setMapperClass(FormatMapper.class);
        averageJob.setCombinerClass(IntSumReducer.class);
        //countJob.setPartitionerClass(HashPartitioner.class);
        averageJob.setMapOutputKeyClass(Text.class);
        averageJob.setMapOutputValueClass(IntWritable.class);

        FileInputFormat.addInputPath(averageJob, in);

        averageJob.setReducerClass(AverageReducer.class);
        //averageJob.setNumReduceTasks(1);
        averageJob.setOutputKeyClass(Text.class);
        averageJob.setOutputValueClass(FloatWritable.class);
        //countJob.setOutputFormatClass(SequenceFileOutputFormat.class);

        FileOutputFormat.setOutputPath(averageJob, out);

        averageJob.waitForCompletion(true);

    }

}

张三    82.0
李四    90.0
王五    82.0
赵六    76.666664