Mapreduce实例——求平均值———代码
MyAverage.java:
package com.ya.mapreduce; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; public class MyAverage { static { try { System.load("D:\\install\\winutils-master\\winutils-master\\hadoop-3.0.0\\bin\\hadoop.dll");//建议采用绝对地址,bin目录下的hadoop.dll文件路径 } catch (UnsatisfiedLinkError e) { System.err.println("Native code library failed to load.\n"+ e); System.exit(1); } } public static class Map extends Mapper<Object, Text, Text, IntWritable> { private static Text newKey = new Text(); //实现map函数 public void map(Object key, Text value, Context context) throws IOException, InterruptedException { // 将输入的纯文本文件的数据转化成String String line = value.toString(); System.out.println(line); String arr[] = line.split("\t"); newKey.set(arr[0]); int click = Integer.parseInt(arr[1]); context.write(newKey, new IntWritable(click)); } } public static class Reduce extends Reducer<Text, IntWritable, Text, IntWritable> { //实现reduce函数 public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { int num = 0; int count = 0; for (IntWritable val : values) { num += val.get();//每个元素求和num count++;//统计元素的次数count } int avg = num / count;//计算平均数 context.write(key, new IntWritable(avg)); } } public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException { Configuration conf = new Configuration(); System.out.println("start"); Job job = new Job(conf, "MyAverage"); job.setJarByClass(MyAverage.class); job.setMapperClass(Map.class); job.setReducerClass(Reduce.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); Path in = new Path("hdfs://node01:8020/kkb/data/mapreduce4/in/goods_click.txt"); Path out = new Path("hdfs://node01:8020/kkb/data/mapreduce4/out"); FileInputFormat.addInputPath(job, in); FileOutputFormat.setOutputPath(job, out); System.exit(job.waitForCompletion(true) ? 0 : 1); } }
结果: