课堂练习-实验六
1,mapper层
import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Mapper; import java.io.IOException; //前两个参数:key是某一行的起始位置相对于文件开头的偏移量 value是指每一行 //后两个参数:单词和单词数目 public class WordCountMapper extends Mapper<LongWritable, Text,Text,LongWritable> { @Override protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { //获取一行的内容 String line = value.toString(); //每一行的单词是以空格隔开的,所以使用空格切割成数组 String[] words = line.split(" "); for (String word:words ) { //输出到reduce context.write(new Text(word),new LongWritable(1)); } } }
2,reduce层
import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.io.Text; import java.io.IOException; //前两个参数分别接收的是mapper的后两个参数,所以类型跟mapper后两个参数的类型一致 //后两个参数为reduce的输出参数 public class WordCountReduce extends Reducer<Text, LongWritable,Text,LongWritable> { @Override protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException { long sum = 0; for (LongWritable value:values ) { sum = sum + value.get(); } //输出 context.write(key,new LongWritable(sum)); } }
3,job层
mport org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; public class WC { public static void main(String[] args) throws Exception{ //1.获取job Configuration conf=new Configuration(); Job job=Job.getInstance(conf); //2.指定job使用的类 job.setJarByClass(WC.class); //3.设置mapper的类以及属性 job.setMapperClass(WordCountMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(LongWritable.class); //4.设置reducer的类以及属 job.setReducerClass(WordCountReduce.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(LongWritable.class); //5.设置输入文件 FileInputFormat.setInputPaths(job, new Path(args[0])); //6.设置输出目录 FileOutputFormat.setOutputPath(job, new Path(args[1])); //7.提交任务 job.waitForCompletion(true); } }