Map Reduce的代码学习
代码引自
https://blog.csdn.net/jorocco/article/details/80142884
关于MapReduce的代码学习
共有三个部分：
- 传输的Value是自定义类型，需要自己实现序列化和反序列化，read()和write()
- 传输的Key是自定义类型，则需要自己实现WritableComparable接口，在compareTo方法中实现排序规则
- 要实现根据Key不同分区，需要重写getPartition方法，并在Job中设置分区类以及Reducer的个数，默认为1，应设为分区个数
//需求：在一个超大文件中(如下图)分别统计出每个电话号码的上行流量、下行流量以及流量总和并输出。
//Value是自己定义的类型，需要序列化和反序列化。
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
//设定需要传输的value类型，需要配置相应属性的set()get()，以及Hadoop的序列化和反序列化write(),read()。
/*
public void write(DataOutput out) throws Exception {
out.writeUTF(key);
out.write类型(属性);
out.write类型(属性);
}
public void read(DataInput in) throws Exception {
key = in.readUTF();
属性 = in.read类型();
属性 = in.read类型();
}
*/
public class FlowBean implements WritableComparable<FlowBean>{
private String phoneNB;
private long up_flow;
private long d_flow;
private long s_flow;
//在反序列化时，反射机制需要调用空参构造函数，所以需要显示的声明一个空参构造函数
public FlowBean() {
}
//为了对象数据的初始化方便，加入一个带参的构造函数
public FlowBean(String phoneNB, long up_flow, long d_flow) {
this.phoneNB = phoneNB;
this.up_flow = up_flow;
this.d_flow = d_flow;
this.s_flow = up_flow+d_flow;
}
public String getPhoneNB() {
return phoneNB;
}
public void setPhoneNB(String phoneNB) {
this.phoneNB = phoneNB;
}
public long getUp_flow() {
return up_flow;
}
public void setUp_flow(long up_flow) {
this.up_flow = up_flow;
}
public long getD_flow() {
return d_flow;
}
public void setD_flow(long d_flow) {
this.d_flow = d_flow;
}
public long getS_flow() {
return s_flow;
}
public void setS_flow(long s_flow) {
this.s_flow = s_flow;
}
//将对象数据序列化到流中
@Override
public void write(DataOutput out) throws IOException {
out.writeUTF(phoneNB);
out.writeLong(up_flow);
out.writeLong(d_flow);
out.writeLong(s_flow);
}
//从数据流中反序列化出对象的数据
//从数据流中读出对象字段时，必须跟序列化时的顺序保持一致
@Override
public void readFields(DataInput in) throws IOException {
phoneNB=in.readUTF();
up_flow=in.readLong();
d_flow=in.readLong();
s_flow=in.readLong();
}
//reduce输出的到文件（磁盘）的时候，会调用这里面的toString方法
@Override
public String toString() {
return ""+up_flow+"\t"+d_flow+"\t"+s_flow;
}
//实现排序输出
@Override
public int compareTo(FlowBean o) {
return s_flow>o.getS_flow()?-1:1;//按总流量大小排序，从大到小
}
}
··············································································································
//Mapper
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import cn.ctgu.hadoop.mr.bean.FlowBean;
/*
* FlowBean是我们自定义的一种数据类型，要在hadoop的各个节点之间传输，应该遵循hadoop的序列化机制
* 就必须实现hadoop相应的序列化接口
*
* */
public class FlowSumMapper extends Mapper<LongWritable,Text,Text,FlowBean>{
//拿到日志中的一行数据，切分成各个字段，抽取出我们需要的字段，手机号，上行流量，下行流量，然后封装成kv发送出去
@Override
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
//拿一行数据
String line=value.toString();
//切分成各个字段
String[] fields=StringUtils.split(line,"\t");
//拿到我们需要的字段
String phoneNB=fields[1];
long u_flow=Long.parseLong(fields[7]);
long d_flow=Long.parseLong(fields[8]);
//封装数据为kv并输出
context.write(new Text(phoneNB), new FlowBean(phoneNB,u_flow,d_flow));
}
}
··············································································································
//Reducer
import java.io.IOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import cn.ctgu.hadoop.mr.bean.FlowBean;
public class FlowSumReducer extends Reducer<Text,FlowBean,Text,FlowBean>{
//框架每传递一组数据就调用一次reduce方法
//reduce中的业务逻辑就是遍历values，对同一个key的values中的上行流量和下行流量进行累加求和再输出
@Override
protected void reduce(Text key, Iterable<FlowBean> values, Context context)
throws IOException, InterruptedException {
long up_flow_counter=0;
long d_flow_counter=0;
for(FlowBean bean:values) {
up_flow_counter+=bean.getUp_flow();
d_flow_counter+=bean.getD_flow();
}
context.write(key, new FlowBean(key.toString(),up_flow_counter,d_flow_counter));
}
}
··············································································································
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import cn.ctgu.hadoop.mr.bean.FlowBean;
//JOb提交类的规范写法
public class FlowSumRunner extends Configured implements Tool{
@Override
public int run(String[] args) throws Exception {
Configuration conf=new Configuration();
Job job=Job.getInstance(conf);
job.setJarByClass(FlowSumRunner.class);
job.setMapperClass(FlowSumMapper.class);
job.setReducerClass(FlowSumReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(FlowBean.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(FlowBean.class);
FileInputFormat.setInputPaths(job,new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
return job.waitForCompletion(true)?0:1;
}
public static void main(String[] args) throws Exception {
int res=ToolRunner.run(new Configuration(), new FlowSumRunner(),args);
System.out.println(res);
}
}
··············································································································
//按总流量大小排序输出
//自定义规则，实现WritableComparable接口，在compareTo方法中实现排序规则
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
public class FlowBean implements WritableComparable<FlowBean>{
private String phoneNB;
private long up_flow;
private long d_flow;
private long s_flow;
//在反序列化时，反射机制需要调用空参构造函数，所以需要显示的声明一个空参构造函数
public FlowBean() {
}
//为了对象数据的初始化方便，加入一个带参的构造函数
public FlowBean(String phoneNB, long up_flow, long d_flow) {
this.phoneNB = phoneNB;
this.up_flow = up_flow;
this.d_flow = d_flow;
this.s_flow = up_flow+d_flow;
}
public String getPhoneNB() {
return phoneNB;
}
public void setPhoneNB(String phoneNB) {
this.phoneNB = phoneNB;
}
public long getUp_flow() {
return up_flow;
}
public void setUp_flow(long up_flow) {
this.up_flow = up_flow;
}
public long getD_flow() {
return d_flow;
}
public void setD_flow(long d_flow) {
this.d_flow = d_flow;
}
public long getS_flow() {
return s_flow;
}
public void setS_flow(long s_flow) {
this.s_flow = s_flow;
}
//将对象数据序列化到流中
@Override
public void write(DataOutput out) throws IOException {
out.writeUTF(phoneNB);
out.writeLong(up_flow);
out.writeLong(d_flow);
out.writeLong(s_flow);
}
//从数据流中反序列化出对象的数据
//从数据流中读出对象字段时，必须跟序列化时的顺序保持一致
@Override
public void readFields(DataInput in) throws IOException {
phoneNB=in.readUTF();
up_flow=in.readLong();
d_flow=in.readLong();
s_flow=in.readLong();
}
//reduce输出的到文件（磁盘）的时候，会调用这里面的toString方法
@Override
public String toString() {
return ""+up_flow+"\t"+d_flow+"\t"+s_flow;
}
//实现排序输出
@Override
public int compareTo(FlowBean o) {
return s_flow>o.getS_flow()?-1:1;//按总流量大小排序，从大到小
}
}
··············································································································
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import cn.ctgu.hadoop.mr.bean.FlowBean;
public class SortMR {
public static class SortMapper extends Mapper<LongWritable,Text,FlowBean,NullWritable>{
//拿到一行数据，先分出各子段，封装成一个flowbean，作为key输出
@Override
protected void map(LongWritable key, Text value,
Mapper<LongWritable, Text, FlowBean, NullWritable>.Context context)
throws IOException, InterruptedException {
String line=value.toString();
String[] fields=StringUtils.split(line, "\t");
String phoneNB=fields[1];
long u_flow=Long.parseLong(fields[7]);
long d_flow=Long.parseLong(fields[8]);
context.write(new FlowBean(phoneNB, u_flow,d_flow),NullWritable.get());
}
}
public static class SortReducer extends Reducer<FlowBean,NullWritable,Text,FlowBean>{
@Override
protected void reduce(FlowBean key, Iterable<NullWritable> values,Context context)
throws IOException, InterruptedException {
String phoneNB=key.getPhoneNB();
context.write(new Text(phoneNB),key);
}
}
public static void main(String[] args) throws Exception {
Configuration conf=new Configuration();
Job job=Job.getInstance(conf);
job.setJarByClass(SortMR.class);
job.setMapperClass(SortMapper.class);
job.setReducerClass(SortReducer.class);
job.setMapOutputKeyClass(FlowBean.class);
job.setMapOutputValueClass(NullWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(FlowBean.class);
FileInputFormat.setInputPaths(job,new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
System.exit(job.waitForCompletion(true)?0:1);
}
}
··············································································································
//根据手机号的不同区分不同省份，并输出到不同的文件
//需要重写getPartition方法，按业务逻辑来
//在main方法中设置Partioner的类，job.setPartitionerClass(DataPartitioner.class);
//设置Reducer的数量，默认是一个，应该与分区个数相同，job.setNumReduceTasks(6);
//Mapper数量根据切片，块的数量，Reducer数量可以用户自定义
import java.util.HashMap;
import org.apache.hadoop.mapreduce.Partitioner;
public class AreaPartitioner<KEY,VALUE> extends Partitioner<KEY,VALUE>{
private static HashMap<String,Integer>areaMap=new HashMap<String,Integer>();
static {
areaMap.put("135", 0);
areaMap.put("136", 1);
areaMap.put("137", 2);
areaMap.put("138", 3);
areaMap.put("139", 4);
}
@Override
public int getPartition(KEY key, VALUE value, int numPartitions) {
//三个参数，键，值，分区号
//从key中拿到手机号，查询手机归属地字典，不同的省份返回不同的组号
int areaCoder=areaMap.get(key.toString().substring(0,3))==null?5:areaMap.get(key.toString().substring(0,3));
return areaCoder;
}
}
··············································································································
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import cn.ctgu.hadoop.mr.bean.FlowBean;
/*
* 对流量原始日志进行流量统计，将不同省份的用户统计结果输出到不同文件
* 需要自定义改造两个机制：
* 1、改造分区的逻辑，自定义一个partition
* 2、自定义reduer、task的并发任务数
*
* */
public class FlowSumArea {
public static class FlowSumAreaMapper extends Mapper<LongWritable,Text,Text,FlowBean>{
@Override
protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, FlowBean>.Context context)
throws IOException, InterruptedException {
//拿一行数据
String line=value.toString();
//切分成各个字段
String[] fields=StringUtils.split(line,"\t");
//拿到我们需要的字段
String phoneNB=fields[1];
long u_flow=Long.parseLong(fields[7]);
long d_flow=Long.parseLong(fields[8]);
//封装数据为kv并输出
context.write(new Text(phoneNB), new FlowBean(phoneNB,u_flow,d_flow));
}
}
public static class FlowSumAreaReducer extends Reducer<Text,FlowBean,Text,FlowBean>{
@Override
protected void reduce(Text key, Iterable<FlowBean> values, Context context)
throws IOException, InterruptedException {
long up_flow_counter=0;
long d_flow_counter=0;
for(FlowBean bean:values) {
up_flow_counter+=bean.getUp_flow();
d_flow_counter+=bean.getD_flow();
}
context.write(key, new FlowBean(key.toString(),up_flow_counter,d_flow_counter));
}
}
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
Configuration conf=new Configuration();
Job job=Job.getInstance(conf);
job.setJarByClass(FlowSumArea.class);
job.setMapperClass(FlowSumAreaMapper.class);
job.setReducerClass(FlowSumAreaReducer.class);
//设置我们自定义的分组逻辑
job.setPartitionerClass(AreaPartitioner.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(FlowBean.class);
//设置reduce的任务并发数（默认是一个），应该跟分组的数量保持一致
job.setNumReduceTasks(6);
FileInputFormat.setInputPaths(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
System.exit(job.waitForCompletion(true)?0:1);
}
}
··············································································································
