实例数据MR统计FLOW

package LogMR;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

import org.apache.hadoop.io.Writable;

public class FlowBean implements Writable {
    
    private String TOS;//服务类型（来自哪个应用程序的请求）
    private long up_flow;//上行流量
    private long d_flow;//下行流量
    private long s_flow;//流量和
    private long count;
    
    public String getTOS() {
        return TOS;
    }

    public void setTOS(String tOS) {
        TOS = tOS;
    }

    public long getUp_flow() {
        return up_flow;
    }

    public void setUp_flow(long up_flow) {
        this.up_flow = up_flow;
    }

    public long getD_folw() {
        return d_flow;
    }

    public void setD_folw(long d_folw) {
        this.d_flow = d_folw;
    }

    public long getS_flow() {
        return s_flow;
    }

    public void setS_flow(long s_flow) {
        this.s_flow = s_flow;
    }
    public long getCount() {
        return count;
    }

    public void setCount(long count) {
        this.count = count;
    }
    
    //在反序列化时，反射机制需要调用构造函数，所以显示定义一个空参构造函数
    public FlowBean(){}
    //为了对象数据的初始化方便，加入一个带参的构造函数
    public FlowBean(String tOS, long up_flow, long d_flow,long count) {
        super();
        TOS = tOS;
        this.up_flow = up_flow;
        this.d_flow = d_flow;
        this.s_flow =  up_flow+d_flow;
        this.count=count;
    }
    
    //将对象数据序列化到流
    @Override
    public void write(DataOutput out) throws IOException {
        // TODO Auto-generated method stub
        out.writeUTF(TOS);
        out.writeLong(up_flow);
        out.writeLong(d_flow);
        out.writeLong(s_flow);    
        out.writeLong(count);
    }
    
    //从数据流中反序列化出对象的数据
    //从数据流中读出对象字段时，必须跟序列化时的顺序保持一致
    //会用到反射机制
    @Override
    public void readFields(DataInput in) throws IOException {
        // TODO Auto-generated method stub
        TOS = in.readUTF();
        up_flow = in.readLong();
        d_flow = in.readLong();
        s_flow = in.readLong();
        count=in.readLong();
    }
    
    @Override
    public String toString() {
        return ""+"\t"+up_flow+"\t"+d_flow+"\t"+s_flow+"\t"+count;
    }
}

package LogMR;

import java.io.IOException;

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

/**
 * FlowBean 是我们自定义的一种数据类型，要在hadoop的各个节点之间传输，应该遵循hadoop的序列化机制
 * 就必须实现hadoop响应的序列化接口
 * 
 *
 */

public class LogMapper extends Mapper<LongWritable, Text, Text, FlowBean>{
    
    //拿到日志中的一行数据，切分各个字段，抽取出我们需要的字段：服务类型，上行流量，下行流量，然后封装成kv发送出去
    @Override
    protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, FlowBean>.Context context)
            throws IOException, InterruptedException {
        // TODO Auto-generated method stub
        //拿一行数据
        String line=new String(value.getBytes(),0,value.getLength(),"GBK");//GBK编码处理
        //切分成各个字段
        String[] fields=StringUtils.split(line,"\t");
        //拿到我们需要的字段
        String TOS=fields[1];
        long up_flow=Long.parseLong(fields[8]);
        long d_flow=Long.parseLong(fields[9]);
        
        //封装数据为kv并输出
        context.write(new Text(TOS), new FlowBean(TOS,up_flow,d_flow,1));
    }
    
/*    public static Text transformTextToUTF8(Text text, String encoding) {
        String value = null;
        try {
        value = new String(text.getBytes(), 0, text.getLength(), encoding);
        } catch (UnsupportedEncodingException e) {
        e.printStackTrace();
        }
        return new Text(value);
        }*/
}

package LogMR;

import java.io.IOException;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class LogReducer extends  Reducer<Text, FlowBean, Text, FlowBean>{
    @Override
    //框架每传递一组数据<服务类型（例如dns，微信聊天），｛flowbean，flowbean，flowbean......｝>
    //调用一次Reduce方法
    //reduce中的业务逻辑就是遍历values，然后进行累加求和
    protected void reduce(Text key, Iterable<FlowBean> values, Reducer<Text, FlowBean, Text, FlowBean>.Context context)
            throws IOException, InterruptedException {
        // TODO Auto-generated method stub
        long up_flow_counter=0;
        long d_flow_counter=0;
        long count=0;
        
        for(FlowBean bean:values){
            up_flow_counter+=bean.getUp_flow();
            d_flow_counter+=bean.getD_folw();
            count+=bean.getCount();
        }
        context.write(key, new FlowBean(key.toString(),up_flow_counter,d_flow_counter,count));
    }

}

package LogMR;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

//这是job描述和提交类的规范写法
public class LogRunner extends Configured implements Tool {

    @Override
    public int run(String[] arg0) throws Exception {
        // TODO Auto-generated method stub
        
        //读取配置文件
        Configuration conf = new Configuration();
        Job job=Job.getInstance(conf);
        
        job.setJarByClass(LogRunner.class);
        
        job.setMapperClass(LogMapper.class);
        job.setReducerClass(LogReducer.class);
        
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(FlowBean.class);
        
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(FlowBean.class);
        
        FileInputFormat.setInputPaths(job,new Path(arg0[0]));
        FileOutputFormat.setOutputPath(job, new Path(arg0[1]));
        
        return job.waitForCompletion(true)?0:1;
    }
    
    public static void main(String[] args) throws Exception {
        int res=ToolRunner.run(new Configuration(),new LogRunner(),args);
        System.exit(res);
    }
}