MapReduce例子_矩阵转置

Mapper

package step1;

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class Mapper1 extends Mapper<LongWritable, Text, Text, Text> {
    
    private Text outKey = new Text();
    private Text outValue = new Text();
    
    /*
     * key : 1 行号
     * value : 1 1_0,2_3,3_-1,4_2,5_-3
     * 
     */
    
    @Override
    protected void map(LongWritable key, Text value,Context context)
            throws IOException, InterruptedException {
        String[] rowAndLine = value.toString().split("\t");
        //矩阵的行号
        String row = rowAndLine[0];
        String[] lines = rowAndLine[1].split(",");
        
        for(int i = 0;i < lines.length;i++){
            String column = lines[i].split("_")[0];
            String valueStr = lines[i].split("_")[1];
            //key:列号     value:行号_值
            outKey.set(column);
            outValue.set(row + "_" + valueStr);
            context.write(outKey,outValue);
        }
        
        
    }
    
    
    
}

Reducer

package step1;

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class Reducer1 extends Reducer<Text, Text, Text, Text> {
    private Text outKey = new Text();
    private Text outValue = new Text();
    
    
    // //key:列号     value:[行号_值,行号_值，行号_值...]
    @Override
    protected void reduce(Text key, Iterable<Text> values, Context context)
            throws IOException, InterruptedException {
        StringBuilder sb = new StringBuilder();
        for(Text text:values){
            sb.append(text + ",");
        }
        String line = null;
        if(sb.toString().endsWith(",")){
            line = sb.substring(0,sb.length()-1);
        }
        
        outKey.set(key);
        outValue.set(line);
        
        context.write(outKey, outValue);
    }
    
    
}

MR

package step1;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import com.sun.jersey.core.impl.provider.entity.XMLJAXBElementProvider.Text;

public class MR1 {

    //输入文件相对路径
    private static String inPath = "/matrix/step1_input/matrix2.txt";
    //输出文件的相对路径
    private static String outPath = "/matrix/step1_output";
    //hdfs地址
    private static String hdfs = "hdfs://master:9000";
    
    public int run(){
        try {
            //设置job配置类
            Configuration conf = new Configuration();
            //设置hdfs的地址
            conf.set("fs.default", hdfs);
            //创建一个job实例
            Job job = Job.getInstance(conf,"step1");
            
            //设置job的主类
            job.setJarByClass(MR1.class);
            //设置job的Mapper类和Reducer类
            job.setMapperClass(Mapper1.class);
            job.setReducerClass(Reducer1.class);
            
            //设置Mapper输出的类型
            job.setMapOutputKeyClass(Text.class);
            job.setMapOutputValueClass(Text.class);
            //设置Reducer输出的类型
            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(Text.class);
            
            FileSystem fs = FileSystem.get(conf);
            //设置输入和输出路径
            Path inputPath = new Path(inPath);
            if(fs.exists(inputPath)){
                FileInputFormat.addInputPath(job, inputPath);
            }
            
            //如果这个路径已经存在就把它删掉
            Path outputPath = new Path(outPath);
            fs.delete(outputPath,true);
            
            FileOutputFormat.setOutputPath(job, outputPath);
            
            try {
                return job.waitForCompletion(true)?1:-1;
            } catch (ClassNotFoundException e) {
                e.printStackTrace();
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
            
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        return -1;
    }
    
    public static void main(String args[]){
        int result = -1;
        result = new MR1().run();
        if(result == 1){
            System.out.println("step1运行成功...");
        }else{
            System.out.println("step1运行失败...");
        }
        
    }
    
}