MapReduce Unit Test

　　以前用java写MR程序总不习惯写单元测试，就是查错也只是在小规模数据上跑一下程序。昨天工作时，遇到一个bug，查了好久也查出来。估计是业务逻辑上的错误。后来没办法，只好写了个单元测试，一步步跟踪，瞬间找到问题所在。所以说，工作中还是要勤快些。

 

import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
import org.apache.hadoop.mrunit.types.Pair;
import org.junit.Before;
import org.junit.Test;
import com.wanda.predict.GenerateCustomerNatureFeature.NatureFeatureMappper;
import com.wanda.predict.GenerateCustomerNatureFeature.NatureReducer;
import com.wanda.predict.pojo.Settings;
 
/**
 * MapReduce 单元测试的模板 , 依赖于junit环境(junit.jar)， mrunit.jar , mockito.jar
 * 
 */
public class MapperReducerUnitTest {
    // 一些设置，与正常的mr程序一样，不过这里主要是加载一些信息。性能优化之类的就不要在单元测试里设置了。
    Configuration conf = new Configuration();
    //Map.class 的测试驱动类
    MapDriver<LongWritable, Text, Text, Text> mapDriver;
    //Reduce.class 的测试驱动类
    ReduceDriver<Text, Text, Text, Text> reduceDriver;
    //Map.calss 、 Reduce.class转接到一起的流程测试驱动
    MapReduceDriver<LongWritable, Text, Text, Text, Text, Text> mapReduceDriver;
    
    @Before
    public void setUp() {
 
        //测试mapreduce
        NatureFeatureMappper mapper = new NatureFeatureMappper();
        NatureReducer reducer = new NatureReducer();
        //添加要测试的map类
        mapDriver = MapDriver.newMapDriver(mapper);
        //添加要测试的reduce类
        reduceDriver = ReduceDriver.newReduceDriver(reducer);
        //添加map类和reduce类
        mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
         
        //测试配置参数
        conf.setInt(Settings.TestDataSize.getName(), 1);
        conf.setInt(Settings.TrainDataSize.getName(), 6);
        //driver之间是独立的，谁用到谁就设置conf 
        reduceDriver.setConfiguration(conf);
        mapReduceDriver.setConfiguration(conf);
    }
 
    @Test
    public void testMapper() throws IOException {
        mapDriver.withInput(new LongWritable(), new Text("map的输入"));
        mapDriver.withOutput(new Text("期望的key"), new Text("期望的value"));
        
        //打印实际结果
        List<Pair<Text , Text>> result = mapDriver.run();
        for(Pair<Text , Text> kv : result){
            System.out.println("mapper : " + kv.getFirst());
            System.out.println("mapper : " + kv.getSecond());
        }
        //进行case测试，对比输入输出结果
        mapDriver.runTest();
    }
 
    @Test
    public void testReducer() throws IOException {
        List<Text> values = new ArrayList<Text>();
        values.add(new Text("输入"));
        reduceDriver.withInput(new Text("输入"), values);
        reduceDriver.withOutput(new Text("期望的输出"), new Text("期望的输出"));
        reduceDriver.runTest();
    }
     
    @Test
    public void testMapperReducer() throws IOException {
        mapReduceDriver.withInput(new LongWritable(), new Text("输入"));
        mapReduceDriver.withOutput(new Text("期望的输出"), new Text("期望的输出"));
        //打印实际结果
        List<Pair<Text, Text>> list =  mapReduceDriver.run();
        System.out.println("mapreducedriver size:" + list.size());
        for(Pair<Text , Text> lst : list){
            System.out.println(lst.getFirst());
            System.out.println(lst.getSecond());
        }
        //进行case测试，对比输入输出结果
        mapReduceDriver.runTest();
    }
 
    @Test
    public void testMapperCount() throws IOException {
        mapDriver.withInput(new LongWritable(), new Text("输入"));
        mapDriver.withOutput(new Text("期望的输出"), new Text("期望的输出"));
        mapDriver.runTest();
        //判断 map中的counter值是否与期望的相同
        assertEquals("Expected 1 counter increment", 1, mapDriver.getCounters().findCounter("data", "suc").getValue());
    }
}