MapReduce-二进制输入

Hadoop的MapReduce不只是可以处理文本信息，它还可以处理二进制格式的数据
1. 关于SequenceFileInputFormat类
Hadoop的顺序文件格式存储二进制的键/值对的序列。由于它们是可分割的(它们有同步点，所以reader可以从文件中的任意一点雨记录边界进行同步，例如分片的起点)，所以它们很符合MapReduce数据的格式要求，并且它们还支持压缩，可以使用一些序列化技术来存储任意类型。
如果要用顺序文件数据作为MapReduce的输入，应用SequenceFileInputFormat。键和值是由顺序文件决定，所以只需要保证map输入的类型匹配。
虽然从名称上看不出来，但SequenceFileInputFormat可以读MapFile(排序后的SequenceFile)和SequenceFile。如果在处理顺序文件时遇到目录，SequenceFileInputFormat类会认为自己正在读MapFile，使用的是其数据文件。

2. 关于SequenceFileAsTextInputFormat类
SequenceFileAsTextInputFormat是SequenceFileInputFormat的变体，它将顺序文件的键和值转换为Text对象。这个转换通过在键和值上调用toString方法实现。这个格式是顺序文件作为Streaming的合适的输入类型。

3. 关于SequenceFileAsBinaryInputFormat类
SequenceFileAsBinaryInputFormat是SequenceFileInputFormat的一种变体，它获取顺序文件的键和值作为二进制对象。它们被封装为BytesWritable对象，因而应用程序可以任意地解释这些字节数组。结合使用SequenceFile.Reader的appendRaw()方法或SequenceFileAsBinaryOutputFormat，它提供了在MapReduce中可以使用任意二进制数据类型的方法。

例子

将数据文件存为SequenceFile

package com.zhen.mapreduce.sequenceToText;
 
import java.io.IOException;
 
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.VLongWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
 
/**
 * @author FengZhen
 * @date 2018年8月18日
 * 输出为SequenceFile
 */
public class TextToSequence {
 
     public static void main(String[] args) throws Exception {        
            Configuration conf = new Configuration();
            Job job = Job.getInstance(conf);
            job.setJarByClass(TextToSequence.class);
 
            job.setMapperClass(WCMapper.class);
            job.setReducerClass(WCReducer.class);
 
            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(VLongWritable.class);        
 
            // 设置输出类
            job.setOutputFormatClass(SequenceFileOutputFormat.class);
 
            /**
             * 设置sequecnfile的格式，对于sequencefile的输出格式，有多种组合方式,
             * 从下面的模式中选择一种，并将其余的注释掉
             */
 
            // 组合方式1：不压缩模式
            SequenceFileOutputFormat.setOutputCompressionType(job, CompressionType.NONE);
 
            //组合方式2：record压缩模式，并指定采用的压缩方式 ：默认、gzip压缩等
            //        SequenceFileOutputFormat.setOutputCompressionType(job,
            //                CompressionType.RECORD);
            //        SequenceFileOutputFormat.setOutputCompressorClass(job,
            //                DefaultCodec.class);
 
 
            //组合方式3：block压缩模式，并指定采用的压缩方式 ：默认、gzip压缩等
            //        SequenceFileOutputFormat.setOutputCompressionType(job,
            //                CompressionType.BLOCK);
            //        SequenceFileOutputFormat.setOutputCompressorClass(job,
            //                DefaultCodec.class);
 
            FileInputFormat.addInputPaths(job, "hdfs://fz/user/hdfs/MapReduce/data/squenceFile/origin");
            SequenceFileOutputFormat.setOutputPath(job, new Path("hdfs://fz/user/hdfs/MapReduce/data/squenceFile/textToSequence/output"));
 
            System.exit(job.waitForCompletion(true)?0:1);
        }
        //map
        public static class WCMapper extends
        Mapper<LongWritable, Text, Text, VLongWritable> {
            public void map(LongWritable key, Text value, Context context)
                    throws IOException, InterruptedException {
                String[] split = value.toString().split("");
                for(String s : split){
                    context.write(new Text(s), new VLongWritable(1L));
                }            
            }
        }
        //reduce
        public static class WCReducer extends Reducer<Text, VLongWritable, Text, VLongWritable>{
            @Override
            protected void reduce(Text key, Iterable<VLongWritable> v2s, Context context)
                    throws IOException, InterruptedException {
 
                long sum=0;
 
                for(VLongWritable vl : v2s){
                    sum += vl.get();                 
                }
                context.write(key, new VLongWritable(sum));
            }
        }
}

读取SequenceFile存为Text

package com.zhen.mapreduce.sequenceToText;
 
import java.io.IOException;
 
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.VLongWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
 
/**
 * @author FengZhen
 * @date 2018年8月18日
 * 输入为SequenceFile
 */
public class SequenceToText extends Configured implements Tool{
 
    static class SequenceToTextMapper extends Mapper<Text, VLongWritable, Text, VLongWritable>{
        @Override
        protected void map(Text key, VLongWritable value,
                Mapper<Text, VLongWritable, Text, VLongWritable>.Context context)
                throws IOException, InterruptedException {
            String contents = value.toString();
            System.out.println(contents);
            context.write(key, value);
        }
    }
     
    static class SequenceToTextReducer extends Reducer<Text, VLongWritable, Text, VLongWritable>{
        @Override
        protected void reduce(Text key, Iterable<VLongWritable> value,
                Reducer<Text, VLongWritable, Text, VLongWritable>.Context context)
                throws IOException, InterruptedException {
            long sum = 0;
            while (value.iterator().hasNext()) {
                sum += Integer.parseInt(value.iterator().next().toString());
            }
            context.write(key, new VLongWritable(sum));
        }
    }
     
    public int run(String[] args) throws Exception {
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);
        job.setJobName("SequenceToText");
        job.setJarByClass(SequenceToText.class);
         
        job.setInputFormatClass(SequenceFileInputFormat.class);
        job.setOutputFormatClass(TextOutputFormat.class);
         
        job.setMapperClass(SequenceToTextMapper.class);
        job.setReducerClass(SequenceToTextReducer.class);
         
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(VLongWritable.class);
         
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(VLongWritable.class);
         
        SequenceFileInputFormat.setInputPaths(job, new Path(args[0]));
        TextOutputFormat.setOutputPath(job, new Path(args[1]));
         
        return job.waitForCompletion(true) ? 0 : 1;
    }
 
    public static void main(String[] args) throws Exception {
        String[] params = new String[]{"hdfs://fz/user/hdfs/MapReduce/data/squenceFile/textToSequence/output","hdfs://fz/user/hdfs/MapReduce/data/squenceFile/sequenceToText/output"};
        int exitCode = ToolRunner.run(new SequenceToText(), params);
        System.out.println(exitCode);
        System.exit(exitCode);
    }
     
}