HDFS小文件处理——Mapper处理
处理小文件的时候,可以通过org.apache.hadoop.io.SequenceFile.Writer类将所有文件写出到一个seq文件中。
大致流程如下:
实现代码:
package study.smallfile.sequence_one; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.SequenceFile.CompressionType; import org.apache.hadoop.io.SequenceFile.Writer; import org.apache.hadoop.io.SequenceFile.Writer.Option; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; public class MapperDemo { private static final String INPUT_PATH = "hdfs://cluster1/smallfile/blankfile"; private static final String OUT_PATH = "hdfs://cluster1/smallfile/combined/map"; static FileSystem fileSystem; public void CombinedFile() throws Exception { Job job = Job.getInstance(); job.setJarByClass(MapperDemo.class); job.setJobName(MapperDemo.class.getSimpleName()); // 设置map类 job.setMapperClass(MapperDemo.CombinedMapper.class); // 设置输出 job.setOutputKeyClass(Text.class); job.setOutputValueClass(BytesWritable.class); // 设置reduce任务数量 job.setNumReduceTasks(0); // 设置输入路径 FileInputFormat.setInputPaths(job, new Path(INPUT_PATH)); // 检查输出路径 Path outdir = new Path(OUT_PATH); fileSystem = FileSystem.get(job.getConfiguration()); if (fileSystem.exists(outdir)) {// 如果已经存在删除 fileSystem.delete(outdir, true); } // 设置输出路径 FileOutputFormat.setOutputPath(job, outdir); job.waitForCompletion(true); } static class CombinedMapper extends Mapper<LongWritable, Text, Text, BytesWritable> { Writer writer = null; FileStatus[] files; Text outKey = new Text(); BytesWritable outValue = new BytesWritable(); FSDataInputStream in; byte[] buffer = null; @Override protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, BytesWritable>.Context context) throws IOException, InterruptedException { // for (FileStatus file : files) { // outKey.set(file.getPath().toString()); // // in = fileSystem.open(file.getPath()); // buffer = new byte[(int) file.getLen()]; // IOUtils.read(in, buffer, 0, buffer.length); // outValue.set(new BytesWritable(buffer)); // writer.append(outKey, outValue); // } } @Override protected void cleanup( Mapper<LongWritable, Text, Text, BytesWritable>.Context context) throws IOException, InterruptedException { for (FileStatus file : files) { outKey.set(file.getPath().toString()); in = fileSystem.open(file.getPath()); buffer = new byte[(int) file.getLen()]; IOUtils.readFully(in, buffer, 0, buffer.length); outValue.set(new BytesWritable(buffer)); writer.append(outKey, outValue); } IOUtils.closeStream(writer); } @Override protected void setup( Mapper<LongWritable, Text, Text, BytesWritable>.Context context) throws IOException, InterruptedException { // 输出文件项 Option fileOption = SequenceFile.Writer.file(new Path(OUT_PATH + "/mapper.seq")); // 压缩选项 Option compressionOption = SequenceFile.Writer .compression(CompressionType.BLOCK); // SequeneFile key类型设置 Option keyClassOption = SequenceFile.Writer.keyClass(Text.class); // SequeneFile value类型设置 Option valueClassOption = SequenceFile.Writer .valueClass(BytesWritable.class); // 构建输出流文件 Configuration conf = new Configuration(); writer = SequenceFile.createWriter(conf, fileOption, compressionOption, keyClassOption, valueClassOption); if (fileSystem == null) { fileSystem = FileSystem.get(conf); } files = fileSystem.listStatus(new Path("hdfs://cluster1/smallfile/logs")); } } }
注意事项:
我原本的逻辑是放到map函数中,将所有文件通过Writer写到HDFS中,但是map在整个mr的执行中被调用的次数是根据输入文件情况确定的,通过控制输入文件的情况,可以通过map函数实现
发现问题:
原本在实现之前,定义了一个FileSystem类型的静态字段,在提交job前已经赋值了,但是,在mapper类中访问到的fileSystem字段,是空值,有知道的大虾,多多指导小弟
SequenceFile介绍:
http://wiki.apache.org/hadoop/SequenceFile
http://www.cnblogs.com/zhenjing/archive/2012/11/02/File-Format.html