【Hadoop离线基础总结】MapReduce自定义InputFormat和OutputFormat案例
MapReduce自定义InputFormat和OutputFormat案例
自定义InputFormat 合并小文件
-
需求
无论hdfs还是mapreduce,存放小文件会占用元数据信息,白白浪费内存,实践中,又难免面临处理大量小文件的场景 -
优化小文件的三种方式
1.在数据采集的时候,就将小文件或小批数据合成大文件再上传HDFS
2.在业务处理之前,在HDFS上使用mapreduce程序对小文件进行合并
3.在mapreduce处理时,可采用combineInputFormat提高效率 -
用代码实现第二种方式
自定义InputFormat
package cn.itcast.demo3;
import jdk.nashorn.internal.ir.Splittable;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import java.io.IOException;
import java.util.List;
public class MyInputFormat extends FileInputFormat<NullWritable, BytesWritable> {
@Override
public RecordReader<NullWritable, BytesWritable> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
MyRecordReader myRecordReader = new MyRecordReader();
myRecordReader.initialize(split, context);
return myRecordReader;
}
/**
* 表示我们的文件是否可切分
* 返回false表示我们的文件不可切分,读取文件时会一次性将文件内容全部读取出来
*
* @param context
* @param filename
* @return
*/
@Override
protected boolean isSplitable(JobContext context, Path filename) {
return false;
}
}
自定义RecordReader
package cn.itcast.demo3;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import java.io.IOException;
import java.io.InputStream;
public class MyRecordReader extends RecordReader<NullWritable, BytesWritable> {
//定义文件切片
private FileSplit fileSplit;
//定义文件configuration
private Configuration configuration;
//定义v2
private BytesWritable bytesWritable = new BytesWritable();
//定义下面nextKeyValue返回值为false
private boolean nextKeyValue = false;
/**
* 初始化方法
* 这里可以拿到文件切片,也就意味着可以拿到文件,将文件转换为字节数组
*
* @param split
* @param context
* @throws IOException
* @throws InterruptedException
*/
@Override
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
//获取文件切片
this.fileSplit = (FileSplit) split;
//获取文件Configuration
this.configuration = context.getConfiguration();
}
/**
* 返回true,表示文件读取完成,不会再往下继续读取文件
* 返回false,表示会继续往下读取文件
*
* @return
* @throws IOException
* @throws InterruptedException
*/
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
if (!nextKeyValue) {
//根据文件的切片,将文件的内容全部读取出来,封装到BytesWritable中
byte[] fileContent = new byte[(int) fileSplit.getLength()];
//获取文件切片路径
Path path = fileSplit.getPath();
//获取文件系统
FileSystem fileSystem = path.getFileSystem(configuration);
//打开文件输入流
FSDataInputStream inputStream = fileSystem.open(path);
//将输入流转到字节数组中
IOUtils.readFully(inputStream, fileContent, 0, (int) fileSplit.getLength());
bytesWritable.set(fileContent, 0, fileContent.length);
//将读取文件的标识设置为true,表示文件已经读取完成,不需要继续读取
nextKeyValue = true;
IOUtils.closeStream(inputStream);
return nextKeyValue;
}
return false;
}
/**
* 用来返回k1的值
*
* @return
* @throws IOException
* @throws InterruptedException
*/
@Override
public NullWritable getCurrentKey() throws IOException, InterruptedException {
return NullWritable.get();
}
/**
* 用来返回v1的值
*
* @return
* @throws IOException
* @throws InterruptedException
*/
@Override
public BytesWritable getCurrentValue() throws IOException, InterruptedException {
return bytesWritable;
}
/**
* 不太需要注意,就是用来读取运行进度的
*
* @return
* @throws IOException
* @throws InterruptedException
*/
@Override
public float getProgress() throws IOException, InterruptedException {
return nextKeyValue ? 1.0F : 0.0F;
}
/**
* 用来读取完后释放资源的,了解即可
*
* @throws IOException
*/
@Override
public void close() throws IOException {
}
}
定义一个Mapper类
package cn.itcast.demo3;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import java.io.IOException;
public class MyMapperInput extends Mapper<NullWritable, BytesWritable, Text, BytesWritable> {
@Override
protected void map(NullWritable key, BytesWritable value, Context context) throws IOException, InterruptedException {
//获取文件切片
FileSplit inputSplit = (FileSplit) context.getInputSplit();
//获取文件名称
String name = inputSplit.getPath().getName();
//输出k2,v2
context.write(new Text(name), value);
}
}
程序main函数入口
package cn.itcast.demo3;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
public class OwnInputFormatMain extends Configured implements Tool {
@Override
public int run(String[] args) throws Exception {
//创建job对象
Job job = Job.getInstance(super.getConf(), "ownInputFormat");
//输入数据,设置输入路径,注意这里是自动以的InputFormat
job.setInputFormatClass(MyInputFormat.class);
MyInputFormat.addInputPath(job, new Path("file:////Volumes/赵壮备份/大数据离线课程资料/5.大数据离线第五天/自定义inputformat_小文件合并/input"));
//自定义map逻辑
job.setMapperClass(MyMapperInput.class);
//设置k2,v2输出类型
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(BytesWritable.class);
//虽然没有reducer,但是不设置reduce输出类型,默认的是<LongWritable,Text>
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(BytesWritable.class);
//输出数据,设置输出路径
job.setOutputFormatClass(SequenceFileOutputFormat.class);
SequenceFileOutputFormat.setOutputPath(job,new Path("file:////Volumes/赵壮备份/大数据离线课程资料/5.大数据离线第五天/自定义inputformat_小文件合并/sequence_output"));
//提交任务到集群
boolean b = job.waitForCompletion(true);
return b ? 0 : 1;
}
public static void main(String[] args) throws Exception {
int run = ToolRunner.run(new Configuration(), new OwnInputFormatMain(), args);
System.exit(run);
}
}
自定义OutputFormat 将一个文件中的数据分发到不同文件
-
需求
将订单的好评与差评区分开来,并将最终的数据发送到不同的文件夹下面去,其中数据第九个字段表示好评,中评,差评。0:好评,1:中评,2:差评 -
代码实现
自定义OutputFormat
package cn.itcast.demo4;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
public class MyOutputFormat extends FileOutputFormat<Text, NullWritable> {
@Override
public RecordWriter<Text, NullWritable> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
//从这个方法里就可以获取一个configuration
Configuration configuration = context.getConfiguration();
//获取文件系统
FileSystem fileSystem = FileSystem.get(configuration);
//设置好评文件的输出路径
Path goodComment = new Path("/Volumes/赵壮备份/大数据离线课程资料/5.大数据离线第五天/自定义outputformat/myGoodComment/1.txt");
//设置差评文件的输出路径
Path badComment = new Path("/Volumes/赵壮备份/大数据离线课程资料/5.大数据离线第五天/自定义outputformat/myBadComment/1.txt");
//获取文件输出流
FSDataOutputStream fsDataOutputStream = fileSystem.create(goodComment);
FSDataOutputStream fsDataOutputStream1 = fileSystem.create(badComment);
MyRecordWriter myRecordWriter = new MyRecordWriter(fsDataOutputStream, fsDataOutputStream1);
return myRecordWriter;
}
}
自定义RecordWriter
package cn.itcast.demo4;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import java.io.IOException;
public class MyRecordWriter extends RecordWriter<Text, NullWritable> {
//使用无参和带参构造调用goodStream和badStream
private FSDataOutputStream goodStream;
private FSDataOutputStream badStream;
public MyRecordWriter() {
}
public MyRecordWriter(FSDataOutputStream goodStream, FSDataOutputStream badStream) {
this.goodStream = goodStream;
this.badStream = badStream;
}
/**
* 这个write方法就是往外写出去数据
*
* @param key 可以根据这个key,来判断文件究竟往哪个目录下写
* @param value
* @throws IOException
* @throws InterruptedException
*/
@Override
public void write(Text key, NullWritable value) throws IOException, InterruptedException {
//分割导入的数据
String[] split = key.toString().split("\t");
//获取评论状态 0:好评 1:中评 2:差评;
//判断评论状态,如果小于等于1则写到好评文件中,否则写到差评文件中
if (Integer.parseInt(split[9]) <= 1) {
goodStream.write(key.toString().getBytes());
goodStream.write("\r\n".getBytes());
} else {
badStream.write(key.toString().getBytes());
badStream.write("\r\n".getBytes());
}
}
@Override
public void close(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
IOUtils.closeStream(goodStream);
IOUtils.closeStream(badStream);
}
}
定义一个Mapper类
package cn.itcast.demo4;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
public class MyOutputMapper extends Mapper<LongWritable, Text, Text, NullWritable> {
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
context.write(value, NullWritable.get());
}
}
程序main函数入口
package cn.itcast.demo4;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
public class MyOutputMain extends Configured implements Tool {
@Override
public int run(String[] args) throws Exception {
//创建job对象
Job job = Job.getInstance(super.getConf(), "OutputFormat");
//输入数据,设置输入路径
job.setInputFormatClass(TextInputFormat.class);
TextInputFormat.setInputPaths(job, new Path("file:////Volumes/赵壮备份/大数据离线课程资料/5.大数据离线第五天/自定义outputformat/input/ordercomment.csv"));
//自定义map逻辑
job.setMapperClass(MyOutputMapper.class);
//设置k2,v2输出类型
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(NullWritable.class);
//输出数据,设置输出路径,这里的输出路径不是真正的输出路径
job.setOutputFormatClass(MyOutputFormat.class);
MyOutputFormat.setOutputPath(job, new Path("file:////Volumes/赵壮备份/大数据离线课程资料/5.大数据离线第五天/自定义outputformat/output"));
//提交任务至集群
boolean b = job.waitForCompletion(true);
return b ? 0 : 1;
}
public static void main(String[] args) throws Exception {
int run = ToolRunner.run(new Configuration(), new MyOutputMain(), args);
System.exit(run);
}
}