每日总结
Mapreduce实例——单表join
依赖:
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>3.2.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-app</artifactId>
<version>3.2.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>3.2.0</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.30</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>3.2.0</version>
</dependency>
实验代码:
package mapreduce;
import java.io.IOException;
import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class DanJoin {
public static class Map extends Mapper<Object, Text, Text, Text> {
public void map(Object key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();
String[] arr = line.split("\t");
String mapkey = arr[0];
String mapvalue = arr[1];
String relationtype = new String();
relationtype = "1";
context.write(new Text(mapkey), new Text(relationtype + "+" + mapvalue));
//System.out.println(relationtype+"+"+mapvalue);
relationtype = "2";
context.write(new Text(mapvalue), new Text(relationtype + "+" + mapkey));
//System.out.println(relationtype+"+"+mapvalue);
}
}
public static class Reduce extends Reducer<Text, Text, Text, Text> {
public void reduce(Text key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
int buyernum = 0;
String[] buyer = new String[20];
int friendsnum = 0;
String[] friends = new String[20];
Iterator ite = values.iterator();
while (ite.hasNext()) {
String record = ite.next().toString();
int len = record.length();
int i = 2;
if (0 == len) {
continue;
}
char relationtype = record.charAt(0);
if ('1' == relationtype) {
buyer[buyernum] = record.substring(i);
buyernum++;
}
if ('2' == relationtype) {
friends[friendsnum] = record.substring(i);
friendsnum++;
}
}
if (0 != buyernum && 0 != friendsnum) {
for (int m = 0; m < buyernum; m++) {
for (int n = 0; n < friendsnum; n++) {
if (buyer[m] != friends[n]) {
context.write(new Text(buyer[m]), new Text(friends[n]));
}
}
}
}
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
String[] otherArgs = new String[2];
otherArgs[0] = "hdfs://hadoop102:8020/mymapreduce2/in/buyer1";
otherArgs[1] = "hdfs://hadoop102:8020/mymapreduce2/out3";
Job job = new Job(conf, " Table join");
job.setJarByClass(DanJoin.class);
job.setMapperClass(Map.class);
job.setReducerClass(Reduce.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}