推荐算法
package com.bw.mr;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import com.bw.advice1.Step5;
import com.bw.advice1.Step5.SMapper;
/**
* @author ghostwowo
* A:B,C,D,E,F,G
* B:C,E,F,D
* C:G:D
* D:F,H,J,K
* 电话号
* 求出每两个人的共同好友
* A-B:C,E,F,D
* B-A:C,E,F,D
*/
public class Friends {
public static class SMapper extends Mapper<LongWritable, Text, Text, Text>{
Pattern p = Pattern.compile("[:,]");
List<String> list = new ArrayList<String>();
// A:B,C,D,E,F,G
@Override
protected void setup(Mapper<LongWritable, Text, Text, Text>.Context context)
throws IOException, InterruptedException {
FileSystem fs = FileSystem.getLocal(context.getConfiguration());
Path[] paths = context.getLocalCacheFiles();
InputStream in = fs.open(paths[0]);
BufferedReader bf = new BufferedReader(new InputStreamReader(in));
String read = "";
while((read=bf.readLine())!=null) {
list.add(read);
}
bf.close();
in.close();
fs.close();
}
@Override
protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, Text>.Context context)
throws IOException, InterruptedException {
// List
// A:B,C,D,E,F,G
// C:G:D
// B:C,E,F,D
// D:F,H,J,K
// B:C,E,F,D
String strs[] = p.split(value.toString());
String uid = strs[0];
for(String s:list) {
if(!s.startsWith(strs[0])) {
//如果不是以这个字母开头的（A:b,c,d A:b,c,d）
String gthy = "";
for(int i=1;i<strs.length;i++) {
if(s.split(":")[1].indexOf(strs[i])>-1) {
//如果包含了，那么就保存起来
gthy+=","+strs[i];
}
}
int uid1Hashcode =s.split(":")[0].hashCode();
int uid2Hashcode = uid.hashCode();
context.write(new Text((uid1Hashcode*uid1Hashcode*uid1Hashcode+uid2Hashcode*uid2Hashcode*uid2Hashcode)+""),
new Text(s.split(":")[0]+"-"+uid+":"+gthy));
//1211323232323 A-B:E,F,G
//1211323232323 B-A:E,F,G
//A-B C
//B-A C
//A.HASHCODE*A.HASHCODE+D.HASHCODE*D.HASHCODE*D.HASHCODE*D.HASHCODE
//100 106
//B.HASHCODE+C.HASHCODE
// 102 104
//A-D D-A C-B B-C
}
}
}
}
public static class SReducer extends Reducer<Text, Text, Text, Text>{
@Override
protected void reduce(Text arg0, Iterable<Text> arg1, Reducer<Text, Text, Text, Text>.Context arg2)
throws IOException, InterruptedException {
//A-B:
String w = arg1.iterator().next().toString();
String strs[] = w.split(":");
if(strs.length>1) {
arg2.write(new Text(strs[0]), new Text(strs[1]));
}
}
}
public static void main(String []args) throws Exception{
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(Friends.class);
job.setMapperClass(SMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setReducerClass(SReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
FileInputFormat.addInputPath(job, new Path("friends.txt"));
job.addCacheFile(new URI("friends.txt"));
FileOutputFormat.setOutputPath(job, new Path("friendsRes"));
job.waitForCompletion(true);
}
}
建一个friend.txt
A:B,C,D,F,E,O
B:A,C,E,K
C:F,A,D,I
D:A,E,F,L
E:B,C,D,M,L
F:A,B,C,D,E,O,M
G:A,C,D,E,F
H:A,C,D,E,O
I:A,O
J:B,O
K:A,C,D
L:D,E,F
M:E,F,G
O:A,H,I,J
结果是
B-D ,A,E,F,L
E-A ,B,C,D,F,E,O
D-C ,F,A,D,I
F-A ,B,C,D,F,E,O
F-B ,A,C,E,K
C-F ,A,B,C,D,E,O,M
G-B ,A,C,E,K
F-D ,A,E,F,L
G-C ,F,A,D,I
B-H ,A,C,D,E,O
G-D ,A,E,F,L
B-I ,A,O
A-J ,B,O
D-H ,A,C,D,E,O
C-I ,A,O
F-G ,A,C,D,E,F
I-D ,A,E,F,L
K-B ,A,C,E,K
H-F ,A,B,C,D,E,O,M
A-M ,E,F,G
H-G ,A,C,D,E,F
I-F ,A,B,C,D,E,O,M
E-J ,B,O
K-D ,A,E,F,L
L-C ,F,A,D,I
B-M ,E,F,G
I-G ,A,C,D,E,F
F-J ,B,O
H-I ,A,O
F-K ,A,C,D
E-L ,D,E,F
K-G ,A,C,D,E,F
O-B ,A,C,E,K
C-O ,A,H,I,J
K-H ,A,C,D,E,O
G-L ,D,E,F
O-D ,A,E,F,L
K-I ,A,O
G-M ,E,F,G
F-O ,A,H,I,J
O-G ,A,C,D,E,F
O-H ,A,C,D,E,O
I-O ,A,H,I,J
O-K ,A,C,D
