import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DefaultStringifier;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordWriter;
import org.apache.hadoop.mapred.TextOutputFormat;
import org.apache.hadoop.mapred.lib.MultipleOutputFormat;
import org.apache.hadoop.mapred.lib.MultipleOutputs;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.Mapper.Context;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.LineReader;
import org.apache.hadoop.util.Progressable;
class Help {
static final boolean DEBUG = false;
public static void debug(Object o, String s) {
if (DEBUG) {
System.out.println(s + ":" + o.toString());
}
}
public static List<ArrayList<Double>> getOldCenters(String inputPath) {
List<ArrayList<Double>> result = new ArrayList<ArrayList<Double>>();
Configuration conf = new Configuration();
try {
FileSystem hdfs = FileSystem.get(conf);
Path inPath = new Path(inputPath);
FSDataInputStream fsIn = hdfs.open(inPath);
LineReader lineIn = new LineReader(fsIn, conf);
Text line = new Text();
while (lineIn.readLine(line) > 0) {
String record = line.toString();
String[] fields = record.split(",");
List<Double> tmpList = new ArrayList<Double>();
for (int i = 0; i < fields.length; i++)
tmpList.add(Double.parseDouble(fields[i]));
result.add((ArrayList<Double>) tmpList);
}
fsIn.close();
} catch (IOException e) {
e.printStackTrace();
}
return result;
}
public static void deleteLastResult(String path) {
Configuration conf = new Configuration();
try {
FileSystem hdfs = FileSystem.get(conf);
Path inPath = new Path(path);
hdfs.delete(inPath);
} catch (IOException e) {
}
}
public static void copyOriginalCenters(String src, String dst) {
Configuration conf = new Configuration();
try {
FileSystem hdfs = FileSystem.get(conf);
hdfs.copyFromLocalFile(new Path(src), new Path(dst));
} catch (IOException e) {
}
}
public static boolean isFinished(String oldPath, String newPath,
String KPath, String dtBegIdxPath, double threshold)
throws IOException {
int dataBeginIndex = Integer.parseInt(dtBegIdxPath);
int K = Integer.parseInt(KPath);
List<ArrayList<Double>> oldCenters = Help.getOldCenters(oldPath);
List<ArrayList<Double>> newCenters = new ArrayList<ArrayList<Double>>();
Configuration conf = new Configuration();
FileSystem hdfs = FileSystem.get(conf);
for (int t = 0; t < K; t++) {
Path inPath = new Path(newPath + t);
if (!hdfs.exists(inPath))
break;
FSDataInputStream fsIn = hdfs.open(inPath);
LineReader lineIn = new LineReader(fsIn, conf);
Text line = new Text();
while (lineIn.readLine(line) > 0) {
String tmp = line.toString();
Help.debug("tmp", tmp);
if(tmp.length()<5)//处理在集群上出现的key与value不在一行的情况
{
lineIn.readLine(line);
tmp = line.toString();
String []fields = tmp.split(",");
List<Double> tmpList = new ArrayList<Double>();
for (int i = 0; i < fields.length; i++)
tmpList.add(Double.parseDouble(fields[i]));
newCenters.add((ArrayList<Double>) tmpList);
continue;
}
String[] tmpLine = tmp.split(" ");
Help.debug(tmpLine[1].toString(), tmpLine.toString());
String record = tmpLine[1];
String[] fields = record.split(",");
List<Double> tmpList = new ArrayList<Double>();
for (int i = 0; i < fields.length; i++)
tmpList.add(Double.parseDouble(fields[i]));
newCenters.add((ArrayList<Double>) tmpList);
}
fsIn.close();
}
// System.out.println("oldCenter size:"+oldCenters.size()+"\nnewCenters size:"+newCenters.size());
double distance = 0;
for (int i = 0; i < K; i++) {
for (int j = dataBeginIndex; j < oldCenters.get(0).size(); j++) {
double t1 = Math.abs(oldCenters.get(i).get(j));
double t2 = Math.abs(newCenters.get(i).get(j));
distance += Math.pow((t1 - t2) / (t1 + t2), 2);
}
}
if (distance <= threshold) {
return true;
}
Help.deleteLastResult(oldPath);
FSDataOutputStream os = hdfs.create(new Path(oldPath));
for (int i = 0; i < newCenters.size(); i++) {
String text = "";
for (int j = 0; j < newCenters.get(i).size(); j++) {
if (j == 0)
text += newCenters.get(i).get(j);
else
text += "," + newCenters.get(i).get(j);
}
text += "\n";
os.write(text.getBytes(), 0, text.length());
}
os.close();
// ///////////////////////////
return false;
}
}
public class Kmeans {
// static List<ArrayList<Double>> centers ;
// static int K;
// static int dataBeginIndex;
public static class KmeansMapper extends
Mapper<Object, Text, IntWritable, Text> {
public void map(Object key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();
String[] fields = line.split(",");
List<ArrayList<Double>> centers = Help.getOldCenters(context
.getConfiguration().get("centersPath"));
int dataBeginIndex = Integer.parseInt(context.getConfiguration()
.get("dtBegIdxPath"));
int K = Integer.parseInt(context.getConfiguration().get("KPath"));
double minDistance = 99999999;
int centerIndex = K;
for (int i = 0; i < K; i++) {
double currentDistance = 0;
for (int j = dataBeginIndex; j < fields.length; j++) {
double t1 = Math.abs(centers.get(i).get(j));
double t2 = Math.abs(Double.parseDouble(fields[j]));
currentDistance += Math.pow((t1 - t2) / (t1 + t2), 2);
}
Help.debug(currentDistance, "currentDistance");
if (minDistance > currentDistance) {
minDistance = currentDistance;
centerIndex = i;
}
}
IntWritable centerId = new IntWritable(centerIndex+1);
Text tValue = new Text();
tValue.set(value);
context.write(centerId, tValue);
}
}
public static class KmeansReducer extends
Reducer<IntWritable, Text, IntWritable, Text> {
public void reduce(IntWritable key, Iterable<Text> values,
Context context) throws IOException, InterruptedException {
List<ArrayList<Double>> helpList = new ArrayList<ArrayList<Double>>();
String tmpResult = "";
for (Text val : values) {
String line = val.toString();
String[] fields = line.split(",");
List<Double> tmpList = new ArrayList<Double>();
for (int i = 0; i < fields.length; i++) {
tmpList.add(Double.parseDouble(fields[i]));
}
helpList.add((ArrayList<Double>) tmpList);
}
// System.out.println(helpList.size());
// for(int i=0;i<helpList.size();i++)
// System.out.println(helpList.get(i));
for (int i = 0; i < helpList.get(0).size(); i++) {
double sum = 0;
for (int j = 0; j < helpList.size(); j++) {
sum += helpList.get(j).get(i);
}
double t = sum / helpList.size();
if (i == 0)
tmpResult += t;
else
tmpResult += "," + t;
}
Text result = new Text();
result.set(tmpResult);
int tmpKey = Integer.parseInt(key.toString());
context.write(new IntWritable(tmpKey), result);
}
}
static void runKmeans(String[] args, boolean isReduce) throws IOException,
ClassNotFoundException, InterruptedException {
Configuration conf = new Configuration();
String[] otherArgs = new GenericOptionsParser(conf, args)
.getRemainingArgs();
if (otherArgs.length != 7) {
System.err
.println("Usage: Kmeans <in> <out> <localOriginalCentersPath> <oldCentersPath> <newCentersPath> <dataBeginIndex> <K>");
System.exit(2);
}
conf.setStrings("centersPath", otherArgs[3]);
conf.setStrings("dtBegIdxPath", otherArgs[5]);
conf.setStrings("KPath", otherArgs[6]);
Job job = new Job(conf, "kmeans");
job.setJarByClass(Kmeans.class);
job.setMapperClass(KmeansMapper.class);
job.setNumReduceTasks(Integer.parseInt(args[6]));
// 判断是否需要执行Reduce
if (isReduce) {
job.setReducerClass(KmeansReducer.class);
}
job.setOutputKeyClass(IntWritable.class);
job.setOutputValueClass(Text.class);
FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
// delete last result
Help.deleteLastResult(otherArgs[1]);
// System.exit(job.waitForCompletion(true)?0:1);
job.waitForCompletion(true);
}
/**
*
* @param in
* - args[0] out - args[1] localOriginalCentersPath - args[2]
* oldCentersPath - args[3] newCentersPath - args[4]
* dataBeginIndex - args[5] K - args[6]
* @throws Exception
*/
public static void main(String[] args) throws Exception {
Help.deleteLastResult(args[3]);
Help.copyOriginalCenters(args[2], args[3]);
int count=1;
// runKmeans(args, true);
while (true) {
System.out.println("迭代的轮次： "+count++);
runKmeans(args, true);
if (Help.isFinished(args[3], args[4], args[6], args[5], 0.0)) {
runKmeans(args, false);
break;
}
}
}
}
