MapReduce实现矩阵相乘
矩阵相乘能够查看百度百科的解释http://baike.baidu.com/view/2455255.htm?fr=aladdin
有a和b两个矩阵
a: 1 2 3
4 5 0
7 8 9
10 11 12
b: 10 15
0 2
11 9
c = a*b
1:将矩阵存到HDFS中:
矩阵a:
行 | 列 | 值 | hdfs存储 |
1 | 1 | 1 | 1,1,1 |
1 | 2 | 2 | 1,2,2 |
1 | 3 | 3 | 1,3,3 |
2 | 1 | 4 | 2,1,4 |
2 | 2 | 5 | 2,2,5 |
2 | 3 | 0 | 0不存储 |
3 | 1 | 7 | 3,1,7 |
3 | 2 | 8 | 3,2,8 |
3 | 3 | 9 | 3,3,9 |
4 | 1 | 10 | 4,1,10 |
4 | 2 | 11 | 4,2,11 |
4 | 3 | 12 | 4,3,12 |
矩阵b:
行 | 列 | 值 | hdfs存储 |
1 | 1 | 10 | 1,1,10 |
1 | 2 | 15 | 1,2,15 |
2 | 1 | 0 | 0不存储 |
2 | 2 | 2 | 2,2,2 |
3 | 1 | 11 | 3,1,11 |
3 | 2 | 9 | 3,2,9 |
2:a的map读取
读取第一个值是1,1,1。它是矩阵a的第一行第一列。那么它要在计算c(1,1) c(1,2)的时候使用(这里c仅仅用2列,假设用n列,那么它的值要在计算c(1,1),c(1,2),c(1,3)...c(1,n)的时候使用)。我们就以 key = 1,1 value = a,1,1 , key= 1,2 value = a,1,1输出两条数据(1,1) (1,2)是 c(1,1) ,c(1,2)的坐标。
b的map读取
读取第一个值是1,1,10。它是矩阵b的第一行第一列。那么它要在计算c(1,1) c(2,1) c(3,1) c(4,1)的时候使用(这里c仅仅用4行,假设用m行,那么它的值要在计算c(1,1),c(2,1),c(3,1)...c(m,1)的时候使用)。我们就以 key = 1,1 value = b,1,10 , key= 2,1 value = b,1,10 , key = 3,1 value = b,1,10 ,
key= 4,1 value = b,1,10输出
3:reduce读取计算
通过mapA和mapB的输出能够得到 key = 1,1 , value=a,1,1 value=b,1,10 value=a,2,2 value=a,3,3 value = b,3,11l来计算c(1,1)的值
代码例如以下:
package MyMatrix; import java.io.IOException; import java.util.Iterator; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.MultipleInputs; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; public class MartrixMultiply{ /** * 最后得到的矩阵的列数 */ public static final int COL_COUNT = 2; /** * 最后得到的矩阵的行数 */ public static final int ROW_COUNT = 4; /** * A矩阵的列数或者是B矩阵的行数 */ public static final int BROW_ACOL= 3; public static class MartrixMaperA extends Mapper<LongWritable, Text, Text, Text>{ @Override protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { String valueStr = value.toString(); String[] items = valueStr.split(","); int rowIndex = Integer.parseInt(items[0]); int colIndex = Integer.parseInt(items[1]); int valueInt = Integer.parseInt(items[2]); Text outKey = null; Text outValue = null; for(int i=0;i<COL_COUNT;i++){ outKey = new Text(rowIndex + "," + (i+1)); outValue = new Text("a,"+colIndex+","+valueInt); context.write(outKey, outValue); } } } public static class MartrixMaperB extends Mapper<LongWritable, Text, Text, Text>{ @Override protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { String valueStr = value.toString(); String[] items = valueStr.split(","); int rowIndex = Integer.parseInt(items[0]); int colIndex = Integer.parseInt(items[1]); int valueInt = Integer.parseInt(items[2]); Text outKey = null; Text outValue = null; for(int i=0;i<ROW_COUNT;i++){ outKey = new Text((i+1) + "," + colIndex); outValue = new Text("b,"+rowIndex+","+valueInt); context.write(outKey, outValue); } } } public static class MartrixReducer extends Reducer<Text,Text,Text,IntWritable>{ @Override protected void reduce(Text key, Iterable<Text> values,Context context) throws IOException, InterruptedException { String[] items = new String[3]; int[] valueA = new int[BROW_ACOL]; int[] valueB = new int[BROW_ACOL]; Iterator<Text> it = values.iterator(); while(it.hasNext()){ items = it.next().toString().split(","); if(items[0].equals("a")){ valueA[Integer.parseInt(items[1])-1] = Integer.parseInt(items[2]); }else if(items[0].equals("b")){ valueB[Integer.parseInt(items[1])-1] = Integer.parseInt(items[2]); } } int result = 0; for(int i=0;i<BROW_ACOL;i++){ result += valueA[i]*valueB[i]; } context.write(key, new IntWritable(result)); } } @SuppressWarnings("deprecation") public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException { Path pathA = new Path("hdfs://localhost:9000/Martrix/a.txt"); Path pathB = new Path("hdfs://localhost:9000/Martrix/b.txt"); Path pathOut = new Path("hdfs://localhost:9000/Martrix/out"); Configuration conf = new Configuration(); Job job = new Job(conf,"MartrixMultiply"); job.setJarByClass(MartrixMultiply.class); MultipleInputs.addInputPath(job, pathA, TextInputFormat.class, MartrixMaperA.class); MultipleInputs.addInputPath(job, pathB, TextInputFormat.class, MartrixMaperB.class); job.setReducerClass(MartrixReducer.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileOutputFormat.setOutputPath(job, pathOut); if(job.waitForCompletion(true)){ System.exit(0); }else{ System.exit(1); } } }