flink PageRank详解（批量迭代的页面排名算法的基本实现）
1、PageRank算法原理
2、基本数据准备
/** * numPages缺省15个测试页面 * * EDGES表示从一个pageId指向相连的另外一个pageId */ public class PageRankData { public static final Object[][] EDGES = { {1L, 2L}, {1L, 15L}, {2L, 3L}, {2L, 4L}, {2L, 5L}, {2L, 6L}, {2L, 7L}, {3L, 13L}, {4L, 2L}, {5L, 11L}, {5L, 12L}, {6L, 1L}, {6L, 7L}, {6L, 8L}, {7L, 1L}, {7L, 8L}, {8L, 1L}, {8L, 9L}, {8L, 10L}, {9L, 14L}, {9L, 1L}, {10L, 1L}, {10L, 13L}, {11L, 12L}, {11L, 1L}, {12L, 1L}, {13L, 14L}, {14L, 12L}, {15L, 1L}, }; private static int numPages = 15; public static DataSet<Tuple2<Long, Long>> getDefaultEdgeDataSet(ExecutionEnvironment env) { List<Tuple2<Long, Long>> edges = new ArrayList<Tuple2<Long, Long>>(); for (Object[] e : EDGES) { edges.add(new Tuple2<Long, Long>((Long) e[0], (Long) e[1])); } return env.fromCollection(edges); } public static DataSet<Long> getDefaultPagesDataSet(ExecutionEnvironment env) { return env.generateSequence(1, 15); } public static int getNumberOfPages() { return numPages; } }
3、算法实现
/**
* @Description: 使用批量迭代的页面排名算法的基本实现。
* 此实现需要一组页面和一组有向链接作为输入，并按如下方式工作。
* 在每次迭代中，每个页面的等级均匀分布到它指向的所有页面。每个页面收集指向它的所有页面的部分等级，对它们求和，并对总和应用阻尼因子。结果是页面的新排名。使用所有页面的新等级开始新的迭代。该实现在固定次数的迭代之后终止。
* 这是页面排名算法的维基百科条目。
*
* 输入文件是纯文本文件，必须格式如下：
*
* 页面表示为由新行字符分隔的（长）ID。
* 例如，"1\n2\n12\n42\n63"给出五个页面ID为1,2,12,42和63的页面。
* 链接表示为页面ID对，由空格字符分隔。链接由换行符分隔。
* 例如，"1 2\n2 12\n1 12\n42 63"给出四个（定向）链接（1） - >（2），（2） - >（12），（1） - >（12）和（42） - >（63）。
* 对于这个简单的实现，要求每个页面至少有一个传入链接和一个传出链接（页面可以指向自身）。
* 用法：PageRankBasic --pages <path> --links <path> --output <path> --numPages <n> --iterations <n>
* 如果未提供参数，则使用{@link PageRankData}中的默认数据和10次迭代运行程序。
*
**/
public class PageRank { //阻尼系数 private static final double DAMPENING_FACTOR = 0.85; //收敛阈值. private static final double EPSILON = 0.0001; private static DataSet<Long> getPagesDataSet(ExecutionEnvironment env, ParameterTool params) { if (params.has("pages")) { return env.readCsvFile(params.get("params")) .fieldDelimiter(" ") .lineDelimiter("\n") .types(Long.class) .map(new MapFunction<Tuple1<Long>, Long>() { @Override public Long map(Tuple1<Long> value) throws Exception { return value.f0; } }); } else { System.out.println("Executing PageRank example with default pages data set."); System.out.println("Use --pages to specify file input."); return PageRankData.getDefaultPagesDataSet(env); } } private static DataSet<Tuple2<Long, Long>> getLinksDataSet(ExecutionEnvironment env, ParameterTool params) { if (params.has("links")) { return env.readCsvFile(params.get("links")) .fieldDelimiter(" ") .lineDelimiter("\n") .types(Long.class, Long.class); } else { System.out.println("Executing PageRank example with default links data set."); System.out.println("Use --links to specify file input."); return PageRankData.getDefaultEdgeDataSet(env); } } public static void main(String[] args) throws Exception { ParameterTool params = ParameterTool.fromArgs(args); final int numPages = params.getInt("numPages", PageRankData.getNumberOfPages()); final int maxIterations = params.getInt("iterations", 10); // set up execution environment final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); // make the parameters available to the web ui env.getConfig().setGlobalJobParameters(params); // get input data DataSet<Long> pagesInput = getPagesDataSet(env, params); DataSet<Tuple2<Long, Long>> linksInput = getLinksDataSet(env, params); // assign initial rank to pages DataSet<Tuple2<Long, Double>> pagesWithRanks = pagesInput .map(new RankAssigner(1.0d / numPages)); // build adjacency list from link input DataSet<Tuple2<Long, Long[]>> adjacencyListInput = linksInput .groupBy(0) .reduceGroup(new BuildOutgoingEdgeList()); // set iterative data set IterativeDataSet<Tuple2<Long, Double>> iteration = pagesWithRanks.iterate(maxIterations); DataSet<Tuple2<Long, Double>> newRanks = iteration.join(adjacencyListInput) .where(0).equalTo(0) //迭代算子 .flatMap(new JoinVertexWithEdgesMatch()) // collect and sum ranks .groupBy(0) .aggregate(Aggregations.SUM, 1) // apply dampening factor .map(new Dampener(PageRank.DAMPENING_FACTOR, numPages)); //如果没有达到收敛条件，循环10次后结束 DataSet<Tuple2<Long, Double>> finalPageRanks = iteration.closeWith( newRanks, newRanks.join(iteration).where(0).equalTo(0) // termination condition .filter(new EpsilonFilter())); // emit result if (params.has("output")) { finalPageRanks.writeAsCsv(params.get("output"), "\n", " "); // execute program env.execute("Basic Page Rank Example"); } else { System.out.println("Printing result to stdout. Use --output to specify output path."); finalPageRanks.print(); } } /** * A map function that assigns an initial rank to all pages. */ public static final class RankAssigner implements MapFunction<Long, Tuple2<Long, Double>> { Tuple2<Long, Double> outPageWithRank; public RankAssigner(double rank) { this.outPageWithRank = new Tuple2<>(-1L, rank); } @Override public Tuple2<Long, Double> map(Long value) throws Exception { outPageWithRank.f0 = value; return outPageWithRank; } } /** * A reduce function that takes a sequence of edges and builds the adjacency list for the vertex where the edges * originate. Run as a pre-processing step. * 将分组后的links数据按Id放入Tuple2<Long, Long[]> * 与hadoop的mapreduce的reduce部分类似，groupby就是shuffle */ @FunctionAnnotation.ForwardedFields("0") public static final class BuildOutgoingEdgeList implements GroupReduceFunction<Tuple2<Long, Long>, Tuple2<Long, Long[]>> { private final ArrayList<Long> neighbors = new ArrayList<Long>(); @Override public void reduce(Iterable<Tuple2<Long, Long>> values, Collector<Tuple2<Long, Long[]>> out) { neighbors.clear(); Long id = 0L; for (Tuple2<Long, Long> n : values) { id = n.f0; neighbors.add(n.f1); System.out.println("id: " + id + " ,neighbors: " + n.f1); } out.collect(new Tuple2<Long, Long[]>(id, neighbors.toArray(new Long[neighbors.size()]))); } } /** * Join function that distributes a fraction of a vertex's rank to all neighbors. * 按照页面id以及其连接页面的数量，重新计算相邻点的rank，迭代10次 * rankToDistribute就是计算了顶点（id）指向边（neighbors）的贡献值，neighbors最终的rank值需要合计后才能算出 */ public static final class JoinVertexWithEdgesMatch implements FlatMapFunction<Tuple2<Tuple2<Long, Double>, Tuple2<Long, Long[]>>, Tuple2<Long, Double>> { @Override public void flatMap(Tuple2<Tuple2<Long, Double>, Tuple2<Long, Long[]>> value, Collector<Tuple2<Long, Double>> out) { System.out.println("before:" + value); Long[] neighbors = value.f1.f1; double rank = value.f0.f1; double rankToDistribute = rank / ((double) neighbors.length); for (Long neighbor : neighbors) { // System.out.println("neighbor:" + neighbor + ",rankToDistribute:" + rankToDistribute); out.collect(new Tuple2<Long, Double>(neighbor, rankToDistribute)); } } } /** * The function that applies the page rank dampening formula. * 阻尼系数公式:PR(A)=(1-d)/N + d(PR(T1)/C(T1)+ ... +PR(Tn)/C(Tn)) * PR(A) 是页面A的PR值 * PR(Ti)是页面Ti的PR值，在这里，页面Ti是指向A的所有页面中的某个页面 * C(Ti)是页面Ti的出度，也就是Ti指向其他页面的边的个数 * d 为阻尼系数，其意义是，在任意时刻，用户到达某页面后并继续向后浏览的概率， * 该数值是根据上网者使用浏览器书签的平均频率估算而得，通常d=0.85 */ @FunctionAnnotation.ForwardedFields("0") public static final class Dampener implements MapFunction<Tuple2<Long, Double>, Tuple2<Long, Double>> { private final double dampening; private final double randomJump; public Dampener(double dampening, double numVertices) { this.dampening = dampening; this.randomJump = (1 - dampening) / numVertices; } @Override public Tuple2<Long, Double> map(Tuple2<Long, Double> value) { value.f1 = (value.f1 * dampening) + randomJump; return value; } } /** * Filter that filters vertices where the rank difference is below a threshold. * 如果迭代之间的参数之和低于此EPSILON，我们将会收敛 * 每次迭代后都会调用Filter判断是否要退出迭代 */ public static final class EpsilonFilter implements FilterFunction<Tuple2<Tuple2<Long, Double>, Tuple2<Long, Double>>> { @Override public boolean filter(Tuple2<Tuple2<Long, Double>, Tuple2<Long, Double>> value) { System.out.println("value:"+value+",math:"+Math.abs(value.f0.f1 - value.f1.f1)); return Math.abs(value.f0.f1 - value.f1.f1) > EPSILON; } } }
