//获取出每天前3的搜索词
ArrayList<String> log = new ArrayList<String>();
log.add("2015-10-01,leo,a1,beijing,android");
log.add("2015-10-01,leo,a1,beijing,android");
log.add("2015-10-01,tom,a1,beijing,android");
log.add("2015-10-01,jack,a1,beijing,android");
log.add("2015-10-01,marry,a1,beijing,android");
log.add("2015-10-01,tom,bbf,beijing,android");
log.add("2015-10-01,jack,bbf,beijing,iphone");
log.add("2015-10-01,jack,bbf,beijing,android");
log.add("2015-10-01,leo,ttyu,beijing,android");
log.add("2015-10-01,leo,ttyu,beijing,android");
log.add("2015-10-01,wede,a1,beijing,android");
log.add("2015-10-01,wede,bbf,beijing,iphone");
log.add("2015-10-02,leo,a2,beijing,android");
log.add("2015-10-02,tom,a2,beijing,android");
log.add("2015-10-02,tom,a2,beijing,android");
log.add("2015-10-02,jack,a1,beijing,android");
log.add("2015-10-02,marry,a1,beijing,android");
log.add("2015-10-02,leo,bbf,beijing,iphone");
log.add("2015-10-02,jack,bbf,beijing,android");
log.add("2015-10-02,wede,bbf,beijing,android");
log.add("2015-10-02,leo,ttyu,beijing,android");
log.add("2015-10-02,leo,ttyu,beijing,android");
log.add("2015-10-02,jack,a1,beijing,android");
log.add("2015-10-02,wede,tour,beijing,android");
SparkConf conf = new SparkConf()
// .setMaster("local")
.setAppName("Top3UV");
JavaSparkContext sc = new JavaSparkContext(conf);
HiveContext sqlContext = new HiveContext(sc.sc());
JavaRDD<String> rdd_list = sc.parallelize(log, 2);
//0条件使用broadcast（每个worker节点共享一个变量）
final org.apache.spark.broadcast.Broadcast<String> bc = sc.broadcast("iphone");
//1条件过滤
JavaRDD<String> rdd_filter_list = rdd_list.filter(new Function<String, Boolean>() {
@Override
public Boolean call(String v1) throws Exception {
String ary[] = v1.split(",");
String platform = ary[4];
if (platform.contains(bc.value()))
return false;
return true;
}
});
//2将每行数据构建成tuple2
JavaPairRDD<String, String> rdd_tuple2_list = rdd_filter_list.mapToPair(new PairFunction<String, String, String>() {
@Override
public Tuple2<String, String> call(String s) throws Exception {
String ary[] = s.split(",");
String time = ary[0];
String word = ary[2];
String userName = ary[1];
return new Tuple2<String, String>(time + "_" + word, userName);
}
});
//3按照tuple._1进行combiner
JavaPairRDD<String, Iterable<String>> rdd_byKey = rdd_tuple2_list.groupByKey();
//4按照tuple._1进行用户数量去重后的统计
JavaPairRDD<String, Integer> rdd_byKey_uv = rdd_byKey.mapToPair(new PairFunction<Tuple2<String, Iterable<String>>, String, Integer>() {//tuple._1仍然为时间_搜索词，而tuple._2变为用户去重后的数量
@Override
public Tuple2<String, Integer> call(Tuple2<String, Iterable<String>> stringIterableTuple2) throws Exception {
String tuple_1 = stringIterableTuple2._1();
Iterable<String> userNames = stringIterableTuple2._2();
Set<String> userNameSet = new HashSet<String>();
for (String item : userNames) {
userNameSet.add(item);//用户名称
}
return new Tuple2<String, Integer>(tuple_1, userNameSet.size());
}
});
//5构建rdd<Row>用来映射DataFrame
JavaRDD<Row> rdd_byKey_row_uv = rdd_byKey_uv.map(new Function<Tuple2<String, Integer>, Row>() {
@Override
public Row call(Tuple2<String, Integer> stringIntegerTuple2) throws Exception {
String ary[] = stringIntegerTuple2._1().split("_");
return RowFactory.create(ary[0], ary[1], stringIntegerTuple2._2());
}
});
List<StructField> list = new ArrayList<StructField>();
list.add(DataTypes.createStructField("date", DataTypes.StringType, true));
list.add(DataTypes.createStructField("word", DataTypes.StringType, true));
list.add(DataTypes.createStructField("uv_num", DataTypes.IntegerType, true));
StructType tmpType = DataTypes.createStructType(list);
DataFrame df_tuple = sqlContext.createDataFrame(rdd_byKey_row_uv, tmpType);
df_tuple.registerTempTable("tuple_keyDS_valUN");
//6使用DataFrame结合开窗函数row_number分组后过滤出访问量前3的搜索词
StringBuilder _sb = new StringBuilder();
_sb.append("select date,word,uv_num from ( ");
_sb.append(" select date,word,uv_num, row_number() OVER (PARTITION BY date ORDER BY uv_num DESC ) as rank from tuple_keyDS_valUN ");
_sb.append(" ) tmp_group_top3 where rank<=3");
DataFrame df_tuple_groupTop3 = sqlContext.sql(_sb.toString()).cache();
//df_tuple_groupTop3.show();//***************在最下面打印
//=====到这里已经获取到每天前3的“搜索词“和“uv数“，并倒叙排序
//在获取每天排名前三“搜索词”的总uv数
//7将结果从DataFrame转换回rdd，并拼接成tuple2（日期,总访问量_访问词）
JavaPairRDD<String, String> rdd_uvKey = df_tuple_groupTop3.javaRDD().mapToPair(new PairFunction<Row, String, String>() {
@Override
public Tuple2<String, String> call(Row row) throws Exception {
String date = row.getString(0);
String word = row.getString(1);
Integer uv_mun = row.getInt(2);
return new Tuple2<String, String>(date, uv_mun + "_" + word);
}
});
//8mapToPair后继续按照key合并
JavaPairRDD<String, Iterable<String>> rdd_dateKey_group = rdd_uvKey.groupByKey();
JavaPairRDD<Integer, String> rdd_uvKey_combiner = rdd_dateKey_group.mapToPair(new PairFunction<Tuple2<String, Iterable<String>>, Integer, String>() {
@Override
public Tuple2<Integer, String> call(Tuple2<String, Iterable<String>> stringIterableTuple2) throws Exception {
Integer uv_sum = 0;
String data_word = "";
Iterable<String> uv_word = stringIterableTuple2._2();
Iterator<String> uv_word_it = uv_word.iterator();
for (; uv_word_it.hasNext(); ) {
String uv_word_str = uv_word_it.next();
String ary[] = uv_word_str.split("_");
Integer uv = Integer.valueOf(ary[0]);
uv_sum += uv;//累加总uv数
String word = ary[1];
data_word += stringIterableTuple2._1() + "_" + word + "|";
}
return new Tuple2<Integer, String>(uv_sum, data_word);
}
});
JavaPairRDD<Integer, String> rdd_uvKey_sort = rdd_uvKey_combiner.sortByKey(false);
List<Tuple2<Integer, String>> ret = rdd_uvKey_sort.collect();
for (Tuple2<Integer, String> item : ret) {
System.out.println(item._1() + "<--->" + item._2());
}
df_tuple_groupTop3.show();
