根据PV统计出前三的热门板块，并统计出热门板块下的用户数--方式二

 

---

根据PV统计出前三的热门板块，并统计出热门板块下的用户数--方式二

测试数据

java代码

package com.hzf.spark.study;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.PairFlatMapFunction;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.api.java.function.VoidFunction;
import org.apache.spark.broadcast.Broadcast;

import scala.Tuple2;

public class HotChannel02 {
    public static void main(String[] args) {
        SparkConf conf = new SparkConf()
                .setAppName("HotChannel")
                .setMaster("local")
                .set("spark.testing.memory", "2147480000");
        JavaSparkContext sc = new JavaSparkContext(conf);
        JavaRDD<String> logRDD = sc.textFile("f:/userLog");
        String str = "View";
        final Broadcast<String> broadcast = sc.broadcast(str);
        hotChannel(sc, logRDD, broadcast);
    }
    private static void hotChannel(JavaSparkContext sc, JavaRDD<String> logRDD, final Broadcast<String> broadcast) {
        JavaRDD<String> filteredLogRDD = logRDD.filter(new Function<String, Boolean>() {
            
            private static final long serialVersionUID = 1L;

            @Override
            public Boolean call(String v1) throws Exception {
                String actionParam = broadcast.value();
                String action = v1.split("\t")[5];
                return actionParam.equals(action);
            }
        });
        
        JavaPairRDD<String, String> channel2nullRDD = filteredLogRDD.mapToPair(new PairFunction<String, String,String>() {
            
            private static final long serialVersionUID = 1L;

            @Override
            public Tuple2<String, String> call(String val) throws Exception {
                String channel = val.split("\t")[4];
                
                return new Tuple2<String, String>(channel,null);
            }
        });
        Map<String, Object> channelPVMap = channel2nullRDD.countByKey();
        Set<String> keySet = channelPVMap.keySet();
        List<SortObj> channels  = new ArrayList<>();
        for(String channel : keySet){ 
            channels.add(new SortObj(channel, Integer.valueOf(channelPVMap.get(channel)+"")));
        }
        Collections.sort(channels, new Comparator<SortObj>() {

            @Override
            public int compare(SortObj o1, SortObj o2) {
                return o2.getValue() - o1.getValue();
            }
        });
        
        List<String> hotChannelList = new ArrayList<>();
        for (int i = 0; i < 3; i++) {
            hotChannelList.add(channels.get(i).getKey());
        }
        
        
        final Broadcast<List<String>> hotChannelListBroadcast = sc.broadcast(hotChannelList);
        
         
        JavaRDD<String> filtedRDD = logRDD.filter(new Function<String, Boolean>() {

            @Override
            public Boolean call(String v1) throws Exception {
                List<String> hostChannels = hotChannelListBroadcast.value();
                String channel = v1.split("\t")[4];
                 String userId = v1.split("\t")[2];
                return hostChannels.contains(channel) && !"null".equals(userId);
            }
        });
        
        JavaPairRDD<String, String> user2ChannelRDD = filtedRDD.mapToPair(new PairFunction<String, String,String>() {

            private static final long serialVersionUID = 1L;

            @Override
            public Tuple2<String, String> call(String val) throws Exception {
                String[] splited = val.split("\t");
                String userId = splited[2];
                String channel = splited[4];
                return new Tuple2<String, String>(userId,channel);
            }
        });
        
        JavaPairRDD<String, String> userVistChannelsRDD = user2ChannelRDD.groupByKey().flatMapToPair(new PairFlatMapFunction<Tuple2<String,Iterable<String>>, String, String>() {

            private static final long serialVersionUID = 1L;

            @Override
            public Iterable<Tuple2<String, String>> call(Tuple2<String, Iterable<String>> tuple) throws Exception {
                String userId = tuple._1;
                Iterator<String> iterator = tuple._2.iterator();
                Map<String, Integer> channelMap = new HashMap<>();
                while (iterator.hasNext()) {
                    String channel = iterator.next();
                    Integer count = channelMap.get(channel);
                    if(count == null)
                        count = 1;
                    else
                        count++;
                    channelMap.put(channel, count);
                }
                
                List<Tuple2<String, String>> list = new ArrayList<>();
                Set<String> keys = channelMap.keySet();
                for(String channel : keys){
                     Integer channelNum  = channelMap.get(channel);
                     list.add(new Tuple2<String, String>(channel, userId + "_" + channelNum));
                }
                return list;
            }
        });
        
        
        userVistChannelsRDD.groupByKey().foreach(new VoidFunction<Tuple2<String,Iterable<String>>>() {

            private static final long serialVersionUID = 1L;

            @Override
            public void call(Tuple2<String, Iterable<String>> tuple) throws Exception {
                String channel = tuple._1;
                Iterator<String> iterator = tuple._2.iterator();
                List<SortObj> list = new ArrayList<>();
                while (iterator.hasNext()) {
                    String ucs = iterator.next();
                    String[] splited = ucs.split("_");
                    String userId = splited[0];
                    Integer num = Integer.valueOf(splited[1]);
                    list.add(new SortObj(userId, num));
                }
                
                Collections.sort(list,new Comparator<SortObj>() {

                    @Override
                    public int compare(SortObj o1, SortObj o2) {
                        return o2.getValue() - o1.getValue();
                    }
                });
                
                System.out.println("HOT_CHANNLE:"+channel);
                for(int i = 0 ; i < 3 ; i++){
                    SortObj sortObj = list.get(i);
                    System.out.println(sortObj.getKey() + "===" + sortObj.getValue());
                }
            }
        });
    }
}

 

result