根据PV统计出前三的热门板块，并统计出热门板块下的用户数--方式一

 

---

根据PV统计出前三的热门板块，并统计出热门板块下的用户数--方式一

测试数据

java代码

package com.hzf.spark.study;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.api.java.function.VoidFunction;
import org.apache.spark.broadcast.Broadcast;

import scala.Tuple2;

public class HotChannel01 {
    public static void main(String[] args) {
        SparkConf conf = new SparkConf()
                .setAppName("HotChannel")
                .setMaster("local")
                .set("spark.testing.memory", "2147480000");
        JavaSparkContext sc = new JavaSparkContext(conf);
        JavaRDD<String> logRDD = sc.textFile("userLog1");
        String str = "View";
        final Broadcast<String> broadcast = sc.broadcast(str);
        hotChannel(sc, logRDD, broadcast);
    }
    private static void hotChannel(JavaSparkContext sc,JavaRDD<String> logRDD, final Broadcast<String> broadcast) {
        JavaRDD<String> filteredLogRDD = logRDD.filter(new Function<String, Boolean>() {
            
            private static final long serialVersionUID = 1L;

            @Override
            public Boolean call(String v1) throws Exception {
                String actionParam = broadcast.value();
                String action = v1.split("\t")[5];
                return actionParam.equals(action);
            }
        });
        
        JavaPairRDD<String, String> channel2nullRDD = filteredLogRDD.mapToPair(new PairFunction<String, String,String>() {

            private static final long serialVersionUID = 1L;

            @Override
            public Tuple2<String, String> call(String val) throws Exception {
                String channel = val.split("\t")[4];
                
                return new Tuple2<String, String>(channel,null);
            }
        });
        Map<String, Object> channelPVMap = channel2nullRDD.countByKey();
        Set<String> keySet = channelPVMap.keySet();
        List<SortObj> channels  = new ArrayList<>();
        for(String channel : keySet){ 
            channels.add(new SortObj(channel, Integer.valueOf(channelPVMap.get(channel)+"")));
        }
        Collections.sort(channels, new Comparator<SortObj>() {

            @Override
            public int compare(SortObj o1, SortObj o2) {
                return o2.getValue() - o1.getValue();
            }
        });
        
        List<String> hotChannelList = new ArrayList<>();
        for (int i = 0; i < 3; i++) {
            hotChannelList.add(channels.get(i).getKey());
        }
        for(String channle : hotChannelList){
            System.out.println("channle:" + channle);
        }
        
        final Broadcast<List<String>> hotChannelListBroadcast = sc.broadcast(hotChannelList);
        
         
        JavaRDD<String> filtedRDD = logRDD.filter(new Function<String, Boolean>() {

            @Override
            public Boolean call(String v1) throws Exception {
                List<String> hostChannels = hotChannelListBroadcast.value();
                String channel = v1.split("\t")[4];
                String userId = v1.split("\t")[2];
                return hostChannels.contains(channel) && !"null".equals(userId);
            }
        });
        
        JavaPairRDD<String, String> channel2UserRDD = filtedRDD.mapToPair(new PairFunction<String, String, String>() {

            @Override
            public Tuple2<String, String> call(String v1) throws Exception {
                String[] splited = v1.split("\t");
                String channel = splited[4];
                String userId = splited[2];
                return new Tuple2<String, String>(channel,userId);
            }
        });
        
        channel2UserRDD.groupByKey().foreach(new VoidFunction<Tuple2<String,Iterable<String>>>() {
            
            private static final long serialVersionUID = 1L;

            @Override
            public void call(Tuple2<String, Iterable<String>> tuple) throws Exception {
                String channel = tuple._1;
                Iterator<String> iterator = tuple._2.iterator();
                Map<String, Integer> userNumMap = new HashMap<>();
                while(iterator.hasNext()){
                    String userId = iterator.next();
                    Integer count = userNumMap.get(userId);
                    if(count == null){
                        count = 1;
                    }else{
                        count ++;
                    }
                    userNumMap.put(userId, count);
                }
                
                List<SortObj> lists = new ArrayList<>();
                Set<String> keys = userNumMap.keySet();
                for(String key : keys){
                    lists.add(new SortObj(key, userNumMap.get(key)));
                }
                
                Collections.sort(lists,new Comparator<SortObj>() {

                    @Override
                    public int compare(SortObj O1, SortObj O2) {
                        return O2.getValue() - O1.getValue();
                    }
                });
                
                System.out.println("HOT_CHANNEL:"+channel);
                for(int i = 0 ; i < 3 ; i++){
                    SortObj sortObj = lists.get(i);
                    System.out.println(sortObj.getKey()+"=="+sortObj.getValue());
                }
            }
        });
    }
}

 

result