WordCount对比spark——java
使用spark编写的WordCount!
package com.test
import org.apache.spark.{SparkConf, SparkContext}
object Spark_Test {
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf().setMaster("local").setAppName("job_name")
val sc = new SparkContext(sparkConf)
val file = sc.textFile("E:\\aa.txt")
val flatMap = file.flatMap{line => line.split(" ")}
val map = flatMap.map(world =>(world,1))
val byKey = map.reduceByKey(_+_).map(tuple => (tuple._2,tuple._1))//使用touple把key value 位置转换(按value排序)
val sortvalue = byKey.sortByKey(true).map(tuple => (tuple._2,tuple._1))//false 倒序 调换位置,使其单词在前,sum在后
sortvalue.foreach(x => println(x))
}
}
使用java编写的WordCount!
package spark; import java.util.Arrays; import java.util.Iterator; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.api.java.function.Function2; import org.apache.spark.api.java.function.PairFunction; import org.apache.spark.api.java.function.VoidFunction; import com.sun.org.apache.bcel.internal.generic.LMUL; import scala.Tuple2; public final class JavaWorldCount { public static void main(String[] args) { SparkConf sparkConf = new SparkConf().setMaster("local").setAppName("job_name"); JavaSparkContext sc = new JavaSparkContext(sparkConf); JavaRDD<String> textFile = sc.textFile("E:/aa.txt"); JavaRDD<String> flatMap = textFile.flatMap(new FlatMapFunction<String, String>() { private static final long serialVersionUID = 1L; @Override public Iterator<String> call(String line) throws Exception { return Arrays.asList(line.split(" ")).iterator(); } }); JavaPairRDD<String,Integer> mapToPair = flatMap.mapToPair(new PairFunction<String, String, Integer>() { private static final long serialVersionUID = 1L; @Override public Tuple2<String, Integer> call(String word) throws Exception { return new Tuple2<String, Integer>(word, 1); } }); //三个int 前两个位计算单词数量 _+_ = INT 最后一个是和 JavaPairRDD<String,Integer> reduceByKey = mapToPair.reduceByKey(new Function2<Integer, Integer, Integer>() { private static final long serialVersionUID = 1L; @Override public Integer call(Integer v1, Integer v2) throws Exception { return v1+v2; } }); JavaPairRDD<Integer,String> mapToPair2 = reduceByKey.mapToPair(new PairFunction<Tuple2<String,Integer>, Integer, String>() { private static final long serialVersionUID = 1L; @Override public Tuple2<Integer, String> call(Tuple2<String, Integer> tuple) throws Exception { return new Tuple2<Integer, String>(tuple._2,tuple._1); } }); JavaPairRDD<String,Integer> sortByKey = mapToPair2.sortByKey(false).mapToPair(new PairFunction<Tuple2<Integer,String>, String, Integer>() { private static final long serialVersionUID = 1L; @Override public Tuple2<String, Integer> call(Tuple2<Integer, String> tuple) throws Exception { return new Tuple2<String, Integer>(tuple._2,tuple._1); } }); sortByKey.foreach(new VoidFunction<Tuple2<String,Integer>>() { private static final long serialVersionUID = 1L; @Override public void call(Tuple2<String, Integer> tuple) throws Exception { System.out.println(tuple); } }); } }