spark实现word count
package com.bjsxt.scala import org.apache.spark.SparkConf import org.apache.spark.SparkContext import org.apache.spark.rdd.RDD import org.apache.spark.rdd.RDD.rddToPairRDDFunctions object test { def main(args: Array[String]): Unit = { val conf = new SparkConf() conf.setMaster("local").setAppName("WC") val sc = new SparkContext(conf) val lines:RDD[String] = sc.textFile("./words") // val word:RDD[String] = lines.flatMap { x => x.split(" ") } // val pairs:RDD[(String, Int)] = word.map{ x => (x,1)} // val result = pairs.reduceByKey{(a,b) => {a+b}} // result.sortBy(_._2,false).foreach(println) //简化写法 lines.flatMap{_.split(" ")}.map{(_,1)}.reduceByKey(_+_).foreach(println) } }