spark scala 例子

object ScalaApp {

  def main(args: Array[String]): Unit = {
    var conf = new SparkConf()
    conf.setMaster("local")
    var sc = new SparkContext(conf)
    val lines: RDD[String] = sc.textFile("./words.txt")
    lines.flatMap(line=>line.split(" ")).map(word=>(word,1)).reduceByKey((v1,v2)=>v1+v2).foreach(println)
    // 简洁版
    // lines.flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_).foreach(println)
  }
}

 

分别取出下面三个班级Top3的分数

class1 90
class2 91
class1 88
class2 99
class1 100
class2 77
class1 77
class2 57
class3 77
class3 88
class3 99
class3 100
class3 22
class3 77

    lines.map(line=>(line.split(" ")(0),line.split(" ")(1).toDouble)).groupByKey()
      .map(group=>{
        (group._1,group._2.toList.sortWith(_>_).take(3))
      })
      .foreach(println)

结果：

(class3,List(100.0, 99.0, 88.0))
(class1,List(100.0, 90.0, 88.0))
(class2,List(99.0, 91.0, 77.0))