spark-二次排序

排序文件：

3,2
5,2
5,3
5,9
6,2
9,1
9,3
8,4

方法一：

package spark.rdd

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

class SecondarySortByKey(val first:Int, val second:Int) extends Ordered[SecondarySortByKey] with Serializable{
  override def compare(that: SecondarySortByKey): Int = {
    if(this.first-that.first != 0){
      this.first - that.first
    } else {
      this.second - that.second
    }
  }
}
object SecondarySortApp {
  // 第一列升序，第二列降序，巧妙使用List的默认排序方法
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("SortByKey").setMaster("local[*]")
    val sc = new SparkContext(conf)
    sc.setLogLevel("ERROR")
    val data = sc.textFile("/test/file/secondarySort.txt")
    val lines = data.map(line => (new SecondarySortByKey(line.split(",")(0).toInt,line.split(",")(1).toInt),line))
    val sorted = lines.sortByKey(true)
    sorted.map(line => line._2).collect().foreach(println(_))
  }


}

第二种方法：

def main(args: Array[String]): Unit = {
  val conf = new SparkConf().setAppName("SortByKey").setMaster("local[*]")
  val sc = new SparkContext(conf)
  sc.setLogLevel("ERROR")
  val data = sc.textFile("/test/file/secondarySort.txt",1)
  //第一个列升序
  val value: RDD[(String, String)] = data.coalesce(1,false).map(line => (line, line)).sortByKey(true)
  val value1: RDD[(String, List[String])] = data.map(line => (line.split(",")(0), line)).groupByKey(1).sortByKey(true).map(line => (line._1, line._2.toList.sortWith(_.compareTo(_) > 0)))
  value1.map(_._2).flatMap(_.mkString("@").split("@")).foreach(println)
}

注意：默认分区产生的影响。