Spark基本函数学习

package cn.itcast.spark.czh

import org.apache.spark.{SparkConf, SparkContext}

object TestFun {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setMaster("local").setAppName("map")
val sc = new SparkContext(conf)
// val rdd = sc.parallelize(1 to 10) //创建RDD
// val map = rdd.map(_*2) //对RDD中的每个元素都乘于2
// map.foreach(x => print(x+" "))
// sc.stop()
// fun1()
// val l = List(("kpop", "female"), ("zorro", "male"), ("mobin", "male"), ("lucy", "female"))
// val rdd = sc.parallelize(l, 2)
// /*写法一*/
// // val mp = rdd.mapPartitions(fun2)
// /*val mp = rdd.mapPartitionsWithIndex(partitionsFun)*/
// // mp.collect.foreach(x => (print(x + " "))) //将分区中的元素转换成Aarray再输出
// /*写法二*/
// // rdd.mapPartitions(x=>x.filter(_._2=="female")).map(_._1).foreach(x=>print(x+" "))
// /*写法三*/
// val mp = rdd.mapPartitionsWithIndex(fun3)
// mp.collect().foreach(x => (print(x + " ")))
// fun4(sc)
fun9(sc)
}

/*map 数据集中的每个元素经过用户自定义的函数转换成一个新的RDD,新的RDD交MappedRDD*/
def fun0(): Unit = {
val conf = new SparkConf().setMaster("local").setAppName("map")
val sc = new SparkContext(conf)
val rdd = sc.parallelize(1 to 10) //创建RDD
val map = rdd.map(_ * 2) //对RDD中的每个元素都乘于2
map.foreach(x => print(x + " "))
sc.stop()
}

/*flatMap 与map类似,但每个元素输入项都可以被映射到0或者多个输出项,最终将结果扁平化输出*/
def fun1(): Unit = {
val conf = new SparkConf().setMaster("local").setAppName("flatMap")
val sc = new SparkContext(conf)
val rdd = sc.parallelize(1 to 10)
val array = rdd.flatMap(x => (1 to x)).collect()
array.foreach(a => print(a))
}

/*mapPartitions 类似于map,map作用于每个分区的每个元素,单mapPartitions作用于每个分区工*/
def fun2(iterable: Iterator[(String, String)]): Iterator[String] = {
var woman = List[String]()
while (iterable.hasNext) {
val next = iterable.next()
next match {
case (_, "female") => woman = next._1 :: woman
case _ =>
}
}
return woman.iterator
}

/*mapPartitionsWithIndex*/
def fun3(index: Int, iterator: Iterator[(String, String)]): Iterator[String] = {
var woman = List[String]()
while (iterator.hasNext) {
val next = iterator.next()
next match {
case (_, "female") => woman = "[" + index + "]" + next._1 :: woman
case _ =>
}
}
return woman.iterator
}

/* sample 对RDD进行抽样
* 参数解释:withReplacement 为true表示抽样之后还放回RDD,可以被多次抽样,false表示不放回。
* fraction 表示抽样比例
* seed为随机数种子,比如时间戳
* */
def fun4(sc: SparkContext): Unit = {
val rdd = sc.parallelize(1 to 10)
val sample1 = rdd.sample(true, 0.5)
sample1.collect().foreach(x => print(x + " "))
sc.stop()
}

/*union 将两个RDD中的数据进行合并,并最终返回两个RDD的并集,若RDD中存在相同的元素也不会去重*/
def fun5(sc: SparkContext): Unit = {
var rdd1 = sc.parallelize(1 to 10)
var rdd2 = sc.parallelize(6 to 15)
rdd1.union(rdd2).collect().foreach(x => print(x + " "))
}

/*intersection 返回两个RDD的交集*/
def fun6(sc: SparkContext): Unit = {
val rdd1 = sc.parallelize(1 to 10)
val rdd2 = sc.parallelize(5 to 20)
rdd1.intersection(rdd2).collect().foreach(x => print(x + " "))
}

/*distinct 对RDD中的元素进行去重*/
def fun7(sc: SparkContext): Unit = {
val rdd1 = sc.parallelize(List(1, 1, 2, 3, 4, 5, 5))
rdd1.distinct().collect().foreach(x => print(x + " "))
}

/*cartesian 对两个RDD中所有的元素进行笛卡尔积操作*/
def fun8(sc : SparkContext): Unit ={
val rdd1 = sc.parallelize(1 to 3)
val rdd2 = sc.parallelize(1 to 4)
rdd1.cartesian(rdd2).collect().foreach(x=>print(x+" "))
}

/*coalesce(numPartitions,shuffle) 对RDD的分区进行重新分区,shuffle默认值为false,当shuffle=false时,不能增加分区数目,但不会报错,只是分区个数还是原来的*/
def fun9(sc :SparkContext): Unit ={
/*shuffle 为 false*/
val rdd1 = sc.parallelize(1 to 16,4)
val rdd2 = rdd1.coalesce(3)
print(rdd2.partitions.size)

}

}
posted @ 2018-08-21 16:24  喜中5000万  阅读(467)  评论(0编辑  收藏  举报