Scala反射创建DataFrame(方式一,已知数据类型)

package sparkSql.方法1创建DataFrame

import org.apache.spark.sql.{SQLContext, SaveMode}
import org.apache.spark.{SparkConf, SparkContext}

object Student {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName(this.getClass.getName).setMaster("local")
val sc = new SparkContext(conf)
val sQLContext = new SQLContext(sc)

val file = sc.textFile("C:\\Program Files\\feiq\\Recv Files\\day11-Spark SQL\\students.txt")

val stuRDD = file.map(t=>{
val arr = t.split(",")
Stu(arr(0).toInt,arr(1),arr(2).toInt);
})

import sQLContext.implicits._
val df = stuRDD.toDF()
df.registerTempTable("stu")
val df2 = sQLContext.sql("select name from stu where age < 18")
//df2.show()
// 保存数据类型
// df2.write.orc()
// df2.write.json()
// df2.write.text()

df2.write.format("text").save("D:\\stuTest")//只能一个字段
//df2.write.mode(SaveMode.Append).save("D:\\stuTest")//只能一个字段


// df.rdd 转javaRDD
// df.show()
}
}
case class Stu(id:Int,name:String,age:Int)


pom.xml文件同java上面
posted @ 2019-02-25 22:34  VIP8cnbl  阅读(654)  评论(0编辑  收藏  举报