Spark 实例
1. parquet
object testSparkReadParquet { def main(args: Array[String]): Unit = { var spark = SparkSession.builder().appName("TestSparkSession").master("local").getOrCreate() val df = spark.read.parquet("D:\\tools\\testSparkFile\\users.parquet"); df.printSchema() df.select("name","favorite_color","favorite_numbers").show() df.select("name","favorite_color").write.mode("overwrite").save("D:\\tools\\testSparkFile\\namesAndFavColors.parquet") val df2 = spark.read.parquet("D:\\tools\\testSparkFile\\namesAndFavColors.parquet"); df2.printSchema() } }
object TestParquet { def main(args: Array[String]): Unit = { val spark = SparkSession .builder() .appName("Java Spark SQL basic example") .config("spark.some.config.option", "some-value") .master("local") .getOrCreate(); import spark.implicits._ val squaresDF = spark.sparkContext.makeRDD(1 to 5).map(i => (i, i * i)).toDF("value", "square") squaresDF.write.mode("append").parquet("D:\\tools\\testSparkFile\\test\\key=1") val cubesDF = spark.sparkContext.makeRDD(6 to 10).map(i => (i, i * i * i)).toDF("value", "cube") cubesDF.write.mode("append").parquet("D:\\tools\\testSparkFile\\test\\key=2") // Read the partitioned table val mergedDF = spark.read.option("mergeSchema", "true").parquet("D:\\tools\\testSparkFile\\test\\") mergedDF.select("value", "square","key").show() mergedDF.printSchema() } }
2. DataFrame
object DFExample { case class Student(id:Int, name:String, phone:String, email:String, age:Int) def main(args: Array[String]): Unit = { val spark = SparkSession.builder().appName("DFExample").master("local").getOrCreate() import spark.implicits._ val studentDF = spark.sparkContext.textFile("D:\\tools\\testSparkFile\\dfTestFile.txt").map(_.split("\\|")) .map(line =>Student(line(0).trim().toInt, line(1).trim(), line(2).trim(),line(3).trim(),line(4).trim().toInt)).toDF() val studentDF2 = studentDF studentDF.show() //过滤掉名字为空的数据 //studentDF.filter("name !='' OR name != 'NULL' " ).show() //没起作用 studentDF.filter("name !='NULL'" ).filter("name !=''").show() //找出名字以l开头的人 studentDF.filter("substr(name,0,1)='l'").show() // spark.sql("show functions").show(2000) import org.apache.spark.sql.functions._ import spark.implicits._ //按照名字排序操作 studentDF.sort("name").show() studentDF.sort(studentDF.col("name").desc).show() //队列进行重新命名 studentDF.select(studentDF.col("name").as("student_name")).show() //join操作 studentDF.join(studentDF2,studentDF.col("id") === studentDF2.col("id")).sort(studentDF.col("id")).show() } }
object ScalaREPL { case class REPL(id_1:Int,id_2:Int,cmp_fname_c1:Double,cmp_fname_c2:String,cmp_lname_c1:Double,cmp_lname_c2:String, cmp_sex:Double,cmp_bd:Double,cmp_bm:Double,cmp_by:Double,cmp_plz:Double,is_match:Boolean) case class REPL2(summary:String,cmp_fname_c1:Double,cmp_fname_c2:Double) def main(args: Array[String]): Unit = { val spark = SparkSession.builder().appName("DFExample").master("local").getOrCreate() import spark.implicits._ import org.apache.spark.sql.functions._ val df = spark.sparkContext.textFile("D:\\tools\\testSparkFile\\scalaREPL.txt").filter(_.length() != 0).map(_.split(",")) .map(line =>REPL(line(0).trim().toInt,line(1).trim().toInt,line(2).trim().toDouble,line(3).trim(), line(4).trim().toDouble,line(5).trim(),line(6).trim().toDouble,line(7).trim().toDouble, line(8).trim().toDouble,line(9).trim().toDouble,line(10).trim().toDouble,line(11).trim().toBoolean)).toDF() df.show() // df.cache() // df.rdd.map(_.getAs[Boolean]("is_match")).countByValue().foreach(println) // df.select(col("is_match")).groupBy(col("is_match")).count().show() // df.groupBy(col("is_match")).count().orderBy(col("is_match").desc).show() // df.agg(count(col("cmp_fname_c1")),mean(col("cmp_fname_c1")), stddev(col("cmp_fname_c1")), min(col("cmp_fname_c1")), max(col("cmp_fname_c1"))).show() // // df.createOrReplaceTempView("linkage") // spark.sql(""" // select is_match, count(*) count // from linkage // group by is_match // order by is_match desc // """).show() val matches = df.where("is_match = true") matches.createOrReplaceTempView("metch_desc") matches.show() // val matchSummary = matches.distinct() val misses = df.filter("is_match = false") misses.createOrReplaceTempView("miss_desc") // val missSummary = misses.distinct() //长表转宽表 /* val df2 = spark.sparkContext.textFile("D:\\tools\\testSparkFile\\scalaREPL2.txt").filter(_.length() != 0).map(_.split(",")) .map(line =>REPL2(line(0).trim(),line(1).trim().toDouble,line(2).trim().toDouble)).toDF() df2.show() val schema = df2.schema val ds = df2.flatMap(row =>{ val metric = row.getString(0) (1 until row.size).map(i =>{ (metric, schema(i).name, row.getDouble(i)) }) }) val longDF = ds.toDF("metric","field","value") longDF.show() longDF.groupBy("field").pivot("metric",Seq("count","mean","suddev","max","min")).agg(first("value")).show()*/ } }
3. //dataSet 转 DataFrame 使用内置函数
object TestMachineLearning { def main(args: Array[String]): Unit = { val spark = SparkSession.builder().appName("DFExample").master("local").getOrCreate() import spark.implicits._ val rawUserArtistData = spark.read.textFile("D:\\tools\\testSparkFile\\profiledata\\user_artist_data.txt") rawUserArtistData.take(5).foreach(println) //dataSet 转 DataFrame 使用内置函数 val userArtistDF = rawUserArtistData.map(line =>{ val Array(user,artist, _*) = line.split(" ") (user, artist.toInt) }).toDF("user","artist") import org.apache.spark.sql.functions._ userArtistDF.agg(max("user"),min("user"),max("artist"),min("artist")).show() } }