spark

shell

sc.textFile("file:///D:///spring///S-park///log///1.txt").flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_).collect # 词统计
sc.textFile("file:///D:///spring///S-park///log///2.txt").map(line => line.split(" ").size).reduce((a, b) => if (a > b) a else b) # 一行中最多的单词数量
sc.textFile("file:///D:///spring///S-park///log///3.txt").filter(line => line.contains("spark")).count() # 有多少行包括字符串"spark"

spark-sql

spark.sql("show databases").show(false)
spark.sql("use default")
spark.sql("show tables").show(false)
spark.sql("select * from test_table").show(false)
spark.sql("select count(1) from test_table").show(false)

spark-submit

spark-submit --class com.cord.StartApplication  --executor-memory 4G --num-executors 8 --master yarn-client spark-example-1.0-SNAPSHOT.jar