spark 解析非结构化数据存储至hive的scala代码
//提交代码包 // /usr/local/spark/bin$ spark-submit --class "getkv" /data/chun/sparktes.jar import org.apache.spark.sql.{DataFrame, Row, SQLContext, SaveMode} import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.sql.hive.HiveContext object split { def main(args:Array[String]) { val cf = new SparkConf().setAppName("ass").setMaster("local") val sc = new SparkContext(cf) val sqlContext = new SQLContext(sc) val hc = new HiveContext(sc) val format=new java.text.SimpleDateFormat("yyyy-MM-dd") val date=format.format(new java.util.Date().getTime-9*24*60*60*1000) val lg= sc.textFile("hdfs://master:9000/data/"+date+"/*/*.gz") val filed1=lg.map(l=>(l.split("android_id\":\"").last.split("\"").head.toString, l.split("anylst_ver\":").last.split(",").head.toString, l.split("area\":\"").last.split("\"").head, l.split("build_CPU_ABI\":\"").last.split("\"").head, l.split("build_board\":\"").last.split("\"").head, l.split("build_model\":\"").last.split("\"").head, l.split("\"city\":\"").last.split("\"").head, l.split("country\":\"").last.split("\"").head, l.split("cpuCount\":").last.split(",").head, l.split("cpuName\":\"").last.split("\"").head, l.split("custom_uuid\":\"").last.split("\"").head, l.split("cid\":\"").last.split("\"").head, l.split("definition\":\"").last.split("\"").head, l.split("firstTitle\":\"").last.split("\"").head, l.split("modeType\":\"").last.split("\"").head, l.split("pageName\":\"").last.split("\"").head, l.split("playIndex\":\"").last.split("\"").head, l.split("rectime\":").last.split(",").head, l.split("time\":\"").last.split("\"").head)) //val F1=filed1.toDF("custom_uuid","region","screenHeight","screenWidth","serial_number","touchMode","umengChannel","vercode","vername","wlan0_mac","rectime","time") val scoreDataFrame1 = hc.createDataFrame(filed1).toDF("android_id","anylst_ver","area","build_CPU_ABI","build_board","build_model","city","country","cpuCount","cpuName","custom_uuid","cid","definition","firstTitle","modeType","pageName","playIndex","rectime","time") scoreDataFrame1.write.mode(SaveMode.Append).saveAsTable("test.f1") val filed2=lg.map(l=>(l.split("custom_uuid\":\"").last.split("\"").head, l.split("playType\":\"").last.split("\"").head, l.split("prevName\":\"").last.split("\"").head, l.split("prevue\":").last.split(",").head, l.split("siteName\":\"").last.split("\"").head, l.split("title\":\"").last.split("\"").head, l.split("uuid\":\"").last.split("\"").head, l.split("vod_seek\":\"").last.split("\"").head, l.split("device_id\":\"").last.split("\"").head, l.split("device_name\":\"").last.split("\"").head, l.split("dpi\":").last.split(",").head, l.split("eth0_mac\":\"").last.split("\"").head, l.split("ip\":\"").last.split("\"").head, l.split("ipaddr\":\"").last.split("\"").head, l.split("isp\":\"").last.split("\"").head, l.split("largeMem\":").last.split(",").head, l.split("limitMem\":").last.split(",").head, l.split("packageName\":\"").last.split("\"").head, l.split("rectime\":").last.split(",").head, l.split("time\":\"").last.split("\"").head)) import sqlContext.implicits._ val scoreDataFrame2 = hc.createDataFrame(filed2).toDF("custom_uuid","playType","prevName","prevue","siteName","title","uuid","vod_seek","device_id","device_name","dpi","eth0_mac","ip","ipaddr","isp","largeMem","limitMem","packageName","rectime","time") scoreDataFrame2.write.mode(SaveMode.Append).saveAsTable("test.f2") // val filed3=lg.map(l=>(l.split("custom_uuid\":\"").last.split("\"").head, l.split("region\":\"").last.split("\"").head, l.split("screenHeight\":").last.split(",").head, l.split("screenWidth\":").last.split(",").head, l.split("serial_number\":\"").last.split("\"").head, l.split("touchMode\":").last.split(",").head, l.split("umengChannel\":\"").last.split("\"").head, l.split("vercode\":").last.split(",").head, l.split("vername\":\"").last.split("\"").head, l.split("wlan0_mac\":\"").last.split("\"").head, l.split("rectime\":").last.split(",").head, l.split("time\":\"").last.split("\"").head )) import sqlContext.implicits._ val scoreDataFrame3= hc.createDataFrame(filed3).toDF("custom_uuid","region","screenHeight","screenWidth","serial_number","touchMode","umengChannel","vercode","vername","wlan0_mac","rectime","time") scoreDataFrame3.write.mode(SaveMode.Append).saveAsTable("test.f3") } }