spark MySQL jar 包
/** * Created by songcl on 2016/6/24. */ import java.sql.DriverManager //val sqlContext = new org.apache.spark.sql.SQLContext(sc) import org.apache.spark.sql.SQLContext import org.apache.spark.{SparkConf, SparkContext} object insertmysql { def main(args:Array[String]): Unit = { //classOf[com.mysql.jdbc.Driver] // Class.forName("com.mysql.jdbc.Driver").newInstance(); val url = "jdbc:mysql://10.0.73.46:3306/falcon?user=data&password=datass" val conf = new SparkConf().setAppName("Simple Application")//.setMaster("spark://10.0.58.21:7077") val sc= new SparkContext(conf) // val conn = DriverManager.getConnection(url) // val conn2=DriverManager.getConnection(url) val format = new java.text.SimpleDateFormat("yyyy/MM/dd") val dat = format.format(new java.util.Date()) val log1 = sc.textFile("hdfs://10.0.58.21:9000/falcon/" + dat + "/*.log") //val log1 =sc.textFile( "hdfs://10.0.58.21:9000/falcon/"+dat+"/*.log") // println(log1.count()) //val sqlContext = new org.apache.spark.sql.SQLContext(sc1) //val conf = new SparkConf().setAppName("Simple Application").setMaster("spark://10.0.58.21:7077") // val sc = new SparkContext(conf) // val log=sc.textFile(logFile) //val t= log.take(2) //val log1=sc.textFile("hdfs://10.0.58.21:9000/falcon/2016/*/*/*.log") val format2 = new java.text.SimpleDateFormat("yyyyMMdd") val dat2 = format2.format(new java.util.Date()) //val log2=sc1.textFile("hdfs://10.0.58.21:9000/user/yejin/logstash."+dat2+".log") val log2 = sc.textFile("hdfs://10.0.58.21:9000/falcon/" + dat + "/*.log") import sqlContext.implicits val rowRDD=log2.map(line=>(line.split("\"message\":\"").last.split(" ").head.trim(),line.split("account: ").last.split(", args:").head)).filter({case(k,v) =>k.length==10 && !k.contains("TypeError:")}).filter({case(k,v)=>v.length==8}).distinct //import sqlContext.implicits val sqlContext = new org.apache.spark.sql.SQLContext(sc) import sqlContext.implicits._ val df=rowRDD.toDF("created","user_id") df.insertIntoJDBC(url,"userlog",false) //println(log2.count()) }}
参考链接
http://www.jianshu.com/p/b1a709a57faa
提交包,前提是要配置环境变量
export SPARK_CLASSPATH=$SPARK_CLASSPATH:/usr/local/spark/mysql/mysql.jar
spark-submit --class insertmysql /home/deploy/mysqltest.jar