Spark+Scalar+Mysql
包:mysql-connector-java-5.1.39-bin.jar
平台:Win8.1
环境:MyEclipse2015
hadoop-2.7.3.tar.gz + winutils.exe
spark-2.1.0-bin-hadoop2.7.tgz
scala-2.11.8.tgz
package org.ah.rcmd import java.sql.{ DriverManager, PreparedStatement, Connection } object Rdbms { def setRcmdResult(iterator: Iterator[(String, String, String, String, String, String)]): Unit = { var conn: Connection = null var ps: PreparedStatement = null val sbSqlUpdate = new StringBuilder; sbSqlUpdate ++= " UPDATE recommond" sbSqlUpdate ++= " SET" sbSqlUpdate ++= " url1 = ?," sbSqlUpdate ++= " url2 = ?," sbSqlUpdate ++= " url3 = ?," sbSqlUpdate ++= " url4 = ?," sbSqlUpdate ++= " url5 = ?" sbSqlUpdate ++= " WHERE uName = ?" val sqlInsert = "insert into recommond(url1,url2,url3,url4,url5,uName) values (?,?,?,?,?,?)" try { conn = DriverManager.getConnection("jdbc:mysql://localhost:3306/ahrecommend", "root", "root") iterator.foreach(data => { ps = conn.prepareStatement(sbSqlUpdate.toString) ps.setString(1, data._2) ps.setString(2, data._3) ps.setString(3, data._4) ps.setString(4, data._5) ps.setString(5, data._6) ps.setString(6, data._1) if (ps.executeUpdate < 1) { ps = conn.prepareStatement(sqlInsert) ps.setString(1, data._2) ps.setString(2, data._3) ps.setString(3, data._4) ps.setString(4, data._5) ps.setString(5, data._6) ps.setString(6, data._1) ps.executeUpdate() } }) } catch { case e: Exception => println(e.fillInStackTrace()) } finally { if (ps != null) { ps.close() ps = null } if (conn != null) { conn.close() conn = null } } } }
package org.ah.rcmd import org.apache.spark.{ SparkContext, SparkConf } object RDDtoMysql { def main(args: Array[String]) { val conf = new SparkConf().setAppName("RDDToMysql").setMaster("local") val sc = new SparkContext(conf) // 获取数据 val data = sc.parallelize(List(("Andy", "W1", "W2", "W3", "W4", "W5"), ("Cat", "W1", "W1", "W1", "W1", "W1"))) // 写入数据库 data.foreachPartition(Rdbms.setRcmdResult) } }