lkl风控.逻辑回归分析模型测试代码spark1.6

/**
  * Created by lkl on 2017/10/31.
  */
import org.apache.spark.sql.hive.HiveContext
import org.apache.spark.SparkConf
import scala.collection.mutable.ArrayBuffer
import org.apache.spark.SparkContext
import org.apache.spark.mllib.classification.{LogisticRegressionWithLBFGS, LogisticRegressionModel}
import org.apache.spark.mllib.evaluation.MulticlassMetrics
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.util.MLUtils
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.sql.SQLContext
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator
object logisticregression {

  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("test") //setMaster("spark://192.168.0.37:7077")
    val sc = new SparkContext(conf)
    val sqlContext = new SQLContext(sc)
    val hc = new HiveContext(sc)
    val data2 = hc.sql("select * from  fin_tec.uvcy2")
    val data = data2.map{ row => val arr = new ArrayBuffer[Double]()
// 第一个变量为身份证号，第二个变量为是否逾期，先过滤
      for(i <- 2 until row.size){
        if(row.isNullAt(i)){
          arr += 0.0}
        else if(row.get(i).isInstanceOf[Double])
          arr += row.getDouble(i)
        else if(row.get(i).isInstanceOf[Long])
          arr += row.getLong(i).toDouble
        else if(row.get(i).isInstanceOf[String])
          arr += row.getString(i).toDouble}
      LabeledPoint(row.getDouble(1), Vectors.dense(arr.toArray))}
    val splits = data.randomSplit(Array(0.7, 0.3))
    val (trainingData, testData) = (splits(0), splits(1))
    val training = splits(0).cache()
    val test = splits(1)
    // Run training algorithm to build the model
    val model = new LogisticRegressionWithLBFGS().setNumClasses(10).run(training)

    // Compute raw scores on the test set.
    val predictionAndLabels = test.map { case LabeledPoint(label, features) =>
      val prediction = model.predict(features)
      (prediction, label)
    }

    // Get evaluation metrics.
    val metrics = new MulticlassMetrics(predictionAndLabels)
    val precision = metrics.precision
    println("Precision = " + precision)

    // Save and load model
    model.save(sc, "myModelPath")
    val sameModel = LogisticRegressionModel.load(sc, "myModelPath")

  }

}