package Spark_MLlib
import org.apache.spark.ml.{Pipeline, PipelineModel}
import org.apache.spark.ml.classification.{LogisticRegression, LogisticRegressionModel}
import org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator
import org.apache.spark.ml.feature.{IndexToString, StringIndexer, VectorIndexer}
import org.apache.spark.sql.SparkSession
import org.apache.spark.ml.linalg.{Vector, Vectors}
import org.apache.spark.ml.tuning.{CrossValidator, ParamGridBuilder}
/**
* 调参+模型选择
*/
case class schema_source(features:Vector,label:String)
object 交叉验证_调参_逻辑回归 {
val spark=SparkSession.builder().master("local[2]").getOrCreate()
import spark.implicits._
def main(args: Array[String]): Unit = {
val data=spark.sparkContext.textFile("file:///home/soyo/桌面/spark编程测试数据/soyo.txt")
.map(_.split(",")).map(x=>schema_source(Vectors.dense(x(0).toDouble,x(1).toDouble,x(2).toDouble,x(3).toDouble),x(4))).toDF()
data.show()
val labelIndexer=new StringIndexer().setInputCol("label").setOutputCol("indexedLabel").fit(data)
val featuresIndexer=new VectorIndexer().setInputCol("features").setOutputCol("indexedFeatures").fit(data)
val Array(trainData,testData)=data.randomSplit(Array(0.7,0.3))
val lr=new LogisticRegression().setLabelCol("indexedLabel").setFeaturesCol("indexedFeatures").setMaxIter(50)
val labelConverter=new IndexToString().setInputCol("prediction").setOutputCol("predictedLabel").setLabels(labelIndexer.labels)
labelIndexer.labels.foreach(println)
//机器学习工作流
val lrPipeline=new Pipeline().setStages(Array(labelIndexer,featuresIndexer,lr,labelConverter))
//交叉验证需要的模型评估
val evaluator=new MulticlassClassificationEvaluator().setLabelCol("indexedLabel").setPredictionCol("prediction")
//构造参数网格
val paramGrid=new ParamGridBuilder().addGrid(lr.regParam,Array(0.01,0.3,0.8)).addGrid(lr.elasticNetParam,Array(0.3,0.9)).build()
//构建机器学习工作流的交叉验证,定义验证模型,模型评估,参数网格,数据集的折叠数(交叉验证原理)
val cv=new CrossValidator().setEstimator(lrPipeline).setEvaluator(evaluator).setEstimatorParamMaps(paramGrid).setNumFolds(3)
//训练模型
val cvModel=cv.fit(trainData)
//测试数据
val lrPrediction=cvModel.transform(testData)
lrPrediction.show()
val evaluator2=new MulticlassClassificationEvaluator().setLabelCol("indexedLabel").setPredictionCol("prediction")
val lrAccuracy=evaluator2.evaluate(lrPrediction)
println("准确率为: "+lrAccuracy)
println("错误率为： "+(1-lrAccuracy))
//获取最优模型
val bestModel=cvModel.bestModel.asInstanceOf[PipelineModel]
val lrModel=bestModel.stages(2).asInstanceOf[LogisticRegressionModel]
println("二项逻辑回归模型系数矩阵: "+lrModel.coefficientMatrix)
println("二项逻辑回归模型的截距向量: "+lrModel.interceptVector)
println("类的数量(标签可以使用的值): "+lrModel.numClasses)
println("模型所接受的特征的数量: "+lrModel.numFeatures)
println("所有参数的设置为： "+lrModel.explainParams())
println("最优的regParam的值为: "+lrModel.explainParam(lrModel.regParam))
println("最优的elasticNetParam的值为: "+lrModel.explainParam(lrModel.elasticNetParam))
}
}
+-----------------+-----+
| features|label|
+-----------------+-----+
|[5.1,3.5,1.4,0.2]|soyo1|
|[4.9,3.0,1.4,0.2]|soyo1|
|[4.7,3.2,1.3,0.2]|soyo1|
|[4.6,3.1,1.5,0.2]|soyo1|
|[5.0,3.6,1.4,0.2]|soyo1|
|[5.4,3.9,1.7,0.4]|soyo1|
|[4.6,3.4,1.4,0.3]|soyo1|
|[5.0,3.4,1.5,0.2]|soyo1|
|[4.4,2.9,1.4,0.2]|soyo1|
|[4.9,3.1,1.5,0.1]|soyo1|
|[5.4,3.7,1.5,0.2]|soyo1|
|[4.8,3.4,1.6,0.2]|soyo1|
|[4.8,3.0,1.4,0.1]|soyo1|
|[4.3,3.0,1.1,0.1]|soyo1|
|[5.8,4.0,1.2,0.2]|soyo1|
|[5.7,4.4,1.5,0.4]|soyo1|
|[5.4,3.9,1.3,0.4]|soyo1|
|[5.1,3.5,1.4,0.3]|soyo1|
|[5.7,3.8,1.7,0.3]|soyo1|
|[5.1,3.8,1.5,0.3]|soyo1|
+-----------------+-----+
only showing top 20 rows
soyo2
soyo1
soyo3
+-----------------+-----+------------+-----------------+--------------------+--------------------+----------+--------------+
| features|label|indexedLabel| indexedFeatures| rawPrediction| probability|prediction|predictedLabel|
+-----------------+-----+------------+-----------------+--------------------+--------------------+----------+--------------+
|[4.3,3.0,1.1,0.1]|soyo1| 1.0|[4.3,3.0,1.1,0.1]|[-0.2949197997435...|[0.00821657808181...| 1.0| soyo1|
|[4.4,2.9,1.4,0.2]|soyo1| 1.0|[4.4,2.9,1.4,0.2]|[-0.1436502505351...|[0.02310764702310...| 1.0| soyo1|
|[4.6,3.1,1.5,0.2]|soyo1| 1.0|[4.6,3.1,1.5,0.2]|[-0.1980725396328...|[0.01584026165726...| 1.0| soyo1|
|[4.8,3.0,1.4,0.1]|soyo1| 1.0|[4.8,3.0,1.4,0.1]|[-0.0360182992158...|[0.01909506488946...| 1.0| soyo1|
|[4.8,3.1,1.6,0.2]|soyo1| 1.0|[4.8,3.1,1.6,0.2]|[-0.0963956817735...|[0.02165865158723...| 1.0| soyo1|
|[4.8,3.4,1.6,0.2]|soyo1| 1.0|[4.8,3.4,1.6,0.2]|[-0.3305444022091...|[0.00764403083532...| 1.0| soyo1|
|[4.9,2.4,3.3,1.0]|soyo2| 0.0|[4.9,2.4,3.3,1.0]|[0.64687664475266...|[0.83588965920895...| 0.0| soyo2|
|[4.9,3.0,1.4,0.2]|soyo1| 1.0|[4.9,3.0,1.4,0.2]|[0.00894554123863...|[0.02696343238302...| 1.0| soyo1|
|[5.0,3.5,1.6,0.6]|soyo1| 1.0|[5.0,3.5,1.6,0.6]|[-0.3209967599706...|[0.01781564148264...| 1.0| soyo1|
|[5.0,3.6,1.4,0.2]|soyo1| 1.0|[5.0,3.6,1.4,0.2]|[-0.4132228265822...|[0.00370148550004...| 1.0| soyo1|
|[5.1,3.7,1.5,0.4]|soyo1| 1.0|[5.1,3.7,1.5,0.4]|[-0.4380550804437...|[0.00533390253840...| 1.0| soyo1|
|[5.1,3.8,1.9,0.4]|soyo1| 1.0|[5.1,3.8,1.9,0.4]|[-0.4784298068885...|[0.00593236888116...| 1.0| soyo1|
|[5.2,2.7,3.9,1.4]|soyo2| 0.0|[5.2,2.7,3.9,1.4]|[0.60296648363520...|[0.65499655703255...| 0.0| soyo2|
|[5.2,3.5,1.5,0.2]|soyo1| 1.0|[5.2,3.5,1.5,0.2]|[-0.2334963952443...|[0.00721300202565...| 1.0| soyo1|
|[5.3,3.7,1.5,0.2]|soyo1| 1.0|[5.3,3.7,1.5,0.2]|[-0.3434664691509...|[0.00396451436269...| 1.0| soyo1|
|[5.4,3.4,1.5,0.4]|soyo1| 1.0|[5.4,3.4,1.5,0.4]|[-0.0655191408567...|[0.02050202848213...| 1.0| soyo1|
|[5.4,3.4,1.7,0.2]|soyo1| 1.0|[5.4,3.4,1.7,0.2]|[-0.0443512521479...|[0.01568504280438...| 1.0| soyo1|
|[5.4,3.9,1.3,0.4]|soyo1| 1.0|[5.4,3.9,1.3,0.4]|[-0.4746044317663...|[0.00285607924154...| 1.0| soyo1|
|[5.4,3.9,1.7,0.4]|soyo1| 1.0|[5.4,3.9,1.7,0.4]|[-0.4369295847326...|[0.00451151133277...| 1.0| soyo1|
|[5.5,2.3,4.0,1.3]|soyo2| 0.0|[5.5,2.3,4.0,1.3]|[1.06413594105520...|[0.51327715648015...| 0.0| soyo2|
+-----------------+-----+------------+-----------------+--------------------+--------------------+----------+--------------+
only showing top 20 rows
准确率为: 0.9418343292582645
错误率为： 0.05816567074173551
二项逻辑回归模型系数矩阵: 0.4612907305046201 -0.7804957347855317 0.09418711758439907 -0.011652325959556013
-0.559055378870932 2.7385209747134933 -1.052922922424876 -2.5223769474140303
-0.07629895224519458 -3.6867236615320547 1.0014498171011217 4.581938360185545
二项逻辑回归模型的截距向量: [-0.039423333303658874,0.0972586768296292,-0.05783534352597033]
类的数量(标签可以使用的值): 3
模型所接受的特征的数量: 4
所有参数的设置为： aggregationDepth: suggested depth for treeAggregate (>= 2) (default: 2)
elasticNetParam: the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty (default: 0.0, current: 0.9)
family: The name of family which is a description of the label distribution to be used in the model. Supported options: auto, binomial, multinomial. (default: auto)
featuresCol: features column name (default: features, current: indexedFeatures)
fitIntercept: whether to fit an intercept term (default: true)
labelCol: label column name (default: label, current: indexedLabel)
lowerBoundsOnCoefficients: The lower bounds on coefficients if fitting under bound constrained optimization. (undefined)
lowerBoundsOnIntercepts: The lower bounds on intercepts if fitting under bound constrained optimization. (undefined)
maxIter: maximum number of iterations (>= 0) (default: 100, current: 50)
predictionCol: prediction column name (default: prediction)
probabilityCol: Column name for predicted class conditional probabilities. Note: Not all models output well-calibrated probability estimates! These probabilities should be treated as confidences, not precise probabilities (default: probability)
rawPredictionCol: raw prediction (a.k.a. confidence) column name (default: rawPrediction)
regParam: regularization parameter (>= 0) (default: 0.0, current: 0.01)
standardization: whether to standardize the training features before fitting the model (default: true)
threshold: threshold in binary classification prediction, in range [0, 1] (default: 0.5)
thresholds: Thresholds in multi-class classification to adjust the probability of predicting each class. Array must have length equal to the number of classes, with values > 0 excepting that at most one value may be 0. The class with largest value p/t is predicted, where p is the original probability of that class and t is the class's threshold (undefined)
tol: the convergence tolerance for iterative algorithms (>= 0) (default: 1.0E-6)
upperBoundsOnCoefficients: The upper bounds on coefficients if fitting under bound constrained optimization. (undefined)
upperBoundsOnIntercepts: The upper bounds on intercepts if fitting under bound constrained optimization. (undefined)
weightCol: weight column name. If this is not set or empty, we treat all instance weights as 1.0 (undefined)
最优的regParam的值为: regParam: regularization parameter (>= 0) (default: 0.0, current: 0.01)
最优的elasticNetParam的值为: elasticNetParam: the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty (default: 0.0, current: 0.9)
