GBT算法在拖动滑块辨别人还是机器中的应用
1.数据源格式：（x,y,t),第一个值x是x坐标范围是1-250的整数，y是1-10的整数，t是滑块从上一个坐标到下一个坐标的时间差，ok是判断是人操作的，Fail是判断是机器操作的，数据看的出，同一个记录里面的同一个点,即x,y都相同，但是t不同，以此分析，如果同一个点只出现一次，则该点记录为1，如果出现n次，重复次数为m次，则设计该点的值为m/n,如该点在该条记录出现的总次数是5，t不同的次数是3，则该点的值是=3、5=0.6，分析的依据是，如果该点重复的次数越多，而且距离上一个时间差越近，说明越接近机器人的轨迹，因为人的轨迹是变化较大的。将模型设计为一个大表，以1-2500这2500个数字为字段，这个表在scala 程序中是一个数组，数组长度为2500,1...250分别对应坐标点中的（0,0,）....(0,250),2-500分别对应坐标点中的(1,0)...(1，250）...所以必然，在每条记录里面的所有点必然在这个数组中能找到，在表中就是一行，这些点必然是在这行中能找到一个列属于该点，如果不存在，则设为0.
[[25,27,0],[6,-1,470],[8,2,20],[14,-1,38],[10,1,28],[3,0,9],[10,-2,28],[9,1,27],[12,-1,38],[12,0,39],[3,2,10],[10,0,35],[3,-1,8],[6,-1,25],[8,1,31],[5,0,18],[2,1,11],[7,0,29],[2,-2,11],[4,2,20],[4,-2,18],[2,1,13],[1,-1,7],[6,2,32],[3,-1,17],[5,0,37],[4,-1,32],[3,0,24],[4,1,32],[4,-1,39],[4,0,40],[3,2,35],[1,0,12],[2,-2,28],[0,1,229]];FAIL
[[27,24,0],[4,1,467],[8,-1,40],[2,-1,7],[2,2,9],[3,-2,15],[6,2,32],[4,-1,20],[7,1,35],[5,-2,26],[2,1,10],[5,1,27],[1,-1,6],[2,0,15],[5,1,26],[4,-2,28],[3,0,23],[1,2,7],[4,-1,26],[3,-1,19],[1,0,7],[3,2,24],[0,0,6],[5,-1,40],[4,-1,39],[1,1,8],[0,0,7],[3,0,35],[3,0,26],[2,-1,32],[2,0,32],[2,2,33],[1,-2,8],[1,1,25],[1,1,14],[1,-1,22],[2,-1,40],[0,0,22],[1,0,1],[0,1,219]];FAIL
[[25,28,0],[8,1,524],[7,-1,27],[4,0,16],[3,-1,16],[3,2,10],[8,-1,35],[8,-1,40],[8,2,38],[2,-1,10],[6,-1,29],[4,0,23],[7,2,36],[5,-2,33],[5,2,26],[4,-1,29],[4,0,26],[1,0,9],[5,-1,38],[2,0,25],[4,1,29],[3,0,37],[2,0,19],[0,-1,6],[3,1,34],[2,0,24],[2,0,27],[2,-1,32],[2,0,40],[1,1,6],[1,-1,22],[1,0,33],[1,0,4],[0,1,299]];OK
2.算法构造scala代码如下：
import org.apache.spark.ml.Pipeline import org.apache.spark.ml.feature.VectorIndexer import org.apache.spark.ml.regression.{GBTRegressionModel, GBTRegressor} import org.apache.spark.SparkContext import org.apache.spark.SparkConf import org.apache.spark.mllib.linalg.Vectors import org.apache.spark.sql.types.{DoubleType, StringType, StructField, StructType} import org.apache.spark.sql.SQLContext import org.apache.spark.mllib.evaluation.BinaryClassificationMetrics import org.apache.spark.mllib.regression.LabeledPoint object GBTForget { /** * Created by lkl on 2017/12/14. */ def main(args: Array[String]): Unit = { val cf = new SparkConf().setAppName("ass").setMaster("local") val sc = new SparkContext(cf) val sqlContext = new SQLContext(sc) import sqlContext.implicits._ val File1 = sc.textFile("20171117PP.txt").filter(_.contains("OK")).map(_.replace(",0],","a[").split("a").last).map(_.replace("OK", "1")).map(_.replace("FAIL", "0")).map(line => (line.split(";").last.toDouble, line.split(";").head)) val File2=sc.textFile("20171117PP.txt").filter(_.contains("FAIL")).map(_.replace(",0],","a[").split("a").last).map(_.replace("OK", "1")).map(_.replace("FAIL", "0")).map(line => (line.split(";").last.toDouble, line.split(";").head)) val b=File2.randomSplit(Array(0.1, 0.9)) val (strainingDatas, stestDatas) = (b(0), b(1)) val File=File1 union(strainingDatas) val ass = File.map { p => { var str = "" val l = p._1 val a = p._2.substring(2, p._2.length - 2) val b = a.replace("],[", "a") val c = b.split("a") for (arr <- c) { val index1 = arr.split(",")(0).toInt + "," val index2 = arr.split(",")(1).toInt + "," val index3 = arr.split(",")(2).toInt + " " val index = index1 + index2 + index3 str += index } (l, str.substring(0, str.length - 1)) } } val rdd = ass.map( p => { val l=p._1 val rowall =new Array[Double](2500) val arr = p._2.split(" ") var map:Map[Int,List[Double]] = Map() var vlist:List[Double] = List() for(a <- arr){ val x = a.split(",")(0).toInt val y = a.split(",")(1).toInt+5 val t = a.split(",")(2).toInt val index = (x*10)+(y+1) val v = t vlist = v :: map.get(index).getOrElse(List()) map += (index -> vlist) } map.foreach(p => { val k = p._1 val v = p._2 val sv = v.toSet.size val rv = sv.toDouble/v.size.toDouble val tmp =f"$rv%1.2f".toDouble rowall(k) = tmp }) (l.toDouble,Vectors.dense(rowall)) }).toDF("label","features") // val label=row.getInt(0).toDouble // val no=row.getString(2) // val feature=Vectors.dense(arr.toArray) // (label,no,feature) // Automatically identify categorical features, and index them. // Set maxCategories so features with > 4 distinct values are treated as continuous. val featureIndexer = new VectorIndexer().setInputCol("features").setOutputCol("indexedFeatures").setMaxCategories(4).fit(rdd) // Split the data into training and test sets (30% held out for testing) val Array(trainingData, testData) = rdd.randomSplit(Array(0.7, 0.3)) // Train a GBT model. val gbt = new GBTRegressor().setLabelCol("label").setFeaturesCol("indexedFeatures").setMaxIter(10) // Chain indexer and GBT in a Pipeline val pipeline = new Pipeline().setStages(Array(featureIndexer, gbt)) // Train model. This also runs the indexer. val model = pipeline.fit(trainingData) // Make predictions. val predictions = model.transform(testData).select("label","prediction").toJavaRDD predictions.repartition(1).saveAsTextFile("/user/hadoop/20171214") val File0=sc.textFile("001.txt").map(_.replace(",0],","a[").split("a").last).map(_.replace("OK", "1")).map(_.replace("FAIL", "0")).map(line => (line.split(";").last.toDouble, line.split(";").head)) val ass001 = File0.map { p => { var str = "" val l = p._1 val a = p._2.substring(2, p._2.length - 2) val b = a.replace("],[", "a") val c = b.split("a") for (arr <- c) { val index1 = arr.split(",")(0).toInt + "," val index2 = arr.split(",")(1).toInt + "," val index3 = arr.split(",")(2).toInt + " " val index = index1 + index2 + index3 str += index } (l, str.substring(0, str.length - 1)) } } val rdd001 = ass001.map( p => { val l=p._1 val rowall =new Array[Double](2500) val arr = p._2.split(" ") var map:Map[Int,List[Double]] = Map() var vlist:List[Double] = List() for(a <- arr){ val x = a.split(",")(0).toInt val y = a.split(",")(1).toInt+5 val t = a.split(",")(2).toInt val index = (x*10)+(y+1) val v = t vlist = v :: map.get(index).getOrElse(List()) map += (index -> vlist) } map.foreach(p => { val k = p._1 val v = p._2 val sv = v.toSet.size val rv = sv.toDouble/v.size.toDouble val tmp =f"$rv%1.2f".toDouble rowall(k) = tmp }) (l.toDouble,Vectors.dense(rowall)) }).toDF("label","features") val predicions001=model.transform(rdd001) // predicions001.repartition(1).saveAsTextFile("/user/hadoop/20171214001") } }
