每日一题 为了工作 2020 0504 第六十二题
package data.bjsj.fjjb;
import org.apache.spark.Accumulator;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.VoidFunction;
import org.apache.spark.mllib.classification.LogisticRegressionModel;
import org.apache.spark.mllib.classification.LogisticRegressionWithLBFGS;
import org.apache.spark.mllib.linalg.Vectors;
import org.apache.spark.mllib.regression.LabeledPoint;
import org.apache.spark.rdd.RDD;
import org.apache.spark.sql.SparkSession;
import scala.Tuple2;
/**
*
* @author 雪瞳
* @Slogan 时钟尚且前行,人怎能就此止步!
* @Function
*
*/
public class LogisticModel {
public static void main(String[] args) {
SparkSession session = SparkSession.builder().appName("logistic").master("local").getOrCreate();
JavaSparkContext jsc = JavaSparkContext.fromSparkContext(session.sparkContext());
SparkContext sc = JavaSparkContext.toSparkContext(jsc);
jsc.setLogLevel("Error");
JavaRDD<String> fileRDD = jsc.textFile("./save/rootData");
JavaRDD<LabeledPoint> labeledPointJavaRDD = fileRDD.map(new Function<String, LabeledPoint>() {
//"2015-11-01 20:20:16" 1.85999330468501 1.22359452534749 2.51578969727773 -0.403918740333512 0.0149184125297424 0
@Override
public LabeledPoint call(String line) throws Exception {
String[] splits = line.split("\t");
String label = splits[splits.length - 1];
double[] wd = new double[splits.length - 3];
for (int i = 0; i < wd.length; i++) {
wd[i] = Double.parseDouble(splits[i+1]);
}
LabeledPoint labeledPoint = new LabeledPoint(Double.parseDouble(label), Vectors.dense(wd));
return labeledPoint;
}
});
double[] doubles = new double[]{0.7,0.3};
RDD<LabeledPoint> rdd = labeledPointJavaRDD.rdd();
RDD<LabeledPoint>[] metaDataSource = rdd.randomSplit(doubles, 100L);
RDD<LabeledPoint> traingData = metaDataSource[0];
RDD<LabeledPoint> testData = metaDataSource[1];
LogisticRegressionWithLBFGS lr = new LogisticRegressionWithLBFGS();
lr.setNumClasses(2);
lr.setIntercept(true);
LogisticRegressionModel model = lr.run(traingData);
JavaRDD<Double> predictRdd = testData.toJavaRDD().map(new Function<LabeledPoint, Double>() {
@Override
public Double call(LabeledPoint labeledPoint) throws Exception {
double predict = model.predict(labeledPoint.features());
return predict;
}
});
JavaPairRDD<Double, Double> zipRdd = predictRdd.zip(testData.toJavaRDD().map(new Function<LabeledPoint, Double>() {
@Override
public Double call(LabeledPoint labeledPoint) throws Exception {
return labeledPoint.label();
}
}));
Accumulator<Integer> accumulator = jsc.accumulator(0);
zipRdd.foreach(new VoidFunction<Tuple2<Double, Double>>() {
@Override
public void call(Tuple2<Double, Double> tp) throws Exception {
Double label = tp._2();
Double predict = tp._1();
if (Double.compare(label,predict)==0){
accumulator.add(1);
}
}
});
long count = zipRdd.count();
Integer value = accumulator.value();
System.err.println("总数目是:"+count);
System.err.println("正确数目是:"+value);
double rate = value / (double) count;
System.err.println("正确率是:"+rate*100+"%");
String path ="./save/model";
double stand = 80.00;
if (Double.compare(rate,stand)<0){
model.save(sc,path);
}
}
}