每日一题 为了工作 2020 0429 第五十八题
//Java版本的线性回归的预测代码
package com.swust.machine;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.mllib.linalg.Vector;
import org.apache.spark.mllib.linalg.Vectors;
import org.apache.spark.mllib.regression.LabeledPoint;
import org.apache.spark.mllib.regression.LinearRegressionModel;
import org.apache.spark.mllib.regression.LinearRegressionWithSGD;
import org.apache.spark.rdd.RDD;
import scala.Tuple2;
import java.util.List;
/**
*
* @author 雪瞳
* @Slogan 时钟尚且前行,人怎能再此止步!
* @Function 线性回归算法实现
*
*/
public class LinearRegression {
public static void main(String[] args) {
SparkConf conf = new SparkConf();
conf.setMaster("local").setAppName("line");
JavaSparkContext jsc = new JavaSparkContext(conf);
jsc.setLogLevel("Error");
// 读取样本数据
JavaRDD<String> data = jsc.textFile("./data/lpsa.data");
JavaRDD<LabeledPoint> examples = data.map(new Function<String, LabeledPoint>() {
@Override
public LabeledPoint call(String line) throws Exception {
String[] splits = line.split(",");
String y = splits[0];
String xs = splits[1];
String[] words = xs.split(" ");
double[] wd = new double[words.length];
for (int i = 0; i < words.length; i++) {
wd[i] = Double.parseDouble(words[i]);
}
return new LabeledPoint(Double.parseDouble(y),
Vectors.dense(wd));
}
});
//将数据集按比例切分为训练集和测试集
double[] doubles = new double[]{0.8,0.2};
RDD<LabeledPoint> rdd = examples.rdd();
RDD<LabeledPoint>[] TestData = rdd.randomSplit(doubles, 1L);
//设置迭代次数
int numIterations = 100;
//设置迭代过程中 梯度下降算法的下降步长大小
// 0.1 0.2 0.3 0.4
int stepSize = 1;
int miniBatchFraction = 1;
LinearRegressionWithSGD lrs = new LinearRegressionWithSGD();
//设置训练模型是否存在截距
lrs.setIntercept(true);
//设置步长
lrs.optimizer().setStepSize(stepSize);
//设置迭代次数
lrs.optimizer().setNumIterations(numIterations);
//计算所有样本的误差值,1代表所有样本,默认1.0
lrs.optimizer().setMiniBatchFraction(miniBatchFraction);
//GeneralizedLinearAlgorithm
LinearRegressionModel model = lrs.run(TestData[0]);
System.err.println(model.weights());
System.err.println(model.intercept());
//对样本的测试
JavaRDD<Double> prediction = model.predict(TestData[1].toJavaRDD().map(new Function<LabeledPoint, Vector>() {
@Override
public Vector call(LabeledPoint labeledPoint) throws Exception {
return labeledPoint.features();
}
}));
//压缩样本
JavaPairRDD<Double, Double> predictionAndLabel = prediction.zip(TestData[1].toJavaRDD().map(new Function<LabeledPoint, Double>() {
@Override
public Double call(LabeledPoint labeledPoint) throws Exception {
return labeledPoint.label();
}
}));
//数据分析 取其中20条
List<Tuple2<Double, Double>> take = predictionAndLabel.take(20);
//预测 标签
System.err.println("prediction"+"\t"+"label");
for (Tuple2<Double, Double> elem:take){
System.out.println(elem._1()+"\t"+elem._2());
}
//计算数据的平均误差
JavaRDD<Double> dataLoss = predictionAndLabel.map(new Function<Tuple2<Double, Double>, Double>() {
@Override
public Double call(Tuple2<Double, Double> one) throws Exception {
double err = one._1() - one._2();
return Math.abs(err);
}
});
Double lossResult = dataLoss.reduce(new Function2<Double, Double, Double>() {
@Override
public Double call(Double aDouble, Double aDouble2) throws Exception {
return aDouble + aDouble2;
}
});
double err = lossResult / TestData[1].count();
System.err.println("Test RMSE"+err);
jsc.stop();
}
}
//由于数据量本身只有100条 所以预测效果相对较差 但是重要的是思路不是嘛
// 有道无术术可求 有术无道止于术 学会一个思想更为重要