垃圾邮件分类(Scala 版本)
import org.apache.log4j.{Level, Logger} import org.apache.spark.mllib.classification.LogisticRegressionWithSGD import org.apache.spark.mllib.feature.HashingTF import org.apache.spark.mllib.regression.LabeledPoint import org.apache.spark.{SparkConf, SparkContext} /** * Created by DengNi on 2016/9/21. * 邮件分类 scala */ object spam_normal { def main(args: Array[String]) { Logger.getLogger("org.apache.spark").setLevel(Level.ERROR) val conf = new SparkConf().setAppName("scala").setMaster("local[*]") val sc = new SparkContext(conf) val spam = sc.textFile("spam.txt") val norm = sc.textFile("noraml.txt") //创建一个HashingTF 实例来把邮件文本映射为包含 10000 个特征的向量 val tf = new HashingTF(numFeatures = 10000) //各个邮件都被切分为单词,每个单词被映射为一个特征 val spamFeatures = spam.map(email =>tf.transform(email.split(" "))) val normFeatures = norm.map(email =>tf.transform(email.split(" "))) //创建lablepoint 数据集分别存放垃圾邮件和正常邮件 val positiveExample =spamFeatures.map(features => LabeledPoint(1,features)) val negativeExample =normFeatures.map(features => LabeledPoint(0,features)) val trainingDatat = positiveExample.union(negativeExample) //因为逻辑回归是迭代算法,所以使用缓存技术 trainingDatat.cache() //使用SGD 算法运行逻辑回归 val model = new LogisticRegressionWithSGD().run(trainingDatat) //使用两组数据测试 val psTest = tf.transform("fuck you love sex cheap by sending money fund".split(" ")) val negTest = tf.transform("Hi hwo do you good to see you want to spark".split(" ")) println(model.predict(psTest)) //should be 1 println(model.predict(negTest)) //should be 0 } }
"C:\Program Files\Java\jdk1.7.0_80\bin\java" -Didea.launcher.port=7533 "-Didea.launcher.bin.path=C:\Program Files (x86)\JetBrains\IntelliJ IDEA Community Edition 2016.1.3\bin" -Dfile.encoding=UTF-8 -classpath "C:\Program Files\Java\jdk1.7.0_80\jre\lib\charsets.jar;C:\Program
Files\Java\jdk1.7.0_80\jre\lib\deploy.jar;C:\Program Files\Java\jdk1.7.0_80\jre\lib\ext\access-bridge-64.jar;C:\Program Files\Java\jdk1.7.0_80\jre\lib\ext\dnsns.jar;C:\Program Files\Java\jdk1.7.0_80\jre\lib\ext\jaccess.jar;C:\Program Files\Java\jdk1.7.0_80\jre\lib\ext\localedata.jar;C:\Program
Files\Java\jdk1.7.0_80\jre\lib\ext\sunec.jar;C:\Program Files\Java\jdk1.7.0_80\jre\lib\ext\sunjce_provider.jar;C:\Program Files\Java\jdk1.7.0_80\jre\lib\ext\sunmscapi.jar;C:\Program Files\Java\jdk1.7.0_80\jre\lib\ext\zipfs.jar;C:\Program Files\Java\jdk1.7.0_80\jre\lib\javaws.jar;C:\Program
Files\Java\jdk1.7.0_80\jre\lib\jce.jar;C:\Program Files\Java\jdk1.7.0_80\jre\lib\jfr.jar;C:\Program Files\Java\jdk1.7.0_80\jre\lib\jfxrt.jar;C:\Program Files\Java\jdk1.7.0_80\jre\lib\jsse.jar;C:\Program Files\Java\jdk1.7.0_80\jre\lib\management-agent.jar;C:\Program
Files\Java\jdk1.7.0_80\jre\lib\plugin.jar;C:\Program Files\Java\jdk1.7.0_80\jre\lib\resources.jar;C:\Program Files\Java\jdk1.7.0_80\jre\lib\rt.jar;D:\bigdataworkspaces\recommder\out\production\recommder;F:\scala\lib\scala-actors-migration.jar;F:\scala\lib\scala-actors.jar;F:\scala\lib\scala-library.jar;F:\scala\lib\scala-reflect.jar;F:\scala\lib\scala-swing.jar;D:\bigdataworkspaces\recommder\lib\spark-assembly-1.6.0-hadoop2.6.0.jar;C:\Program
Files (x86)\JetBrains\IntelliJ IDEA Community Edition 2016.1.3\lib\idea_rt.jar" com.intellij.rt.execution.application.AppMain spam_normal
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
16/09/21 22:16:14 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
16/09/21 22:16:15 INFO Slf4jLogger: Slf4jLogger started
16/09/21 22:16:15 INFO Remoting: Starting remoting
16/09/21 22:16:16 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://sparkDriverActorSystem@192.168.184.1:3070]
16/09/21 22:16:18 WARN : Your hostname, root resolves to a loopback/non-reachable address: fe80:0:0:0:0:5efe:c0a8:8c01%17, but we couldn't find any external IP address!
16/09/21 22:16:19 INFO FileInputFormat: Total input paths to process : 1
16/09/21 22:16:19 INFO FileInputFormat: Total input paths to process : 1
16/09/21 22:16:20 INFO deprecation: mapred.tip.id is deprecated. Instead, use mapreduce.task.id
16/09/21 22:16:20 INFO deprecation: mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id
16/09/21 22:16:20 INFO deprecation: mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap
16/09/21 22:16:20 INFO deprecation: mapred.task.partition is deprecated. Instead, use mapreduce.task.partition
16/09/21 22:16:20 INFO deprecation: mapred.job.id is deprecated. Instead, use mapreduce.job.id
16/09/21 22:16:20 WARN BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS
16/09/21 22:16:20 WARN BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS
1.0
0.0
16/09/21 22:16:24 INFO RemoteActorRefProvider$RemotingTerminator: Shutting down remote daemon.
Process finished with exit code 0