（六）Spark-Eclipse开发环境WordCount-Java&Python版Spark

Spark-Eclipse开发环境WordCount

视频教程：

1、优酷

2、YouTube

 

安装eclipse

解压eclipse-jee-mars-2-win32-x86_64.zip

 

JavaWordcount

解压spark-2.0.0-bin-hadoop2.6.tgz

创建 Java Project-->Spark

将spark-2.0.0-bin-hadoop2.6下的jars里面的jar全部复制到Spark项目下的lib下

Add Build Path

package com.bean.spark.wordcount;

 

import java.util.Arrays;

import java.util.Iterator;

 

import org.apache.spark.SparkConf;

import org.apache.spark.api.java.JavaPairRDD;

import org.apache.spark.api.java.JavaRDD;

import org.apache.spark.api.java.JavaSparkContext;

import org.apache.spark.api.java.function.FlatMapFunction;

import org.apache.spark.api.java.function.Function2;

import org.apache.spark.api.java.function.PairFunction;

import org.apache.spark.api.java.function.VoidFunction;

 

import scala.Tuple2;

 

public class WordCount {

public static void main(String[] args) {

//创建SparkConf对象，设置Spark应用程序的配置信息

SparkConf conf = new SparkConf();

conf.setMaster("local");

conf.setAppName("wordcount");

 

//创建SparkContext对象，Java开发使用JavaSparkContext；Scala开发使用SparkContext

//SparkContext负责连接Spark集群，创建RDD、累积量和广播量等

JavaSparkContext sc = new JavaSparkContext(conf);

 

//sc中提供了textFile方法是SparkContext中定义的,用来读取HDFS上的

//文本文件、集群中节点的本地文本文件或任何支持Hadoop的文件系统上的文本文件，它的返回值是JavaRDD[String]，是文本文件每一行

JavaRDD<String> lines = sc.textFile("D:/tools/data/wordcount/wordcount.txt");

//将每一行文本内容拆分为多个单词

//lines调用flatMap这个transformation算子（参数类型是FlatMapFunction接口实现类）返回每一行的每个单词

JavaRDD<String> words = lines.flatMap(new FlatMapFunction<String, String>() {

 

private static final long serialVersionUID = 1L;

 

@Override

public Iterator<String> call(String s) throws Exception {

// TODO Auto-generated method stub

return Arrays.asList(s.split(" ")).iterator();

}

});

//将每个单词的初始数量都标记为1个

//words调用mapToPair这个transformation算子（参数类型是PairFunction接口实现类，

//PairFunction<String, String, Integer>的三个参数是<输入单词, Tuple2的key, Tuple2的value>），

//返回一个新的RDD，即JavaPairRDD

JavaPairRDD<String, Integer> word = words.mapToPair(new PairFunction<String, String, Integer>() {

 

private static final long serialVersionUID = 1L;

 

@Override

public Tuple2<String, Integer> call(String s) throws Exception {

// TODO Auto-generated method stub

return new Tuple2<String, Integer>(s, 1);

}

});

//计算每个相同单词出现的次数

//pairs调用reduceByKey这个transformation算子（参数是Function2接口实现类）对每个key的value进行reduce操作，

//返回一个JavaPairRDD，这个JavaPairRDD中的每一个Tuple的key是单词、value则是相同单词次数的和

JavaPairRDD<String, Integer> counts = word.reduceByKey(new Function2<Integer, Integer, Integer>() {

 

private static final long serialVersionUID = 1L;

 

@Override

public Integer call(Integer s1, Integer s2) throws Exception {

// TODO Auto-generated method stub

return s1 + s2;

}

});

counts.foreach(new VoidFunction<Tuple2<String,Integer>>() {

 

private static final long serialVersionUID = 1L;

 

@Override

public void call(Tuple2<String, Integer> wordcount) throws Exception {

// TODO Auto-generated method stub

System.out.println(wordcount._1+" : "+wordcount._2);

}

});

//将计算结果文件输出到文件系统

/*

 * HDFS

 * 新版的API

 * org.apache.hadoop.mapreduce.lib.output.TextOutputFormat

 * counts.saveAsNewAPIHadoopFile("hdfs://master:9000/data/wordcount/output", Text.class, IntWritable.class, TextOutputFormat.class, new Configuration());

 * 使用默认TextOutputFile写入到HDFS(注意写入HDFS权限，如无权限则执行：hdfs dfs -chmod -R 777 /data/wordCount/output)

         * wordCount.saveAsTextFile("hdfs://soy1:9000/data/wordCount/output");

         *

 *

 * */

counts.saveAsTextFile("D:/tools/data/wordcount/output");

 

 

//关闭SparkContext容器，结束本次作业

sc.close();

}

}

 

 

运行出错

在代码中加入：只要式加在JavaSparkContext初始化之前就可以

System.setProperty("hadoop.home.dir", "D:/tools/spark-2.0.0-bin-hadoop2.6");

将hadoop2.6(x64)工具.zip解压到D:\tools\spark-2.0.0-bin-hadoop2.6\bin目录下

 

PythonWordcount

eclipse集成python插件

解压pydev.zip将features和plugins中的包复制到eclipse的对应目录

#-*- coding:utf-8-*-

 

from __future__ import print_function

from operator import add

import os

from pyspark.context import SparkContext

'''

wordcount

'''

if __name__ == "__main__":

    os.environ["HADOOP_HOME"] = "D:/tools/spark-2.0.0-bin-hadoop2.6"

    sc = SparkContext()

    lines = sc.textFile("file:///D:/tools/data/wordcount/wordcount.txt").map(lambda r: r[0:])

    counts = lines.flatMap(lambda x: x.split(' ')) \

                  .map(lambda x: (x, 1)) \

                  .reduceByKey(add)

    output = counts.collect()

    for (word, count) in output:

        print("%s: %i" % (word, count))

 

 

提交代码到集群上运行

java：

[hadoop@master application]$ spark-submit --master spark://master:7077 --class com.bean.spark.wordcount.WordCount spark.jar

 python：

[hadoop@master application]$ spark-submit --master spark://master:7077 wordcount.py