大三寒假学习 spark学习利用Idea编写spark程序

1.打开file打开settings，找到plugins，在里面搜索scala并下载

2.新建maven项目

3.将pom.xml文件清空粘贴下面内容：

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>dblab</groupId>
    <artifactId>WordCount</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <spark.version>2.1.0</spark.version>
        <scala.version>2.11</scala.version>
    </properties>


    <dependencies>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_${scala.version}</artifactId>
            <version>${spark.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-streaming_${scala.version}</artifactId>
            <version>${spark.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_${scala.version}</artifactId>
            <version>${spark.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-hive_${scala.version}</artifactId>
            <version>${spark.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-mllib_${scala.version}</artifactId>
            <version>${spark.version}</version>
        </dependency>

    </dependencies>

    <build>
        <plugins>

            <plugin>
                <groupId>org.scala-tools</groupId>
                <artifactId>maven-scala-plugin</artifactId>
                <version>2.15.2</version>
                <executions>
                    <execution>
                        <goals>
                            <goal>compile</goal>
                            <goal>testCompile</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>

            <plugin>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.6.0</version>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                </configuration>
            </plugin>

            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-surefire-plugin</artifactId>
                <version>2.19</version>
                <configuration>
                    <skip>true</skip>
                </configuration>
            </plugin>

        </plugins>
    </build>

</project>

4.新建scala class文件选择object

5.粘贴程序测试;

package org.example
import java.io.File
import scala.io.Source

object MyScalaTest {
  def main(args: Array[String]): Unit ={
    val dirfile = new File("F://english")
    val files = dirfile.listFiles// 获取文件列表
    for(file <- files) println(file)//遍历文件列表
    val listFiles = files.toList
    val wordsMap = scala.collection.mutable.Map[String,Int]()//单词映射，统计数目
    //对每个文件遍历
    listFiles.foreach(file => Source.fromFile(file).getLines().foreach(line => line.split(" ").
      //Scoure.fromFile(file).getLines()读取文件获取每一行
      //line => line.split(" ")分割成单词
      //对line.split(" ")集合遍历
      foreach(
        word=>{
          if(wordsMap.contains(word)){
            wordsMap(word)+=1
          }else{
            wordsMap+=(word->1)
          }
        }//进行单词统计
      )
    ))
    println(wordsMap)
    for((key,value)<-wordsMap) println(key+": "+value)
  }
}