【spark】IDEA建立基于scala语言的spark项目

1.新建一个Spark项目

2.选择maven，用模板创建项目

如果没有这个模板，我们需要添加一个

我们这里使用的是1.6版本
Archetype Group Id : net.alchim31.maven
Archetype Artifact Id : scala-archetype-simple
Archetype Version : 1.6

3.填写GoupId等。

4.选择本地的maven配置文件和仓库

5.创建完毕

这里我们要注意项目pom.xml文件中的配置

核对scala版本

并在pom.xml文件中添加如下信息

<properties>
  <scala.version>2.12.3</scala.version>
  <spark.version>2.2.0</spark.version>
  <hadoop.version>2.6.0</hadoop.version>
  <hbase.version>1.2.0</hbase.version>
</properties>

<dependencies>
  <!--scala-->
  <dependency>
    <groupId>org.scala-lang</groupId>
    <artifactId>scala-library</artifactId>
    <version>${scala.version}</version>
  </dependency>
  <!-- spark -->
  <dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-core_2.11</artifactId>
    <version>${spark.version}</version>
  </dependency>
  <dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-sql_2.11</artifactId>
    <version>${spark.version}</version>
  </dependency>
  <dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-streaming_2.11</artifactId>
    <version>${spark.version}</version>
    <scope>provided</scope>
  </dependency>

  <!-- hadoop -->
  <dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-client</artifactId>
    <version>${hadoop.version}</version>
  </dependency>
  <dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-common</artifactId>
    <version>${hadoop.version}</version>
  </dependency>
  <dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-hdfs</artifactId>
    <version>${hadoop.version}</version>
  </dependency>

  <!--hbase-->
  <dependency>
    <groupId>org.apache.hbase</groupId>
    <artifactId>hbase-client</artifactId>
    <version>${hbase.version}</version>
  </dependency>
  <dependency>
    <groupId>org.apache.hbase</groupId>
    <artifactId>hbase-server</artifactId>
    <version>${hbase.version}</version>
  </dependency>
</dependencies>