搭建一个IntelliJ的Spark项目
之前发现创建一个新项目之后，无法添加scala class
创建新项目
选择maven项目，然后选择simple或者quickstart；
进入项目后，在Project Structure里面，在global libraries面板中，删除已有的scala-sdk，然后再添加；
然后再工程中添加scala文件夹，添加一个package，在package里面就可以添加scala文件了。
Maven
1 <?xml version="1.0" encoding="UTF-8"?> 2 <project xmlns="http://maven.apache.org/POM/4.0.0" 3 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 4 xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 5 <modelVersion>4.0.0</modelVersion> 6 7 <groupId>com.cmiot</groupId> 8 <artifactId>parquetToHbase</artifactId> 9 <packaging>jar</packaging> 10 <version>3.2.1-diy-thinrow</version> 11 12 <repositories> 13 <repository> 14 <id>cloudera</id> 15 <url>https://repository.cloudera.com/cloudera/cloudera-repos</url> 16 </repository> 17 <repository> 18 <id>nexus</id> 19 <url>https://maven.atlassian.com/3rdparty/</url> 20 </repository> 21 </repositories> 22 23 <dependencies> 24 <dependency> 25 <groupId>com.typesafe</groupId> 26 <artifactId>config</artifactId> 27 <version>1.2.1</version> 28 </dependency> 29 30 31 <dependency> 32 <groupId>org.apache.spark</groupId> 33 <artifactId>spark-core_${spark.tool.version}</artifactId> 34 <version>${spark.version}</version> 35 <!--scope>provided</scope--> 36 <exclusions> 37 <exclusion> 38 <groupId>org.scala-lang</groupId> 39 <artifactId>scala-library</artifactId> 40 </exclusion> 41 <exclusion> 42 <groupId>org.scala-lang</groupId> 43 <artifactId>scala-lang</artifactId> 44 </exclusion> 45 <exclusion> 46 <groupId>org.scala-lang</groupId> 47 <artifactId>scalap</artifactId> 48 </exclusion> 49 <exclusion> 50 <artifactId>slf4j-log4j12</artifactId> 51 <groupId>org.slf4j</groupId> 52 </exclusion> 53 </exclusions> 54 </dependency> 55 <dependency> 56 <groupId>org.apache.spark</groupId> 57 <artifactId>spark-sql_${spark.tool.version}</artifactId> 58 <version>${spark.version}</version> 59 <!--scope>provided</scope--> 60 </dependency> 61 <!--<dependency>--> 62 <!--<groupId>com.oracle</groupId>--> 63 <!--<artifactId>ojdbc6</artifactId>--> 64 <!--<version>11.2.0.4.0-atlassian-hosted</version>--> 65 <!--</dependency> 66 <dependency> 67 <groupId>oracle</groupId> 68 <artifactId>ojdbc</artifactId> 69 <version>8</version> 70 </dependency>--> 71 <dependency> 72 <groupId>com.github.nscala-time</groupId> 73 <artifactId>nscala-time_${scala.version.tools}</artifactId> 74 <version>2.0.0</version> 75 </dependency> 76 <dependency> 77 <groupId>org.scala-lang</groupId> 78 <artifactId>scala-library</artifactId> 79 <version>${scala.version}</version> 80 </dependency> 81 82 <dependency> 83 <groupId>org.scala-lang</groupId> 84 <artifactId>scala-reflect</artifactId> 85 <version>${scala.version}</version> 86 </dependency> 87 88 <dependency> 89 <groupId>org.scala-lang</groupId> 90 <artifactId>scala-compiler</artifactId> 91 <version>${scala.version}</version> 92 </dependency> 93 <dependency> 94 <groupId>org.scalatest</groupId> 95 <artifactId>scalatest_${scala.version.tools}</artifactId> 96 <version>${scala.test.version}</version> 97 <scope>test</scope> 98 <exclusions> 99 <exclusion> 100 <groupId>org.scala-lang</groupId> 101 <artifactId>scala-library</artifactId> 102 </exclusion> 103 </exclusions> 104 </dependency> 105 <dependency> 106 <groupId>org.scalacheck</groupId> 107 <artifactId>scalacheck_${scala.version.tools}</artifactId> 108 <version>${scala.check.version}</version> 109 <scope>test</scope> 110 <exclusions> 111 <exclusion> 112 <groupId>org.scala-lang</groupId> 113 <artifactId>scala-library</artifactId> 114 </exclusion> 115 </exclusions> 116 </dependency> 117 <dependency> 118 <groupId>junit</groupId> 119 <artifactId>junit</artifactId> 120 <version>${junit.version}</version> 121 <scope>test</scope> 122 </dependency> 123 <dependency> 124 <groupId>org.slf4j</groupId> 125 <artifactId>slf4j-log4j12</artifactId> 126 <version>${slf4j.log4j12.version}</version> 127 <!--<scope>test</scope>--> 128 </dependency> 129 <dependency> 130 <groupId>org.alluxio</groupId> 131 <artifactId>alluxio-core-client</artifactId> 132 <version>${alluxio.client.version}</version> 133 </dependency> 134 <dependency> 135 <groupId>com.databricks</groupId> 136 <artifactId>spark-xml_${spark.tool.version}</artifactId> 137 <version>${spark.xml.version}</version> 138 </dependency> 139 <dependency> 140 <groupId>org.apache.hbase</groupId> 141 <artifactId>hbase-client</artifactId> 142 <version>${hbase.version}</version> 143 <exclusions> 144 <exclusion> 145 <groupId>javax.servlet</groupId> 146 <artifactId>*</artifactId> 147 </exclusion> 148 </exclusions> 149 </dependency> 150 <dependency> 151 <groupId>org.apache.hbase</groupId> 152 <artifactId>hbase-server</artifactId> 153 <version>${hbase.version}</version> 154 <exclusions> 155 <exclusion> 156 <groupId>javax.servlet</groupId> 157 <artifactId>*</artifactId> 158 </exclusion> 159 </exclusions> 160 </dependency> 161 <!-- 162 <dependency> 163 <groupId>org.apache.hbase</groupId> 164 <artifactId>hbase-spark</artifactId> 165 <version>1.2.0-cdh5.10.2</version> 166 </dependency> 167 --> 168 </dependencies> 169 170 <properties> 171 <hbase.version>1.2.0-cdh5.11.2</hbase.version> 172 <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> 173 <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> 174 <maven.compiler.encoding>UTF-8</maven.compiler.encoding> 175 <maven.surefire.version>2.7</maven.surefire.version> 176 <maven.compiler.version>3.1</maven.compiler.version> 177 <maven.assembly.version>2.4</maven.assembly.version> 178 <java.version>1.8</java.version> 179 <scala.version>2.11.8</scala.version> 180 <scala.version.tools>2.11</scala.version.tools> 181 <scala.maven.version>3.2.0</scala.maven.version> 182 <scala.check.version>1.11.4</scala.check.version> 183 <scala.test.version>3.0.1</scala.test.version> 184 <scala.test.maven.version>1.0</scala.test.maven.version> 185 <junit.version>4.11</junit.version> 186 <spark.version>2.1.0.cloudera2</spark.version> 187 <spark.tool.version>2.11</spark.tool.version> 188 <spark.xml.version>0.4.1</spark.xml.version> 189 <alluxio.client.version>1.0.0</alluxio.client.version> 190 <cobertura.version>2.7</cobertura.version> 191 <slf4j.log4j12.version>1.7.10</slf4j.log4j12.version> 192 <test.arg.line>-Xms1024m -Xmx1024m -XX:PermSize=512m -XX:MaxPermSize=1024m</test.arg.line> 193 </properties> 194 195 <profiles> 196 <profile> 197 <id>dev</id> 198 <activation> 199 <activeByDefault>true</activeByDefault> 200 </activation> 201 <properties> 202 <filters.env>dev</filters.env> 203 </properties> 204 </profile> 205 206 <profile> 207 <id>test</id> 208 209 <properties> 210 <filters.env>test</filters.env> 211 </properties> 212 </profile> 213 214 <profile> 215 <id>prod</id> 216 <properties> 217 <filters.env>prod</filters.env> 218 </properties> 219 </profile> 220 </profiles> 221 222 <build> 223 <resources> 224 <resource> 225 <directory>${project.basedir}/src/main/resources</directory> 226 <filtering>true</filtering> 227 </resource> 228 </resources> 229 <filters> 230 <filter>${project.basedir}/filters/${filters.env}/conf.properties</filter> 231 </filters> 232 <plugins> 233 234 <plugin> 235 <groupId>net.alchim31.maven</groupId> 236 <artifactId>scala-maven-plugin</artifactId> 237 <version>${scala.maven.version}</version> 238 <executions> 239 <execution> 240 <id>scala-compile</id> 241 <phase>process-resources</phase> 242 <goals> 243 <goal>add-source</goal> 244 <goal>compile</goal> 245 </goals> 246 </execution> 247 <execution> 248 <id>scala-test-compile</id> 249 <phase>process-test-resources</phase> 250 <goals> 251 <goal>testCompile</goal> 252 </goals> 253 </execution> 254 </executions> 255 </plugin> 256 <!----> 257 <plugin> 258 <groupId>org.apache.maven.plugins</groupId> 259 <artifactId>maven-compiler-plugin</artifactId> 260 <version>${maven.compiler.version}</version> 261 <configuration> 262 <source>${java.version}</source> 263 <target>${java.version}</target> 264 </configuration> 265 </plugin> 266 <!-- 267 <plugin> 268 <groupId>org.apache.maven.plugins</groupId> 269 <artifactId>maven-assembly-plugin</artifactId> 270 <version>${maven.assembly.version}</version> 271 <configuration> 272 <descriptorRefs> 273 <descriptorRef>jar-with-dependencies</descriptorRef> 274 </descriptorRefs> 275 <archive> 276 <manifest> 277 <mainClass>com.cmiot.StatsApp</mainClass> 278 </manifest> 279 </archive> 280 </configuration> 281 <executions> 282 <execution> 283 <phase>package</phase> 284 <goals> 285 <goal>single</goal> 286 </goals> 287 </execution> 288 </executions> 289 </plugin> 290 <plugin> 291 <groupId>org.apache.maven.plugins</groupId> 292 <artifactId>maven-surefire-plugin</artifactId> 293 <version>${maven.surefire.version}</version> 294 <configuration> 295 <argLine>${test.arg.line}</argLine> 296 </configuration> 297 </plugin> 298 <plugin> 299 <groupId>org.scalatest</groupId> 300 <artifactId>scalatest-maven-plugin</artifactId> 301 <version>${scala.test.maven.version}</version> 302 <configuration> 303 <skipTests>false</skipTests> 304 <reportsDirectory>${project.build.directory}/test-reports</reportsDirectory> 305 <junitxml>.</junitxml> 306 <filereports>WDF TestSuite.txt</filereports> 307 <argLine>${test.arg.line}</argLine> 308 </configuration> 309 <executions> 310 <execution> 311 <id>test</id> 312 <goals> 313 <goal>test</goal> 314 </goals> 315 </execution> 316 </executions> 317 </plugin> 318 <plugin> 319 <groupId>org.codehaus.mojo</groupId> 320 <artifactId>cobertura-maven-plugin</artifactId> 321 <version>${cobertura.version}</version> 322 <configuration> 323 <formats> 324 <format>xml</format> 325 <format>html</format> 326 </formats> 327 <check/> 328 <instrumentation> 329 <ignores> 330 <ignore>com.cmiot.StatsApp</ignore> 331 <ignore>com.cmiot.app</ignore> 332 </ignores> 333 <excludes> 334 <exclude>com/cmiot/StatsApp*</exclude> 335 <exclude>com/cmiot/app/Stats*</exclude> 336 </excludes> 337 </instrumentation> 338 </configuration> 339 <executions> 340 <execution> 341 <phase>test</phase> 342 <goals> 343 <goal>cobertura</goal> 344 </goals> 345 </execution> 346 </executions> 347 </plugin> 348 --> 349 </plugins> 350 </build> 351 </project>
