java操作spark1.2.0

---

虽然推荐的是scala，但是还是试一下

---

package org.admln.java7OperateSpark;

import java.util.Arrays;
import java.util.List;
import java.util.regex.Pattern;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;

import scala.Tuple2;

public class OperateSpark {
    //单词切分分隔符
    private static final Pattern SPACE = Pattern.compile(" ");
    
    public static void main(String[] args) {
        //初始化
        SparkConf sparkConf = new SparkConf().setAppName("JavaWordCount").setMaster("spark://hadoop:7077");
        JavaSparkContext ctx = new JavaSparkContext(sparkConf);
        
        //第二个参数是文件的最小切分
        JavaRDD<String> lines = ctx.textFile("hdfs://hadoop:8020/in/spark/javaOperateSpark/wordcount.txt");
        JavaRDD<String> words = lines.flatMap(new FlatMapFunction<String,String>() {
            public Iterable<String> call(String s) {
                return Arrays.asList(SPACE.split(s));
            }
        });
        
        //划成键值对
        JavaPairRDD<String,Integer> ones = words.mapToPair(new PairFunction<String,String,Integer>() {
            public Tuple2<String, Integer> call(String t) {
                return new Tuple2<String,Integer>(t,1);
            }
        });
        
        JavaPairRDD<String,Integer> counts = ones.reduceByKey(new Function2<Integer,Integer,Integer>() {
            public Integer call(Integer v1, Integer v2) {
                return v1 + v2;
            }
        });
        
        List<Tuple2<String,Integer>> output = counts.collect();
        for(Tuple2<?,?> tuple : output) {
            System.out.println(tuple._1() + ":" +tuple._2());
        }
        counts.saveAsTextFile("hdfs://hadoop:8020/out/spark/javaOperateSpark2/");
        ctx.stop();
    }
}

---

运行的时候出现了错误

eclipse中为：

Exception in thread "main" java.lang.NoSuchMethodError: com.google.common.hash.HashFunction.hashInt(I)Lcom/google/common/hash/HashCode;
    at org.apache.spark.util.collection.OpenHashSet.org$apache$spark$util$collection$OpenHashSet$$hashcode(OpenHashSet.scala:261)
    at org.apache.spark.util.collection.OpenHashSet$mcI$sp.getPos$mcI$sp(OpenHashSet.scala:165)
    at org.apache.spark.util.collection.OpenHashSet$mcI$sp.contains$mcI$sp(OpenHashSet.scala:102)
    at org.apache.spark.util.SizeEstimator$$anonfun$visitArray$2.apply$mcVI$sp(SizeEstimator.scala:214)
    at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141)
    at org.apache.spark.util.SizeEstimator$.visitArray(SizeEstimator.scala:210)
    at org.apache.spark.util.SizeEstimator$.visitSingleObject(SizeEstimator.scala:169)
    at org.apache.spark.util.SizeEstimator$.org$apache$spark$util$SizeEstimator$$estimate(SizeEstimator.scala:161)
    at org.apache.spark.util.SizeEstimator$.estimate(SizeEstimator.scala:155)
    at org.apache.spark.util.collection.SizeTracker$class.takeSample(SizeTracker.scala:78)
    at org.apache.spark.util.collection.SizeTracker$class.afterUpdate(SizeTracker.scala:70)
    at org.apache.spark.util.collection.SizeTrackingVector.$plus$eq(SizeTrackingVector.scala:31)
    at org.apache.spark.storage.MemoryStore.unrollSafely(MemoryStore.scala:249)
    at org.apache.spark.storage.MemoryStore.putIterator(MemoryStore.scala:136)
    at org.apache.spark.storage.MemoryStore.putIterator(MemoryStore.scala:114)
    at org.apache.spark.storage.BlockManager.doPut(BlockManager.scala:787)
    at org.apache.spark.storage.BlockManager.putIterator(BlockManager.scala:638)
    at org.apache.spark.storage.BlockManager.putSingle(BlockManager.scala:992)
    at org.apache.spark.broadcast.TorrentBroadcast.writeBlocks(TorrentBroadcast.scala:98)
    at org.apache.spark.broadcast.TorrentBroadcast.<init>(TorrentBroadcast.scala:84)
    at org.apache.spark.broadcast.TorrentBroadcastFactory.newBroadcast(TorrentBroadcastFactory.scala:34)
    at org.apache.spark.broadcast.TorrentBroadcastFactory.newBroadcast(TorrentBroadcastFactory.scala:29)
    at org.apache.spark.broadcast.BroadcastManager.newBroadcast(BroadcastManager.scala:62)
    at org.apache.spark.SparkContext.broadcast(SparkContext.scala:945)
    at org.apache.spark.SparkContext.hadoopFile(SparkContext.scala:695)
    at org.apache.spark.SparkContext.textFile(SparkContext.scala:540)
    at org.apache.spark.api.java.JavaSparkContext.textFile(JavaSparkContext.scala:184)
    at org.admln.java7OperateSpark.OperateSpark.main(OperateSpark.java:27)

ｓｈｅｌｌ中为：

Exception in thread "main" java.lang.VerifyError: class org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$AddBlockRequestProto overrides final method getUnknownFields.()Lcom/google/protobuf/UnknownFieldSet;
        at java.lang.ClassLoader.defineClass1(Native Method)
        at java.lang.ClassLoader.defineClass(ClassLoader.java:800)
        at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
        at java.net.URLClassLoader.defineClass(URLClassLoader.java:449)

        ．．．　．．．

        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:606)
        at org.apache.spark.deploy.SparkSubmit$.launch(SparkSubmit.scala:358)
        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:75)
        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)

可以看到是ｐｒｏｔｏｂｕｆ版本和hadoop的冲突了

默认ｓｐａｒｋ１．２．０的ｐｒｏｔｏｂｕｆ版本为

而hadoop２.２.０的为ｐｒｏｔｏｂｕｆ２.５.０

所以修改spark中ｐｏｍ．ｘｍｌ后重新编译生成部署包（花费一个多小时）

再运行的话ｓｈｅｌｌ端成功。但是eclipse端仍然报那个错误

这是因为我用的ｍａｖｅｎ引用的spark包，存在ｇｕａｖａ版本冲突，默认为

单独加一个依赖

　　<dependency>
        <groupId>com.google.guava</groupId>
        <artifactId>guava</artifactId>
        <version>14.0.1</version>
    </dependency>

---

然后eclipse提交的话不报错了，不过任务一直循环不执行，报告资源不够

WARN TaskSchedulerImpl: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient memory

然后把核数加到２，内存加到１５００Ｍ，可是仍然报

INFO SparkDeploySchedulerBackend: Granted executor ID app-20150111003236-0000/3 on hostPort hadoop:34766 with 2 cores, 512.0 MB RAM

也就是说核数改了，但是执行内存改不了，不知道为什么，还有就是同样的程序ｓｈｅｌｌ端提交就正常执行，ｅｃｌｉｐｓｅ外部提交就报内存不足

改驱动的内存也不行。

我推测有两种可能的原因

１.spark的ＢＵＧ，SPARK_DRIVER_MEMORY变量默认是５１２Ｍ，但是外部修改不生效；

２.ｃｅｎｔｏｓ的资源和本机windows的资源混乱了，因为我看到了

ERROR SparkDeploySchedulerBackend: Asked to remove non-existent executor 2

的错误，我本机是４核，虚拟机是２核。

---

 

不知道为什么网上没有eclipse提交的示例，应该要不就是本身就不支持，会和客户端资源混乱，要不就是还没人摸透。

---