随笔- 266 文章- 0 评论- 6 阅读- 24万

spark Transformations算子

在java中，RDD分为javaRDDs和javaPairRDDs。下面分两大类来进行。

都必须要进行的一步。

1 2	`SparkConf conf =` `new` `SparkConf().setMaster("local").setAppName("test");` `JavaSparkContext sc =` `new` `JavaSparkContext(conf);`

一。javaRDDs

 1         String[] ayys = {"a","b","c"};
 2         List<String> strings = Arrays.asList(ayys);
 3         
 4         JavaRDD<String> rdd1 = sc.parallelize(strings);
 5         strings.add("d");
 6         JavaRDD<String> rdd2 = sc.parallelize(strings);
 7 
 8 
 9         JavaRDD<Tuple2<String, Integer>> parallelize = sc.parallelize(Arrays.asList(
10                 new Tuple2<String, Integer>("asd", 11),
11                 new Tuple2<String, Integer>("asd", 11),
12                 new Tuple2<String, Integer>("asd", 11)
13         ));
14 
15         rdd1.map(new Function<String, String>() {
16             public String call(String s) throws Exception {
17                 return s.replace("a","qqq");
18             }
19         }).foreach(new VoidFunction<String>() {
20             public void call(String s) throws Exception {
21                 System.out.println(s);
22             }
23         });
24 
25 
26         List<String> a = rdd1.filter(new Function<String, Boolean>() {
27             public Boolean call(String s) throws Exception {
28                 return s.contains("a");
29             }
30         }).collect();
31 
32         System.out.println(a);
33 
34         JavaRDD<String> rdd22 = rdd1.flatMap(new FlatMapFunction<String, String>() {
35             public Iterable<String> call(String s) throws Exception {
36                 return Arrays.asList(s.split(" "));
37             }
38         });
39 
40         JavaPairRDD<String, Integer> rdd4 = rdd2.mapToPair(new PairFunction<String, String, Integer>() {
41             public Tuple2<String, Integer> call(String s) throws Exception {
42                 return new Tuple2<String, Integer>(s, 1);
43             }
44         });
45 
46          JavaRDD<String> rdd11 = rdd2.mapPartitions(new FlatMapFunction<Iterator<String>, String>() {
47             public Iterable<String> call(Iterator<String> stringIterator) throws Exception {
48                 ArrayList<String> strings = new ArrayList<String>();
49                 while (stringIterator.hasNext()){
50                     strings.add(stringIterator.next());
51                 }
52                 return strings;
53             }
54         });
55 
56         JavaRDD<String> stringJavaRDD = rdd1.mapPartitionsWithIndex(new Function2<Integer, Iterator<String>, Iterator<String>>() {
57             public Iterator<String> call(Integer integer, Iterator<String> stringIterator) throws Exception {
58                 ArrayList<String> strings = new ArrayList<String>();
59                 while (stringIterator.hasNext()){
60                     strings.add(stringIterator.next());
61                 }
62                 return strings.iterator();
63             }
64         },false);
65 
66         JavaRDD<String> sample = rdd1.sample(false, 0.3);
67 
68         JavaRDD<String> union = rdd1.union(rdd2);
69 
70         JavaRDD<String> intersection = rdd1.intersection(rdd2);
71 
72         JavaRDD<String> distinct = rdd1.distinct();

二。JavaPairRDDs.

JavaPairRDD<String, Integer> rdd1 = sc.parallelizePairs(Arrays.asList(
        new Tuple2<String, Integer>("asd", 111),
        new Tuple2<String, Integer>("asd", 111),
        new Tuple2<String, Integer>("asd", 111)
));
 
JavaPairRDD<String, Integer> rdd2 = sc.parallelizePairs(Arrays.asList(
        new Tuple2<String, Integer>("sdfsd", 222),
        new Tuple2<String, Integer>("sdfsd", 222),
        new Tuple2<String, Integer>("sdfsd", 222)
));
 
JavaPairRDD<String, Iterable<Integer>> stringIterableJavaPairRDD = rdd1.groupByKey();
 
JavaPairRDD<String, Integer> rdd = rdd1.reduceByKey(new Function2<Integer, Integer, Integer>() {
    public Integer call(Integer integer, Integer integer2) throws Exception {
        return integer + integer2;
    }
});
 
JavaPairRDD<String, Integer> rdd3 = rdd1.aggregateByKey(0, new Function2<Integer, Integer, Integer>() {
    public Integer call(Integer integer, Integer integer2) throws Exception {
        return max(integer,integer2);
    }
}, new Function2<Integer, Integer, Integer>() {
    public Integer call(Integer integer, Integer integer2) throws Exception {
        return integer + integer2;
    }
});
 
JavaPairRDD<String, Integer> rdd111 = rdd1.sortByKey();
 
JavaPairRDD<String, Tuple2<Integer, Integer>> join = rdd1.join(rdd2);
JavaPairRDD<String, Tuple2<Integer, Optional<Integer>>> stringTuple2JavaPairRDD = rdd1.leftOuterJoin(rdd2);
JavaPairRDD<String, Tuple2<Optional<Integer>, Integer>> stringTuple2JavaPairRDD1 = rdd1.rightOuterJoin(rdd2);
JavaPairRDD<String, Tuple2<Optional<Integer>, Optional<Integer>>> stringTuple2JavaPairRDD2 = rdd1.fullOuterJoin(rdd2);
 
JavaPairRDD<String, Tuple2<Iterable<Integer>, Iterable<Integer>>> cogroup = rdd1.cogroup(rdd2);
 
JavaPairRDD<String, Integer> coalesce = rdd1.coalesce(3, false);
 
JavaPairRDD<String, Integer> repartition = rdd1.repartition(3);
 
JavaPairRDD<String, Integer> rdd5 = rdd1.repartitionAndSortWithinPartitions(new HashPartitioner(2));
 
JavaPairRDD<Tuple2<String, Integer>, Tuple2<String, Integer>> cartesian = rdd1.cartesian(rdd2);
 
JavaRDD<String> pipe = rdd1.pipe("");

zip:

JavaPairRDD<Tuple2<String, Integer>, Tuple2<String, Integer>> zip = rdd1.zip(rdd2);
 
JavaPairRDD<Tuple2<String, Integer>, Long> tuple2LongJavaPairRDD =     rdd1.zipWithIndex();

最后都要加上