spark streaming
discretized 离散化的 http://spark.apache.org/docs/1.6.0/streaming-programming-guide.html#overview
[root@node5 ~]# yum install nc
[root@node5 ~]# nc -lk 9999
package com.bjsxt.spark;
import java.util.Arrays;
import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.api.java.function.VoidFunction;
import org.apache.spark.broadcast.Broadcast;
import org.apache.spark.streaming.Duration;
import org.apache.spark.streaming.Durations;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaPairDStream;
import org.apache.spark.streaming.api.java.JavaReceiverInputDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
import scala.Tuple2;
public class SparkStreamingTest {
public static void main(String[] args) {
// local[1] 时只有一个task，被占用为接收数据，打印输出没有任务线程执行
SparkConf conf = new SparkConf().setMaster("local[2]").setAppName("SparkStreamingTest");
final JavaSparkContext sc = new JavaSparkContext(conf);
sc.setLogLevel("WARN");
JavaStreamingContext jsc = new JavaStreamingContext(sc,Durations.seconds(5));
JavaReceiverInputDStream<String> socketTextStream = jsc.socketTextStream("node5",9999);
JavaDStream<String> words = socketTextStream.flatMap(new FlatMapFunction<String,String>() {
/**
*
*/
private static final long serialVersionUID = 1L;
@Override
public Iterable<String> call(String lines) throws Exception {
return Arrays.asList(lines.split(" "));
}
});
JavaPairDStream<String,Integer> pairWords = words.mapToPair(new PairFunction<String, String, Integer>() {
/**
*
*/
private static final long serialVersionUID = 1L;
@Override
public Tuple2<String, Integer> call(String word) throws Exception {
return new Tuple2<String, Integer>(word,1);
}
});
JavaPairDStream<String, Integer> reduceByKey = pairWords.reduceByKey(new Function2<Integer, Integer, Integer>() {
/**
*
*/
private static final long serialVersionUID = 1L;
@Override
public Integer call(Integer v1, Integer v2) throws Exception {
System.out.println("rdd reduceByKey************************");
return v1 + v2;
}
});
// reduceByKey.print(1000);
/**
* foreachRDD 可以拿到DStream中的RDD,对拿到的RDD可以使用RDD的transformation类算子转换，要对拿到的RDD使用action算子触发执行，否则，foreachRDD也不会执行。
* foreachRDD中call方法内，拿到的RDD的算子外，代码是在Driver端执行；可以使用这个算子动态改变广播变量（使用配置文件的方式）
*/
reduceByKey.foreachRDD(new VoidFunction<JavaPairRDD<String,Integer>>() {
/**
*
*/
private static final long serialVersionUID = 1L;
@Override
public void call(JavaPairRDD<String, Integer> rdd)
throws Exception {
// Driver中获取SparkContext的正确方式； 中获取广播变量的正确方式。
SparkContext context = rdd.context();
JavaSparkContext sc = new JavaSparkContext(context);
Broadcast<String> broadcast = sc.broadcast("hello");
String value = broadcast.value();
System.out.println("Driver.........");
JavaPairRDD<String, Integer> mapToPair = rdd.mapToPair(new PairFunction<Tuple2<String,Integer>, String, Integer>() {
/**
*
*/
private static final long serialVersionUID = 1L;
@Override
public Tuple2<String, Integer> call(
Tuple2<String, Integer> tuple) throws Exception {
System.out.println("Executor.........");
return new Tuple2<String,Integer>(tuple._1,tuple._2);
}
});
mapToPair.foreach(new VoidFunction<Tuple2<String,Integer>>() {
/**
*
*/
private static final long serialVersionUID = 1L;
@Override
public void call(Tuple2<String, Integer> arg0)
throws Exception {
System.out.println(arg0);
}
});
}
});
jsc.start();
jsc.awaitTermination();
jsc.stop();
}
}
[root@node5 ~]# nc -lk 9999 ## linux 发送socket数据。
hello sxt
hello bj
hello
hello
zhongguo
zhngguo
zhongguo
* foreachRDD能够动态获取广播变量， * 能顾获取到JavaSparkContext，说明是在Driver端执行
checkpoint 在内存中也存在一份，磁盘中保存一份。
package com.bjsxt.sparkstreaming;
import java.util.Arrays;
import java.util.List;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.streaming.Durations;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaPairDStream;
import org.apache.spark.streaming.api.java.JavaReceiverInputDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
import com.google.common.base.Optional;
import scala.Tuple2;
/**
* UpdateStateByKey的主要功能:
* 1、为Spark Streaming中每一个Key维护一份state状态，state类型可以是任意类型的， 可以是一个自定义的对象，那么更新函数也可以是自定义的。
* 2、通过更新函数对该key的状态不断更新，对于每个新的batch而言，Spark Streaming会在使用updateStateByKey的时候为已经存在的key进行state的状态更新
*
* hello,3
* bjsxt,2
*
* 如果要不断的更新每个key的state，就一定涉及到了状态的保存和容错，这个时候就需要开启checkpoint机制和功能
*
* 全面的广告点击分析
* @author root
*
* 有何用？ 统计广告点击流量，统计这一天的车流量，统计点击量
*/
public class UpdateStateByKeyOperator {
public static void main(String[] args) {
SparkConf conf = new SparkConf().setMaster("local[2]").setAppName("UpdateStateByKeyDemo");
JavaStreamingContext jsc = new JavaStreamingContext(conf, Durations.seconds(5));
/**
* 设置checkpoint目录
*
* 多久会将内存中的数据（每一个key所对应的状态）写入到磁盘上一份呢？
* 如果你的batchInterval小于10s 那么10s会将内存中的数据写入到磁盘一份
* 如果bacthInterval 大于10s，那么就以bacthInterval为准
*
* 这样做是为了防止频繁的写HDFS
*/
// JavaSparkContext sc = jsc.sparkContext();
// sc.setCheckpointDir("./checkpoint");
// jsc.checkpoint("hdfs://node1:9000/spark/checkpoint");
jsc.checkpoint("./checkpoint");
JavaReceiverInputDStream<String> lines = jsc.socketTextStream("node5", 9999);
JavaDStream<String> words = lines.flatMap(new FlatMapFunction<String, String>() {
/**
*
*/
private static final long serialVersionUID = 1L;
@Override
public Iterable<String> call(String s) {
return Arrays.asList(s.split(" "));
}
});
JavaPairDStream<String, Integer> ones = words.mapToPair(new PairFunction<String, String, Integer>() {
/**
*
*/
private static final long serialVersionUID = 1L;
@Override
public Tuple2<String, Integer> call(String s) {
return new Tuple2<String, Integer>(s, 1);
}
});
JavaPairDStream<String, Integer> counts =
ones.updateStateByKey(new Function2<List<Integer>, Optional<Integer>, Optional<Integer>>() {
/**
*
*/
private static final long serialVersionUID = 1L;
@Override
public Optional<Integer> call(List<Integer> values, Optional<Integer> state) throws Exception {
/**
* values:经过分组最后 这个key所对应的value [1,1,1,1,1]
* state:这个key在本次之前之前的状态
*/
Integer updateValue = 0 ;
if(state.isPresent()){
updateValue = state.get();
}
for(Integer value : values) {
updateValue += value;
}
return Optional.of(updateValue);
}
});
//output operator
counts.print();
jsc.start();
jsc.awaitTermination();
jsc.close();
}
}
// 窗口长度和滑动间隔必须是批次的整数倍
package com.bjsxt.sparkstreaming;
import java.util.Arrays;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.streaming.Durations;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaPairDStream;
import org.apache.spark.streaming.api.java.JavaReceiverInputDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
import scala.Tuple2;
/**
* 基于滑动窗口的热点搜索词实时统计
* @author root
*
*/
public class WindowOperator {
public static void main(String[] args) {
SparkConf conf = new SparkConf()
.setMaster("local[2]")
.setAppName("WindowHotWord");
JavaStreamingContext jssc = new JavaStreamingContext(conf, Durations.seconds(5));
/**
* 设置日志级别为WARN
*
*/
jssc.sparkContext().setLogLevel("WARN");
/**
* 注意：
* 没有优化的窗口函数可以不设置checkpoint目录
* 优化的窗口函数必须设置checkpoint目录
*/
// jssc.checkpoint("hdfs://node1:9000/spark/checkpoint");
jssc.checkpoint("./checkpoint");
JavaReceiverInputDStream<String> searchLogsDStream = jssc.socketTextStream("node5", 9999);
// JavaDStream<String> window = searchLogsDStream.window(Durations.seconds(15), Durations.seconds(5));
//word 1
JavaDStream<String> searchWordsDStream = searchLogsDStream.flatMap(new FlatMapFunction<String, String>() {
/**
*
*/
private static final long serialVersionUID = 1L;
@Override
public Iterable<String> call(String t) throws Exception {
return Arrays.asList(t.split(" "));
}
});
// 将搜索词映射为(searchWord, 1)的tuple格式
JavaPairDStream<String, Integer> searchWordPairDStream = searchWordsDStream.mapToPair(
new PairFunction<String, String, Integer>() {
private static final long serialVersionUID = 1L;
@Override
public Tuple2<String, Integer> call(String searchWord)
throws Exception {
return new Tuple2<String, Integer>(searchWord, 1);
}
});
/**
* 每隔10秒，计算最近60秒内的数据，那么这个窗口大小就是60秒，里面有12个rdd，在没有计算之前，这些rdd是不会进行计算的。
* 那么在计算的时候会将这12个rdd聚合起来，然后一起执行reduceByKeyAndWindow操作 ，
* reduceByKeyAndWindow是针对窗口操作的而不是针对DStream操作的。
*/
// JavaPairDStream<String, Integer> searchWordCountsDStream =
//
// searchWordPairDStream.reduceByKeyAndWindow(new Function2<Integer, Integer, Integer>() {
//
// private static final long serialVersionUID = 1L;
//
// @Override
// public Integer call(Integer v1, Integer v2) throws Exception {
// return v1 + v2;
// }
// }, Durations.seconds(15), Durations.seconds(5));
/**
* window窗口操作优化：
*/
JavaPairDStream<String, Integer> searchWordCountsDStream =
searchWordPairDStream.reduceByKeyAndWindow(new Function2<Integer, Integer, Integer>() {
private static final long serialVersionUID = 1L;
@Override
public Integer call(Integer v1, Integer v2) throws Exception {
return v1 + v2;
}
},new Function2<Integer, Integer, Integer>() {
private static final long serialVersionUID = 1L;
@Override
public Integer call(Integer v1, Integer v2) throws Exception {
return v1 - v2;
}
}, Durations.seconds(15), Durations.seconds(5));
searchWordCountsDStream.print();
jssc.start();
jssc.awaitTermination();
jssc.close();
}
}
// 设置本地运维模式启动两个线程运行程序
SparkConf conf = new SparkConf()
.setMaster("local[2]")
.setAppName("WindowHotWord");
普通机制不需要设置checkpoint; 窗口滑动的优化机制，必须设置checkpoint; 相当于减去出去的批次，增加新进入的批次。
package com.bjsxt.sparkstreaming;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.broadcast.Broadcast;
import org.apache.spark.streaming.Durations;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaPairDStream;
import org.apache.spark.streaming.api.java.JavaReceiverInputDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
import scala.Tuple2;
/**
* 过滤黑名单
* transform操作
* DStream可以通过transform做RDD到RDD的任意操作。
* @author root
*
*/
public class TransformOperator {
public static void main(String[] args) {
SparkConf conf = new SparkConf();
conf.setMaster("local[2]").setAppName("transform");
JavaStreamingContext jsc = new JavaStreamingContext(conf,Durations.seconds(5));
//黑名单
List<String> list = Arrays.asList("zhangsan");
final Broadcast<List<String>> bcBlackList = jsc.sparkContext().broadcast(list);
//接受socket数据源
JavaReceiverInputDStream<String> nameList = jsc.socketTextStream("node5", 9999);
JavaPairDStream<String, String> pairNameList =
nameList.mapToPair(new PairFunction<String, String, String>() {
/**
*
*/
private static final long serialVersionUID = 1L;
@Override
public Tuple2<String, String> call(String s) throws Exception {
return new Tuple2<String, String>(s.split(" ")[1], s);
}
});
/**
* transform 可以拿到DStream中的RDD，做RDD到RDD之间的转换，不需要Action算子触发，需要返回RDD类型。
* 注意：transform call方法内，拿到RDD 算子外的代码 在Driver端执行，也可以做到动态改变广播变量。
*/
JavaDStream<String> transFormResult =
pairNameList.transform(new Function<JavaPairRDD<String,String>, JavaRDD<String>>() {
/**
*
*/
private static final long serialVersionUID = 1L;
@Override
public JavaRDD<String> call(JavaPairRDD<String, String> nameRDD)
throws Exception {
JavaPairRDD<String, String> filter = nameRDD.filter(new Function<Tuple2<String,String>, Boolean>() {
/**
*
*/
private static final long serialVersionUID = 1L;
@Override
public Boolean call(Tuple2<String, String> tuple) throws Exception {
return !bcBlackList.value().contains(tuple._1);
}
});
JavaRDD<String> map = filter.map(new Function<Tuple2<String,String>, String>() {
/**
*
*/
private static final long serialVersionUID = 1L;
@Override
public String call(Tuple2<String, String> tuple) throws Exception {
return tuple._2;
}
});
//返回过滤好的结果
return map;
}
});
transFormResult.print();
jsc.start();
jsc.awaitTermination();
jsc.stop();
}
}
监控socket需要2个task，因为spark streaming 底层监控socket需要有一个task一致receive.另外一个负责运行程序。setMaster("local[2]")
监控文件夹则可以不需要两个task,一个task也可。setMaster("local")
spark streaming 监控文件夹。
package streamingOperate.transformations;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.streaming.Durations;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
/**
* 通过func函数过滤返回为true的记录，返回一个新的Dstream
* @author root
*
*/
public class Operate_filter {
public static void main(String[] args) {
SparkConf conf = new SparkConf().setMaster("local[2]").setAppName("Operate_filter");
JavaStreamingContext jsc = new JavaStreamingContext(conf,Durations.seconds(5));
JavaDStream<String> textFileStream = jsc.textFileStream("data");
/**
* 下面的例子是每次读入一行数据，通过观察文件，看到第一行为“crosses the repetition to duplicate daily,”开头
* 所以下面将过滤只显示以“crosses the repetition to duplicate daily,”开头的行
*/
textFileStream.filter(new Function<String,Boolean>(){
/**
*
*/
private static final long serialVersionUID = 1L;
public Boolean call(String line) throws Exception {
return line.startsWith("a 100");
}
}).print(1000);
jsc.start();
jsc.awaitTermination();
jsc.close();
}
}
// 不能监控手动修改的文件，需要保证文件操作的原子性。
package streamingOperate.util;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.UUID;
/**
* 此复制文件的程序是模拟在data目录下动态生成相同格式的txt文件，用于给sparkstreaming 中 textFileStream提供输入流。
* @author root
*
*/
public class CopyFile_data1 {
public static void main(String[] args) throws IOException, InterruptedException {
while(true){
Thread.sleep(5000);
String uuid = UUID.randomUUID().toString();
System.out.println(uuid);
copyFile(new File("data1.txt"),new File(".\\data\\"+uuid+"----data1.txt"));
}
}
public static void copyFile(File fromFile, File toFile) throws IOException {
FileInputStream ins = new FileInputStream(fromFile);
FileOutputStream out = new FileOutputStream(toFile);
byte[] b = new byte[1024];
@SuppressWarnings("unused")
int n = 0;
while ((n = ins.read(b)) != -1) {
out.write(b, 0, b.length);
}
ins.close();
out.close();
}
}
package streamingOperate.output;
import java.util.Arrays;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.streaming.Durations;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
import org.apache.spark.streaming.dstream.DStream;
/**
* saveAsTextFiles(prefix, [suffix])：
* 将此DStream的内容另存为文本文件。每批次数据产生的文件名称格式基于：prefix和suffix: "prefix-TIME_IN_MS[.suffix]".
*
* 注意：
* saveAsTextFile是调用saveAsHadoopFile实现的
* spark中普通rdd可以直接只用saveAsTextFile(path)的方式，保存到本地，但是此时DStream的只有saveAsTextFiles()方法，没有传入路径的方法，
* 其参数只有prefix, suffix
* 其实：DStream中的saveAsTextFiles方法中又调用了rdd中的saveAsTextFile方法，我们需要将path包含在prefix中
*
*/
public class Operate_saveAsTextFiles {
public static void main(String[] args) {
SparkConf conf = new SparkConf().setMaster("local").setAppName("Operate_saveAsTextFiles");
JavaStreamingContext jsc = new JavaStreamingContext(conf,Durations.seconds(5));
JavaDStream<String> textFileStream = jsc.textFileStream("./data");
JavaDStream<String> flatMap = textFileStream.flatMap(new FlatMapFunction<String, String>() {
/**
*
*/
private static final long serialVersionUID = 1L;
public Iterable<String> call(String t) throws Exception {
return Arrays.asList(t.split(" "));
}
});
//保存在当前路径中savedata路径下，以prefix开头，以suffix结尾的文件。
DStream<String> dstream = flatMap.dstream();
// dstream.saveAsTextFiles(".\\savedata\\prefix", "suffix");
dstream.saveAsTextFiles(".\\savedata\\spark\\mydate", "aaaa");
jsc.start();
jsc.awaitTermination();
jsc.close();
}
}
