saprk2 structed streaming
netcat (windows) >nc -L -p 9999
import java.sql.Timestamp
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.functions._
/**
*/
object Test extends App {
val host = "localhost"
val port = 9999
val windowSize = 10
val slideSize = 5
if (slideSize > windowSize) {
System.err.println("<slide duration> must be less than or equal to <window duration>")
}
val windowDuration = s"$windowSize seconds"
val slideDuration = s"$slideSize seconds"
val spark = SparkSession
.builder
.appName("StructuredNetworkWordCountWindowed")
.master("local[3]")
.config("spark.sql.shuffle.partitions", 3)
.getOrCreate()
spark.sparkContext.setLogLevel("ERROR")
import spark.implicits._
// Create DataFrame representing the stream of input lines from connection to host:port
val lines = spark.readStream
.format("socket")
.option("host", host)
.option("port", port)
.option("includeTimestamp", true)
.load()
// Split the lines into words, retaining timestamps
val words = lines.as[(String, Timestamp)].flatMap(line =>
line._1.split(" ").map(word => (word, line._2))
).toDF("word", "timestamp")
// Group the data by window and word and compute the count of each group
val windowedCounts = words.groupBy(
window($"timestamp", windowDuration, slideDuration), $"word"
).count().orderBy($"window".desc)
// Start running the query that prints the windowed word counts to the console
val query = windowedCounts.writeStream
.outputMode("complete")
.format("console")
.option("truncate", "false")
.start()
query.awaitTermination()
}
Result:
-------------------------------------------
Batch: 1
-------------------------------------------
+---------------------------------------------+----+-----+
|window |word|count|
+---------------------------------------------+----+-----+
|[2017-10-24 16:09:30.0,2017-10-24 16:09:40.0]|b |3 |
|[2017-10-24 16:09:30.0,2017-10-24 16:09:40.0]|a |3 |
|[2017-10-24 16:09:30.0,2017-10-24 16:09:40.0]|c |1 |
|[2017-10-24 16:09:30.0,2017-10-24 16:09:40.0]|d |1 |
|[2017-10-24 16:06:40.0,2017-10-24 16:06:50.0]|a |4 |
|[2017-10-24 16:06:35.0,2017-10-24 16:06:45.0]|a |8 |
|[2017-10-24 16:06:30.0,2017-10-24 16:06:40.0]|a |4 |
+---------------------------------------------+----+-----+
窗口移动5秒,窗口宽度10秒。
聚合维度: window, {world}
http://asyncified.io/2017/07/30/exploring-stateful-streaming-with-spark-structured-streaming/