package stream.scala
import java.io.PrintWriter
import java.net.ServerSocket
class LoggerSimulation {
}
object LoggerSimulation{
/**
* 生成一个字母
* @param 字母的下标
* @return 生成的字母
*/
def gennerateContent(index:Int):String = {
import scala.collection.mutable.ListBuffer
val charList = ListBuffer[Char]();
for (i<- 65 to 90){
charList += i.toChar
}
val charArray = charList.toArray
charArray(index).toString();
}
/**
* 生成随机下标
* @return 返回一个下标
*/
def index = {
import java.util.Random
val rdm = new Random()
rdm.nextInt(7)
}
/**
* 启动一个main方法来创建一个serversockt发送消息
* @param args 端口，发送的时间间隔
*/
def main(args: Array[String]): Unit = {
// if (args.length !=2){
// System.err.println("Usage:<port><millisecond>")
// System.exit(1);
// }
val listener = new ServerSocket(8888.toInt)
println("已经做好连接的准备-------")
while(true){
val socket = listener.accept()
new Thread(){
override def run(): Unit = {
println("Got client connected from:"+socket.getInetAddress)
val out = new PrintWriter(socket.getOutputStream,true)
while(true){
Thread.sleep(1000.toLong)
val content = gennerateContent(index)
println(content)
out.write(content+"\n")
out.flush()
}
socket.close()
}
}.start()
}
}
}
--------------------------------------------------
package stream.scala
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}
/**
* 这是一个接收来自网络端口的信息
* 参数 spark集群的主节点地址，网络通信的节点地址，网络通信的端口，每个多长时间作为一个单位进行执行任务
* local[*] localhost 8888 5
*/
class NetWorkWordCount {
}
object NetWorkWordCount{
def main(args: Array[String]): Unit = {
// if(args.length!=4){
// System.err.println("Usage:NetWorkWordCount<master> <hostname> <port> <seconds>\n+" +
// "In local modle,<master> should be 'local[n]' with n >1 ");
// System.exit(1);
// }
val config = new SparkConf().setAppName("NetWorkWordCount").setMaster("local[*]");
val ssc = new StreamingContext(config,Seconds(5.toInt));
val lines = ssc.socketTextStream("localhost",8888.toInt,StorageLevel.MEMORY_ONLY_SER)
val words = lines.flatMap(line => line.split(" "));
val wordCount = words.map(x=>(x,1)).reduceByKey(_+_);
wordCount.print()
ssc.start();
ssc.awaitTermination();
}
}
*************************************************************************
18/01/12 10:52:25 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=112292, maxMem=490356080
18/01/12 10:52:25 INFO MemoryStore: Block broadcast_136 stored as values in memory (estimated size 2.3 KB, free 467.5 MB)
18/01/12 10:52:25 INFO MemoryStore: ensureFreeSpace(1417) called with curMem=114596, maxMem=490356080
18/01/12 10:52:25 INFO MemoryStore: Block broadcast_136_piece0 stored as bytes in memory (estimated size 1417.0 B, free 467.5 MB)
18/01/12 10:52:25 INFO BlockManagerInfo: Added broadcast_136_piece0 in memory on localhost:54439 (size: 1417.0 B, free: 467.6 MB)
18/01/12 10:52:25 INFO SparkContext: Created broadcast 136 from broadcast at DAGScheduler.scala:874
18/01/12 10:52:25 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 182 (ShuffledRDD[184] at reduceByKey at NetWorkWordCount.scala:28)
18/01/12 10:52:25 INFO TaskSchedulerImpl: Adding task set 182.0 with 1 tasks
18/01/12 10:52:25 INFO TaskSetManager: Starting task 0.0 in stage 182.0 (TID 402, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:25 INFO Executor: Running task 0.0 in stage 182.0 (TID 402)
18/01/12 10:52:25 INFO ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 5 blocks
18/01/12 10:52:25 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/01/12 10:52:25 INFO Executor: Finished task 0.0 in stage 182.0 (TID 402). 1028 bytes result sent to driver
18/01/12 10:52:25 INFO TaskSetManager: Finished task 0.0 in stage 182.0 (TID 402) in 2 ms on localhost (1/1)
18/01/12 10:52:25 INFO TaskSchedulerImpl: Removed TaskSet 182.0, whose tasks have all completed, from pool
18/01/12 10:52:25 INFO DAGScheduler: ResultStage 182 (print at NetWorkWordCount.scala:29) finished in 0.002 s
18/01/12 10:52:25 INFO DAGScheduler: Job 91 finished: print at NetWorkWordCount.scala:29, took 0.058539 s
18/01/12 10:52:25 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29
18/01/12 10:52:25 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 45 is 167 bytes
18/01/12 10:52:25 INFO DAGScheduler: Got job 92 (print at NetWorkWordCount.scala:29) with 3 output partitions (allowLocal=true)
18/01/12 10:52:25 INFO DAGScheduler: Final stage: ResultStage 184(print at NetWorkWordCount.scala:29)
18/01/12 10:52:25 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 183)
18/01/12 10:52:25 INFO DAGScheduler: Missing parents: List()
18/01/12 10:52:25 INFO DAGScheduler: Submitting ResultStage 184 (ShuffledRDD[184] at reduceByKey at NetWorkWordCount.scala:28), which has no missing parents
18/01/12 10:52:25 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=116013, maxMem=490356080
18/01/12 10:52:25 INFO MemoryStore: Block broadcast_137 stored as values in memory (estimated size 2.3 KB, free 467.5 MB)
18/01/12 10:52:25 INFO MemoryStore: ensureFreeSpace(1417) called with curMem=118317, maxMem=490356080
18/01/12 10:52:25 INFO MemoryStore: Block broadcast_137_piece0 stored as bytes in memory (estimated size 1417.0 B, free 467.5 MB)
18/01/12 10:52:25 INFO BlockManagerInfo: Added broadcast_137_piece0 in memory on localhost:54439 (size: 1417.0 B, free: 467.6 MB)
18/01/12 10:52:25 INFO SparkContext: Created broadcast 137 from broadcast at DAGScheduler.scala:874
18/01/12 10:52:25 INFO DAGScheduler: Submitting 3 missing tasks from ResultStage 184 (ShuffledRDD[184] at reduceByKey at NetWorkWordCount.scala:28)
18/01/12 10:52:25 INFO TaskSchedulerImpl: Adding task set 184.0 with 3 tasks
18/01/12 10:52:25 INFO TaskSetManager: Starting task 0.0 in stage 184.0 (TID 403, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:25 INFO TaskSetManager: Starting task 1.0 in stage 184.0 (TID 404, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:25 INFO TaskSetManager: Starting task 2.0 in stage 184.0 (TID 405, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:25 INFO Executor: Running task 1.0 in stage 184.0 (TID 404)
18/01/12 10:52:25 INFO Executor: Running task 2.0 in stage 184.0 (TID 405)
18/01/12 10:52:25 INFO Executor: Running task 0.0 in stage 184.0 (TID 403)
18/01/12 10:52:25 INFO ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 5 blocks
18/01/12 10:52:25 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/01/12 10:52:25 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 5 blocks
18/01/12 10:52:25 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/01/12 10:52:25 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 5 blocks
18/01/12 10:52:25 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/01/12 10:52:25 INFO Executor: Finished task 2.0 in stage 184.0 (TID 405). 882 bytes result sent to driver
18/01/12 10:52:25 INFO Executor: Finished task 1.0 in stage 184.0 (TID 404). 1028 bytes result sent to driver
18/01/12 10:52:25 INFO Executor: Finished task 0.0 in stage 184.0 (TID 403). 1028 bytes result sent to driver
18/01/12 10:52:25 INFO TaskSetManager: Finished task 2.0 in stage 184.0 (TID 405) in 2 ms on localhost (1/3)
18/01/12 10:52:25 INFO TaskSetManager: Finished task 1.0 in stage 184.0 (TID 404) in 2 ms on localhost (2/3)
18/01/12 10:52:25 INFO TaskSetManager: Finished task 0.0 in stage 184.0 (TID 403) in 2 ms on localhost (3/3)
18/01/12 10:52:25 INFO DAGScheduler: ResultStage 184 (print at NetWorkWordCount.scala:29) finished in 0.002 s
18/01/12 10:52:25 INFO TaskSchedulerImpl: Removed TaskSet 184.0, whose tasks have all completed, from pool
18/01/12 10:52:25 INFO DAGScheduler: Job 92 finished: print at NetWorkWordCount.scala:29, took 0.005325 s
-------------------------------------------
Time: 1515725545000 ms
-------------------------------------------
18/01/12 10:52:25 INFO JobScheduler: Finished job streaming job 1515725545000 ms.0 from job set of time 1515725545000 ms
18/01/12 10:52:25 INFO JobScheduler: Total delay: 0.090 s for time 1515725545000 ms (execution: 0.066 s)
18/01/12 10:52:25 INFO ShuffledRDD: Removing RDD 180 from persistence list
18/01/12 10:52:25 INFO BlockManager: Removing RDD 180
18/01/12 10:52:25 INFO MapPartitionsRDD: Removing RDD 179 from persistence list
18/01/12 10:52:25 INFO BlockManager: Removing RDD 179
18/01/12 10:52:25 INFO MapPartitionsRDD: Removing RDD 178 from persistence list
18/01/12 10:52:25 INFO BlockManager: Removing RDD 178
18/01/12 10:52:25 INFO BlockRDD: Removing RDD 177 from persistence list
18/01/12 10:52:25 INFO BlockManager: Removing RDD 177
18/01/12 10:52:25 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[177] at socketTextStream at NetWorkWordCount.scala:26 of time 1515725545000 ms
(D,2)
(E,1)
(F,2)
18/01/12 10:52:25 INFO BlockManagerInfo: Removed input-0-1515725535600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:25 INFO BlockManagerInfo: Removed input-0-1515725536600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:25 INFO BlockManagerInfo: Removed input-0-1515725537600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:25 INFO BlockManagerInfo: Removed input-0-1515725538600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:25 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1515725535000 ms)
18/01/12 10:52:25 INFO InputInfoTracker: remove old batch metadata: 1515725535000 ms
18/01/12 10:52:25 INFO BlockManagerInfo: Removed input-0-1515725539600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:25 INFO MemoryStore: ensureFreeSpace(8) called with curMem=119694, maxMem=490356080
18/01/12 10:52:25 INFO MemoryStore: Block input-0-1515725545600 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB)
18/01/12 10:52:25 INFO BlockManagerInfo: Added input-0-1515725545600 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:25 INFO BlockGenerator: Pushed block input-0-1515725545600
18/01/12 10:52:26 INFO MemoryStore: ensureFreeSpace(8) called with curMem=119702, maxMem=490356080
18/01/12 10:52:26 INFO MemoryStore: Block input-0-1515725546600 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB)
18/01/12 10:52:26 INFO BlockManagerInfo: Added input-0-1515725546600 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:26 INFO BlockGenerator: Pushed block input-0-1515725546600
18/01/12 10:52:27 INFO MemoryStore: ensureFreeSpace(8) called with curMem=119710, maxMem=490356080
18/01/12 10:52:27 INFO MemoryStore: Block input-0-1515725547600 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB)
18/01/12 10:52:27 INFO BlockManagerInfo: Added input-0-1515725547600 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:27 INFO BlockGenerator: Pushed block input-0-1515725547600
18/01/12 10:52:28 INFO MemoryStore: ensureFreeSpace(8) called with curMem=119718, maxMem=490356080
18/01/12 10:52:28 INFO MemoryStore: Block input-0-1515725548600 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB)
18/01/12 10:52:28 INFO BlockManagerInfo: Added input-0-1515725548600 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:28 INFO BlockGenerator: Pushed block input-0-1515725548600
18/01/12 10:52:29 INFO MemoryStore: ensureFreeSpace(8) called with curMem=119726, maxMem=490356080
18/01/12 10:52:29 INFO MemoryStore: Block input-0-1515725549600 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB)
18/01/12 10:52:29 INFO BlockManagerInfo: Added input-0-1515725549600 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:29 INFO BlockGenerator: Pushed block input-0-1515725549600
18/01/12 10:52:30 INFO JobScheduler: Added jobs for time 1515725550000 ms
18/01/12 10:52:30 INFO JobScheduler: Starting job streaming job 1515725550000 ms.0 from job set of time 1515725550000 ms
18/01/12 10:52:30 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29
18/01/12 10:52:30 INFO DAGScheduler: Registering RDD 187 (map at NetWorkWordCount.scala:28)
18/01/12 10:52:30 INFO DAGScheduler: Got job 93 (print at NetWorkWordCount.scala:29) with 1 output partitions (allowLocal=true)
18/01/12 10:52:30 INFO DAGScheduler: Final stage: ResultStage 186(print at NetWorkWordCount.scala:29)
18/01/12 10:52:30 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 185)
18/01/12 10:52:30 INFO DAGScheduler: Missing parents: List(ShuffleMapStage 185)
18/01/12 10:52:30 INFO DAGScheduler: Submitting ShuffleMapStage 185 (MapPartitionsRDD[187] at map at NetWorkWordCount.scala:28), which has no missing parents
18/01/12 10:52:30 INFO MemoryStore: ensureFreeSpace(2560) called with curMem=119734, maxMem=490356080
18/01/12 10:52:30 INFO MemoryStore: Block broadcast_138 stored as values in memory (estimated size 2.5 KB, free 467.5 MB)
18/01/12 10:52:30 INFO MemoryStore: ensureFreeSpace(1564) called with curMem=122294, maxMem=490356080
18/01/12 10:52:30 INFO MemoryStore: Block broadcast_138_piece0 stored as bytes in memory (estimated size 1564.0 B, free 467.5 MB)
18/01/12 10:52:30 INFO BlockManagerInfo: Added broadcast_138_piece0 in memory on localhost:54439 (size: 1564.0 B, free: 467.6 MB)
18/01/12 10:52:30 INFO SparkContext: Created broadcast 138 from broadcast at DAGScheduler.scala:874
18/01/12 10:52:30 INFO DAGScheduler: Submitting 5 missing tasks from ShuffleMapStage 185 (MapPartitionsRDD[187] at map at NetWorkWordCount.scala:28)
18/01/12 10:52:30 INFO TaskSchedulerImpl: Adding task set 185.0 with 5 tasks
18/01/12 10:52:30 INFO TaskSetManager: Starting task 0.0 in stage 185.0 (TID 406, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:30 INFO TaskSetManager: Starting task 1.0 in stage 185.0 (TID 407, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:30 INFO TaskSetManager: Starting task 2.0 in stage 185.0 (TID 408, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:30 INFO Executor: Running task 1.0 in stage 185.0 (TID 407)
18/01/12 10:52:30 INFO Executor: Running task 2.0 in stage 185.0 (TID 408)
18/01/12 10:52:30 INFO Executor: Running task 0.0 in stage 185.0 (TID 406)
18/01/12 10:52:30 INFO BlockManager: Found block input-0-1515725546600 locally
18/01/12 10:52:30 INFO BlockManager: Found block input-0-1515725545600 locally
18/01/12 10:52:30 INFO BlockManager: Found block input-0-1515725547600 locally
18/01/12 10:52:30 INFO Executor: Finished task 2.0 in stage 185.0 (TID 408). 882 bytes result sent to driver
18/01/12 10:52:30 INFO TaskSetManager: Starting task 3.0 in stage 185.0 (TID 409, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:30 INFO TaskSetManager: Finished task 2.0 in stage 185.0 (TID 408) in 24 ms on localhost (1/5)
18/01/12 10:52:30 INFO Executor: Running task 3.0 in stage 185.0 (TID 409)
18/01/12 10:52:30 INFO Executor: Finished task 1.0 in stage 185.0 (TID 407). 882 bytes result sent to driver
18/01/12 10:52:30 INFO BlockManager: Found block input-0-1515725548600 locally
18/01/12 10:52:30 INFO Executor: Finished task 0.0 in stage 185.0 (TID 406). 882 bytes result sent to driver
18/01/12 10:52:30 INFO TaskSetManager: Starting task 4.0 in stage 185.0 (TID 410, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:30 INFO Executor: Running task 4.0 in stage 185.0 (TID 410)
18/01/12 10:52:30 INFO TaskSetManager: Finished task 0.0 in stage 185.0 (TID 406) in 28 ms on localhost (2/5)
18/01/12 10:52:30 INFO BlockManager: Found block input-0-1515725549600 locally
18/01/12 10:52:30 INFO TaskSetManager: Finished task 1.0 in stage 185.0 (TID 407) in 31 ms on localhost (3/5)
18/01/12 10:52:30 INFO Executor: Finished task 3.0 in stage 185.0 (TID 409). 882 bytes result sent to driver
18/01/12 10:52:30 INFO TaskSetManager: Finished task 3.0 in stage 185.0 (TID 409) in 9 ms on localhost (4/5)
18/01/12 10:52:30 INFO Executor: Finished task 4.0 in stage 185.0 (TID 410). 882 bytes result sent to driver
18/01/12 10:52:30 INFO TaskSetManager: Finished task 4.0 in stage 185.0 (TID 410) in 13 ms on localhost (5/5)
18/01/12 10:52:30 INFO TaskSchedulerImpl: Removed TaskSet 185.0, whose tasks have all completed, from pool
18/01/12 10:52:30 INFO DAGScheduler: ShuffleMapStage 185 (map at NetWorkWordCount.scala:28) finished in 0.041 s
18/01/12 10:52:30 INFO DAGScheduler: looking for newly runnable stages
18/01/12 10:52:30 INFO DAGScheduler: running: Set(ResultStage 0)
18/01/12 10:52:30 INFO DAGScheduler: waiting: Set(ResultStage 186)
18/01/12 10:52:30 INFO DAGScheduler: failed: Set()
18/01/12 10:52:30 INFO DAGScheduler: Missing parents for ResultStage 186: List()
18/01/12 10:52:30 INFO DAGScheduler: Submitting ResultStage 186 (ShuffledRDD[188] at reduceByKey at NetWorkWordCount.scala:28), which is now runnable
18/01/12 10:52:30 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=123858, maxMem=490356080
18/01/12 10:52:30 INFO MemoryStore: Block broadcast_139 stored as values in memory (estimated size 2.3 KB, free 467.5 MB)
18/01/12 10:52:30 INFO MemoryStore: ensureFreeSpace(1417) called with curMem=126162, maxMem=490356080
18/01/12 10:52:30 INFO MemoryStore: Block broadcast_139_piece0 stored as bytes in memory (estimated size 1417.0 B, free 467.5 MB)
18/01/12 10:52:30 INFO BlockManagerInfo: Added broadcast_139_piece0 in memory on localhost:54439 (size: 1417.0 B, free: 467.6 MB)
18/01/12 10:52:30 INFO SparkContext: Created broadcast 139 from broadcast at DAGScheduler.scala:874
18/01/12 10:52:30 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 186 (ShuffledRDD[188] at reduceByKey at NetWorkWordCount.scala:28)
18/01/12 10:52:30 INFO TaskSchedulerImpl: Adding task set 186.0 with 1 tasks
18/01/12 10:52:30 INFO TaskSetManager: Starting task 0.0 in stage 186.0 (TID 411, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:30 INFO Executor: Running task 0.0 in stage 186.0 (TID 411)
18/01/12 10:52:30 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 5 blocks
18/01/12 10:52:30 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/01/12 10:52:30 INFO Executor: Finished task 0.0 in stage 186.0 (TID 411). 882 bytes result sent to driver
18/01/12 10:52:30 INFO TaskSetManager: Finished task 0.0 in stage 186.0 (TID 411) in 1 ms on localhost (1/1)
18/01/12 10:52:30 INFO TaskSchedulerImpl: Removed TaskSet 186.0, whose tasks have all completed, from pool
18/01/12 10:52:30 INFO DAGScheduler: ResultStage 186 (print at NetWorkWordCount.scala:29) finished in 0.002 s
18/01/12 10:52:30 INFO DAGScheduler: Job 93 finished: print at NetWorkWordCount.scala:29, took 0.049675 s
18/01/12 10:52:30 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29
18/01/12 10:52:30 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 46 is 158 bytes
18/01/12 10:52:30 INFO DAGScheduler: Got job 94 (print at NetWorkWordCount.scala:29) with 3 output partitions (allowLocal=true)
18/01/12 10:52:30 INFO DAGScheduler: Final stage: ResultStage 188(print at NetWorkWordCount.scala:29)
18/01/12 10:52:30 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 187)
18/01/12 10:52:30 INFO DAGScheduler: Missing parents: List()
18/01/12 10:52:30 INFO DAGScheduler: Submitting ResultStage 188 (ShuffledRDD[188] at reduceByKey at NetWorkWordCount.scala:28), which has no missing parents
18/01/12 10:52:30 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=127579, maxMem=490356080
18/01/12 10:52:30 INFO MemoryStore: Block broadcast_140 stored as values in memory (estimated size 2.3 KB, free 467.5 MB)
18/01/12 10:52:30 INFO MemoryStore: ensureFreeSpace(1417) called with curMem=129883, maxMem=490356080
18/01/12 10:52:30 INFO MemoryStore: Block broadcast_140_piece0 stored as bytes in memory (estimated size 1417.0 B, free 467.5 MB)
18/01/12 10:52:30 INFO BlockManagerInfo: Added broadcast_140_piece0 in memory on localhost:54439 (size: 1417.0 B, free: 467.6 MB)
18/01/12 10:52:30 INFO SparkContext: Created broadcast 140 from broadcast at DAGScheduler.scala:874
18/01/12 10:52:30 INFO DAGScheduler: Submitting 3 missing tasks from ResultStage 188 (ShuffledRDD[188] at reduceByKey at NetWorkWordCount.scala:28)
18/01/12 10:52:30 INFO TaskSchedulerImpl: Adding task set 188.0 with 3 tasks
18/01/12 10:52:30 INFO TaskSetManager: Starting task 0.0 in stage 188.0 (TID 412, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:30 INFO TaskSetManager: Starting task 1.0 in stage 188.0 (TID 413, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:30 INFO TaskSetManager: Starting task 2.0 in stage 188.0 (TID 414, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:30 INFO Executor: Running task 0.0 in stage 188.0 (TID 412)
18/01/12 10:52:30 INFO Executor: Running task 2.0 in stage 188.0 (TID 414)
18/01/12 10:52:30 INFO Executor: Running task 1.0 in stage 188.0 (TID 413)
18/01/12 10:52:30 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 5 blocks
18/01/12 10:52:30 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
18/01/12 10:52:30 INFO Executor: Finished task 0.0 in stage 188.0 (TID 412). 882 bytes result sent to driver
18/01/12 10:52:30 INFO ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 5 blocks
18/01/12 10:52:30 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/01/12 10:52:30 INFO TaskSetManager: Finished task 0.0 in stage 188.0 (TID 412) in 3 ms on localhost (1/3)
18/01/12 10:52:30 INFO ShuffleBlockFetcherIterator: Getting 3 non-empty blocks out of 5 blocks
18/01/12 10:52:30 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 3 ms
18/01/12 10:52:30 INFO Executor: Finished task 1.0 in stage 188.0 (TID 413). 1048 bytes result sent to driver
18/01/12 10:52:30 INFO TaskSetManager: Finished task 1.0 in stage 188.0 (TID 413) in 4 ms on localhost (2/3)
18/01/12 10:52:30 INFO Executor: Finished task 2.0 in stage 188.0 (TID 414). 1048 bytes result sent to driver
18/01/12 10:52:30 INFO TaskSetManager: Finished task 2.0 in stage 188.0 (TID 414) in 4 ms on localhost (3/3)
18/01/12 10:52:30 INFO TaskSchedulerImpl: Removed TaskSet 188.0, whose tasks have all completed, from pool
18/01/12 10:52:30 INFO DAGScheduler: ResultStage 188 (print at NetWorkWordCount.scala:29) finished in 0.006 s
18/01/12 10:52:30 INFO DAGScheduler: Job 94 finished: print at NetWorkWordCount.scala:29, took 0.009241 s
-------------------------------------------
Time: 1515725550000 ms
-------------------------------------------
(B,1)
(F,1)
(G,2)
(C,1)
18/01/12 10:52:30 INFO JobScheduler: Finished job streaming job 1515725550000 ms.0 from job set of time 1515725550000 ms
18/01/12 10:52:30 INFO JobScheduler: Total delay: 0.069 s for time 1515725550000 ms (execution: 0.063 s)
18/01/12 10:52:30 INFO ShuffledRDD: Removing RDD 184 from persistence list
18/01/12 10:52:30 INFO MapPartitionsRDD: Removing RDD 183 from persistence list
18/01/12 10:52:30 INFO BlockManager: Removing RDD 184
18/01/12 10:52:30 INFO MapPartitionsRDD: Removing RDD 182 from persistence list
18/01/12 10:52:30 INFO BlockManager: Removing RDD 183
18/01/12 10:52:30 INFO BlockRDD: Removing RDD 181 from persistence list
18/01/12 10:52:30 INFO BlockManager: Removing RDD 182
18/01/12 10:52:30 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[181] at socketTextStream at NetWorkWordCount.scala:26 of time 1515725550000 ms
18/01/12 10:52:30 INFO BlockManager: Removing RDD 181
18/01/12 10:52:30 INFO BlockManagerInfo: Removed input-0-1515725540800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:30 INFO BlockManagerInfo: Removed input-0-1515725541600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:30 INFO BlockManagerInfo: Removed input-0-1515725542600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:30 INFO BlockManagerInfo: Removed input-0-1515725543600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:30 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1515725540000 ms)
18/01/12 10:52:30 INFO InputInfoTracker: remove old batch metadata: 1515725540000 ms
18/01/12 10:52:30 INFO BlockManagerInfo: Removed input-0-1515725544600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:30 INFO MemoryStore: ensureFreeSpace(8) called with curMem=131260, maxMem=490356080
18/01/12 10:52:30 INFO MemoryStore: Block input-0-1515725550600 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB)
18/01/12 10:52:30 INFO BlockManagerInfo: Added input-0-1515725550600 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:30 INFO BlockGenerator: Pushed block input-0-1515725550600
18/01/12 10:52:32 INFO MemoryStore: ensureFreeSpace(8) called with curMem=131268, maxMem=490356080
18/01/12 10:52:32 INFO MemoryStore: Block input-0-1515725551800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB)
18/01/12 10:52:32 INFO BlockManagerInfo: Added input-0-1515725551800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:32 INFO BlockGenerator: Pushed block input-0-1515725551800
18/01/12 10:52:33 INFO MemoryStore: ensureFreeSpace(8) called with curMem=131276, maxMem=490356080
18/01/12 10:52:33 INFO MemoryStore: Block input-0-1515725552800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB)
18/01/12 10:52:33 INFO BlockManagerInfo: Added input-0-1515725552800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:33 INFO BlockGenerator: Pushed block input-0-1515725552800
18/01/12 10:52:34 INFO MemoryStore: ensureFreeSpace(8) called with curMem=131284, maxMem=490356080
18/01/12 10:52:34 INFO MemoryStore: Block input-0-1515725553800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB)
18/01/12 10:52:34 INFO BlockManagerInfo: Added input-0-1515725553800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:34 INFO BlockGenerator: Pushed block input-0-1515725553800
18/01/12 10:52:35 INFO MemoryStore: ensureFreeSpace(8) called with curMem=131292, maxMem=490356080
18/01/12 10:52:35 INFO MemoryStore: Block input-0-1515725554800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB)
18/01/12 10:52:35 INFO BlockManagerInfo: Added input-0-1515725554800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO BlockGenerator: Pushed block input-0-1515725554800
18/01/12 10:52:35 INFO JobScheduler: Added jobs for time 1515725555000 ms
18/01/12 10:52:35 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29
18/01/12 10:52:35 INFO JobScheduler: Starting job streaming job 1515725555000 ms.0 from job set of time 1515725555000 ms
18/01/12 10:52:35 INFO DAGScheduler: Registering RDD 191 (map at NetWorkWordCount.scala:28)
18/01/12 10:52:35 INFO DAGScheduler: Got job 95 (print at NetWorkWordCount.scala:29) with 1 output partitions (allowLocal=true)
18/01/12 10:52:35 INFO DAGScheduler: Final stage: ResultStage 190(print at NetWorkWordCount.scala:29)
18/01/12 10:52:35 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 189)
18/01/12 10:52:35 INFO DAGScheduler: Missing parents: List(ShuffleMapStage 189)
18/01/12 10:52:35 INFO DAGScheduler: Submitting ShuffleMapStage 189 (MapPartitionsRDD[191] at map at NetWorkWordCount.scala:28), which has no missing parents
18/01/12 10:52:35 INFO MemoryStore: ensureFreeSpace(2560) called with curMem=131300, maxMem=490356080
18/01/12 10:52:35 INFO MemoryStore: Block broadcast_141 stored as values in memory (estimated size 2.5 KB, free 467.5 MB)
18/01/12 10:52:35 INFO MemoryStore: ensureFreeSpace(1564) called with curMem=133860, maxMem=490356080
18/01/12 10:52:35 INFO MemoryStore: Block broadcast_141_piece0 stored as bytes in memory (estimated size 1564.0 B, free 467.5 MB)
18/01/12 10:52:35 INFO BlockManagerInfo: Added broadcast_141_piece0 in memory on localhost:54439 (size: 1564.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO SparkContext: Created broadcast 141 from broadcast at DAGScheduler.scala:874
18/01/12 10:52:35 INFO DAGScheduler: Submitting 4 missing tasks from ShuffleMapStage 189 (MapPartitionsRDD[191] at map at NetWorkWordCount.scala:28)
18/01/12 10:52:35 INFO TaskSchedulerImpl: Adding task set 189.0 with 4 tasks
18/01/12 10:52:35 INFO TaskSetManager: Starting task 0.0 in stage 189.0 (TID 415, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:35 INFO TaskSetManager: Starting task 1.0 in stage 189.0 (TID 416, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:35 INFO TaskSetManager: Starting task 2.0 in stage 189.0 (TID 417, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:35 INFO Executor: Running task 2.0 in stage 189.0 (TID 417)
18/01/12 10:52:35 INFO BlockManager: Found block input-0-1515725552800 locally
18/01/12 10:52:35 INFO Executor: Running task 0.0 in stage 189.0 (TID 415)
18/01/12 10:52:35 INFO BlockManager: Found block input-0-1515725550600 locally
18/01/12 10:52:35 INFO Executor: Running task 1.0 in stage 189.0 (TID 416)
18/01/12 10:52:35 INFO BlockManager: Found block input-0-1515725551800 locally
18/01/12 10:52:35 INFO Executor: Finished task 2.0 in stage 189.0 (TID 417). 882 bytes result sent to driver
18/01/12 10:52:35 INFO TaskSetManager: Starting task 3.0 in stage 189.0 (TID 418, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:35 INFO Executor: Running task 3.0 in stage 189.0 (TID 418)
18/01/12 10:52:35 INFO TaskSetManager: Finished task 2.0 in stage 189.0 (TID 417) in 7 ms on localhost (1/4)
18/01/12 10:52:35 INFO BlockManager: Found block input-0-1515725553800 locally
18/01/12 10:52:35 INFO Executor: Finished task 0.0 in stage 189.0 (TID 415). 882 bytes result sent to driver
18/01/12 10:52:35 INFO TaskSetManager: Finished task 0.0 in stage 189.0 (TID 415) in 9 ms on localhost (2/4)
18/01/12 10:52:35 INFO Executor: Finished task 3.0 in stage 189.0 (TID 418). 882 bytes result sent to driver
18/01/12 10:52:35 INFO TaskSetManager: Finished task 3.0 in stage 189.0 (TID 418) in 4 ms on localhost (3/4)
18/01/12 10:52:35 INFO Executor: Finished task 1.0 in stage 189.0 (TID 416). 882 bytes result sent to driver
18/01/12 10:52:35 INFO TaskSetManager: Finished task 1.0 in stage 189.0 (TID 416) in 12 ms on localhost (4/4)
18/01/12 10:52:35 INFO DAGScheduler: ShuffleMapStage 189 (map at NetWorkWordCount.scala:28) finished in 0.012 s
18/01/12 10:52:35 INFO DAGScheduler: looking for newly runnable stages
18/01/12 10:52:35 INFO TaskSchedulerImpl: Removed TaskSet 189.0, whose tasks have all completed, from pool
18/01/12 10:52:35 INFO DAGScheduler: running: Set(ResultStage 0)
18/01/12 10:52:35 INFO DAGScheduler: waiting: Set(ResultStage 190)
18/01/12 10:52:35 INFO DAGScheduler: failed: Set()
18/01/12 10:52:35 INFO DAGScheduler: Missing parents for ResultStage 190: List()
18/01/12 10:52:35 INFO DAGScheduler: Submitting ResultStage 190 (ShuffledRDD[192] at reduceByKey at NetWorkWordCount.scala:28), which is now runnable
18/01/12 10:52:35 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=135424, maxMem=490356080
18/01/12 10:52:35 INFO MemoryStore: Block broadcast_142 stored as values in memory (estimated size 2.3 KB, free 467.5 MB)
18/01/12 10:52:35 INFO MemoryStore: ensureFreeSpace(1413) called with curMem=137728, maxMem=490356080
18/01/12 10:52:35 INFO MemoryStore: Block broadcast_142_piece0 stored as bytes in memory (estimated size 1413.0 B, free 467.5 MB)
18/01/12 10:52:35 INFO BlockManagerInfo: Added broadcast_142_piece0 in memory on localhost:54439 (size: 1413.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO SparkContext: Created broadcast 142 from broadcast at DAGScheduler.scala:874
18/01/12 10:52:35 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 190 (ShuffledRDD[192] at reduceByKey at NetWorkWordCount.scala:28)
18/01/12 10:52:35 INFO TaskSchedulerImpl: Adding task set 190.0 with 1 tasks
18/01/12 10:52:35 INFO TaskSetManager: Starting task 0.0 in stage 190.0 (TID 419, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:35 INFO Executor: Running task 0.0 in stage 190.0 (TID 419)
18/01/12 10:52:35 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 4 blocks
18/01/12 10:52:35 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
18/01/12 10:52:35 INFO Executor: Finished task 0.0 in stage 190.0 (TID 419). 882 bytes result sent to driver
18/01/12 10:52:35 INFO TaskSetManager: Finished task 0.0 in stage 190.0 (TID 419) in 1 ms on localhost (1/1)
18/01/12 10:52:35 INFO TaskSchedulerImpl: Removed TaskSet 190.0, whose tasks have all completed, from pool
18/01/12 10:52:35 INFO DAGScheduler: ResultStage 190 (print at NetWorkWordCount.scala:29) finished in 0.002 s
18/01/12 10:52:35 INFO DAGScheduler: Job 95 finished: print at NetWorkWordCount.scala:29, took 0.022178 s
18/01/12 10:52:35 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29
18/01/12 10:52:35 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 47 is 165 bytes
18/01/12 10:52:35 INFO DAGScheduler: Got job 96 (print at NetWorkWordCount.scala:29) with 3 output partitions (allowLocal=true)
18/01/12 10:52:35 INFO DAGScheduler: Final stage: ResultStage 192(print at NetWorkWordCount.scala:29)
18/01/12 10:52:35 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 191)
18/01/12 10:52:35 INFO DAGScheduler: Missing parents: List()
18/01/12 10:52:35 INFO DAGScheduler: Submitting ResultStage 192 (ShuffledRDD[192] at reduceByKey at NetWorkWordCount.scala:28), which has no missing parents
18/01/12 10:52:35 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=139141, maxMem=490356080
18/01/12 10:52:35 INFO MemoryStore: Block broadcast_143 stored as values in memory (estimated size 2.3 KB, free 467.5 MB)
18/01/12 10:52:35 INFO MemoryStore: ensureFreeSpace(1413) called with curMem=141445, maxMem=490356080
18/01/12 10:52:35 INFO MemoryStore: Block broadcast_143_piece0 stored as bytes in memory (estimated size 1413.0 B, free 467.5 MB)
18/01/12 10:52:35 INFO BlockManagerInfo: Added broadcast_143_piece0 in memory on localhost:54439 (size: 1413.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO SparkContext: Created broadcast 143 from broadcast at DAGScheduler.scala:874
18/01/12 10:52:35 INFO DAGScheduler: Submitting 3 missing tasks from ResultStage 192 (ShuffledRDD[192] at reduceByKey at NetWorkWordCount.scala:28)
18/01/12 10:52:35 INFO TaskSchedulerImpl: Adding task set 192.0 with 3 tasks
18/01/12 10:52:35 INFO ContextCleaner: Cleaned shuffle 40
18/01/12 10:52:35 INFO ContextCleaner: Cleaned shuffle 41
18/01/12 10:52:35 INFO TaskSetManager: Starting task 0.0 in stage 192.0 (TID 420, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:35 INFO TaskSetManager: Starting task 1.0 in stage 192.0 (TID 421, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:35 INFO TaskSetManager: Starting task 2.0 in stage 192.0 (TID 422, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:35 INFO Executor: Running task 1.0 in stage 192.0 (TID 421)
18/01/12 10:52:35 INFO ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 4 blocks
18/01/12 10:52:35 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_125_piece0 on localhost:54439 in memory (size: 1417.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO Executor: Running task 0.0 in stage 192.0 (TID 420)
18/01/12 10:52:35 INFO Executor: Finished task 1.0 in stage 192.0 (TID 421). 1028 bytes result sent to driver
18/01/12 10:52:35 INFO Executor: Running task 2.0 in stage 192.0 (TID 422)
18/01/12 10:52:35 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 4 blocks
18/01/12 10:52:35 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/01/12 10:52:35 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 4 blocks
18/01/12 10:52:35 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
18/01/12 10:52:35 INFO Executor: Finished task 0.0 in stage 192.0 (TID 420). 1028 bytes result sent to driver
18/01/12 10:52:35 INFO Executor: Finished task 2.0 in stage 192.0 (TID 422). 1028 bytes result sent to driver
18/01/12 10:52:35 INFO TaskSetManager: Finished task 1.0 in stage 192.0 (TID 421) in 5 ms on localhost (1/3)
18/01/12 10:52:35 INFO TaskSetManager: Finished task 0.0 in stage 192.0 (TID 420) in 6 ms on localhost (2/3)
18/01/12 10:52:35 INFO TaskSetManager: Finished task 2.0 in stage 192.0 (TID 422) in 5 ms on localhost (3/3)
18/01/12 10:52:35 INFO TaskSchedulerImpl: Removed TaskSet 192.0, whose tasks have all completed, from pool
18/01/12 10:52:35 INFO DAGScheduler: ResultStage 192 (print at NetWorkWordCount.scala:29) finished in 0.006 s
18/01/12 10:52:35 INFO DAGScheduler: Job 96 finished: print at NetWorkWordCount.scala:29, took 0.021802 s
-------------------------------------------18/01/12 10:52:35 INFO ContextCleaner: Cleaned shuffle 42
Time: 1515725555000 ms
-------------------------------------------
(A,1)
(B,2)
(C,1)
18/01/12 10:52:35 INFO JobScheduler: Finished job streaming job 1515725555000 ms.0 from job set of time 1515725555000 ms
18/01/12 10:52:35 INFO JobScheduler: Total delay: 0.056 s for time 1515725555000 ms (execution: 0.047 s)
18/01/12 10:52:35 INFO ShuffledRDD: Removing RDD 188 from persistence list
18/01/12 10:52:35 INFO MapPartitionsRDD: Removing RDD 187 from persistence list
18/01/12 10:52:35 INFO BlockManager: Removing RDD 188
18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_126_piece0 on localhost:54439 in memory (size: 1564.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO MapPartitionsRDD: Removing RDD 186 from persistence list
18/01/12 10:52:35 INFO BlockManager: Removing RDD 187
18/01/12 10:52:35 INFO BlockRDD: Removing RDD 185 from persistence list
18/01/12 10:52:35 INFO BlockManager: Removing RDD 186
18/01/12 10:52:35 INFO BlockManager: Removing RDD 185
18/01/12 10:52:35 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[185] at socketTextStream at NetWorkWordCount.scala:26 of time 1515725555000 ms
18/01/12 10:52:35 INFO BlockManagerInfo: Removed input-0-1515725547600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO BlockManagerInfo: Removed input-0-1515725545600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1515725545000 ms)
18/01/12 10:52:35 INFO InputInfoTracker: remove old batch metadata: 1515725545000 ms
18/01/12 10:52:35 INFO BlockManagerInfo: Removed input-0-1515725546600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO BlockManagerInfo: Removed input-0-1515725549600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO BlockManagerInfo: Removed input-0-1515725548600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_127_piece0 on localhost:54439 in memory (size: 1417.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_128_piece0 on localhost:54439 in memory (size: 1417.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO ContextCleaner: Cleaned shuffle 43
18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_129_piece0 on localhost:54439 in memory (size: 1564.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_130_piece0 on localhost:54439 in memory (size: 1417.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_131_piece0 on localhost:54439 in memory (size: 1417.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO ContextCleaner: Cleaned shuffle 44
18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_132_piece0 on localhost:54439 in memory (size: 1563.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_142_piece0 on localhost:54439 in memory (size: 1413.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_141_piece0 on localhost:54439 in memory (size: 1564.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_140_piece0 on localhost:54439 in memory (size: 1417.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_139_piece0 on localhost:54439 in memory (size: 1417.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_138_piece0 on localhost:54439 in memory (size: 1564.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_137_piece0 on localhost:54439 in memory (size: 1417.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_136_piece0 on localhost:54439 in memory (size: 1417.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_135_piece0 on localhost:54439 in memory (size: 1564.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO ContextCleaner: Cleaned shuffle 45
18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_134_piece0 on localhost:54439 in memory (size: 1417.0 B, free: 467.6 MB)
18/01/12 10:52:35 INFO BlockManagerInfo: Removed broadcast_133_piece0 on localhost:54439 in memory (size: 1417.0 B, free: 467.6 MB)
18/01/12 10:52:36 INFO MemoryStore: ensureFreeSpace(8) called with curMem=73427, maxMem=490356080
18/01/12 10:52:36 INFO MemoryStore: Block input-0-1515725555800 stored as bytes in memory (estimated size 8.0 B, free 467.6 MB)
18/01/12 10:52:36 INFO BlockManagerInfo: Added input-0-1515725555800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:36 INFO BlockGenerator: Pushed block input-0-1515725555800
18/01/12 10:52:37 INFO MemoryStore: ensureFreeSpace(8) called with curMem=73435, maxMem=490356080
18/01/12 10:52:37 INFO MemoryStore: Block input-0-1515725556800 stored as bytes in memory (estimated size 8.0 B, free 467.6 MB)
18/01/12 10:52:37 INFO BlockManagerInfo: Added input-0-1515725556800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:37 INFO BlockGenerator: Pushed block input-0-1515725556800
18/01/12 10:52:38 INFO MemoryStore: ensureFreeSpace(8) called with curMem=73443, maxMem=490356080
18/01/12 10:52:38 INFO MemoryStore: Block input-0-1515725557800 stored as bytes in memory (estimated size 8.0 B, free 467.6 MB)
18/01/12 10:52:38 INFO BlockManagerInfo: Added input-0-1515725557800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:38 INFO BlockGenerator: Pushed block input-0-1515725557800
18/01/12 10:52:39 INFO MemoryStore: ensureFreeSpace(8) called with curMem=73451, maxMem=490356080
18/01/12 10:52:39 INFO MemoryStore: Block input-0-1515725558800 stored as bytes in memory (estimated size 8.0 B, free 467.6 MB)
18/01/12 10:52:39 INFO BlockManagerInfo: Added input-0-1515725558800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:39 INFO BlockGenerator: Pushed block input-0-1515725558800
18/01/12 10:52:40 INFO MemoryStore: ensureFreeSpace(8) called with curMem=73459, maxMem=490356080
18/01/12 10:52:40 INFO MemoryStore: Block input-0-1515725559800 stored as bytes in memory (estimated size 8.0 B, free 467.6 MB)
18/01/12 10:52:40 INFO BlockManagerInfo: Added input-0-1515725559800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:40 INFO BlockGenerator: Pushed block input-0-1515725559800
18/01/12 10:52:40 INFO JobScheduler: Added jobs for time 1515725560000 ms
18/01/12 10:52:40 INFO JobScheduler: Starting job streaming job 1515725560000 ms.0 from job set of time 1515725560000 ms
18/01/12 10:52:40 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29
18/01/12 10:52:40 INFO DAGScheduler: Registering RDD 195 (map at NetWorkWordCount.scala:28)
18/01/12 10:52:40 INFO DAGScheduler: Got job 97 (print at NetWorkWordCount.scala:29) with 1 output partitions (allowLocal=true)
18/01/12 10:52:40 INFO DAGScheduler: Final stage: ResultStage 194(print at NetWorkWordCount.scala:29)
18/01/12 10:52:40 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 193)
18/01/12 10:52:40 INFO DAGScheduler: Missing parents: List(ShuffleMapStage 193)
18/01/12 10:52:40 INFO DAGScheduler: Submitting ShuffleMapStage 193 (MapPartitionsRDD[195] at map at NetWorkWordCount.scala:28), which has no missing parents
18/01/12 10:52:40 INFO MemoryStore: ensureFreeSpace(2560) called with curMem=73467, maxMem=490356080
18/01/12 10:52:40 INFO MemoryStore: Block broadcast_144 stored as values in memory (estimated size 2.5 KB, free 467.6 MB)
18/01/12 10:52:40 INFO MemoryStore: ensureFreeSpace(1564) called with curMem=76027, maxMem=490356080
18/01/12 10:52:40 INFO MemoryStore: Block broadcast_144_piece0 stored as bytes in memory (estimated size 1564.0 B, free 467.6 MB)
18/01/12 10:52:40 INFO BlockManagerInfo: Added broadcast_144_piece0 in memory on localhost:54439 (size: 1564.0 B, free: 467.6 MB)
18/01/12 10:52:40 INFO SparkContext: Created broadcast 144 from broadcast at DAGScheduler.scala:874
18/01/12 10:52:40 INFO DAGScheduler: Submitting 5 missing tasks from ShuffleMapStage 193 (MapPartitionsRDD[195] at map at NetWorkWordCount.scala:28)
18/01/12 10:52:40 INFO TaskSchedulerImpl: Adding task set 193.0 with 5 tasks
18/01/12 10:52:40 INFO TaskSetManager: Starting task 0.0 in stage 193.0 (TID 423, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:40 INFO TaskSetManager: Starting task 1.0 in stage 193.0 (TID 424, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:40 INFO TaskSetManager: Starting task 2.0 in stage 193.0 (TID 425, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:40 INFO Executor: Running task 2.0 in stage 193.0 (TID 425)
18/01/12 10:52:40 INFO BlockManager: Found block input-0-1515725556800 locally
18/01/12 10:52:40 INFO Executor: Running task 0.0 in stage 193.0 (TID 423)
18/01/12 10:52:40 INFO BlockManager: Found block input-0-1515725554800 locally
18/01/12 10:52:40 INFO Executor: Running task 1.0 in stage 193.0 (TID 424)
18/01/12 10:52:40 INFO BlockManager: Found block input-0-1515725555800 locally
18/01/12 10:52:40 INFO Executor: Finished task 2.0 in stage 193.0 (TID 425). 882 bytes result sent to driver
18/01/12 10:52:40 INFO TaskSetManager: Starting task 3.0 in stage 193.0 (TID 426, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:40 INFO Executor: Running task 3.0 in stage 193.0 (TID 426)
18/01/12 10:52:40 INFO TaskSetManager: Finished task 2.0 in stage 193.0 (TID 425) in 4 ms on localhost (1/5)
18/01/12 10:52:40 INFO Executor: Finished task 0.0 in stage 193.0 (TID 423). 882 bytes result sent to driver
18/01/12 10:52:40 INFO BlockManager: Found block input-0-1515725557800 locally
18/01/12 10:52:40 INFO TaskSetManager: Starting task 4.0 in stage 193.0 (TID 427, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:40 INFO TaskSetManager: Finished task 0.0 in stage 193.0 (TID 423) in 6 ms on localhost (2/5)
18/01/12 10:52:40 INFO Executor: Running task 4.0 in stage 193.0 (TID 427)
18/01/12 10:52:40 INFO Executor: Finished task 1.0 in stage 193.0 (TID 424). 882 bytes result sent to driver
18/01/12 10:52:40 INFO BlockManager: Found block input-0-1515725558800 locally
18/01/12 10:52:40 INFO Executor: Finished task 3.0 in stage 193.0 (TID 426). 882 bytes result sent to driver
18/01/12 10:52:40 INFO TaskSetManager: Finished task 1.0 in stage 193.0 (TID 424) in 8 ms on localhost (3/5)
18/01/12 10:52:40 INFO TaskSetManager: Finished task 3.0 in stage 193.0 (TID 426) in 4 ms on localhost (4/5)
18/01/12 10:52:40 INFO Executor: Finished task 4.0 in stage 193.0 (TID 427). 882 bytes result sent to driver
18/01/12 10:52:40 INFO TaskSetManager: Finished task 4.0 in stage 193.0 (TID 427) in 5 ms on localhost (5/5)
18/01/12 10:52:40 INFO TaskSchedulerImpl: Removed TaskSet 193.0, whose tasks have all completed, from pool
18/01/12 10:52:40 INFO DAGScheduler: ShuffleMapStage 193 (map at NetWorkWordCount.scala:28) finished in 0.011 s
18/01/12 10:52:40 INFO DAGScheduler: looking for newly runnable stages
18/01/12 10:52:40 INFO DAGScheduler: running: Set(ResultStage 0)
18/01/12 10:52:40 INFO DAGScheduler: waiting: Set(ResultStage 194)
18/01/12 10:52:40 INFO DAGScheduler: failed: Set()
18/01/12 10:52:40 INFO DAGScheduler: Missing parents for ResultStage 194: List()
18/01/12 10:52:40 INFO DAGScheduler: Submitting ResultStage 194 (ShuffledRDD[196] at reduceByKey at NetWorkWordCount.scala:28), which is now runnable
18/01/12 10:52:40 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=77591, maxMem=490356080
18/01/12 10:52:40 INFO MemoryStore: Block broadcast_145 stored as values in memory (estimated size 2.3 KB, free 467.6 MB)
18/01/12 10:52:40 INFO MemoryStore: ensureFreeSpace(1417) called with curMem=79895, maxMem=490356080
18/01/12 10:52:40 INFO MemoryStore: Block broadcast_145_piece0 stored as bytes in memory (estimated size 1417.0 B, free 467.6 MB)
18/01/12 10:52:40 INFO BlockManagerInfo: Added broadcast_145_piece0 in memory on localhost:54439 (size: 1417.0 B, free: 467.6 MB)
18/01/12 10:52:40 INFO SparkContext: Created broadcast 145 from broadcast at DAGScheduler.scala:874
18/01/12 10:52:40 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 194 (ShuffledRDD[196] at reduceByKey at NetWorkWordCount.scala:28)
18/01/12 10:52:40 INFO TaskSchedulerImpl: Adding task set 194.0 with 1 tasks
18/01/12 10:52:40 INFO TaskSetManager: Starting task 0.0 in stage 194.0 (TID 428, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:40 INFO Executor: Running task 0.0 in stage 194.0 (TID 428)
18/01/12 10:52:40 INFO ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 5 blocks
18/01/12 10:52:40 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/01/12 10:52:40 INFO Executor: Finished task 0.0 in stage 194.0 (TID 428). 1028 bytes result sent to driver
18/01/12 10:52:40 INFO TaskSetManager: Finished task 0.0 in stage 194.0 (TID 428) in 3 ms on localhost (1/1)
18/01/12 10:52:40 INFO TaskSchedulerImpl: Removed TaskSet 194.0, whose tasks have all completed, from pool
18/01/12 10:52:40 INFO DAGScheduler: ResultStage 194 (print at NetWorkWordCount.scala:29) finished in 0.005 s
18/01/12 10:52:40 INFO DAGScheduler: Job 97 finished: print at NetWorkWordCount.scala:29, took 0.026568 s
18/01/12 10:52:40 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29
18/01/12 10:52:40 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 48 is 161 bytes
18/01/12 10:52:40 INFO DAGScheduler: Got job 98 (print at NetWorkWordCount.scala:29) with 3 output partitions (allowLocal=true)
18/01/12 10:52:40 INFO DAGScheduler: Final stage: ResultStage 196(print at NetWorkWordCount.scala:29)
18/01/12 10:52:40 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 195)
18/01/12 10:52:40 INFO DAGScheduler: Missing parents: List()
18/01/12 10:52:40 INFO DAGScheduler: Submitting ResultStage 196 (ShuffledRDD[196] at reduceByKey at NetWorkWordCount.scala:28), which has no missing parents
18/01/12 10:52:40 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=81312, maxMem=490356080
18/01/12 10:52:40 INFO MemoryStore: Block broadcast_146 stored as values in memory (estimated size 2.3 KB, free 467.6 MB)
18/01/12 10:52:40 INFO MemoryStore: ensureFreeSpace(1417) called with curMem=83616, maxMem=490356080
18/01/12 10:52:40 INFO MemoryStore: Block broadcast_146_piece0 stored as bytes in memory (estimated size 1417.0 B, free 467.6 MB)
18/01/12 10:52:40 INFO BlockManagerInfo: Added broadcast_146_piece0 in memory on localhost:54439 (size: 1417.0 B, free: 467.6 MB)
18/01/12 10:52:40 INFO SparkContext: Created broadcast 146 from broadcast at DAGScheduler.scala:874
18/01/12 10:52:40 INFO DAGScheduler: Submitting 3 missing tasks from ResultStage 196 (ShuffledRDD[196] at reduceByKey at NetWorkWordCount.scala:28)
18/01/12 10:52:40 INFO TaskSchedulerImpl: Adding task set 196.0 with 3 tasks
18/01/12 10:52:40 INFO TaskSetManager: Starting task 0.0 in stage 196.0 (TID 429, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:40 INFO TaskSetManager: Starting task 1.0 in stage 196.0 (TID 430, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:40 INFO TaskSetManager: Starting task 2.0 in stage 196.0 (TID 431, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:40 INFO Executor: Running task 1.0 in stage 196.0 (TID 430)
18/01/12 10:52:40 INFO Executor: Running task 0.0 in stage 196.0 (TID 429)
18/01/12 10:52:40 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 5 blocks
18/01/12 10:52:40 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/01/12 10:52:40 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 5 blocks
18/01/12 10:52:40 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/01/12 10:52:40 INFO Executor: Finished task 1.0 in stage 196.0 (TID 430). 882 bytes result sent to driver
18/01/12 10:52:40 INFO Executor: Finished task 0.0 in stage 196.0 (TID 429). 882 bytes result sent to driver
18/01/12 10:52:40 INFO TaskSetManager: Finished task 1.0 in stage 196.0 (TID 430) in 2 ms on localhost (1/3)
18/01/12 10:52:40 INFO Executor: Running task 2.0 in stage 196.0 (TID 431)
18/01/12 10:52:40 INFO TaskSetManager: Finished task 0.0 in stage 196.0 (TID 429) in 2 ms on localhost (2/3)
18/01/12 10:52:40 INFO ShuffleBlockFetcherIterator: Getting 3 non-empty blocks out of 5 blocks
18/01/12 10:52:40 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/01/12 10:52:40 INFO Executor: Finished task 2.0 in stage 196.0 (TID 431). 1028 bytes result sent to driver
18/01/12 10:52:40 INFO TaskSetManager: Finished task 2.0 in stage 196.0 (TID 431) in 4 ms on localhost (3/3)
18/01/12 10:52:40 INFO TaskSchedulerImpl: Removed TaskSet 196.0, whose tasks have all completed, from pool
18/01/12 10:52:40 INFO DAGScheduler: ResultStage 196 (print at NetWorkWordCount.scala:29) finished in 0.004 s
18/01/12 10:52:40 INFO DAGScheduler: Job 98 finished: print at NetWorkWordCount.scala:29, took 0.007002 s
-------------------------------------------
Time: 1515725560000 ms
-------------------------------------------
18/01/12 10:52:40 INFO JobScheduler: Finished job streaming job 1515725560000 ms.0 from job set of time 1515725560000 ms
18/01/12 10:52:40 INFO JobScheduler: Total delay: 0.043 s for time 1515725560000 ms (execution: 0.037 s)
18/01/12 10:52:40 INFO ShuffledRDD: Removing RDD 192 from persistence list
(D,2)
(G,3)
18/01/12 10:52:40 INFO BlockManager: Removing RDD 192
18/01/12 10:52:40 INFO MapPartitionsRDD: Removing RDD 191 from persistence list
18/01/12 10:52:40 INFO BlockManager: Removing RDD 191
18/01/12 10:52:40 INFO MapPartitionsRDD: Removing RDD 190 from persistence list
18/01/12 10:52:40 INFO BlockManager: Removing RDD 190
18/01/12 10:52:40 INFO BlockRDD: Removing RDD 189 from persistence list
18/01/12 10:52:40 INFO BlockManager: Removing RDD 189
18/01/12 10:52:40 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[189] at socketTextStream at NetWorkWordCount.scala:26 of time 1515725560000 ms
18/01/12 10:52:40 INFO BlockManagerInfo: Removed input-0-1515725550600 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:40 INFO BlockManagerInfo: Removed input-0-1515725551800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:40 INFO BlockManagerInfo: Removed input-0-1515725552800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:40 INFO BlockManagerInfo: Removed input-0-1515725553800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:40 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1515725550000 ms)
18/01/12 10:52:40 INFO InputInfoTracker: remove old batch metadata: 1515725550000 ms
18/01/12 10:52:41 INFO MemoryStore: ensureFreeSpace(8) called with curMem=85001, maxMem=490356080
18/01/12 10:52:41 INFO MemoryStore: Block input-0-1515725560800 stored as bytes in memory (estimated size 8.0 B, free 467.6 MB)
18/01/12 10:52:41 INFO BlockManagerInfo: Added input-0-1515725560800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:41 INFO BlockGenerator: Pushed block input-0-1515725560800
18/01/12 10:52:42 INFO MemoryStore: ensureFreeSpace(8) called with curMem=85009, maxMem=490356080
18/01/12 10:52:42 INFO MemoryStore: Block input-0-1515725561800 stored as bytes in memory (estimated size 8.0 B, free 467.6 MB)
18/01/12 10:52:42 INFO BlockManagerInfo: Added input-0-1515725561800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:42 INFO BlockGenerator: Pushed block input-0-1515725561800
18/01/12 10:52:43 INFO MemoryStore: ensureFreeSpace(8) called with curMem=85017, maxMem=490356080
18/01/12 10:52:43 INFO MemoryStore: Block input-0-1515725562800 stored as bytes in memory (estimated size 8.0 B, free 467.6 MB)
18/01/12 10:52:43 INFO BlockManagerInfo: Added input-0-1515725562800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:43 INFO BlockGenerator: Pushed block input-0-1515725562800
18/01/12 10:52:44 INFO MemoryStore: ensureFreeSpace(8) called with curMem=85025, maxMem=490356080
18/01/12 10:52:44 INFO MemoryStore: Block input-0-1515725563800 stored as bytes in memory (estimated size 8.0 B, free 467.6 MB)
18/01/12 10:52:44 INFO BlockManagerInfo: Added input-0-1515725563800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:44 INFO BlockGenerator: Pushed block input-0-1515725563800
18/01/12 10:52:45 INFO MemoryStore: ensureFreeSpace(8) called with curMem=85033, maxMem=490356080
18/01/12 10:52:45 INFO MemoryStore: Block input-0-1515725564800 stored as bytes in memory (estimated size 8.0 B, free 467.6 MB)
18/01/12 10:52:45 INFO BlockManagerInfo: Added input-0-1515725564800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:45 INFO BlockGenerator: Pushed block input-0-1515725564800
18/01/12 10:52:45 INFO JobScheduler: Added jobs for time 1515725565000 ms
18/01/12 10:52:45 INFO JobScheduler: Starting job streaming job 1515725565000 ms.0 from job set of time 1515725565000 ms
18/01/12 10:52:45 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29
18/01/12 10:52:45 INFO DAGScheduler: Registering RDD 199 (map at NetWorkWordCount.scala:28)
18/01/12 10:52:45 INFO DAGScheduler: Got job 99 (print at NetWorkWordCount.scala:29) with 1 output partitions (allowLocal=true)
18/01/12 10:52:45 INFO DAGScheduler: Final stage: ResultStage 198(print at NetWorkWordCount.scala:29)
18/01/12 10:52:45 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 197)
18/01/12 10:52:45 INFO DAGScheduler: Missing parents: List(ShuffleMapStage 197)
18/01/12 10:52:45 INFO DAGScheduler: Submitting ShuffleMapStage 197 (MapPartitionsRDD[199] at map at NetWorkWordCount.scala:28), which has no missing parents
18/01/12 10:52:45 INFO MemoryStore: ensureFreeSpace(2560) called with curMem=85041, maxMem=490356080
18/01/12 10:52:45 INFO MemoryStore: Block broadcast_147 stored as values in memory (estimated size 2.5 KB, free 467.6 MB)
18/01/12 10:52:45 INFO MemoryStore: ensureFreeSpace(1565) called with curMem=87601, maxMem=490356080
18/01/12 10:52:45 INFO MemoryStore: Block broadcast_147_piece0 stored as bytes in memory (estimated size 1565.0 B, free 467.6 MB)
18/01/12 10:52:45 INFO BlockManagerInfo: Added broadcast_147_piece0 in memory on localhost:54439 (size: 1565.0 B, free: 467.6 MB)
18/01/12 10:52:45 INFO SparkContext: Created broadcast 147 from broadcast at DAGScheduler.scala:874
18/01/12 10:52:45 INFO DAGScheduler: Submitting 5 missing tasks from ShuffleMapStage 197 (MapPartitionsRDD[199] at map at NetWorkWordCount.scala:28)
18/01/12 10:52:45 INFO TaskSchedulerImpl: Adding task set 197.0 with 5 tasks
18/01/12 10:52:45 INFO TaskSetManager: Starting task 0.0 in stage 197.0 (TID 432, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:45 INFO TaskSetManager: Starting task 1.0 in stage 197.0 (TID 433, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:45 INFO TaskSetManager: Starting task 2.0 in stage 197.0 (TID 434, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:45 INFO Executor: Running task 2.0 in stage 197.0 (TID 434)
18/01/12 10:52:45 INFO BlockManager: Found block input-0-1515725561800 locally
18/01/12 10:52:45 INFO Executor: Running task 0.0 in stage 197.0 (TID 432)
18/01/12 10:52:45 INFO BlockManager: Found block input-0-1515725559800 locally
18/01/12 10:52:45 INFO Executor: Running task 1.0 in stage 197.0 (TID 433)
18/01/12 10:52:45 INFO BlockManager: Found block input-0-1515725560800 locally
18/01/12 10:52:45 INFO Executor: Finished task 2.0 in stage 197.0 (TID 434). 882 bytes result sent to driver
18/01/12 10:52:45 INFO TaskSetManager: Starting task 3.0 in stage 197.0 (TID 435, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:45 INFO Executor: Running task 3.0 in stage 197.0 (TID 435)
18/01/12 10:52:45 INFO BlockManager: Found block input-0-1515725562800 locally
18/01/12 10:52:45 INFO TaskSetManager: Finished task 2.0 in stage 197.0 (TID 434) in 14 ms on localhost (1/5)
18/01/12 10:52:45 INFO Executor: Finished task 1.0 in stage 197.0 (TID 433). 882 bytes result sent to driver
18/01/12 10:52:45 INFO TaskSetManager: Starting task 4.0 in stage 197.0 (TID 436, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:45 INFO TaskSetManager: Finished task 1.0 in stage 197.0 (TID 433) in 17 ms on localhost (2/5)
18/01/12 10:52:45 INFO Executor: Running task 4.0 in stage 197.0 (TID 436)
18/01/12 10:52:45 INFO BlockManager: Found block input-0-1515725563800 locally
18/01/12 10:52:45 INFO Executor: Finished task 3.0 in stage 197.0 (TID 435). 882 bytes result sent to driver
18/01/12 10:52:45 INFO TaskSetManager: Finished task 3.0 in stage 197.0 (TID 435) in 14 ms on localhost (3/5)
18/01/12 10:52:45 INFO Executor: Finished task 0.0 in stage 197.0 (TID 432). 882 bytes result sent to driver
18/01/12 10:52:45 INFO TaskSetManager: Finished task 0.0 in stage 197.0 (TID 432) in 25 ms on localhost (4/5)
18/01/12 10:52:45 INFO Executor: Finished task 4.0 in stage 197.0 (TID 436). 882 bytes result sent to driver
18/01/12 10:52:45 INFO TaskSetManager: Finished task 4.0 in stage 197.0 (TID 436) in 12 ms on localhost (5/5)
18/01/12 10:52:45 INFO TaskSchedulerImpl: Removed TaskSet 197.0, whose tasks have all completed, from pool
18/01/12 10:52:45 INFO DAGScheduler: ShuffleMapStage 197 (map at NetWorkWordCount.scala:28) finished in 0.028 s
18/01/12 10:52:45 INFO DAGScheduler: looking for newly runnable stages
18/01/12 10:52:45 INFO DAGScheduler: running: Set(ResultStage 0)
18/01/12 10:52:45 INFO DAGScheduler: waiting: Set(ResultStage 198)
18/01/12 10:52:45 INFO DAGScheduler: failed: Set()
18/01/12 10:52:45 INFO DAGScheduler: Missing parents for ResultStage 198: List()
18/01/12 10:52:45 INFO DAGScheduler: Submitting ResultStage 198 (ShuffledRDD[200] at reduceByKey at NetWorkWordCount.scala:28), which is now runnable
18/01/12 10:52:45 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=89166, maxMem=490356080
18/01/12 10:52:45 INFO MemoryStore: Block broadcast_148 stored as values in memory (estimated size 2.3 KB, free 467.6 MB)
18/01/12 10:52:45 INFO MemoryStore: ensureFreeSpace(1415) called with curMem=91470, maxMem=490356080
18/01/12 10:52:45 INFO MemoryStore: Block broadcast_148_piece0 stored as bytes in memory (estimated size 1415.0 B, free 467.6 MB)
18/01/12 10:52:45 INFO BlockManagerInfo: Added broadcast_148_piece0 in memory on localhost:54439 (size: 1415.0 B, free: 467.6 MB)
18/01/12 10:52:45 INFO SparkContext: Created broadcast 148 from broadcast at DAGScheduler.scala:874
18/01/12 10:52:45 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 198 (ShuffledRDD[200] at reduceByKey at NetWorkWordCount.scala:28)
18/01/12 10:52:45 INFO TaskSchedulerImpl: Adding task set 198.0 with 1 tasks
18/01/12 10:52:45 INFO TaskSetManager: Starting task 0.0 in stage 198.0 (TID 437, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:45 INFO Executor: Running task 0.0 in stage 198.0 (TID 437)
18/01/12 10:52:45 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 5 blocks
18/01/12 10:52:45 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/01/12 10:52:45 INFO Executor: Finished task 0.0 in stage 198.0 (TID 437). 1028 bytes result sent to driver
18/01/12 10:52:45 INFO TaskSetManager: Finished task 0.0 in stage 198.0 (TID 437) in 2 ms on localhost (1/1)
18/01/12 10:52:45 INFO TaskSchedulerImpl: Removed TaskSet 198.0, whose tasks have all completed, from pool
18/01/12 10:52:45 INFO DAGScheduler: ResultStage 198 (print at NetWorkWordCount.scala:29) finished in 0.002 s
18/01/12 10:52:45 INFO DAGScheduler: Job 99 finished: print at NetWorkWordCount.scala:29, took 0.039103 s
18/01/12 10:52:45 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29
18/01/12 10:52:45 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 49 is 170 bytes
18/01/12 10:52:45 INFO DAGScheduler: Got job 100 (print at NetWorkWordCount.scala:29) with 3 output partitions (allowLocal=true)
18/01/12 10:52:45 INFO DAGScheduler: Final stage: ResultStage 200(print at NetWorkWordCount.scala:29)
18/01/12 10:52:45 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 199)
18/01/12 10:52:45 INFO DAGScheduler: Missing parents: List()
18/01/12 10:52:45 INFO DAGScheduler: Submitting ResultStage 200 (ShuffledRDD[200] at reduceByKey at NetWorkWordCount.scala:28), which has no missing parents
18/01/12 10:52:45 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=92885, maxMem=490356080
18/01/12 10:52:45 INFO MemoryStore: Block broadcast_149 stored as values in memory (estimated size 2.3 KB, free 467.5 MB)
18/01/12 10:52:45 INFO MemoryStore: ensureFreeSpace(1415) called with curMem=95189, maxMem=490356080
18/01/12 10:52:45 INFO MemoryStore: Block broadcast_149_piece0 stored as bytes in memory (estimated size 1415.0 B, free 467.5 MB)
18/01/12 10:52:45 INFO BlockManagerInfo: Added broadcast_149_piece0 in memory on localhost:54439 (size: 1415.0 B, free: 467.6 MB)
18/01/12 10:52:45 INFO SparkContext: Created broadcast 149 from broadcast at DAGScheduler.scala:874
18/01/12 10:52:45 INFO DAGScheduler: Submitting 3 missing tasks from ResultStage 200 (ShuffledRDD[200] at reduceByKey at NetWorkWordCount.scala:28)
18/01/12 10:52:45 INFO TaskSchedulerImpl: Adding task set 200.0 with 3 tasks
18/01/12 10:52:45 INFO TaskSetManager: Starting task 0.0 in stage 200.0 (TID 438, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:45 INFO TaskSetManager: Starting task 1.0 in stage 200.0 (TID 439, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:45 INFO TaskSetManager: Starting task 2.0 in stage 200.0 (TID 440, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:45 INFO Executor: Running task 2.0 in stage 200.0 (TID 440)
18/01/12 10:52:45 INFO Executor: Running task 1.0 in stage 200.0 (TID 439)
18/01/12 10:52:45 INFO Executor: Running task 0.0 in stage 200.0 (TID 438)
18/01/12 10:52:45 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 5 blocks
18/01/12 10:52:45 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/01/12 10:52:45 INFO ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 5 blocks
18/01/12 10:52:45 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
18/01/12 10:52:45 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 5 blocks
18/01/12 10:52:45 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
18/01/12 10:52:45 INFO Executor: Finished task 2.0 in stage 200.0 (TID 440). 1028 bytes result sent to driver
18/01/12 10:52:45 INFO Executor: Finished task 0.0 in stage 200.0 (TID 438). 1028 bytes result sent to driver
18/01/12 10:52:45 INFO Executor: Finished task 1.0 in stage 200.0 (TID 439). 1028 bytes result sent to driver
18/01/12 10:52:45 INFO TaskSetManager: Finished task 0.0 in stage 200.0 (TID 438) in 1 ms on localhost (1/3)
18/01/12 10:52:45 INFO TaskSetManager: Finished task 2.0 in stage 200.0 (TID 440) in 2 ms on localhost (2/3)
18/01/12 10:52:45 INFO TaskSetManager: Finished task 1.0 in stage 200.0 (TID 439) in 2 ms on localhost (3/3)
18/01/12 10:52:45 INFO TaskSchedulerImpl: Removed TaskSet 200.0, whose tasks have all completed, from pool
18/01/12 10:52:45 INFO DAGScheduler: ResultStage 200 (print at NetWorkWordCount.scala:29) finished in 0.003 s
18/01/12 10:52:45 INFO DAGScheduler: Job 100 finished: print at NetWorkWordCount.scala:29, took 0.005057 s
-------------------------------------------
Time: 1515725565000 ms
-------------------------------------------
(D,1)
(A,1)
(F,2)
(G,1)
18/01/12 10:52:45 INFO JobScheduler: Finished job streaming job 1515725565000 ms.0 from job set of time 1515725565000 ms
18/01/12 10:52:45 INFO JobScheduler: Total delay: 0.061 s for time 1515725565000 ms (execution: 0.048 s)
18/01/12 10:52:45 INFO ShuffledRDD: Removing RDD 196 from persistence list
18/01/12 10:52:45 INFO MapPartitionsRDD: Removing RDD 195 from persistence list
18/01/12 10:52:45 INFO BlockManager: Removing RDD 196
18/01/12 10:52:45 INFO BlockManager: Removing RDD 195
18/01/12 10:52:45 INFO MapPartitionsRDD: Removing RDD 194 from persistence list
18/01/12 10:52:45 INFO BlockManager: Removing RDD 194
18/01/12 10:52:45 INFO BlockRDD: Removing RDD 193 from persistence list
18/01/12 10:52:45 INFO BlockManager: Removing RDD 193
18/01/12 10:52:45 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[193] at socketTextStream at NetWorkWordCount.scala:26 of time 1515725565000 ms
18/01/12 10:52:45 INFO BlockManagerInfo: Removed input-0-1515725554800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:45 INFO BlockManagerInfo: Removed input-0-1515725555800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:45 INFO BlockManagerInfo: Removed input-0-1515725556800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:45 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1515725555000 ms)
18/01/12 10:52:45 INFO BlockManagerInfo: Removed input-0-1515725557800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:45 INFO InputInfoTracker: remove old batch metadata: 1515725555000 ms
18/01/12 10:52:45 INFO BlockManagerInfo: Removed input-0-1515725558800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:46 INFO MemoryStore: ensureFreeSpace(8) called with curMem=96564, maxMem=490356080
18/01/12 10:52:46 INFO MemoryStore: Block input-0-1515725565800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB)
18/01/12 10:52:46 INFO BlockManagerInfo: Added input-0-1515725565800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:46 INFO BlockGenerator: Pushed block input-0-1515725565800
18/01/12 10:52:47 INFO MemoryStore: ensureFreeSpace(8) called with curMem=96572, maxMem=490356080
18/01/12 10:52:47 INFO MemoryStore: Block input-0-1515725566800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB)
18/01/12 10:52:47 INFO BlockManagerInfo: Added input-0-1515725566800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:47 INFO BlockGenerator: Pushed block input-0-1515725566800
18/01/12 10:52:48 INFO MemoryStore: ensureFreeSpace(8) called with curMem=96580, maxMem=490356080
18/01/12 10:52:48 INFO MemoryStore: Block input-0-1515725567800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB)
18/01/12 10:52:48 INFO BlockManagerInfo: Added input-0-1515725567800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:48 INFO BlockGenerator: Pushed block input-0-1515725567800
18/01/12 10:52:49 INFO MemoryStore: ensureFreeSpace(8) called with curMem=96588, maxMem=490356080
18/01/12 10:52:49 INFO MemoryStore: Block input-0-1515725568800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB)
18/01/12 10:52:49 INFO BlockManagerInfo: Added input-0-1515725568800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:49 INFO BlockGenerator: Pushed block input-0-1515725568800
18/01/12 10:52:50 INFO MemoryStore: ensureFreeSpace(8) called with curMem=96596, maxMem=490356080
18/01/12 10:52:50 INFO MemoryStore: Block input-0-1515725569800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB)
18/01/12 10:52:50 INFO BlockManagerInfo: Added input-0-1515725569800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:50 INFO BlockGenerator: Pushed block input-0-1515725569800
18/01/12 10:52:50 INFO JobScheduler: Added jobs for time 1515725570000 ms
18/01/12 10:52:50 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29
18/01/12 10:52:50 INFO JobScheduler: Starting job streaming job 1515725570000 ms.0 from job set of time 1515725570000 ms
18/01/12 10:52:50 INFO DAGScheduler: Registering RDD 203 (map at NetWorkWordCount.scala:28)
18/01/12 10:52:50 INFO DAGScheduler: Got job 101 (print at NetWorkWordCount.scala:29) with 1 output partitions (allowLocal=true)
18/01/12 10:52:50 INFO DAGScheduler: Final stage: ResultStage 202(print at NetWorkWordCount.scala:29)
18/01/12 10:52:50 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 201)
18/01/12 10:52:50 INFO DAGScheduler: Missing parents: List(ShuffleMapStage 201)
18/01/12 10:52:50 INFO DAGScheduler: Submitting ShuffleMapStage 201 (MapPartitionsRDD[203] at map at NetWorkWordCount.scala:28), which has no missing parents
18/01/12 10:52:50 INFO MemoryStore: ensureFreeSpace(2560) called with curMem=96604, maxMem=490356080
18/01/12 10:52:50 INFO MemoryStore: Block broadcast_150 stored as values in memory (estimated size 2.5 KB, free 467.5 MB)
18/01/12 10:52:50 INFO MemoryStore: ensureFreeSpace(1564) called with curMem=99164, maxMem=490356080
18/01/12 10:52:50 INFO MemoryStore: Block broadcast_150_piece0 stored as bytes in memory (estimated size 1564.0 B, free 467.5 MB)
18/01/12 10:52:50 INFO BlockManagerInfo: Added broadcast_150_piece0 in memory on localhost:54439 (size: 1564.0 B, free: 467.6 MB)
18/01/12 10:52:50 INFO SparkContext: Created broadcast 150 from broadcast at DAGScheduler.scala:874
18/01/12 10:52:50 INFO DAGScheduler: Submitting 5 missing tasks from ShuffleMapStage 201 (MapPartitionsRDD[203] at map at NetWorkWordCount.scala:28)
18/01/12 10:52:50 INFO TaskSchedulerImpl: Adding task set 201.0 with 5 tasks
18/01/12 10:52:50 INFO TaskSetManager: Starting task 0.0 in stage 201.0 (TID 441, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:50 INFO TaskSetManager: Starting task 1.0 in stage 201.0 (TID 442, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:50 INFO TaskSetManager: Starting task 2.0 in stage 201.0 (TID 443, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:50 INFO Executor: Running task 2.0 in stage 201.0 (TID 443)
18/01/12 10:52:50 INFO BlockManager: Found block input-0-1515725566800 locally
18/01/12 10:52:50 INFO Executor: Running task 0.0 in stage 201.0 (TID 441)
18/01/12 10:52:50 INFO BlockManager: Found block input-0-1515725564800 locally
18/01/12 10:52:50 INFO Executor: Running task 1.0 in stage 201.0 (TID 442)
18/01/12 10:52:50 INFO BlockManager: Found block input-0-1515725565800 locally
18/01/12 10:52:50 INFO Executor: Finished task 2.0 in stage 201.0 (TID 443). 882 bytes result sent to driver
18/01/12 10:52:50 INFO TaskSetManager: Starting task 3.0 in stage 201.0 (TID 444, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:50 INFO TaskSetManager: Finished task 2.0 in stage 201.0 (TID 443) in 12 ms on localhost (1/5)
18/01/12 10:52:50 INFO Executor: Running task 3.0 in stage 201.0 (TID 444)
18/01/12 10:52:50 INFO Executor: Finished task 0.0 in stage 201.0 (TID 441). 882 bytes result sent to driver
18/01/12 10:52:50 INFO TaskSetManager: Starting task 4.0 in stage 201.0 (TID 445, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:50 INFO TaskSetManager: Finished task 0.0 in stage 201.0 (TID 441) in 14 ms on localhost (2/5)
18/01/12 10:52:50 INFO Executor: Running task 4.0 in stage 201.0 (TID 445)
18/01/12 10:52:50 INFO Executor: Finished task 1.0 in stage 201.0 (TID 442). 882 bytes result sent to driver
18/01/12 10:52:50 INFO TaskSetManager: Finished task 1.0 in stage 201.0 (TID 442) in 15 ms on localhost (3/5)
18/01/12 10:52:50 INFO BlockManager: Found block input-0-1515725568800 locally
18/01/12 10:52:50 INFO BlockManager: Found block input-0-1515725567800 locally
18/01/12 10:52:50 INFO Executor: Finished task 4.0 in stage 201.0 (TID 445). 882 bytes result sent to driver
18/01/12 10:52:50 INFO Executor: Finished task 3.0 in stage 201.0 (TID 444). 882 bytes result sent to driver
18/01/12 10:52:50 INFO TaskSetManager: Finished task 4.0 in stage 201.0 (TID 445) in 4 ms on localhost (4/5)
18/01/12 10:52:50 INFO TaskSetManager: Finished task 3.0 in stage 201.0 (TID 444) in 6 ms on localhost (5/5)
18/01/12 10:52:50 INFO TaskSchedulerImpl: Removed TaskSet 201.0, whose tasks have all completed, from pool
18/01/12 10:52:50 INFO DAGScheduler: ShuffleMapStage 201 (map at NetWorkWordCount.scala:28) finished in 0.018 s
18/01/12 10:52:50 INFO DAGScheduler: looking for newly runnable stages
18/01/12 10:52:50 INFO DAGScheduler: running: Set(ResultStage 0)
18/01/12 10:52:50 INFO DAGScheduler: waiting: Set(ResultStage 202)
18/01/12 10:52:50 INFO DAGScheduler: failed: Set()
18/01/12 10:52:50 INFO DAGScheduler: Missing parents for ResultStage 202: List()
18/01/12 10:52:50 INFO DAGScheduler: Submitting ResultStage 202 (ShuffledRDD[204] at reduceByKey at NetWorkWordCount.scala:28), which is now runnable
18/01/12 10:52:50 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=100728, maxMem=490356080
18/01/12 10:52:50 INFO MemoryStore: Block broadcast_151 stored as values in memory (estimated size 2.3 KB, free 467.5 MB)
18/01/12 10:52:50 INFO MemoryStore: ensureFreeSpace(1417) called with curMem=103032, maxMem=490356080
18/01/12 10:52:50 INFO MemoryStore: Block broadcast_151_piece0 stored as bytes in memory (estimated size 1417.0 B, free 467.5 MB)
18/01/12 10:52:50 INFO BlockManagerInfo: Added broadcast_151_piece0 in memory on localhost:54439 (size: 1417.0 B, free: 467.6 MB)
18/01/12 10:52:50 INFO SparkContext: Created broadcast 151 from broadcast at DAGScheduler.scala:874
18/01/12 10:52:50 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 202 (ShuffledRDD[204] at reduceByKey at NetWorkWordCount.scala:28)
18/01/12 10:52:50 INFO TaskSchedulerImpl: Adding task set 202.0 with 1 tasks
18/01/12 10:52:50 INFO TaskSetManager: Starting task 0.0 in stage 202.0 (TID 446, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:50 INFO Executor: Running task 0.0 in stage 202.0 (TID 446)
18/01/12 10:52:50 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 5 blocks
18/01/12 10:52:50 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/01/12 10:52:50 INFO Executor: Finished task 0.0 in stage 202.0 (TID 446). 1028 bytes result sent to driver
18/01/12 10:52:50 INFO TaskSetManager: Finished task 0.0 in stage 202.0 (TID 446) in 2 ms on localhost (1/1)
18/01/12 10:52:50 INFO DAGScheduler: ResultStage 202 (print at NetWorkWordCount.scala:29) finished in 0.002 s
18/01/12 10:52:50 INFO TaskSchedulerImpl: Removed TaskSet 202.0, whose tasks have all completed, from pool
18/01/12 10:52:50 INFO DAGScheduler: Job 101 finished: print at NetWorkWordCount.scala:29, took 0.028394 s
18/01/12 10:52:50 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29
18/01/12 10:52:50 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 50 is 165 bytes
18/01/12 10:52:50 INFO DAGScheduler: Got job 102 (print at NetWorkWordCount.scala:29) with 3 output partitions (allowLocal=true)
18/01/12 10:52:50 INFO DAGScheduler: Final stage: ResultStage 204(print at NetWorkWordCount.scala:29)
18/01/12 10:52:50 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 203)
18/01/12 10:52:50 INFO DAGScheduler: Missing parents: List()
18/01/12 10:52:50 INFO DAGScheduler: Submitting ResultStage 204 (ShuffledRDD[204] at reduceByKey at NetWorkWordCount.scala:28), which has no missing parents
18/01/12 10:52:50 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=104449, maxMem=490356080
18/01/12 10:52:50 INFO MemoryStore: Block broadcast_152 stored as values in memory (estimated size 2.3 KB, free 467.5 MB)
18/01/12 10:52:50 INFO MemoryStore: ensureFreeSpace(1417) called with curMem=106753, maxMem=490356080
18/01/12 10:52:50 INFO MemoryStore: Block broadcast_152_piece0 stored as bytes in memory (estimated size 1417.0 B, free 467.5 MB)
18/01/12 10:52:50 INFO BlockManagerInfo: Added broadcast_152_piece0 in memory on localhost:54439 (size: 1417.0 B, free: 467.6 MB)
18/01/12 10:52:50 INFO SparkContext: Created broadcast 152 from broadcast at DAGScheduler.scala:874
18/01/12 10:52:50 INFO DAGScheduler: Submitting 3 missing tasks from ResultStage 204 (ShuffledRDD[204] at reduceByKey at NetWorkWordCount.scala:28)
18/01/12 10:52:50 INFO TaskSchedulerImpl: Adding task set 204.0 with 3 tasks
18/01/12 10:52:50 INFO TaskSetManager: Starting task 0.0 in stage 204.0 (TID 447, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:50 INFO TaskSetManager: Starting task 1.0 in stage 204.0 (TID 448, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:50 INFO TaskSetManager: Starting task 2.0 in stage 204.0 (TID 449, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:50 INFO Executor: Running task 0.0 in stage 204.0 (TID 447)
18/01/12 10:52:50 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 5 blocks
18/01/12 10:52:50 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
18/01/12 10:52:50 INFO Executor: Finished task 0.0 in stage 204.0 (TID 447). 882 bytes result sent to driver
18/01/12 10:52:50 INFO Executor: Running task 2.0 in stage 204.0 (TID 449)
18/01/12 10:52:50 INFO Executor: Running task 1.0 in stage 204.0 (TID 448)
18/01/12 10:52:50 INFO ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 5 blocks
18/01/12 10:52:50 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/01/12 10:52:50 INFO TaskSetManager: Finished task 0.0 in stage 204.0 (TID 447) in 4 ms on localhost (1/3)
18/01/12 10:52:50 INFO ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 5 blocks
18/01/12 10:52:50 INFO Executor: Finished task 1.0 in stage 204.0 (TID 448). 1048 bytes result sent to driver
18/01/12 10:52:50 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 2 ms
18/01/12 10:52:50 INFO TaskSetManager: Finished task 1.0 in stage 204.0 (TID 448) in 5 ms on localhost (2/3)
18/01/12 10:52:50 INFO Executor: Finished task 2.0 in stage 204.0 (TID 449). 1028 bytes result sent to driver
18/01/12 10:52:50 INFO TaskSetManager: Finished task 2.0 in stage 204.0 (TID 449) in 6 ms on localhost (3/3)
18/01/12 10:52:50 INFO TaskSchedulerImpl: Removed TaskSet 204.0, whose tasks have all completed, from pool
18/01/12 10:52:50 INFO DAGScheduler: ResultStage 204 (print at NetWorkWordCount.scala:29) finished in 0.006 s
18/01/12 10:52:50 INFO DAGScheduler: Job 102 finished: print at NetWorkWordCount.scala:29, took 0.009907 s
-------------------------------------------
Time: 1515725570000 ms
-------------------------------------------
18/01/12 10:52:50 INFO JobScheduler: Finished job streaming job 1515725570000 ms.0 from job set of time 1515725570000 ms
18/01/12 10:52:50 INFO JobScheduler: Total delay: 0.054 s for time 1515725570000 ms (execution: 0.046 s)
18/01/12 10:52:50 INFO ShuffledRDD: Removing RDD 200 from persistence list
(D,1)
(B,1)
(F,1)
(C,2)
18/01/12 10:52:50 INFO BlockManager: Removing RDD 200
18/01/12 10:52:50 INFO MapPartitionsRDD: Removing RDD 199 from persistence list
18/01/12 10:52:50 INFO BlockManager: Removing RDD 199
18/01/12 10:52:50 INFO MapPartitionsRDD: Removing RDD 198 from persistence list
18/01/12 10:52:50 INFO BlockManager: Removing RDD 198
18/01/12 10:52:50 INFO BlockRDD: Removing RDD 197 from persistence list
18/01/12 10:52:50 INFO BlockManager: Removing RDD 197
18/01/12 10:52:50 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[197] at socketTextStream at NetWorkWordCount.scala:26 of time 1515725570000 ms
18/01/12 10:52:50 INFO BlockManagerInfo: Removed input-0-1515725559800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:50 INFO BlockManagerInfo: Removed input-0-1515725560800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:50 INFO BlockManagerInfo: Removed input-0-1515725561800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:50 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1515725560000 ms)
18/01/12 10:52:50 INFO InputInfoTracker: remove old batch metadata: 1515725560000 ms
18/01/12 10:52:50 INFO BlockManagerInfo: Removed input-0-1515725563800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:50 INFO BlockManagerInfo: Removed input-0-1515725562800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:51 INFO MemoryStore: ensureFreeSpace(8) called with curMem=108130, maxMem=490356080
18/01/12 10:52:51 INFO MemoryStore: Block input-0-1515725570800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB)
18/01/12 10:52:51 INFO BlockManagerInfo: Added input-0-1515725570800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:51 INFO BlockGenerator: Pushed block input-0-1515725570800
18/01/12 10:52:52 INFO MemoryStore: ensureFreeSpace(8) called with curMem=108138, maxMem=490356080
18/01/12 10:52:52 INFO MemoryStore: Block input-0-1515725571800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB)
18/01/12 10:52:52 INFO BlockManagerInfo: Added input-0-1515725571800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:52 INFO BlockGenerator: Pushed block input-0-1515725571800
18/01/12 10:52:53 INFO MemoryStore: ensureFreeSpace(8) called with curMem=108146, maxMem=490356080
18/01/12 10:52:53 INFO MemoryStore: Block input-0-1515725572800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB)
18/01/12 10:52:53 INFO BlockManagerInfo: Added input-0-1515725572800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:53 INFO BlockGenerator: Pushed block input-0-1515725572800
18/01/12 10:52:54 INFO MemoryStore: ensureFreeSpace(8) called with curMem=108154, maxMem=490356080
18/01/12 10:52:54 INFO MemoryStore: Block input-0-1515725573800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB)
18/01/12 10:52:54 INFO BlockManagerInfo: Added input-0-1515725573800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:54 INFO BlockGenerator: Pushed block input-0-1515725573800
18/01/12 10:52:55 INFO MemoryStore: ensureFreeSpace(8) called with curMem=108162, maxMem=490356080
18/01/12 10:52:55 INFO MemoryStore: Block input-0-1515725574800 stored as bytes in memory (estimated size 8.0 B, free 467.5 MB)
18/01/12 10:52:55 INFO BlockManagerInfo: Added input-0-1515725574800 in memory on localhost:54439 (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:55 INFO BlockGenerator: Pushed block input-0-1515725574800
18/01/12 10:52:55 INFO JobScheduler: Added jobs for time 1515725575000 ms
18/01/12 10:52:55 INFO JobScheduler: Starting job streaming job 1515725575000 ms.0 from job set of time 1515725575000 ms
18/01/12 10:52:55 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29
18/01/12 10:52:55 INFO DAGScheduler: Registering RDD 207 (map at NetWorkWordCount.scala:28)
18/01/12 10:52:55 INFO DAGScheduler: Got job 103 (print at NetWorkWordCount.scala:29) with 1 output partitions (allowLocal=true)
18/01/12 10:52:55 INFO DAGScheduler: Final stage: ResultStage 206(print at NetWorkWordCount.scala:29)
18/01/12 10:52:55 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 205)
18/01/12 10:52:55 INFO DAGScheduler: Missing parents: List(ShuffleMapStage 205)
18/01/12 10:52:55 INFO DAGScheduler: Submitting ShuffleMapStage 205 (MapPartitionsRDD[207] at map at NetWorkWordCount.scala:28), which has no missing parents
18/01/12 10:52:55 INFO MemoryStore: ensureFreeSpace(2560) called with curMem=108170, maxMem=490356080
18/01/12 10:52:55 INFO MemoryStore: Block broadcast_153 stored as values in memory (estimated size 2.5 KB, free 467.5 MB)
18/01/12 10:52:55 INFO MemoryStore: ensureFreeSpace(1564) called with curMem=110730, maxMem=490356080
18/01/12 10:52:55 INFO MemoryStore: Block broadcast_153_piece0 stored as bytes in memory (estimated size 1564.0 B, free 467.5 MB)
18/01/12 10:52:55 INFO BlockManagerInfo: Added broadcast_153_piece0 in memory on localhost:54439 (size: 1564.0 B, free: 467.6 MB)
18/01/12 10:52:55 INFO SparkContext: Created broadcast 153 from broadcast at DAGScheduler.scala:874
18/01/12 10:52:55 INFO DAGScheduler: Submitting 5 missing tasks from ShuffleMapStage 205 (MapPartitionsRDD[207] at map at NetWorkWordCount.scala:28)
18/01/12 10:52:55 INFO TaskSchedulerImpl: Adding task set 205.0 with 5 tasks
18/01/12 10:52:55 INFO TaskSetManager: Starting task 0.0 in stage 205.0 (TID 450, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:55 INFO TaskSetManager: Starting task 1.0 in stage 205.0 (TID 451, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:55 INFO TaskSetManager: Starting task 2.0 in stage 205.0 (TID 452, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:55 INFO Executor: Running task 0.0 in stage 205.0 (TID 450)
18/01/12 10:52:55 INFO Executor: Running task 1.0 in stage 205.0 (TID 451)
18/01/12 10:52:55 INFO Executor: Running task 2.0 in stage 205.0 (TID 452)
18/01/12 10:52:55 INFO BlockManager: Found block input-0-1515725569800 locally
18/01/12 10:52:55 INFO BlockManager: Found block input-0-1515725571800 locally
18/01/12 10:52:55 INFO BlockManager: Found block input-0-1515725570800 locally
18/01/12 10:52:55 INFO Executor: Finished task 0.0 in stage 205.0 (TID 450). 882 bytes result sent to driver
18/01/12 10:52:55 INFO TaskSetManager: Starting task 3.0 in stage 205.0 (TID 453, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:55 INFO Executor: Running task 3.0 in stage 205.0 (TID 453)
18/01/12 10:52:55 INFO TaskSetManager: Finished task 0.0 in stage 205.0 (TID 450) in 5 ms on localhost (1/5)
18/01/12 10:52:55 INFO BlockManager: Found block input-0-1515725572800 locally
18/01/12 10:52:55 INFO Executor: Finished task 2.0 in stage 205.0 (TID 452). 882 bytes result sent to driver
18/01/12 10:52:55 INFO Executor: Finished task 3.0 in stage 205.0 (TID 453). 882 bytes result sent to driver
18/01/12 10:52:55 INFO TaskSetManager: Starting task 4.0 in stage 205.0 (TID 454, localhost, NODE_LOCAL, 1277 bytes)
18/01/12 10:52:55 INFO Executor: Running task 4.0 in stage 205.0 (TID 454)
18/01/12 10:52:55 INFO BlockManager: Found block input-0-1515725573800 locally
18/01/12 10:52:55 INFO TaskSetManager: Finished task 2.0 in stage 205.0 (TID 452) in 12 ms on localhost (2/5)
18/01/12 10:52:55 INFO TaskSetManager: Finished task 3.0 in stage 205.0 (TID 453) in 9 ms on localhost (3/5)
18/01/12 10:52:55 INFO Executor: Finished task 4.0 in stage 205.0 (TID 454). 882 bytes result sent to driver
18/01/12 10:52:55 INFO TaskSetManager: Finished task 4.0 in stage 205.0 (TID 454) in 4 ms on localhost (4/5)
18/01/12 10:52:55 INFO Executor: Finished task 1.0 in stage 205.0 (TID 451). 882 bytes result sent to driver
18/01/12 10:52:55 INFO TaskSetManager: Finished task 1.0 in stage 205.0 (TID 451) in 15 ms on localhost (5/5)
18/01/12 10:52:55 INFO TaskSchedulerImpl: Removed TaskSet 205.0, whose tasks have all completed, from pool
18/01/12 10:52:55 INFO DAGScheduler: ShuffleMapStage 205 (map at NetWorkWordCount.scala:28) finished in 0.015 s
18/01/12 10:52:55 INFO DAGScheduler: looking for newly runnable stages
18/01/12 10:52:55 INFO DAGScheduler: running: Set(ResultStage 0)
18/01/12 10:52:55 INFO DAGScheduler: waiting: Set(ResultStage 206)
18/01/12 10:52:55 INFO DAGScheduler: failed: Set()
18/01/12 10:52:55 INFO DAGScheduler: Missing parents for ResultStage 206: List()
18/01/12 10:52:55 INFO DAGScheduler: Submitting ResultStage 206 (ShuffledRDD[208] at reduceByKey at NetWorkWordCount.scala:28), which is now runnable
18/01/12 10:52:55 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=112294, maxMem=490356080
18/01/12 10:52:55 INFO MemoryStore: Block broadcast_154 stored as values in memory (estimated size 2.3 KB, free 467.5 MB)
18/01/12 10:52:55 INFO MemoryStore: ensureFreeSpace(1417) called with curMem=114598, maxMem=490356080
18/01/12 10:52:55 INFO MemoryStore: Block broadcast_154_piece0 stored as bytes in memory (estimated size 1417.0 B, free 467.5 MB)
18/01/12 10:52:55 INFO BlockManagerInfo: Added broadcast_154_piece0 in memory on localhost:54439 (size: 1417.0 B, free: 467.6 MB)
18/01/12 10:52:55 INFO SparkContext: Created broadcast 154 from broadcast at DAGScheduler.scala:874
18/01/12 10:52:55 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 206 (ShuffledRDD[208] at reduceByKey at NetWorkWordCount.scala:28)
18/01/12 10:52:55 INFO TaskSchedulerImpl: Adding task set 206.0 with 1 tasks
18/01/12 10:52:55 INFO TaskSetManager: Starting task 0.0 in stage 206.0 (TID 455, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:55 INFO Executor: Running task 0.0 in stage 206.0 (TID 455)
18/01/12 10:52:55 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 5 blocks
18/01/12 10:52:55 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/01/12 10:52:55 INFO Executor: Finished task 0.0 in stage 206.0 (TID 455). 882 bytes result sent to driver
18/01/12 10:52:55 INFO TaskSetManager: Finished task 0.0 in stage 206.0 (TID 455) in 2 ms on localhost (1/1)
18/01/12 10:52:55 INFO TaskSchedulerImpl: Removed TaskSet 206.0, whose tasks have all completed, from pool
18/01/12 10:52:55 INFO DAGScheduler: ResultStage 206 (print at NetWorkWordCount.scala:29) finished in 0.002 s
18/01/12 10:52:55 INFO DAGScheduler: Job 103 finished: print at NetWorkWordCount.scala:29, took 0.025503 s
18/01/12 10:52:55 INFO SparkContext: Starting job: print at NetWorkWordCount.scala:29
18/01/12 10:52:55 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 51 is 165 bytes
18/01/12 10:52:55 INFO DAGScheduler: Got job 104 (print at NetWorkWordCount.scala:29) with 3 output partitions (allowLocal=true)
18/01/12 10:52:55 INFO DAGScheduler: Final stage: ResultStage 208(print at NetWorkWordCount.scala:29)
18/01/12 10:52:55 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 207)
18/01/12 10:52:55 INFO DAGScheduler: Missing parents: List()
18/01/12 10:52:55 INFO DAGScheduler: Submitting ResultStage 208 (ShuffledRDD[208] at reduceByKey at NetWorkWordCount.scala:28), which has no missing parents
18/01/12 10:52:55 INFO MemoryStore: ensureFreeSpace(2304) called with curMem=116015, maxMem=490356080
18/01/12 10:52:55 INFO MemoryStore: Block broadcast_155 stored as values in memory (estimated size 2.3 KB, free 467.5 MB)
18/01/12 10:52:55 INFO MemoryStore: ensureFreeSpace(1417) called with curMem=118319, maxMem=490356080
18/01/12 10:52:55 INFO MemoryStore: Block broadcast_155_piece0 stored as bytes in memory (estimated size 1417.0 B, free 467.5 MB)
18/01/12 10:52:55 INFO BlockManagerInfo: Added broadcast_155_piece0 in memory on localhost:54439 (size: 1417.0 B, free: 467.6 MB)
18/01/12 10:52:55 INFO SparkContext: Created broadcast 155 from broadcast at DAGScheduler.scala:874
18/01/12 10:52:55 INFO DAGScheduler: Submitting 3 missing tasks from ResultStage 208 (ShuffledRDD[208] at reduceByKey at NetWorkWordCount.scala:28)
18/01/12 10:52:55 INFO TaskSchedulerImpl: Adding task set 208.0 with 3 tasks
18/01/12 10:52:55 INFO TaskSetManager: Starting task 0.0 in stage 208.0 (TID 456, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:55 INFO TaskSetManager: Starting task 1.0 in stage 208.0 (TID 457, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:55 INFO TaskSetManager: Starting task 2.0 in stage 208.0 (TID 458, localhost, PROCESS_LOCAL, 1165 bytes)
18/01/12 10:52:55 INFO Executor: Running task 1.0 in stage 208.0 (TID 457)
18/01/12 10:52:55 INFO ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 5 blocks
18/01/12 10:52:55 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/01/12 10:52:55 INFO Executor: Finished task 1.0 in stage 208.0 (TID 457). 1048 bytes result sent to driver
18/01/12 10:52:55 INFO Executor: Running task 2.0 in stage 208.0 (TID 458)
18/01/12 10:52:55 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 5 blocks
18/01/12 10:52:55 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/01/12 10:52:55 INFO Executor: Finished task 2.0 in stage 208.0 (TID 458). 1028 bytes result sent to driver
18/01/12 10:52:55 INFO Executor: Running task 0.0 in stage 208.0 (TID 456)
18/01/12 10:52:55 INFO ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 5 blocks
18/01/12 10:52:55 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/01/12 10:52:55 INFO TaskSetManager: Finished task 1.0 in stage 208.0 (TID 457) in 5 ms on localhost (1/3)
18/01/12 10:52:55 INFO Executor: Finished task 0.0 in stage 208.0 (TID 456). 1028 bytes result sent to driver
18/01/12 10:52:55 INFO TaskSetManager: Finished task 2.0 in stage 208.0 (TID 458) in 6 ms on localhost (2/3)
18/01/12 10:52:55 INFO TaskSetManager: Finished task 0.0 in stage 208.0 (TID 456) in 6 ms on localhost (3/3)
18/01/12 10:52:55 INFO TaskSchedulerImpl: Removed TaskSet 208.0, whose tasks have all completed, from pool
18/01/12 10:52:55 INFO DAGScheduler: ResultStage 208 (print at NetWorkWordCount.scala:29) finished in 0.008 s
-------------------------------------------18/01/12 10:52:55 INFO DAGScheduler: Job 104 finished: print at NetWorkWordCount.scala:29, took 0.010832 s
18/01/12 10:52:55 INFO JobScheduler: Finished job streaming job 1515725575000 ms.0 from job set of time 1515725575000 ms
18/01/12 10:52:55 INFO JobScheduler: Total delay: 0.051 s for time 1515725575000 ms (execution: 0.042 s)
Time: 1515725575000 ms
-------------------------------------------
(A,2)
(B,1)
(F,1)
(C,1)
18/01/12 10:52:55 INFO ShuffledRDD: Removing RDD 204 from persistence list
18/01/12 10:52:55 INFO MapPartitionsRDD: Removing RDD 203 from persistence list
18/01/12 10:52:55 INFO BlockManager: Removing RDD 204
18/01/12 10:52:55 INFO MapPartitionsRDD: Removing RDD 202 from persistence list
18/01/12 10:52:55 INFO BlockRDD: Removing RDD 201 from persistence list
18/01/12 10:52:55 INFO SocketInputDStream: Removing blocks of RDD BlockRDD[201] at socketTextStream at NetWorkWordCount.scala:26 of time 1515725575000 ms
18/01/12 10:52:55 INFO BlockManagerInfo: Removed input-0-1515725564800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:55 INFO BlockManager: Removing RDD 202
18/01/12 10:52:55 INFO BlockManagerInfo: Removed input-0-1515725565800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:55 INFO BlockManagerInfo: Removed input-0-1515725566800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:55 INFO BlockManagerInfo: Removed input-0-1515725567800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
18/01/12 10:52:55 INFO BlockManager: Removing RDD 203
18/01/12 10:52:55 INFO BlockManager: Removing RDD 201
18/01/12 10:52:55 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer(1515725565000 ms)
18/01/12 10:52:55 INFO InputInfoTracker: remove old batch metadata: 1515725565000 ms
18/01/12 10:52:55 INFO BlockManagerInfo: Removed input-0-1515725568800 on localhost:54439 in memory (size: 8.0 B, free: 467.6 MB)
