Spark Streaming程序优雅关闭

流式任务需要 7*24 小时执行，但是有时涉及到升级代码需要主动停止程序，但是分布式程序，没办法做到一个个进程去杀死，所有配置优雅的关闭就显得至关重要了。使用外部文件系统来控制内部程序关闭。

其实就是单独起一个线程专门去专门查找程序是否停止的标志

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26

import java.net.URI import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.spark.streaming.{StreamingContext, StreamingContextState} class MonitorStop(ssc: StreamingContext) extends Runnable {  override def run(): Unit = {  val fs: FileSystem = FileSystem.get(new URI("hdfs://linux1:9000"), new Configuration(), "atguigu")  while (true) {  try  Thread.sleep(5000)  catch {  case e: InterruptedException =>  e.printStackTrace()  }  val state: StreamingContextState = ssc.getState  val bool: Boolean = fs.exists(new Path("hdfs://linux1:9000/stopSpark"))  if (bool) {  if (state == StreamingContextState.ACTIVE) {  ssc.stop(stopSparkContext = true, stopGracefully = true)  System.exit(0)  }  }  }  } }<br><br>import org.apache.spark.SparkConf<br>import org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream}<br>import org.apache.spark.streaming.{Seconds, StreamingContext}<br>object SparkTest {<br> def createSSC(): _root_.org.apache.spark.streaming.StreamingContext = {<br> val update: (Seq[Int], Option[Int]) => Some[Int] = (values: Seq[Int], status: <br>Option[Int]) => {<br> //当前批次内容的计算<br> val sum: Int = values.sum<br> //取出状态信息中上一次状态<br>  val lastStatu: Int = status.getOrElse(0)<br> Some(sum + lastStatu)<br> }<br> val sparkConf: SparkConf = new <br>SparkConf().setMaster("local[4]").setAppName("SparkTest")<br> //设置优雅的关闭<br> sparkConf.set("spark.streaming.stopGracefullyOnShutdown", "true")<br> val ssc = new StreamingContext(sparkConf, Seconds(5))<br> ssc.checkpoint("./ck")<br> val line: ReceiverInputDStream[String] = ssc.socketTextStream("linux1", 9999)<br> val word: DStream[String] = line.flatMap(_.split(" "))<br> val wordAndOne: DStream[(String, Int)] = word.map((_, 1))<br> val wordAndCount: DStream[(String, Int)] = wordAndOne.updateStateByKey(update)<br> wordAndCount.print()<br> ssc<br> }<br> def main(args: Array[String]): Unit = {<br> val ssc: StreamingContext = StreamingContext.getActiveOrCreate("./ck", () => <br>createSSC())<br> new Thread(new MonitorStop(ssc)).start()<br> ssc.start()<br> ssc.awaitTermination()<br> }<br>}