流式任務需要 7*24 小時執行,但是有時涉及到升級代碼需要主動停止程序,但是分佈式程序,沒辦法做到一個個進程去殺死,所有配置優雅的關閉就顯得至關重要了。使用外部文件系統來控制內部程序關閉。
其實就是單獨起一個線程專門去專門查找程序是否停止的標誌
import java.net.URI import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.spark.streaming.{StreamingContext, StreamingContextState} class MonitorStop(ssc: StreamingContext) extends Runnable { override def run(): Unit = { val fs: FileSystem = FileSystem.get(new URI("hdfs://linux1:9000"), new Configuration(), "atguigu") while (true) { try Thread.sleep(5000) catch { case e: InterruptedException => e.printStackTrace() } val state: StreamingContextState = ssc.getState val bool: Boolean = fs.exists(new Path("hdfs://linux1:9000/stopSpark")) if (bool) { if (state == StreamingContextState.ACTIVE) { ssc.stop(stopSparkContext = true, stopGracefully = true) System.exit(0) } } } } }
import org.apache.spark.SparkConf
import org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream}
import org.apache.spark.streaming.{Seconds, StreamingContext}
object SparkTest {
def createSSC(): _root_.org.apache.spark.streaming.StreamingContext = {
val update: (Seq[Int], Option[Int]) => Some[Int] = (values: Seq[Int], status:
Option[Int]) => {
//當前批次內容的計算
val sum: Int = values.sum
//取出狀態信息中上一次狀態
val lastStatu: Int = status.getOrElse(0)
Some(sum + lastStatu)
}
val sparkConf: SparkConf = new
SparkConf().setMaster("local[4]").setAppName("SparkTest")
//設置優雅的關閉
sparkConf.set("spark.streaming.stopGracefullyOnShutdown", "true")
val ssc = new StreamingContext(sparkConf, Seconds(5))
ssc.checkpoint("./ck")
val line: ReceiverInputDStream[String] = ssc.socketTextStream("linux1", 9999)
val word: DStream[String] = line.flatMap(_.split(" "))
val wordAndOne: DStream[(String, Int)] = word.map((_, 1))
val wordAndCount: DStream[(String, Int)] = wordAndOne.updateStateByKey(update)
wordAndCount.print()
ssc
}
def main(args: Array[String]): Unit = {
val ssc: StreamingContext = StreamingContext.getActiveOrCreate("./ck", () =>
createSSC())
new Thread(new MonitorStop(ssc)).start()
ssc.start()
ssc.awaitTermination()
}
}