分析一下newMasterWebUI(MasterRpcEndPoint,8080)：它的主要作用就是將每個頁面的html以scala.xml.Node的形勢封裝放在serlvet中，然後再將servlet放到servletContextHandler中，供jetty.Server使用
/**
* Web UI server for the standalonemaster.
* 實現抽象類WebUI的子類有:MasterWebUI,HistoryServer,MesosClusterUI, SparkUI, WorkerWebUI。這些都是真正提供web ui服務的。
*/
private[master]
class MasterWebUI(
val master:Master,
requestedPort: Int,
customMasterPage: Option[MasterPage] = None)
extends WebUI(master.securityMgr, requestedPort, master.conf, name = "MasterUI") with Logging
with UIRoot{
val masterEndpointRef= master.self
val killEnabled= master.conf.getBoolean("spark.ui.killEnabled", true)
val masterPage= customMasterPage.getOrElse(new MasterPage(this))
initialize()
/** Initialize all components of the server.
* 初始化Master UI的web組件: ServletContextHandler
* ===
* MasterPage、ApplicationPage、HistoryNotFoundPage他們父類是
* WebUIPage有兩個抽象函數render(request: HttpServletRequest): Seq[Node]和renderJson(request: HttpServletRequest): JValue。
* 它兩個方法都被attachPage（）裏面的方法，進行調用
* */
def initialize() {
//MasterPage路徑： / ，可以跟進去看WebUIPage主構造參數就是它的路徑
val masterPage= new MasterPage(this) //MasterPage:渲染workers, 活躍的appliction,活躍的drivers
//ApplicationPage路徑：/app
attachPage(new ApplicationPage(this)) //ApplicationPage根據request中的appId參數, 展示這個活躍的application
attachPage(new HistoryNotFoundPage(this)) //HistoryNotFoundPage路徑 /history/not-found
attachPage(masterPage)
//加載靜態資源, 見這裏:resources/org/apache/spark/ui/static/
attachHandler(createStaticHandler(MasterWebUI.STATIC_RESOURCE_DIR, "/static"))
attachHandler(ApiRootResource.getServletHandler(this))
//把這倆kill請求轉發給master page處理, Master異步kill，也就是從頁面上可以停止app和driver, 路徑：/app/kill, /driver/kill
attachHandler(createRedirectHandler(
"/app/kill", "/", masterPage.handleAppKillRequest, httpMethods = Set("POST")))
attachHandler(createRedirectHandler(
"/driver/kill", "/", masterPage.handleDriverKillRequest, httpMethods= Set("POST")))
}
1,分析一下new MasterPage(this),它的render()方法用scala.xml的方式將html構造出來
/**
* WebUIPage有兩個抽象函數render(request:HttpServletRequest): Seq[Node]和renderJson(request:HttpServletRequest): JValue。
* 需要注意的是成員參數prefix,這個就是這個頁面對應的http path了
* 看參數可以猜出, 接收servlet request, 處理請求, 返回Node或者Json結果。
* 它的子類有MasterPage、ApplicationPage等
*/
private[ui] class MasterPage(parent:MasterWebUI) extends WebUIPage("") {
private val master =parent.masterEndpointRef
….
/** Index view listing applications and executors
* 一大堆的html代碼, 補上所需的變量, 加上公共部分頁頭UIUtils.basicSparkPage, 就是一個完整的渲染後的頁面了
* 它會被MasterWebUI初始化後，被initialize（）方法的attachPage()調用==>再被createServletHandler方法調用（它是創建jetty的servletContextHandler用的）
* ==》然後這個過程使用createServlet創建servlet
* */
def render(request: HttpServletRequest): Seq[Node] = {
val state= getMasterState
//worker列表上的列字段
val workerHeaders= Seq("WorkerId", "Address", "State", "Cores", "Memory")
val workers= state.workers.sortBy(_.id)
val aliveWorkers= state.workers.filter(_.state== WorkerState.ALIVE)
val workerTable= UIUtils.listingTable(workerHeaders, workerRow, workers)
val appHeaders= Seq("ApplicationID", "Name", "Cores", "Memory per Node", "Submitted Time",
"User", "State", "Duration")
val activeApps= state.activeApps.sortBy(_.startTime).reverse
val activeAppsTable= UIUtils.listingTable(appHeaders, appRow, activeApps)
val completedApps= state.completedApps.sortBy(_.endTime).reverse
val completedAppsTable= UIUtils.listingTable(appHeaders, appRow, completedApps)
val driverHeaders= Seq("SubmissionID", "Submitted Time", "Worker", "State", "Cores",
"Memory", "MainClass")
val activeDrivers= state.activeDrivers.sortBy(_.startTime).reverse
val activeDriversTable= UIUtils.listingTable(driverHeaders, driverRow, activeDrivers)
val completedDrivers= state.completedDrivers.sortBy(_.startTime).reverse
val completedDriversTable= UIUtils.listingTable(driverHeaders, driverRow, completedDrivers)
// For now we only show driver information if the userhas submitted drivers to the cluster.
// This is until we integrate thenotion of drivers and applications in the UI.
def hasDrivers: Boolean =activeDrivers.length > 0 || completedDrivers.length > 0
//scala的xml和函數一樣是一等公民
//使用花括號操作scala成員
val content=
<div class="row-fluid">
<div class="span12">
<ul class="unstyled">
<li><strong>URL:</strong> {state.uri}</li>
{
state.restUri.map { uri=>
<li>
<strong>REST URL:</strong> {uri}
<span class="rest-uri"> (cluster mode)</span>
</li>
}.getOrElse { Seq.empty}
}
。。。。 </ul>
</div>
</div>
。。。。。 </div>;
UIUtils.basicSparkPage(content, "Spark Master at " + state.uri)
}
2，再回MasterWebUI的initialize（）中attachPage(masterPage)
/** Attach apage to this UI.
* WebUIPage的子類有:MasterPage\ApplicationPage\HistoryNotFoundPage
* */
def attachPage(page: WebUIPage) {
//pagePath是servlet的訪問路徑
val pagePath= "/" + page.prefix
val renderHandler= createServletHandler(pagePath,
(request: HttpServletRequest) => page.render(request), securityManager, conf, basePath)
val renderJsonHandler= createServletHandler(pagePath.stripSuffix("/") + "/json",
(request: HttpServletRequest) =>page.renderJson(request), securityManager, conf, basePath)
attachHandler(renderHandler)
attachHandler(renderJsonHandler)
pageToHandlers.getOrElseUpdate(page, ArrayBuffer[ServletContextHandler]())
.append(renderHandler)
}
===> createServletHandler()就是用創建ServletContextHandler
/** Create acontext handler that responds to a request with the given path prefix
* 傳進來的第一個參數就是servlet的url
* 第二個參數是(request:HttpServletRequest) => page.render(request)。它變成ServletParams是因爲有隱式轉換，
private[spark] object JettyUtils extendsLogging {
type Responder[T] =HttpServletRequest => T
class ServletParams[T <% AnyRef](valresponder: Responder[T],
val contentType: String,
val extractFn: T => String= (in: Any) => in.toString) {}
implicit def htmlResponderToServlet(responder:Responder[Seq[Node]]): ServletParams[Seq[Node]] =
new ServletParams(responder,"text/html", (in: Seq[Node]) => "<!DOCTYPE html>"+ in.toString)
將這個匿名函數隱式成了ServletParams目的是爲了得到ServletContextHandler，給jetty的server使用
* */
def createServletHandler[T <% AnyRef](
path: String,
servletParams: ServletParams[T],
securityMgr: SecurityManager,
conf: SparkConf,
basePath: String= ""): ServletContextHandler = {
createServletHandler(path, createServlet(servletParams, securityMgr, conf), basePath)
}
/** Create acontext handler that responds to a request with the given path prefix
* 真正創建ServletContextHandler的地方，這個ServletContextHandler，可以按setContextPath()設置上下文路徑
* */
def createServletHandler(
path: String,
servlet: HttpServlet,
basePath: String): ServletContextHandler = {
val prefixedPath= if (basePath == "" && path== "/") {
path
} else {
//將傳進來是項目路徑最後一個“/”去掉
(basePath + path).stripSuffix("/")
}
val contextHandler= new ServletContextHandler
val holder= new ServletHolder(servlet) //ServletHolder就是用來承載servlet
contextHandler.setContextPath(prefixedPath)
contextHandler.addServlet(holder, "/") //即通過prefixedPath+“/”就可以訪問到這個servlet
contextHandler
}
===》再看一下createServlet(servletParams, securityMgr, conf)是如何創建Servlet的
def createServlet[T <% AnyRef](
servletParams: ServletParams[T],
securityMgr: SecurityManager,
conf: SparkConf): HttpServlet = {
// SPARK-10589 avoid frame-related click-jackingvulnerability, using X-Frame-Options
// (seehttp://tools.ietf.org/html/rfc7034). By default allow framing only from the
// same origin, but allow framing for aspecific named URI.
// Example: spark.ui.allowFramingFrom =https://example.com/
//X-Frame-Options用於設置頁面是否可以被放在iframe中，其中SAMEORIGIN表示只能本網站進行iframe
/**
* Java代碼:
response.addHeader("x-frame-options","SAMEORIGIN");
Nginx配置:
add_header X-Frame-OptionsSAMEORIGIN
Apache配置:
Header always appendX-Frame-Options SAMEORIGIN
*/
val allowFramingFrom= conf.getOption("spark.ui.allowFramingFrom")
val xFrameOptionsValue=
allowFramingFrom.map(uri => s"ALLOW-FROM $uri").getOrElse("SAMEORIGIN")
new HttpServlet{
override def doGet(request:HttpServletRequest, response: HttpServletResponse) {
try {
//默認是true,SecurityManager是關於用戶權限和ssl相關的管理類
if (securityMgr.checkUIViewPermissions(request.getRemoteUser)){
//這個ServletParams是隱式轉換過來的，如果"text/html"表示解析成html，text/plain解析成源碼，text/json解析成json格式
response.setContentType("%s;charset=utf-8".format(servletParams.contentType))
response.setStatus(HttpServletResponse.SC_OK) //響應狀態
//調用(request:HttpServletRequest) => page.render(request)，將會得到Seq[scala.xml.Node]
val result = servletParams.responder(request)
response.setHeader("Cache-Control", "no-cache, no-store, must-revalidate")
response.setHeader("X-Frame-Options", xFrameOptionsValue)
// scalastyle:off println
//通過response管道輸出Seq[scala.xml.Node],extractFn():可以將Seq[Node]變成字符串
response.getWriter.println(servletParams.extractFn(result))
// scalastyle:on println
} else{
response.setStatus(HttpServletResponse.SC_UNAUTHORIZED)
response.setHeader("Cache-Control", "no-cache, no-store, must-revalidate")
response.sendError(HttpServletResponse.SC_UNAUTHORIZED,
"User is not authorized to access this page.")
}
} catch {
case e: IllegalArgumentException =>
response.sendError(HttpServletResponse.SC_BAD_REQUEST, e.getMessage)
case e: Exception =>
logWarning(s"GET ${request.getRequestURI} failed: $e", e)
throw e
}
}
// SPARK-5983 ensure TRACE is not supported
protected override def doTrace(req:HttpServletRequest, res: HttpServletResponse): Unit = {
res.sendError(HttpServletResponse.SC_METHOD_NOT_ALLOWED)
}
}
}
3，MasterWebUI的initialize（）中attachPage(masterPage){…attachHandler(renderHandler)。。}這個attachHandler（）就是將返回的ServletContextHandler加到一個ArrayBuffer[ServletContextHandler]
====》到此，MasterWebUI初始化代碼就結束了，還回到Master中
webUi.bind() //啓動一個jetty ,生成server實例, 監聽web端口, jetty server綁定handler
五、看如何啓動的jetty.Server的
/** Bind to theHTTP server behind this web interface. */
def bind() {
assert(!serverInfo.isDefined, "Attempted to bind %s more thanonce!".format(className))
try {
serverInfo = Some(startJettyServer("0.0.0.0", port, handlers, conf, name))
logInfo("Started %s at http://%s:%d".format(className, publicHostName, boundPort))
} catch {
case e: Exception =>
logError("Failed to bind %s".format(className), e)
System.exit(1)
}
}
/**
* 嘗試使用給定的上下文處理程序啓動綁定到提供的hostName：port的Jetty服務器。
如果所需的端口號被爭用，則繼續增加端口直到找到空閒端口。 返回碼頭服務器對象，選定的端口以及可變的處理程序集合。
*/
def startJettyServer(
hostName: String,
port: Int,
handlers: Seq[ServletContextHandler],
conf: SparkConf,
serverName: String = ""): ServerInfo = {
addFilters(handlers, conf)
//這個ContextHandlerCollection會對每個Handler進行執行，即便某個Handler失敗，後面的Handler也會執行
val collection= new ContextHandlerCollection
/**
* 壓縮內容可以極大地提高網絡帶寬使用率，但是會以內存和CPU週期爲代價。如果此處理程序用於靜態內容， 那麼可以避免使用高效的直接NIO，因此建議使用<code>org.eclipse.jetty.servlet.DefaultServlet</ code>的gzip機制
*/
val gzipHandlers = handlers.map { h =>
val gzipHandler= new GzipHandler
gzipHandler.setHandler(h)
gzipHandler
}
collection.setHandlers(gzipHandlers.toArray)
// Bind to the given port, or throw ajava.net.BindException if the port is occupied
def connect(currentPort: Int):(Server, Int) = {
//給server設置InetSocketAddress對應的host及port
val server= new Server(new InetSocketAddress(hostName, currentPort))
val pool= new QueuedThreadPool
pool.setDaemon(true)
server.setThreadPool(pool) //給server定義聯接池
val errorHandler= new ErrorHandler()
errorHandler.setShowStacks(true)
server.addBean(errorHandler)
server.setHandler(collection)
try {
server.start()
(server, server.getConnectors.head.getLocalPort)
} catch {
case e: Exception =>
server.stop()
pool.stop()
throw e
}
}
//就是將上面的(server, server.getConnectors.head.getLocalPort)返回
val (server, boundPort) = Utils.startServiceOnPort[Server](port, connect, conf, serverName)
ServerInfo(server, boundPort, collection)
}
}
到此JettyServer啓動結束,再回到Master中看如何清理超時的Woker及zk是如何選舉