spark源码（九）Worker receive 方法

    一.case msg: RegisterWorkerResponse 
    二.case SendHeartbeat
    三.case WorkDirCleanup
    四.case MasterChanged
    五.case ReconnectWorker
    六.case LaunchExecutor
    七.case executorStateChanged: ExecutorStateChanged
    八.case KillExecutor(masterUrl, appId, execId)
    九.case LaunchDriver(driverId, driverDesc, resources_)
    十.case KillDriver(driverId)
    十一.case driverStateChanged @ DriverStateChanged(driverId, state, exception)
    十二.case ReregisterWithMaster
    十三.case ApplicationFinished(id)
    十四.case DecommissionWorker
    十五.case WorkerSigPWRReceived

八. KillExecutor(masterUrl, appId, execId) 详解

    if (masterUrl != activeMasterUrl) {
      logWarning("Invalid Master (" + masterUrl + ") attempted to kill executor " + execId)
    } else {
      val fullId = appId + "/" + execId
      executors.get(fullId) match {
        case Some(executor) =>
          logInfo("Asked to kill executor " + fullId)
          executor.kill()//循环找到executor 杀掉   
          //这只是找到杀掉 没有真正的回收资源 删除shuffle文件
          //好像这体现出了 WorkDirCleanup 的作用  但是从哪调用呢
          //handleRegisterResponse 定时发送   感觉这一块应该放在心跳以后啊
        case None =>
          logInfo("Asked to kill unknown executor " + fullId)
      }
    }

九. LaunchDriver(driverId, driverDesc, resources_) 详解

    logInfo(s"Asked to launch driver $driverId")
    val driver = new DriverRunner(conf,driverId,workDir,sparkHome,
      driverDesc.copy(command = Worker.maybeUpdateSSLSettings(driverDesc.command, conf)),
      self,workerUri,workerWebUiUrl,securityMgr,resources_)
    drivers(driverId) = driver
    driver.start()
    coresUsed += driverDesc.cores
    memoryUsed += driverDesc.mem
    addResourcesUsed(resources_)

9.1 driver.start 详解

    private[worker] def start() = {
      new Thread("DriverRunner for " + driverId) {
        override def run(): Unit = {
          var shutdownHook: AnyRef = null
          try {
            shutdownHook = ShutdownHookManager.addShutdownHook { () =>
              logInfo(s"Worker shutting down, killing driver $driverId")
              kill()
            }

            val exitCode = prepareAndRunDriver()/*启动Driver*/

            finalState = if (exitCode == 0) {
              Some(DriverState.FINISHED)
            } else if (killed) {
              Some(DriverState.KILLED)
            } else {
              Some(DriverState.FAILED)
            }
          } catch {
            case e: Exception =>
              kill()
              finalState = Some(DriverState.ERROR)
              finalException = Some(e)
          } finally {
            if (shutdownHook != null) {
              ShutdownHookManager.removeShutdownHook(shutdownHook)
            }
          }
          /*发送Driver启动成功的消息*/
          worker.send(DriverStateChanged(driverId, finalState.get, finalException))
        }
      }.start()
    }

9.1.1 prepareAndRunDriver 详解

    private[worker] def prepareAndRunDriver(): Int = {
      //感觉这一块的代码比 executor的好多了
      val driverDir = createWorkingDirectory()//创建工作目录
      val localJarFilename = downloadUserJar(driverDir)//下载jar包  
      //准备资源文件
      val resourceFileOpt = prepareResourcesFile(SPARK_DRIVER_PREFIX, resources, driverDir)

      def substituteVariables(argument: String): String = argument match {
        case "{{WORKER_URL}}" => workerUrl
        case "{{USER_JAR}}" => localJarFilename
        case other => other
      }

      val javaOpts = driverDesc.command.javaOpts ++ resourceFileOpt.map(f =>
        Seq(s"-D${DRIVER_RESOURCES_FILE.key}=${f.getAbsolutePath}")).getOrElse(Seq.empty)
      val builder = CommandUtils.buildProcessBuilder(driverDesc.command.copy(javaOpts = javaOpts),
        securityManager, driverDesc.mem, sparkHome.getAbsolutePath, substituteVariables)

      // webui 
      val reverseProxy = conf.get(UI_REVERSE_PROXY)
      val workerUrlRef = UIUtils.makeHref(reverseProxy, driverId, workerWebUiUrl)
      builder.environment.put("SPARK_DRIVER_LOG_URL_STDOUT",
        s"$workerUrlRef/logPage?driverId=$driverId&logType=stdout")
      builder.environment.put("SPARK_DRIVER_LOG_URL_STDERR",
        s"$workerUrlRef/logPage?driverId=$driverId&logType=stderr")

      runDriver(builder, driverDir, driverDesc.supervise)
    }

9.1.1.1 runDriver 详解

    private def runDriver(builder: ProcessBuilder, baseDir: File, supervise: Boolean): Int = {
      builder.directory(baseDir)
      def initialize(process: Process): Unit = {/*shell方式启动Driver*/
        // Redirect stdout and stderr to files
        val stdout = new File(baseDir, "stdout")
        CommandUtils.redirectStream(process.getInputStream, stdout)

        val stderr = new File(baseDir, "stderr")
        //添加参数 -d 定义的参数
        val redactedCommand = Utils.redactCommandLineArgs(conf, builder.command.asScala.toSeq)
          .mkString("\"", "\" \"", "\"")
        val header = "Launch Command: %s\n%s\n\n".format(redactedCommand, "=" * 40)
        Files.append(header, stderr, StandardCharsets.UTF_8)
        CommandUtils.redirectStream(process.getErrorStream, stderr)
      }
      runCommandWithRetry(ProcessBuilderLike(builder), initialize, supervise)
    }

十. KillDriver(driverId) 详解

    logInfo(s"Asked to kill driver $driverId")
    drivers.get(driverId) match {
      case Some(runner) =>
        runner.kill()//driver 也是一样的没有回收资源就是杀掉任务
      case None =>
        logError(s"Asked to kill unknown driver $driverId")
    }