|NO.Z.00089|——————————|BigDataEnd|——|Hadoop&Spark.V05|——|Spark.v05|Spark 原理 源码|作业执行原理&Stage划分&调用dagScheduler.handleJobSumitted|

一、Stage划分

### --- 调用 dagScheduler.handleJobSubmitted

~~~     # 源码提取说明：DAGScheduler.scala
~~~     # 952行~1015行
  // 处理Job的提交
  private[scheduler] def handleJobSubmitted(
     jobId: Int, 
     finalRDD: RDD[_], 
     func: (TaskContext, Iterator[_]) => _,
     partitions: Array[Int],
     callSite: CallSite,
     listener: JobListener,
     properties: Properties) {
    var finalStage: ResultStage = null
    try {
      // New stage creation may throw an exception if, for example, jobs are run on a
      // HadoopRDD whose underlying HDFS files have been deleted.
      // 创建ResultStage
      finalStage = createResultStage(finalRDD, func, partitions, jobId, callSite)
    } catch {
      case e: Exception =>
        logWarning("Creating new stage failed due to exception - job: " + jobId, e)
        listener.jobFailed(e)
        return
    }
    // 创建ActiveJob
    val job = new ActiveJob(jobId, finalStage, callSite, listener, properties)
    // 清空缓存的各个RDD的所有分区的位置信息
    clearCacheLocs()
    logInfo("Got job %s (%s) with %d output partitions".format(
      job.jobId, callSite.shortForm, partitions.length))
    logInfo("Final stage: " + finalStage + " (" + finalStage.name + ")")
    logInfo("Parents of final stage: " + finalStage.parents)
    logInfo("Missing parents: " + getMissingParentStages(finalStage))
    // 生成Job提交时间
    val jobSubmissionTime = clock.getTimeMillis()
    // 记录jobId与ActiveJob的映射
    jobIdToActiveJob(jobId) = job
    activeJobs += job
    // 将finalStage的ActiveJob设置为当前提交的Job
    finalStage.setActiveJob(job)
    // 获取Job所有Stage的StageInfo对象
    val stageIds = jobIdToStageIds(jobId).toArray
    val stageInfos = stageIds.flatMap(id => stageIdToStage.get(id).map(_.latestInfo))
    // 向事件总线投递SparkListenerJobStart事件
    listenerBus.post(
      SparkListenerJobStart(job.jobId, jobSubmissionTime, stageInfos, properties))
    // 提交ResultStage
    submitStage(finalStage)
  }

---

 

 

 

 

 

 

 

 

 

---

Walter Savage Landor:strove with none,for none was worth my strife.Nature I loved and, next to Nature, Art:I warm'd both hands before the fire of life.It sinks, and I am ready to depart

                                                                                                                                                   ——W.S.Landor

---