099-Spark-源码-SparkSubmit源码和任务执行流程

SparkSubmit

-- main

	-- doSubmit
	
		// 解析参数
		-- parseArguments
		
			// master => --master => yarn
			// mainClass => --class => SparkPi(WordCount)
			-- parse
			
		-- submit
		
			-- doRunMain
			
			-- runMain
			
				// (childArgs, childClasspath, sparkConf, childMainClass)
				// 【Cluster】childMainClass => org.apache.spark.deploy.yarn.YarnClusterApplication
				// 【Client 】childMainClass => SparkPi(WordCount)
				-- prepareSubmitEnvironment
				
				// Class.forName("xxxxxxx")
				-- mainClass : Class = Utils.classForName(childMainClass)
				
				// classOf[SparkApplication].isAssignableFrom(mainClass)
				-- 【Cluster】a). mainClass.getConstructor().newInstance().asInstanceOf[SparkApplication]
				-- 【Client 】b). new JavaMainApplication(mainClass)
				
				-- app.start

YarnClusterApplication

-- start

	// new ClientArguments
	// --class => userClass => SparkPI(WordCount)
	-- new Client
	
	-- client.run
	
		-- submitApplication
		
			// 【Cluster】 org.apache.spark.deploy.yarn.ApplicationMaster
			// 【Client 】 org.apache.spark.deploy.yarn.ExecutorLauncher
			--createContainerLaunchContext
			--createApplicationSubmissionContext

启动ApplicationMaster和Driver

-- main

	// --class => userClass => SparkPi(WordCount)
	-- new ApplicationMasterArguments
	
	-- master = new ApplicationMaster
	
	-- master.run()
	
	-- 【Client 】runExecutorLauncher
	-- 【Cluster】runDriver
	
		-- userClassThread = startUserApplication()
		
			-- ClassLoader.loadClass(args.userClass)
				getMethod("main")
				
			-- new Thread().start()
			
				-- run
					-- mainMethod.invoke(null, userArgs.toArray)
					
						-- WordCount
						
							-- new SparkContext() // 创建Spark的环境
							-- 【blocking....................】
		
		// Blocking -----------------------------------------
		-- val sc = ThreadUtils.awaitResult
		
		// 注册
		-- registerAM
		
		// 资源分配
		-- createAllocator
		
			-- allocator.allocateResources
			
				-- handleAllocatedContainers
				
					-- runAllocatedContainers
					
						// bin/java org.apache.spark.executor.YarnCoarseGrainedExecutorBackend
						-- prepareCommand
		
		-- resumeDriver // 执行Spark作业
		-- userClassThread.join()

启动ExecutorBacked，创建计算对象、任务的调度和执行
YarnCoarseGrainedExecutorBackend

-- main

	-- CoarseGrainedExecutorBackend.run
	
		-- val env = SparkEnv.createExecutorEnv
		
		-- env.rpcEnv.setupEndpoint("Executor")