Flink – JobManager.submitJob
JobManager作为actor，
case SubmitJob(jobGraph, listeningBehaviour) => val client = sender() val jobInfo = new JobInfo(client, listeningBehaviour, System.currentTimeMillis(), jobGraph.getSessionTimeout) submitJob(jobGraph, jobInfo)
submitJob，做3件事、
根据JobGraph生成ExecuteGraph
恢复状态CheckpointedState，或者Savepoint
提交ExecuteGraph给Scheduler进行调度
ExecuteGraph
executionGraph = ExecutionGraphBuilder.buildGraph( executionGraph, //currentJobs.get(jobGraph.getJobID)，对应的jobid是否有现存的ExecuteGraph jobGraph, flinkConfiguration, //配置 futureExecutor, //Executors.newFixedThreadPool(numberProcessors, new NamedThreadFactory("jobmanager-future-", "-thread-"))，根据cpu核数创建的线程池 ioExecutor, // Executors.newFixedThreadPool(numberProcessors, new NamedThreadFactory("jobmanager-io-", "-thread-")) userCodeLoader, //libraryCacheManager.getClassLoader(jobGraph.getJobID)，从jar中加载 checkpointRecoveryFactory, //用于createCheckpointStore和createCheckpointIDCounter，standalone和zk两种 Time.of(timeout.length, timeout.unit), restartStrategy, //job重启策略 jobMetrics, numSlots, //scheduler.getTotalNumberOfSlots(),注册到该JM上的instances一共有多少slots log.logger)
ExecutionGraphBuilder.buildGraph
New
// create a new execution graph, if none exists so far final ExecutionGraph executionGraph; try { executionGraph = (prior != null) ? prior : new ExecutionGraph( futureExecutor, ioExecutor, jobId, jobName, jobGraph.getJobConfiguration(), jobGraph.getSerializedExecutionConfig(), timeout, restartStrategy, jobGraph.getUserJarBlobKeys(), jobGraph.getClasspaths(), classLoader, metrics); } catch (IOException e) { throw new JobException("Could not create the execution graph.", e); }
attachJobGraph，生成Graph的节点和边
// topologically sort the job vertices and attach the graph to the existing one List<JobVertex> sortedTopology = jobGraph.getVerticesSortedTopologicallyFromSources(); executionGraph.attachJobGraph(sortedTopology);
ExecutionGraph.attachJobGraph
for (JobVertex jobVertex : topologiallySorted) { // create the execution job vertex and attach it to the graph ExecutionJobVertex ejv = new ExecutionJobVertex(this, jobVertex, 1, timeout, createTimestamp); ejv.connectToPredecessors(this.intermediateResults); //All job vertices that are part of this graph, ConcurrentHashMap<JobVertexID, ExecutionJobVertex> tasks ExecutionJobVertex previousTask = this.tasks.putIfAbsent(jobVertex.getID(), ejv); for (IntermediateResult res : ejv.getProducedDataSets()) { //All intermediate results that are part of this graph //ConcurrentHashMap<IntermediateDataSetID, IntermediateResult> intermediateResults IntermediateResult previousDataSet = this.intermediateResults.putIfAbsent(res.getId(), res); } //All vertices, in the order in which they were created //List<ExecutionJobVertex> verticesInCreationOrder this.verticesInCreationOrder.add(ejv); }
将JobVertex封装成ExecutionJobVertex
会依次创建出ExecutionJobVertex，ExecutionVertex, Execution; IntermediateResult, IntermediateResultPartition
ExecutionJobVertex
public ExecutionJobVertex( ExecutionGraph graph, JobVertex jobVertex, int defaultParallelism, Time timeout, long createTimestamp) throws JobException { if (graph == null || jobVertex == null) { throw new NullPointerException(); } //并发度，决定有多少ExecutionVertex int vertexParallelism = jobVertex.getParallelism(); int numTaskVertices = vertexParallelism > 0 ? vertexParallelism : defaultParallelism; //产生ExecutionVertex this.taskVertices = new ExecutionVertex[numTaskVertices]; this.inputs = new ArrayList<>(jobVertex.getInputs().size()); // take the sharing group this.slotSharingGroup = jobVertex.getSlotSharingGroup(); this.coLocationGroup = jobVertex.getCoLocationGroup(); // create the intermediate results this.producedDataSets = new IntermediateResult[jobVertex.getNumberOfProducedIntermediateDataSets()]; //创建用于存放中间结果的IntermediateResult for (int i = 0; i < jobVertex.getProducedDataSets().size(); i++) { final IntermediateDataSet result = jobVertex.getProducedDataSets().get(i); this.producedDataSets[i] = new IntermediateResult( //将JobGraph中的IntermediateDataSet封装成IntermediateResult result.getId(), this, numTaskVertices, result.getResultType()); } // create all task vertices for (int i = 0; i < numTaskVertices; i++) { ExecutionVertex vertex = new ExecutionVertex( //初始化ExecutionVertex this, i, this.producedDataSets, timeout, createTimestamp, maxPriorAttemptsHistoryLength); this.taskVertices[i] = vertex; // } finishedSubtasks = new boolean[parallelism]; }
ExecutionVertex
public ExecutionVertex( ExecutionJobVertex jobVertex, int subTaskIndex, //第几个task，task和ExecutionVertex对应 IntermediateResult[] producedDataSets, Time timeout, long createTimestamp, int maxPriorExecutionHistoryLength) { this.jobVertex = jobVertex; this.subTaskIndex = subTaskIndex; this.taskNameWithSubtask = String.format("%s (%d/%d)", jobVertex.getJobVertex().getName(), subTaskIndex + 1, jobVertex.getParallelism()); this.resultPartitions = new LinkedHashMap<IntermediateResultPartitionID, IntermediateResultPartition>(producedDataSets.length, 1); //用于记录IntermediateResultPartition for (IntermediateResult result : producedDataSets) { IntermediateResultPartition irp = new IntermediateResultPartition(result, this, subTaskIndex); //初始化IntermediateResultPartition result.setPartition(subTaskIndex, irp); resultPartitions.put(irp.getPartitionId(), irp); } this.inputEdges = new ExecutionEdge[jobVertex.getJobVertex().getInputs().size()][]; this.priorExecutions = new EvictingBoundedList<>(maxPriorExecutionHistoryLength); this.currentExecution = new Execution( //创建Execution getExecutionGraph().getFutureExecutor(), this, 0, createTimestamp, timeout); this.timeout = timeout; }
connectToPredecessors，把节点用edge相连
public void connectToPredecessors(Map<IntermediateDataSetID, IntermediateResult> intermediateDataSets) throws JobException { List<JobEdge> inputs = jobVertex.getInputs(); //JobVertex的输入 for (int num = 0; num < inputs.size(); num++) { JobEdge edge = inputs.get(num); //对应的JobEdge IntermediateResult ires = intermediateDataSets.get(edge.getSourceId()); //取出JobEdge的source IntermediateResult this.inputs.add(ires); //List<IntermediateResult> inputs; int consumerIndex = ires.registerConsumer(); //将当前vertex作为consumer注册到IntermediateResult的每个IntermediateResultPartition for (int i = 0; i < parallelism; i++) { ExecutionVertex ev = taskVertices[i]; ev.connectSource(num, ires, edge, consumerIndex); //为每个ExecutionVertex建立到具体IntermediateResultPartition的ExecutionEdge } } }
connectSource
public void connectSource(int inputNumber, IntermediateResult source, JobEdge edge, int consumerNumber) { final DistributionPattern pattern = edge.getDistributionPattern(); // 获取edge的distribution pattern final IntermediateResultPartition[] sourcePartitions = source.getPartitions(); // 获取souce的partitions ExecutionEdge[] edges; switch (pattern) { case POINTWISE: edges = connectPointwise(sourcePartitions, inputNumber); break; case ALL_TO_ALL: edges = connectAllToAll(sourcePartitions, inputNumber); break; default: throw new RuntimeException("Unrecognized distribution pattern."); } this.inputEdges[inputNumber] = edges; // add the consumers to the source // for now (until the receiver initiated handshake is in place), we need to register the // edges as the execution graph for (ExecutionEdge ee : edges) { ee.getSource().addConsumer(ee, consumerNumber); } }
看下connectPointwise
private ExecutionEdge[] connectPointwise(IntermediateResultPartition[] sourcePartitions, int inputNumber) { final int numSources = sourcePartitions.length; //Partitions的个数 final int parallelism = getTotalNumberOfParallelSubtasks(); //subTasks的并发度 // simple case same number of sources as targets if (numSources == parallelism) { //如果1比1，简单 return new ExecutionEdge[] { new ExecutionEdge(sourcePartitions[subTaskIndex], this, inputNumber) }; //取sourcePartitions中和subTaskIndex对应的那个partition } else if (numSources < parallelism) { //如果subTasks的并发度高，那一个source会对应于多个task int sourcePartition; // check if the pattern is regular or irregular // we use int arithmetics for regular, and floating point with rounding for irregular if (parallelism % numSources == 0) { //整除的情况下，比如2个source，6个task，那么第3个task应该对应于第一个source // same number of targets per source int factor = parallelism / numSources; sourcePartition = subTaskIndex / factor; } else { // different number of targets per source float factor = ((float) parallelism) / numSources; sourcePartition = (int) (subTaskIndex / factor); } return new ExecutionEdge[] { new ExecutionEdge(sourcePartitions[sourcePartition], this, inputNumber) }; } else { //...... } }
配置checkpoint
executionGraph.enableSnapshotCheckpointing(
snapshotSettings.getCheckpointInterval(),
snapshotSettings.getCheckpointTimeout(),
snapshotSettings.getMinPauseBetweenCheckpoints(),
snapshotSettings.getMaxConcurrentCheckpoints(),
snapshotSettings.getExternalizedCheckpointSettings(),
triggerVertices,
ackVertices,
confirmVertices,
checkpointIdCounter,
completedCheckpoints,
externalizedCheckpointsDir,
checkpointStatsTracker);
启动CheckpointCoordinator，参考专门讨论Checkpoint机制的blog
Scheduler
下面看看如何将生成好的ExecutionGraph进行调度
future { //异步
try {
submittedJobGraphs.putJobGraph(new SubmittedJobGraph(jobGraph, jobInfo)) //放入submittedJobGraphs
} catch {
//
}
}
jobInfo.notifyClients(
decorateMessage(JobSubmitSuccess(jobGraph.getJobID))) //通知用户提交成功
if (leaderElectionService.hasLeadership) {
executionGraph.scheduleForExecution(scheduler) //调度
}
} catch {
//
}
}(context.dispatcher)
}
executionGraph.scheduleForExecution
public void scheduleForExecution(SlotProvider slotProvider) throws JobException { switch (scheduleMode) { case LAZY_FROM_SOURCES: // simply take the vertices without inputs. for (ExecutionJobVertex ejv : this.tasks.values()) { //ConcurrentHashMap<JobVertexID, ExecutionJobVertex> tasks,这个tasks的命名不科学 if (ejv.getJobVertex().isInputVertex()) { ejv.scheduleAll(slotProvider, allowQueuedScheduling); } } break; case EAGER: for (ExecutionJobVertex ejv : getVerticesTopologically()) { ejv.scheduleAll(slotProvider, allowQueuedScheduling); } break; default: throw new JobException("Schedule mode is invalid."); } }
对于流默认是EAGER，
public JobGraph createJobGraph() { jobGraph = new JobGraph(streamGraph.getJobName()); // make sure that all vertices start immediately jobGraph.setScheduleMode(ScheduleMode.EAGER);
ExecutionJobVertex.scheduleAll
public void scheduleAll(SlotProvider slotProvider, boolean queued) throws NoResourceAvailableException { ExecutionVertex[] vertices = this.taskVertices; // kick off the tasks for (ExecutionVertex ev : vertices) { ev.scheduleForExecution(slotProvider, queued); } }
ExecutionVertex.scheduleForExecution
//The current or latest execution attempt of this vertex's task public boolean scheduleForExecution(SlotProvider slotProvider, boolean queued) throws NoResourceAvailableException { return this.currentExecution.scheduleForExecution(slotProvider, queued); }
Execution.scheduleForExecution
public boolean scheduleForExecution(SlotProvider slotProvider, boolean queued) throws NoResourceAvailableException { final SlotSharingGroup sharingGroup = vertex.getJobVertex().getSlotSharingGroup(); final CoLocationConstraint locationConstraint = vertex.getLocationConstraint(); if (transitionState(CREATED, SCHEDULED)) { ScheduledUnit toSchedule = locationConstraint == null ? //生成ScheduledUnit new ScheduledUnit(this, sharingGroup) : new ScheduledUnit(this, sharingGroup, locationConstraint); final Future<SimpleSlot> slotAllocationFuture = slotProvider.allocateSlot(toSchedule, queued); //从slotProvider获取slot final Future<Void> deploymentFuture = slotAllocationFuture.handle(new BiFunction<SimpleSlot, Throwable, Void>() { @Override public Void apply(SimpleSlot simpleSlot, Throwable throwable) { if (simpleSlot != null) { //slot分配成功 try { deployToSlot(simpleSlot); //deploy } catch (Throwable t) { try { simpleSlot.releaseSlot(); } finally { markFailed(t); } } } else { markFailed(throwable); } return null; } }); }
slotProvider，参考Flink - Scheduler
deployToSlot，核心就是往TaskManager提交submitTask请求
public void deployToSlot(final SimpleSlot slot) throws JobException { ExecutionState previous = this.state; if (previous == SCHEDULED || previous == CREATED) { if (!transitionState(previous, DEPLOYING)) { //状态迁移成Deploying throw new IllegalStateException("Cannot deploy task: Concurrent deployment call race."); } } try { // good, we are allowed to deploy if (!slot.setExecutedVertex(this)) { //设置slot和ExecuteVertex关系 throw new JobException("Could not assign the ExecutionVertex to the slot " + slot); } this.assignedResource = slot; final TaskDeploymentDescriptor deployment = vertex.createDeploymentDescriptor( //创建DeploymentDescriptor attemptId, slot, taskState, attemptNumber); // register this execution at the execution graph, to receive call backs vertex.getExecutionGraph().registerExecution(this); final TaskManagerGateway taskManagerGateway = slot.getTaskManagerGateway(); final Future<Acknowledge> submitResultFuture = taskManagerGateway.submitTask(deployment, timeout); //向TaskMananger的Actor发送请求 submitResultFuture.exceptionallyAsync(new ApplyFunction<Throwable, Void>() {......} }
