Hadoop 2.2.0源码浏览:2. ResourceManager
基本流程
public static void main(String argv[]) {
// 设置发生未捕获异常时的处理Handler
Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
// 同namenode相同,设置启动和关闭时记录日志信息
StringUtils.startupShutdownMessage(ResourceManager.class, argv, LOG);
try {
/*
* 1. 初始化配置对象和ResourceManager对象
* 2. 设置ResourceManager关闭时,清理composite service的Hook
* 3. 设置http策略
* 4. 初始化配置
* 5. 启动
*/
Configuration conf = new YarnConfiguration();
ResourceManager resourceManager = new ResourceManager();
ShutdownHookManager.get().addShutdownHook(
new CompositeServiceShutdownHook(resourceManager),
SHUTDOWN_HOOK_PRIORITY);
setHttpPolicy(conf);
resourceManager.init(conf);
resourceManager.start();
} catch (Throwable t) {
LOG.fatal("Error starting ResourceManager", t);
System.exit(-1);
}
}
// 设置发生未捕获异常时的处理Handler
Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
// 同namenode相同,设置启动和关闭时记录日志信息
StringUtils.startupShutdownMessage(ResourceManager.class, argv, LOG);
try {
/*
* 1. 初始化配置对象和ResourceManager对象
* 2. 设置ResourceManager关闭时,清理composite service的Hook
* 3. 设置http策略
* 4. 初始化配置
* 5. 启动
*/
Configuration conf = new YarnConfiguration();
ResourceManager resourceManager = new ResourceManager();
ShutdownHookManager.get().addShutdownHook(
new CompositeServiceShutdownHook(resourceManager),
SHUTDOWN_HOOK_PRIORITY);
setHttpPolicy(conf);
resourceManager.init(conf);
resourceManager.start();
} catch (Throwable t) {
LOG.fatal("Error starting ResourceManager", t);
System.exit(-1);
}
}
创建和初始化ResourceManager
1. 创建命名为ResourceManager的CompositeService
2. 设置状态为INIT,服务初始化
3. 服务初始化
protected void serviceInit(Configuration conf) throws Exception {
validateConfigs(conf);
this.conf = conf;
// 设置发生错误时分发器退出还是crash,测试时建议设置为false(crash),线上相反
this.conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);
// 创建分发器(异步),并加入服务列表中
this.rmDispatcher = createDispatcher();
addIfService(this.rmDispatcher);
// 资源管理器中面向应用的token管理器
this.amRmTokenSecretManager = createAMRMTokenSecretManager(conf);
// 为容器分配处理过期情况的成员,并加入服务列表
this.containerAllocationExpirer = new ContainerAllocationExpirer(
this.rmDispatcher);
addService(this.containerAllocationExpirer);
// 应用生存周期监控器,监控生存情况,加入服务列表
AMLivelinessMonitor amLivelinessMonitor = createAMLivelinessMonitor();
addService(amLivelinessMonitor);
// 应用生存周期监控器,监控结束情况,加入服务列表
AMLivelinessMonitor amFinishingMonitor = createAMLivelinessMonitor();
addService(amFinishingMonitor);
// 容器token管理和命名空间token管理
this.containerTokenSecretManager = createContainerTokenSecretManager(conf);
this.nmTokenSecretManager = createNMTokenSecretManager(conf);
boolean isRecoveryEnabled = conf.getBoolean(
YarnConfiguration.RECOVERY_ENABLED,
YarnConfiguration.DEFAULT_RM_RECOVERY_ENABLED);
// 资源管理器状态信息存储
RMStateStore rmStore = null;
if(isRecoveryEnabled) {
recoveryEnabled = true;
rmStore = RMStateStoreFactory.getStore(conf);
} else {
recoveryEnabled = false;
rmStore = new NullRMStateStore();
}
try {
rmStore.init(conf);
rmStore.setRMDispatcher(rmDispatcher);
} catch (Exception e) {
// the Exception from stateStore.init() needs to be handled for
// HA and we need to give up master status if we got fenced
LOG.error("Failed to init state store", e);
ExitUtil.terminate(1, e);
}
if (UserGroupInformation.isSecurityEnabled()) {
this.delegationTokenRenewer = createDelegationTokenRenewer();
}
this.rmContext =
new RMContextImpl(this.rmDispatcher, rmStore,
this.containerAllocationExpirer, amLivelinessMonitor,
amFinishingMonitor, delegationTokenRenewer, this.amRmTokenSecretManager,
this.containerTokenSecretManager, this.nmTokenSecretManager,
this.clientToAMSecretManager);
// Register event handler for NodesListManager
this.nodesListManager = new NodesListManager(this.rmContext);
this.rmDispatcher.register(NodesListManagerEventType.class,
this.nodesListManager);
addService(nodesListManager);
// Initialize the scheduler
this.scheduler = createScheduler();
this.schedulerDispatcher = createSchedulerEventDispatcher();
addIfService(this.schedulerDispatcher);
this.rmDispatcher.register(SchedulerEventType.class,
this.schedulerDispatcher);
// Register event handler for RmAppEvents
this.rmDispatcher.register(RMAppEventType.class,
new ApplicationEventDispatcher(this.rmContext));
// Register event handler for RmAppAttemptEvents
this.rmDispatcher.register(RMAppAttemptEventType.class,
new ApplicationAttemptEventDispatcher(this.rmContext));
// Register event handler for RmNodes
this.rmDispatcher.register(RMNodeEventType.class,
new NodeEventDispatcher(this.rmContext));
this.nmLivelinessMonitor = createNMLivelinessMonitor();
addService(this.nmLivelinessMonitor);
this.resourceTracker = createResourceTrackerService();
addService(resourceTracker);
DefaultMetricsSystem.initialize("ResourceManager");
JvmMetrics.initSingleton("ResourceManager", null);
try {
this.scheduler.reinitialize(conf, this.rmContext);
} catch (IOException ioe) {
throw new RuntimeException("Failed to initialize scheduler", ioe);
}
// creating monitors that handle preemption
createPolicyMonitors();
masterService = createApplicationMasterService();
addService(masterService) ;
this.applicationACLsManager = new ApplicationACLsManager(conf);
queueACLsManager = createQueueACLsManager(scheduler, conf);
this.rmAppManager = createRMAppManager();
// Register event handler for RMAppManagerEvents
this.rmDispatcher.register(RMAppManagerEventType.class,
this.rmAppManager);
this.rmDTSecretManager = createRMDelegationTokenSecretManager(this.rmContext);
rmContext.setRMDelegationTokenSecretManager(this.rmDTSecretManager);
clientRM = createClientRMService();
rmContext.setClientRMService(clientRM);
addService(clientRM);
adminService = createAdminService(clientRM, masterService, resourceTracker);
addService(adminService);
this.applicationMasterLauncher = createAMLauncher();
this.rmDispatcher.register(AMLauncherEventType.class,
this.applicationMasterLauncher);
addService(applicationMasterLauncher);
if (UserGroupInformation.isSecurityEnabled()) {
addService(delegationTokenRenewer);
delegationTokenRenewer.setRMContext(rmContext);
}
new RMNMInfo(this.rmContext, this.scheduler);
super.serviceInit(conf);
validateConfigs(conf);
this.conf = conf;
// 设置发生错误时分发器退出还是crash,测试时建议设置为false(crash),线上相反
this.conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);
// 创建分发器(异步),并加入服务列表中
this.rmDispatcher = createDispatcher();
addIfService(this.rmDispatcher);
// 资源管理器中面向应用的token管理器
this.amRmTokenSecretManager = createAMRMTokenSecretManager(conf);
// 为容器分配处理过期情况的成员,并加入服务列表
this.containerAllocationExpirer = new ContainerAllocationExpirer(
this.rmDispatcher);
addService(this.containerAllocationExpirer);
// 应用生存周期监控器,监控生存情况,加入服务列表
AMLivelinessMonitor amLivelinessMonitor = createAMLivelinessMonitor();
addService(amLivelinessMonitor);
// 应用生存周期监控器,监控结束情况,加入服务列表
AMLivelinessMonitor amFinishingMonitor = createAMLivelinessMonitor();
addService(amFinishingMonitor);
// 容器token管理和命名空间token管理
this.containerTokenSecretManager = createContainerTokenSecretManager(conf);
this.nmTokenSecretManager = createNMTokenSecretManager(conf);
boolean isRecoveryEnabled = conf.getBoolean(
YarnConfiguration.RECOVERY_ENABLED,
YarnConfiguration.DEFAULT_RM_RECOVERY_ENABLED);
// 资源管理器状态信息存储
RMStateStore rmStore = null;
if(isRecoveryEnabled) {
recoveryEnabled = true;
rmStore = RMStateStoreFactory.getStore(conf);
} else {
recoveryEnabled = false;
rmStore = new NullRMStateStore();
}
try {
rmStore.init(conf);
rmStore.setRMDispatcher(rmDispatcher);
} catch (Exception e) {
// the Exception from stateStore.init() needs to be handled for
// HA and we need to give up master status if we got fenced
LOG.error("Failed to init state store", e);
ExitUtil.terminate(1, e);
}
if (UserGroupInformation.isSecurityEnabled()) {
this.delegationTokenRenewer = createDelegationTokenRenewer();
}
this.rmContext =
new RMContextImpl(this.rmDispatcher, rmStore,
this.containerAllocationExpirer, amLivelinessMonitor,
amFinishingMonitor, delegationTokenRenewer, this.amRmTokenSecretManager,
this.containerTokenSecretManager, this.nmTokenSecretManager,
this.clientToAMSecretManager);
// Register event handler for NodesListManager
this.nodesListManager = new NodesListManager(this.rmContext);
this.rmDispatcher.register(NodesListManagerEventType.class,
this.nodesListManager);
addService(nodesListManager);
// Initialize the scheduler
this.scheduler = createScheduler();
this.schedulerDispatcher = createSchedulerEventDispatcher();
addIfService(this.schedulerDispatcher);
this.rmDispatcher.register(SchedulerEventType.class,
this.schedulerDispatcher);
// Register event handler for RmAppEvents
this.rmDispatcher.register(RMAppEventType.class,
new ApplicationEventDispatcher(this.rmContext));
// Register event handler for RmAppAttemptEvents
this.rmDispatcher.register(RMAppAttemptEventType.class,
new ApplicationAttemptEventDispatcher(this.rmContext));
// Register event handler for RmNodes
this.rmDispatcher.register(RMNodeEventType.class,
new NodeEventDispatcher(this.rmContext));
this.nmLivelinessMonitor = createNMLivelinessMonitor();
addService(this.nmLivelinessMonitor);
this.resourceTracker = createResourceTrackerService();
addService(resourceTracker);
DefaultMetricsSystem.initialize("ResourceManager");
JvmMetrics.initSingleton("ResourceManager", null);
try {
this.scheduler.reinitialize(conf, this.rmContext);
} catch (IOException ioe) {
throw new RuntimeException("Failed to initialize scheduler", ioe);
}
// creating monitors that handle preemption
createPolicyMonitors();
masterService = createApplicationMasterService();
addService(masterService) ;
this.applicationACLsManager = new ApplicationACLsManager(conf);
queueACLsManager = createQueueACLsManager(scheduler, conf);
this.rmAppManager = createRMAppManager();
// Register event handler for RMAppManagerEvents
this.rmDispatcher.register(RMAppManagerEventType.class,
this.rmAppManager);
this.rmDTSecretManager = createRMDelegationTokenSecretManager(this.rmContext);
rmContext.setRMDelegationTokenSecretManager(this.rmDTSecretManager);
clientRM = createClientRMService();
rmContext.setClientRMService(clientRM);
addService(clientRM);
adminService = createAdminService(clientRM, masterService, resourceTracker);
addService(adminService);
this.applicationMasterLauncher = createAMLauncher();
this.rmDispatcher.register(AMLauncherEventType.class,
this.applicationMasterLauncher);
addService(applicationMasterLauncher);
if (UserGroupInformation.isSecurityEnabled()) {
addService(delegationTokenRenewer);
delegationTokenRenewer.setRMContext(rmContext);
}
new RMNMInfo(this.rmContext, this.scheduler);
super.serviceInit(conf);
}
。。。待续