Hadoop 2.6.0 Namenode HA,ResourceManager HA

 

 

先启动所有的zookeeper

zkServer.sh start

 

在所有节点上启动JournalNode：

sbin/hadoop-daemon.sh start journalnode

 

格式化第一个NameNode

bin/hdfs namenode –format

启动第一个的NameNode

sbin/hadoop-daemon.sh start namenode

在第二个NameNode上同步元数据

bin/hdfs namenode -bootstrapStandby

启动第二个NameNode

sbin/hadoop-daemon.sh start namenode

 

 

ZKFC

在第一个NameNode上执行格式化zkfc:

bin/hdfs zkfc -formatZK

运行这个命令后，会在zookeeper上创建一个/hadoop-ha/mycluster/的znode，用来存放automatic failover的数据。

启动zkfc(zookeeper failover controller)

在第一个NameNode节点上启动zkfc

sbin/hadoop-daemon.sh start zkfc

此时有一个namenode已经变为active了

在第二个NameNode节点上启动zkfc

sbin/hadoop-daemon.sh start zkfc

 

两个NameNode上都启动了zkfc，此时停止第一个NamNode后第二个会自动变成active状态。

 

启动datanode

在namenode上启动：sbin/hadoop-daemons.sh start datanode

 

启动yarn

在namenode上启动：sbin/start-yarn.sh

 

 

停止集群：

sbin/stop-yarn.sh

sbin/stop-dfs.sh

再每台机器上执行：zkServer.sh stop

第二次启动：

sbin/start-dfs.sh

sbin/start-yarn.sh

 

 

在代码里加载如下代码，可实现自动切换：

<property>

      <name>dfs.defaultFS</name>

      <value>hdfs://mycluster</value>

   </property>

   <property>

      <name>dfs.nameservices</name>

      <value>mycluster</value>

   </property>

   <property>

      <name>dfs.ha.namenodes.mycluster</name>

      <value>nn0,nn1</value>

   </property>

   <property>

      <name>dfs.namenode.rpc-address.mycluster.nn0</name>

      <value>192.168.1.126:8020</value>

   </property>

   <property>

      <name>dfs.namenode.rpc-address.mycluster.nn1</name>

      <value>192.168.1.127:8020</value>

   </property>

   <property>

      <name>dfs.client.failover.proxy.provider.mycluster</name>

   <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider

      </value>

   </property>

 

 配置文件：

1 core-site.xml:

<configuration>

<property>
<name>fs.defaultFS</name>
<value>hdfs://test23:8020</value>
</property>

<property>
<name>hadoop.tmp.dir</name>
<value>/hadoop/tmp</value>
</property>

</configuration>

 

2 yarn-site.xml:

<configuration>

<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
<description></description>
</property>

<property>
<name>yarn.nodemanager.vmem-pmem-ratio</name>
<value>10</value>
</property>

<property>
<name>yarn.log-aggregation-enable</name>
<value>true</value>
</property>

<!--
<property>
<name>yarn.resourcemanager.scheduler.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
</property>
<property>
<name>yarn.scheduler.fair.allocation.file</name>
<value>${yarn.home.dir}/etc/hadoop/fair-scheduler.xml</value>
</property>
-->

<!-- rm ha-->
<property>
<name>yarn.resourcemanager.ha.enabled</name>
<value>true</value>
</property>
<property>
<name>yarn.resourcemanager.cluster-id</name>
<value>rm-ha</value>
</property>
<property>
<name>yarn.resourcemanager.ha.rm-ids</name>
<value>rm1,rm2</value>
</property>
<property>
<name>yarn.resourcemanager.hostname.rm1</name>
<value>test23</value>
</property>
<property>
<name>yarn.resourcemanager.hostname.rm2</name>
<value>test24</value>
</property>
<property>
<name>yarn.resourcemanager.zk-address</name>
<value>test23:2181,test24:2181,test25:2181</value>
</property>
<property>
<name>yarn.resourcemanager.ha.automatic-failover.enabled</name>
<value>true</value>
</property>

<!-- RM1 configs -->
<property>
<name>yarn.resourcemanager.address.rm1</name>
<value>test23:8032</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address.rm1</name>
<value>test23:8030</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.https.address.rm1</name>
<value>test23:8090</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address.rm1</name>
<value>test23:8088</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address.rm1</name>
<value>test23:8031</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address.rm1</name>
<value>test23:8033</value>
</property>
<!-- RM2 configs -->
<property>
<name>yarn.resourcemanager.address.rm2</name>
<value>test24:8032</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address.rm2</name>
<value>test24:8030</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.https.address.rm2</name>
<value>test24:8090</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address.rm2</name>
<value>test24:8088</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address.rm2</name>
<value>test24:8031</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address.rm2</name>
<value>test24:8033</value>
</property>

<!--for optimize-->
<property>
<description>Amount of physical memory, in MB, that can be allocated
for containers.default is 8192MB</description>
<name>yarn.nodemanager.resource.memory-mb</name>
<value>8192</value>
</property>

<property>
<description>Number of CPU cores that can be allocated
for containers.</description>
<name>yarn.nodemanager.resource.cpu-vcores</name>
<value>20</value>
</property>

</configuration>

 

3 hdfs-site.xml

<configuration>
<property>
<name>dfs.replication</name>
<value>2</value>
</property>
<property>
<name>dfs.permissions</name>
<value>false</value>
</property>

<property>
<name>hadoop.datahome</name>
<value>/hadoop</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>${hadoop.datahome}/name</value>
</property>

<property>
<name>dfs.datanode.data.dir</name>
<value>${hadoop.datahome}/data</value>
</property>

<property>
<name>dfs.nameservices</name>
<value>mycluster</value>
</property>
<property>
<name>dfs.ha.namenodes.mycluster</name>
<value>nn0,nn1</value>
</property>
<property>
<name>dfs.namenode.rpc-address.mycluster.nn0</name>
<value>test23:8020</value>
</property>
<property>
<name>dfs.namenode.rpc-address.mycluster.nn1</name>
<value>test24:8020</value>
</property>
<property>
<name>dfs.namenode.http-address.mycluster.nn0</name>
<value>test23:50070</value>
</property>
<property>
<name>dfs.namenode.http-address.mycluster.nn1</name>
<value>test24:50070</value>
</property>
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://test23:8485;test24:8485;test25:8485/mycluster</value>
</property>
<property>
<name>dfs.journalnode.edits.dir</name>
<value>${hadoop.datahome}/journal</value>
</property>
<property>
<name>dfs.client.failover.proxy.provider.mycluster</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>

<property>
<name>ha.zookeeper.quorum</name>
<value>test23:2181,test24:2181,test25:2181</value>
</property>
<property>
<name>dfs.ha.fencing.methods</name>
<value>shell(/bin/true)</value>
</property>
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
</configuration>

 

4 mapred-site.xml

 

<configuration>

<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>

<property>
<name>hadoop.datahome</name>
<value>/hadoop</value>
</property>

<property>
<name>mapreduce.jobhistory.address</name>
<value>test23:10020</value>
</property>
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>test23:19888</value>
</property>
<property>
<name>mapreduce.jobhistory.done-dir</name>
<value>/history/done</value>
</property>

<property>
<name>mapreduce.jobhistory.intermediate-done-dir</name>
<value>/history/done_intermediate</value>
</property>

<!-- for optimize-->
<property>
<name>mapreduce.input.fileinputformat.split.minsize</name>
<value></value>
</property>

<property>
<name>mapreduce.cluster.local.dir</name>
<value>${hadoop.datahome}/local</value>
</property>
<property>
<name>fs.trash.interval</name>
<value>2880</value>
</property>
<property>
<name>mapreduce.task.io.sort.factor</name>
<value>100</value>
</property>
<property>
<name>mapreduce.task.io.sort.mb</name>
<value>200</value>
</property>
<property>
<name>dfs.namenode.handler.count</name>
<value>50</value>
</property>
<!--
<property>
<name>mapred.child.java.opts</name>
<value>-Xmx10240m</value>
</property>
-->
<property>
<name>mapreduce.map.java.opts</name>
<value>-Xmx1024m</value>
</property>
<property>
<name>mapreduce.reduce.java.opts</name>
<value>-Xmx1024m</value>
</property>

<property>
<name>mapreduce.jobtracker.handler.count</name>
<value>15</value>
</property>

<property>
<name>mapreduce.reduce.shuffle.parallelcopies</name>
<value>10</value>
</property>

<property>
<name>mapreduce.tasktracker.http.threads</name>
<value>50</value>
</property>
<property>
<name>mapreduce.tasktracker.map.tasks.maximum</name>
<value>4</value>
</property>
<property>
<name>mapreduce.tasktracker.reduce.tasks.maximum</name>
<value>4</value>
</property>

</configuration>

 

5 slaves

test24
test25

 附上所有配置文件：https://files.cnblogs.com/files/JavaSmart/hadoop2_ha.rar