hadoop-2.0.0-cdh4.2.1安装手册
2013-06-21 10:45 java20130722 阅读(615) 评论(0) 编辑 收藏 举报1. 角色分配：
- ocdata16: 10.1.253.26 :NameNode JournalNode Zookeeper DFSZKFailoverController
- ocdata17: 10.1.253.27 :NameNode JournalNode Zookeeper DFSZKFailoverController
- ocdata18: 10.1.253.28 :DataNode JournalNode Zookeeper
- ocdata19: 10.1.253.29 :DataNode
2. 添加SSH互信
- 登录10.1.253.26\10.1.253.27\10.1.253.28\10.1.253.29
ssh-keygen -t rsa -P "" cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys chmod go-w $HOME $HOME/.ssh chmod 600 $HOME/.ssh/authorized_keys chown `whoami` $HOME/.ssh/authorized_keys - 登录10.1.253.26,执行
ssh stream_dev@ocdata17 cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys ssh stream_dev@ocdata18 cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys ssh stream_dev@ocdata19 cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys scp ~/.ssh/authorized_keys stream_dev@ocdata17:~/.ssh/authorized_keys scp ~/.ssh/authorized_keys stream_dev@ocdata18:~/.ssh/authorized_keys scp ~/.ssh/authorized_keys stream_dev@ocdata19:~/.ssh/authorized_keys
3. 安装
- 下载安装包
- 解压，配置
- 配置zookeeper
cd /home/stream_dev/shaoaq/zookeeper-3.4.5-cdh4.2.1/conf cp zoo_sample.cfg zoo.cfg 修改： dataDir=/home/stream_dev/shaoaq/zookeeper clientPort=2183 server.16=ocdata16:2898:3898 server.17=ocdata17:2898:3898 server.18=ocdata18:2898:3898 在 ocdata16\ocdata17\ocdata18上， mkdir /home/stream_dev/shaoaq/zookeeper, 然后vim /home/stream_dev/shaoaq/zookeeper/myid,分别输入16，17，18 - 配置hadoop
- hdfs-site.xml
<property> <name>dfs.nameservices</name> <value>mycluster</value> </property> <property> <name>dfs.ha.namenodes.mycluster</name> <value>nn1,nn2</value> </property> <property> <name>dfs.namenode.rpc-address.mycluster.nn1</name> <value>ocdata16:8020</value> </property> <property> <name>dfs.namenode.rpc-address.mycluster.nn2</name> <value>ocdata17:8020</value> </property> <property> <name>dfs.namenode.http-address.mycluster.nn1</name> <value>ocdata16:50070</value> </property> <property> <name>dfs.namenode.http-address.mycluster.nn2</name> <value>ocdata17:50070</value> </property> <property> <name>dfs.namenode.shared.edits.dir</name> <value>qjournal://ocdata16:9485;ocdata17:9485;ocdata18:9485/mycluster</value> </property> <property> <name>dfs.client.failover.proxy.provider.mycluster</name> <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value> </property> <property> <name>dfs.ha.fencing.methods</name> <value>shell(/bin/true)</value> </property> <property> <name>dfs.ha.fencing.ssh.connect-timeout</name> <value>10000</value> </property> <property> <name>dfs.journalnode.edits.dir</name> <value>/home/stream_dev/shaoaq/journal/node/local/data</value> </property> <property> <name>dfs.journalnode.http-address</name> <value>0.0.0.0:9480</value> </property> <property> <name>dfs.journalnode.rpc-address</name> <value>0.0.0.0:9485</value> </property> <property> <name>dfs.ha.automatic-failover.enabled</name> <value>true</value> </property> <property> <name>ha.zookeeper.quorum</name> <value>ocdata16:2183,ocdata17:2183,ocdata18:2183</value> </property> - core-site.xml
<property> <name>fs.defaultFS</name> <value>hdfs://mycluster</value> </property> - slaves
ocdata16 ocdata17 ocdata18 ocdata19
- hadoop-env.sh
export JAVA_HOME=/home/stream_dev/stream/soft/jdk1.7.0_21
- 新建sbin/start-dfs-journalnode.sh
#!/usr/bin/env bash bin=`dirname "${BASH_SOURCE-$0}"` bin=`cd "$bin"; pwd` DEFAULT_LIBEXEC_DIR="$bin"/../libexec HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} . $HADOOP_LIBEXEC_DIR/hdfs-config.sh #--------------------------------------------------------- # quorumjournal nodes (if any) SHARED_EDITS_DIR=$($HADOOP_PREFIX/bin/hdfs getconf -confKey dfs.namenode.shared.edits.dir 2>&-) case "$SHARED_EDITS_DIR" in qjournal://*) JOURNAL_NODES=$(echo "$SHARED_EDITS_DIR" | sed 's,qjournal://\([^/]*\)/.*,\1,g; s/;/ /g; s/:[0-9]*//g') echo "Starting journal nodes [$JOURNAL_NODES]" "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \ --config "$HADOOP_CONF_DIR" \ --hostnames "$JOURNAL_NODES" \ --script "$bin/hdfs" start journalnode ;; esac - 新建sbin/stop-dfs-journalnode.sh
#!/usr/bin/env bash bin=`dirname "${BASH_SOURCE-$0}"` bin=`cd "$bin"; pwd` DEFAULT_LIBEXEC_DIR="$bin"/../libexec HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} . $HADOOP_LIBEXEC_DIR/hdfs-config.sh #--------------------------------------------------------- # quorumjournal nodes (if any) SHARED_EDITS_DIR=$($HADOOP_PREFIX/bin/hdfs getconf -confKey dfs.namenode.shared.edits.dir 2>&-) case "$SHARED_EDITS_DIR" in qjournal://*) JOURNAL_NODES=$(echo "$SHARED_EDITS_DIR" | sed 's,qjournal://\([^/]*\)/.*,\1,g; s/;/ /g; s/:[0-9]*//g') echo "Stopping journal nodes [$JOURNAL_NODES]" "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \ --config "$HADOOP_CONF_DIR" \ --hostnames "$JOURNAL_NODES" \ --script "$bin/hdfs" stop journalnode ;; esac
- hdfs-site.xml
- 配置zookeeper
- 分发到其他机器
scp -r ~/shaoaq stream_dev@ocdata17:~/ scp -r ~/shaoaq stream_dev@ocdata18:~/ scp -r ~/shaoaq stream_dev@ocdata19:~/
4. 初始化
- 启动HDFS
- 登录ocdata16,执行
./hdfs namenode -format ./hdfs namenode - 登录ocdata17,执行
./hdfs namenode -bootstrapStandby
- 在ocdata16上，ctrl+c 停止namenode
- 停止journalnode
./stop-dfs-journalnode.sh
- 登录ocdata16,执行
5. 日常启动停止¶
- 启动Zookeeper
- 分别在ocdata16\ocdata17\ocdata18上执行：
./zkServer.sh start
- 判断是否执行成功：
echo ruok | nc ocdata16 2183 echo ruok | nc ocdata17 2183 echo ruok | nc ocdata18 2183如果都返回imok,则代表启动成功
- 分别在ocdata16\ocdata17\ocdata18上执行：
- 启动HDFS
cd sbin ./start-dfs.sh
- 停止HDFS
cd sbin ./stop-dfs.sh
6. Yarn
mapred-site.xm
cp ../etc/hadoop/mapred-site.xml.template ../etc/hadoop/mapred-site.xm
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
yarn-site.xml
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce.shuffle</value>
</property>
<property>
<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<property>
<description>The address of the applications manager interface in the RM.</description>
<name>yarn.resourcemanager.address</name>
<value>ocnosql76:8032</value>
</property>
<property>
<description>The address of the scheduler interface.</description>
<name>yarn.resourcemanager.scheduler.address</name>
<value>ocnosql76:8030</value>
</property>
<property>
<description>The address of the RM web application.</description>
<name>yarn.resourcemanager.webapp.address</name>
<value>ocnosql76:8088</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>ocnosql76:8031</value>
</property>
<property>
<description>The address of the RM admin interface.</description>
<name>yarn.resourcemanager.admin.address</name>
<value>ocnosql76:8033</value>
</property>
Yarn的启动停止
cd sbin ./start-yarn.sh ./stop-yarn.sh
7. 其他
- 在安装前，最好在每台机器上确认hostname -i 返回该机器的IP地址
- 在安装前，确保每个机器上hosts里包含其他机器的主机名和IP的对应关系
- 如果节点一直在备节点，而不是自动切换到主节点，请重新格式化zkfc：./hdfs zkfc -formatZK
- 注意：主机名不能含有下划线
