#Hadoop集群部署
#
1. 配置JAVA环境（vi/etc/profile.d/java.sh）
[root@master ~]# yum install java-1.8.0-openjdk-devel
[root@master ~]# vi /etc/profile.d/java.sh 添加JAVA环境变量 export JAVA_HOME=/usr
或者编译安装。
export JAVA_HOME=/usr/java/jdk1.8.0_151
export JRE_HOME=$JAVA_HOME/jre
export CLASSPATH=$JAVA_HOME/lib:$JAVA_HOME/lib/tools.jar
export PATH=$PATH:$JAVA_HOME/bin:$JAVA_HOME/jre/bin
node节点也需要配置JAVA环境。
2.创建hadoop用户
[root@master hadoop]# useradd hadoop
[root@master hadoop]#echo 'hadoop'|passwd -stdin hadoop
node2，node3，node4都需要创建。
3. 配置hosts（/etc/hosts）
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 192.168.15.128 master192.168.15.129 node2 192.168.15.130 node3
192.168.15.131 node4
node2，node3，node4都需要配置。
4. 配置master免密登录node节点
[root@master ~]# su - hadoop
[hadoop@master ~]$ ssh-keygen -t rsa -P ''
[hadoop@master ~]$ for i in 2 3 4;do ssh-copy-id -i .ssh/id_rsa_pub hadoop@node${i};done
5. 解压hadoop安装包
[root@master ~]# tar -zxvf hadoop-2.8.2.tar.gz -C /usr/local/
[root@master ~]# cd /usr/local
[root@master ~]# ln -sv hadoop-2.8.2 hadoop
6. 创建数据目录和日志目录
[root@master ~]# mkdir -pv /data/hadoop/hdfs/{name,sname,data}
[root@master ~]# chown -R hadoop:hadoop /data
创建日志目录
[root@master ~]# mkdir /usr/local/hadoop/logs
[root@master ~]# chown -R hadoop:hadoop /usr/local/hadoop/
[root@master ~]# chmod g+x /usr/local/hadoop/logs
7. 配置hadoop配置文件（/usr/local/hadoop/etc/hadoop/）下面的配置Node节点也需要同样的配置，master配置完，
可以通过命令[hadoop@master ~]$ scp /usr/local/hadoop/etc/hadoop/* hadoop@node2:/usr/local/hadoop/etc/hadoop/
core-site.xml
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://master:8082</value>
</property>
</configuration>
hdfs-site.xml
<configuration>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:///data/hadoop/hdfs/name</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:///data/hadoop/hdfs/data</value>
</property>
<property>
<name>dfs.replication</name>
<value>2</value>
</property>
<property>
<name>fs.checkpoint.dir</name>
<value>file:///data/hadoop/hdfs/sname</value>
</property>
<property>
<name>fs.checkpoint.edits.dir</name>
<value>file:///data/hadoop/hdfs/sname</value>
</property>
</configuration>
yarn-site.xml
<configuration>
<!-- Site specific YARN configuration properties -->
<property>
<name>yarn.resourcemanager.address</name>
<value>master:8032</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address</name>
<value>master:8030</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>master:8031</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address</name>
<value>master:8033</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address</name>
<value>master:8088</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.auxservices.mapreduce_shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
</property>
</configuration>
mapred-site.xml
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
</configuration>
slaves
node2
node3
node4
hadoop-env.sh(需要添加JAVA环境变量)
export JAVA_HOME=/usr/java/jdk1.8.0_151
8. 格式化HDFS
[root@master ~]# su - hadoop
[hadoop@master ~]$ hdfs namenode -format
输出结果为 17/11/13 23:13:47 INFO common.Storage: Storage directory /data/hadoop/hdfs/name has been successfully formatted.
说明格式化成功了。
9. 启动dfs
[hadoop@master ~]$ start-dfs.sh
Starting namenodes on [master]
master: starting namenode, logging to /bdapps/hadoop/logs/hadoop-hadoop-namenode-master.out
node2: starting datanode, logging to /bdapps/hadoop-2.8.2/logs/hadoop-hadoop-datanode-node2.out
node3: starting datanode, logging to /bdapps/hadoop-2.8.2/logs/hadoop-hadoop-datanode-node3.out
node4: starting datanode, logging to /bdapps/hadoop-2.8.2/logs/hadoop-hadoop-datanode-node4.out
Starting secondary namenodes [0.0.0.0]
0.0.0.0: starting secondarynamenode, logging to /bdapps/hadoop/logs/hadoop-hadoop-secondarynamenode-master.out
10. 上传一个文件（/etc/fstab）
[hadoop@master ~]$ hdfs dfs -mkdir /test
[hadoop@master ~]$ hdfs dfs -put /etc/fstab /test
[hadoop@master ~]$ hdfs dfs -ls /test
Found 1 items
-rw-r--r-- 2 hadoop supergroup 779 2017-11-14 00:03 /test/fstab
[hadoop@master ~]$ hdfs dfs -cat /test/fstab
#
# /etc/fstab
# Created by anaconda on Mon Nov 13 08:43:43 2017
#
# Accessible filesystems, by reference, are maintained under '/dev/disk'
# See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info
#
/dev/mapper/VolGroup-lv_root / ext4 defaults 1 1
UUID=98ed038f-f133-4809-bad8-c4139402b11b /boot ext4 defaults 1 2
/dev/mapper/VolGroup-lv_swap swap swap defaults 0 0
tmpfs /dev/shm tmpfs defaults 0 0
devpts /dev/pts devpts gid=5,mode=620 0 0
sysfs /sys sysfs defaults 0 0
proc /proc proc defaults 0 0
[hadoop@master ~]$
11. 启动yarn
[hadoop@master ~]$ start-yarn.sh
starting yarn daemons
starting resourcemanager, logging to /bdapps/hadoop/logs/yarn-hadoop-resourcemanager-master.out
node4: starting nodemanager, logging to /bdapps/hadoop/logs/yarn-hadoop-nodemanager-node4.out
node3: starting nodemanager, logging to /bdapps/hadoop/logs/yarn-hadoop-nodemanager-node3.out
node2: starting nodemanager, logging to /bdapps/hadoop/logs/yarn-hadoop-nodemanager-node2.out
可以通过命令jps查看java进程
Master节点
[hadoop@master ~]$ jps
7600 ResourceManager
7106 NameNode
8483 Jps
7304 SecondaryNameNode
Node节点
[root@node4 ~]# jps
2802 Jps
2678 NodeManager
2553 DataNode
12. 访问
hadoop总览
http://192.168.15.128:50070
查看hadoop的应用
http://192.168.15.128:8088
#
