Hadoop集群搭建

环境变量：

JAVA_HOME=/usr/local/jdk1.8.0_171
JRE_HOME=$JAVA_HOME/jre
CLASS_PATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib
PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin
export JAVA_HOME JRE_HOME CLASS_PATH PATH

export HADOOP_HOME=/root/hadoop-3.1.0
export PATH=.:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$JAVA_HOME/bin:$PATH

 

1.修改host

vim /etc/hosts

192.168.1.70 slave
192.168.1.106 master

 

关闭防火墙：

firewall-cmd --state

systemctl stop firewalld && systemctl disable firewalld

 

2.master免密登陆slave

master将公钥发送到slave：

scp ~/.ssh/id_rsa.pub slave:~/

 

slave将授权登陆

mv id_rsa.pub ~/.ssh

cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

chmod 0600 ~/.ssh/authorized_keys

 

3.vim core-site.xml 

<configuration>
    <property>
        <name>fs.default.name</name>
        <value>hdfs://master:9000</value>
    </property>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://master:9000</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/root/hadoop-3.1.0/tmp</value>
    </property>
</configuration>

 

4.vim hdfs-site.xml

<configuration>
    <property>
        <name>dfs.replication</name>
        <value>2</value>
    </property>
    <property>
        <name>dfs.namenode.secondary.http-address</name>
        <value>slave:50090</value>
    </property>
</configuration>

 

5.vim mapred-site.xml

    <!-- 通知框架MR使用YARN -->
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <property>  
        <name>mapreduce.application.classpath</name>  
        <value>  
            /root/hadoop-3.1.0/etc/hadoop,  
            /root/hadoop-3.1.0/share/hadoop/common/*,  
            /root/hadoop-3.1.0/share/hadoop/common/lib/*,  
            /root/hadoop-3.1.0/share/hadoop/hdfs/*,  
            /root/hadoop-3.1.0/share/hadoop/hdfs/lib/*,  
            /root/hadoop-3.1.0/share/hadoop/mapreduce/*,  
            /root/hadoop-3.1.0/share/hadoop/mapreduce/lib/*,  
            /root/hadoop-3.1.0/share/hadoop/yarn/*,  
            /root/hadoop-3.1.0/share/hadoop/yarn/lib/*  
        </value>  
    </property>

 

6.vim yarn-site.xml

<configuration>
    <property>  
        <name>yarn.nodemanager.aux-services</name>  
        <value>mapreduce_shuffle</value>  
    </property>  
    <property>  
        <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>  
        <value>org.apache.hadoop.mapred.ShuffleHandle</value>  
    </property>  
    <property>  
        <name>yarn.resourcemanager.resource-tracker.address</name>  
        <value>master:8025</value>  
    </property>  
    <property>  
        <name>yarn.resourcemanager.scheduler.address</name>  
        <value>master:8030</value>  
    </property>  
    <property>  
        <name>yarn.resourcemanager.address</name>  
        <value>master:8040</value>  
    </property>  
</configuration>