hadoop伪分布式部署

官网信息

# 官网
https://hadoop.apache.org/

# 指定版本文档
https://hadoop.apache.org/docs/r3.3.6/

# 单机部署文档
https://hadoop.apache.org/docs/r3.3.6/hadoop-project-dist/hadoop-common/SingleCluster.html

# 集群部署文档
https://hadoop.apache.org/docs/r3.3.6/hadoop-project-dist/hadoop-common/ClusterSetup.html

# 官网下载hadoop-3.1.0安装包
https://archive.apache.org/dist/hadoop/common/hadoop-3.1.0/hadoop-3.1.0.tar.gz

# 官网hadoop-3.1.0文档
https://hadoop.apache.org/docs/r3.1.0/hadoop-project-dist/hadoop-common/SingleCluster.html#Pseudo-Distributed_Operation

部署规划

// yarn的nodeManager节点不用指定，datanode的节点都会存在nodeManager
          namenode    secondNameNode    datanode
node1     *           *                 *

使用hadoop-3.1.0版本

新建操作用户

#添加hadoop用户
useradd hadoop

#切换到hadoop用户操作，设置hadoop用户免密登录
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 0600 ~/.ssh/authorized_keys

#测试hadoop用户可以免密登录
切换到hadoop用户操作
ssh localhost直接登录成功

1、配置环境变量/etc/profile

export HADOOP_HOME=/opt/hadoop/hadoop-2.10.1
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
source /etc/profile

2、修改配置etc/hadoop/hadoop-env.sh

# set to the root of your Java installation
export JAVA_HOME=jdk地址

3、修改配置etc/hadoop/core-site.xml

<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://node1:9000</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>设置数据存储目录</value>
    </property>

    <!--配置root(超级用户)允许通过代理访问的主机节点-->
    <property>
        <name>hadoop.proxyuser.root.hosts</name>
        <value>*</value>
    </property>
    <property>
        <name>hadoop.proxyuser.root.groups</name>
        <value>*</value>
    </property>
    <property>
        <name>hadoop.proxyuser.root.user</name>
        <value>*</value>
    </property>

    <property>
 　　    <name>dfs.permissions.enabled</name>
 　　    <value>false</value>
 　　    <description>如果为"true"，则在HDFS中启用权限检查;如果为"false"，则关闭权限检查;默认值为"true"。</description>
    </property>
</configuration>

4、修改配置etc/hadoop/hdfs-site.xml

<configuration>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
    <property>
        <name>dfs.namenode.secondary.http-address</name>
        <value>node1:50090</value>
    </property>
</configuration>

5、修改配置etc/hadoop/slaves

node1

6、格式化namenode

bin/hdfs namenode -format

7、启动、停止 hdfs

sbin/start-dfs.sh
sbin/stop-dfs.sh

8、hadoop访问界面

http://IP:9870