macos下安装单机版hadoop

mac单机

brew install hadoop

Configure

默认配置目录为/usr/local/cellar/hadoop/3.3.0/libexec/etc/hadoop/

hadoop-env.sh

# 放开并设置
export JAVA_HOME="/Library/Java/JavaVirtualMachines/adoptopenjdk-11.jdk/Contents/Home"
export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true -Dsun.security.krb5.debug=true -Dsun.security.spnego.debug"

core-site.xml

<configuration>
  <property>
    <name>fs.defaultFS</name>
    <value>hdfs://localhost:9000</value>
  </property>
</configuration>

hdfs-site.xml

<configuration>
  <property>
    <name>dfs.replication</name>
    <value>1</value>
  </property>
</configuration>

mapred-site.xml

<configuration>
  <property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
  </property>
  <property>
      <name>mapreduce.application.classpath</name>   
      <value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
  </property>
</configuration>

yarn-site.xml

<configuration>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.nodemanager.env-whitelist</name>
        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
    </property>
</configuration>

ssh

开启ssh服务

ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa    # 生成ID
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys    # 配置证书登录
ssh localhost      # 测试登录

Format NameNode

# /usr/local/Cellar/hadoop/3.3.0/bin
hdfs namenode -format

start

# /usr/local/Cellar/hadoop/3.3.0/sbin
./start-all.sh
jps

http://localhost:9870 查看当前的配置

stop

./stop-all.sh