Hadoop安装
Hadoop安装步骤
1.安装JDK
2.下载并解压hadoop安装包,免密通信,/etc/hosts文件
3.修改配置文件
hadoop-env.sh
export JAVA_HOME=/home/buyabs.corp/biadmin/java/jdk8
export HADOOP_PID_DIR=/home/buyabs.corp/biadmin/hadoop/tmp
mapred-env.sh:
#进程号保存地址
export HADOOP_MAPRED_PID_DIR=/home/buyabs.corp/biadmin/hadoop/tmp
yarn-env.sh:
#进程号保存地址
export YARN_PID_DIR=/home/buyabs.corp/biadmin/hadoop/tmp
slaves:
hadoop-env.sh:
# The java implementation to use.
export JAVA_HOME=/home/buyabs.corp/biadmin/java/jdk8
hdfs-site.xml:
<configuration>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
<property>
<name>dis.namenode.name.dir</name>
<value>/home/buyabs.corp/biadmin/hadoop/name</value>
</property>
<property>
<name>dis.datanode.data.dir</name>
<value>/home/buyabs.corp/biadmin/hadoop/data</value>
</property>
<property>
<name>dfs.secondary.http.address</name>
<value>127.0.0.1:50090</value>
</property>
</configuration>
core-site.xml:
<configuration>
<property>
<name>fs.default.name</name>
<value>hdfs://scc1bi02:9000</value>
</property>
<property>
<name>hadoop.temp.dir</name>
<value>/home/buyabs.corp/biadmin/hadoop/tmp</value>
</property>
</configuration>
yarn-site.xml
<configuration>
<!-- Site specific YARN configuration properties -->
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<property>
<name>yarn.log-aggregation-enable</name>
<value>true</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address</name>
<value>scc1bi02:18088</value>
</property>
<property>
<name>yarn.resourcemanager.hostname</name>
<value>scc1bi02</value>
</property>
</configuration>
mapred-site.xml:
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
</configuration>
环境变量 vi /etc/profile:
export JAVA_HOME=/home/buyabs.corp/biadmin/java/jdk8
export HADOOP_HOME=/home/buyabs.corp/biadmin/hadoop/hadoop277
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
export HADOOP_PREFIX=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_PREFIX
export HADOOP_CONF_DIR=$HADOOP_PREFIX/etc/hadoop
export HADOOP_HDFS_HOME=$HADOOP_PREFIX
export HADOOP_MAPRED_HOME=$HADOOP_PREFIX
export HADOOP_YARN_HOME=$HADOOP_PREFIX