hadoop生态搭建（3节点）-10.spark配置
# https://www.scala-lang.org/download/2.12.4.html
# ==================================================================安装 scala
tar -zxvf ~/scala-2.12.4.tgz -C /usr/local rm –r ~/scala-2.12.4.tgz
# http://archive.apache.org/dist/spark/spark-2.3.0/
# ==================================================================安装 spark
tar -zxf ~/spark-2.3.0-bin-hadoop2.7.tgz -C /usr/local mv /usr/local/spark-2.3.0-bin-hadoop2.7 /usr/local/spark-2.3.0 rm –r ~/spark-2.3.0-bin-hadoop2.7.tgz
# 环境变量
# ==================================================================node1 node2 node3
vi /etc/profile # 在export PATH USER LOGNAME MAIL HOSTNAME HISTSIZE HISTCONTROL下添加 export JAVA_HOME=/usr/java/jdk1.8.0_111 export ZOOKEEPER_HOME=/usr/local/zookeeper-3.4.12 export HADOOP_HOME=/usr/local/hadoop/hadoop-2.7.6 export MYSQL_HOME=/usr/local/mysql export HBASE_HOME=/usr/local/hbase-1.2.4 export HIVE_HOME=/usr/local/hive-2.1.1 export SCALA_HOME=/usr/local/scala-2.12.4 export KAFKA_HOME=/usr/local/kafka_2.12-0.10.2.1 export FLUME_HOME=/usr/local/flume-1.8.0 export SPARK_HOME=/usr/local/spark-2.3.0 export PATH=$PATH:$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$ZOOKEEPER_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$MYSQL_HOME/bin:$HBASE_HOME/bin:$HIVE_HOME/bin:$SCALA_HOME/bin:$KAFKA_HOME/bin:$FLUME_HOME/bin:$SPARK_HOME/bin:$SPARK_HOME/sbin export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar export HADOOP_INSTALL=$HADOOP_HOME export HADOOP_MAPRED_HOME=$HADOOP_HOME export HADOOP_COMMON_HOME=$HADOOP_HOME export HADOOP_HDFS_HOME=$HADOOP_HOME export YARN_HOME=$HADOOP_HOME export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
# ==================================================================node1
# 使环境变量生效 source /etc/profile # 查看配置结果 echo $SPARK_HOME
# ==================================================================node1
cp $SPARK_HOME/conf/docker.properties.template $SPARK_HOME/conf/docker.properties vi $SPARK_HOME/conf/docker.properties spark.mesos.executor.home: /usr/local/spark-2.3.0 cp $SPARK_HOME/conf/fairscheduler.xml.template $SPARK_HOME/conf/fairscheduler.xml cp $SPARK_HOME/conf/log4j.properties.template $SPARK_HOME/conf/log4j.properties cp $SPARK_HOME/conf/metrics.properties.template $SPARK_HOME/conf/metrics.properties cp $SPARK_HOME/conf/slaves.template $SPARK_HOME/conf/slaves vi $SPARK_HOME/conf/slaves node1 node2 node3 cp $SPARK_HOME/conf/spark-defaults.conf.template $SPARK_HOME/conf/spark-defaults.conf vi $SPARK_HOME/conf/spark-defaults.conf spark.eventLog.enabled true spark.eventLog.dir hdfs://appcluster/spark/eventslog # 监控页面需要监控的目录，需要先启用和指定事件日志目录，配合上面两项使用 spark.history.fs.logDirectory hdfs://appcluster/spark spark.eventLog.compress true # 如果想 YARN ResourceManager 访问 Spark History Server ，则添加一行： # spark.yarn.historyServer.address http://node1:19888 cp $SPARK_HOME/conf/spark-env.sh.template $SPARK_HOME/conf/spark-env.sh vi $SPARK_HOME/conf/spark-env.sh export SPARK_MASTER_PORT=7077 #提交任务的端口，默认是7077 export SPARK_MASTER_WEBUI_PORT=8070 #masster节点的webui端口 默认8080改为8070 export SPARK_WORKER_CORES=1 #每个worker从节点能够支配的core的个数 export SPARK_WORKER_MEMORY=1g #每个worker从节点能够支配的内存数 export SPARK_WORKER_PORT=7078 #每个worker从节点的端口（可选配置） export SPARK_WORKER_WEBUI_PORT=8071 #每个worker从节点的wwebui端口（可选配置） export SPARK_WORKER_INSTANCES=1 #每个worker从节点的实例（可选配置） export JAVA_HOME=/usr/java/jdk1.8.0_111 export SCALA_HOME=/usr/local/scala-2.12.4 export HADOOP_HOME=/usr/local/hadoop-2.7.6 export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop export YARN_CONF_DIR=$HADOOP_HOME/etc/Hadoop export SPARK_PID_DIR=/usr/local/spark-2.3.0/pids export SPARK_LOCAL_DIR=/usr/local/spark-2.3.0/tmp export LD_LIBRARY_PATH=$HADOOP_HOME/lib/native export SPARK_DAEMON_JAVA_OPTS="-Dspark.deploy.recoveryMode=ZOOKEEPER -Dspark.deploy.zookeeper.url=node1:2181,node2:2181,node3:2181 -Dspark.deploy.zookeeper.dir=/spark" vi $SPARK_HOME/sbin/start-master.sh SPARK_MASTER_WEBUI_PORT=8070 cp $HADOOP_HOME/etc/hadoop/hdfs-site.xml $SPARK_HOME/conf/ vi $HADOOP_HOME/etc/hadoop/log4j.properties log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR scp -r $HADOOP_HOME/etc/hadoop/log4j.properties node2:$HADOOP_HOME/etc/hadoop/ scp -r $HADOOP_HOME/etc/hadoop/log4j.properties node3:$HADOOP_HOME/etc/hadoop/
# ==================================================================node1
scp -r $SPARK_HOME node2:/usr/local/ scp -r $SPARK_HOME node3:/usr/local/
# ==================================================================node2 node3
# 使环境变量生效 source /etc/profile # 查看配置结果 echo $FLUME_HOME
# 启动
# ==================================================================node1 node2 node3
# 先启动zookeeper 和 hdfs zkServer.sh start zkServer.sh status # ==================================================================node1 zkCli.sh create /spark '' $HADOOP_HOME/sbin/start-all.sh $HADOOP_HOME/sbin/hadoop-daemon.sh start zkfc # ==================================================================node2 $HADOOP_HOME/sbin/hadoop-daemon.sh start zkfc $HADOOP_HOME/sbin/yarn-daemon.sh start resourcemanager
# 启动spark
# ==================================================================node1 $SPARK_HOME/sbin/start-master.sh $SPARK_HOME/sbin/start-slaves.sh # ==================================================================node2 $SPARK_HOME/sbin/start-master.sh # ==================================================================node1 # 获取安全模式的状态: hdfs dfsadmin -safemode get # 安全模式打开 # hdfs dfsadmin -safemode enter # 安全模式关闭 # hdfs dfsadmin -safemode leave hdfs dfs -mkdir -p /spark/eventslog $SPARK_HOME/bin/spark-shell # http://node1:4040 # http://node1:8070 > :quit
# test
# 需保证hdfs上该目录不存在
# hdfs dfs -mkdir -p /spark/output
# hdfs dfs -rmr /spark/output
vi ~/sparkdata.txt
hello man
what are you doing now
my running
hello
kevin
hi man
hdfs dfs -mkdir -p /usr/file/input
hdfs dfs -put ~/sparkdata.txt /usr/file/input
hdfs dfs -ls /usr/file/input
val file1 = sc.textFile("file:///root/sparkdata.txt")
val count1=file1.flatMap(line => line.split(" ")).map(word => (word,1)).reduceByKey(_+_)
count1.saveAsTextFile("hdfs://node1:8020/spark/output1")
val file=sc.textFile("hdfs://appcluster/usr/file/input/sparkdata.txt")
val count=file.flatMap(line => line.split(" ")).map(word => (word,1)).reduceByKey(_+_)
count.saveAsTextFile("hdfs://node1:8020/spark/output")
hdfs dfs -ls /spark/output
hdfs dfs -cat /spark/output/part-00000
# stop已经启动的进程
# ==================================================================node1 $SPARK_HOME/sbin/stop-slaves.sh $SPARK_HOME/sbin/stop-master.sh $HADOOP_HOME/sbin/stop-all.sh # ==================================================================node1 node2 node3 # 停止 zookeeper zkServer.sh stop # ==================================================================node2 $HADOOP_HOME/sbin/yarn-daemon.sh stop resourcemanager $HADOOP_HOME/sbin/hadoop-daemon.sh stop zkfc # ==================================================================node1 $HADOOP_HOME/sbin/hadoop-daemon.sh stop zkfc shutdown -h now # 快照 spark
