1.安装前准备
安装scala https://www.cnblogs.com/yoyo1216/p/13365935.html
安装java https://www.cnblogs.com/yoyo1216/p/12668926.html
mkdir -p /usr/local/spark/data cd /usr/local/spark
2.下载 http://spark.apache.org/downloads.html
wget https://apache.website-solution.net/spark/spark-2.4.6/spark-2.4.6-bin-hadoop2.6.tgz
3.解压
tar -zvxf spark-1.6.0-cdh5.9.0-src.tar.gz # 解压
4.配置环境变量
vim /etc/profile export SPARK_HOME=/usr/local/spark/spark-2.4.6-bin-hadoop2.6 export PATH=SPARK_HOME/bin:SPARK_HOME/sbin:$PATH source /etc/profile
5.修改配置文件
cd /usr/local/spark/spark-1.6.0-cdh5.9.0/conf
cp spark-env.sh.template spark-env.sh
vim spark-env.sh
export JAVA_HOME=/usr/local/java1.8/jdk1.8.0_241
export SPARK_HOME=/u/usr/local/spark/spark-2.4.6-bin-hadoop2.6
export SPARK_MASTER_IP=192.168.104.94
export SPARK_LOCAL_IP=192.168.104.94
export SPARK_EXECUTOR_MEMORY=1G
export SCALA_HOME=/usr/local/scala/scala-2.13.3
export HADOOP_HOME=/usr/local/hadoop2.6/hadoop-2.6.0-cdh5.9.0
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export SPARK_MASTER_PORT=7077
export SPARK_MASTER_WEBUI_PORT=7070
export SPARK_WORKER_CORES=2
export SPARK_WORKER_MEMORY=1024m
export SPARK_WORKER_INSTANCES=2
export SPARK_YARN_USER_ENV="CLASSPATH=/usr/local/hadoop2.6/hadoop-2.6.0-cdh5.9.0/etc/hadoop"
export SPARK_CLASSPATH=$HBASE_HOME/lib/hbase-protocol-1.2.0-cdh5.9.0.jar:$HBASE_HOME/lib/hbase-common-1.2.0-cdh5.9.0.jar:$HBASE_HOME/lib/htrace-core-3.2.0-incubating.jar:$HBASE_HOME/lib/hbase-server-1.2.0-cdh5.9.0.jar:$HBASE_HOME/lib/hbase-client-1.2.0-cdh5.9.0.jar:$HBASE_HOME/lib/metrics-core-2.2.0.jar:$SPARK_CLASSPATH
export SPARK_LOCAL_DIR="/mnt/spark/tmp"
export YARN_CONF_DIR=$HADOOP_HOME/etc/hadoop
export SPARK_JAVA_OPTS="-Dspark.storage.blockManagerHeartBeatMs=60000-Dspark.local.dir=$SPARK_LOCAL_DIR -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:$SPARK_HOME/logs/gc.log -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=60"
export SPARK_DAEMON_JAVA_OPTS="-Dspark.deploy.recoveryMode=ZOOKEEPER -Dspark.deploy.zookeeper.url=192.168.104.94:2181,192.168.104.96:2181,192.168.104.97:2181 -Dspark.deploy.zookeeper.dir=/spark"
export SPARK_DIST_CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath)
cd /usr/local/spark/spark-1.6.0-cdh5.9.0/conf
cp log4j.properties.template log4j.properties
vim log4j.properties
log4j.logger.org.apache.spark.repl.Main=INFO
6.启动
cd /usr/local/spark/spark-1.6.0-cdh5.9.0/sbin bash start-all.sh # 启动 bash stop-all.sh # 停止 bash spark-daemon.sh start/stop/restart regionserver # 启动/停止/重启
7.查看是否启动成功
jps Master Worker 出现这个两个启动正常
8.Spark监控页面
http://192.168.104.94:7070/
9.验证是否安装成功
cd /usr/local/spark/spark-2.4.6-bin-hadoop2.6/bin ./spark-shell --master local # 本地模式运行 http://192.168.104.94:4040/jobs/ # 监控页面
10.安装过程中的报错
Spark 启动java.lang.NoClassDefFoundError: com/fasterxml/jackson/databind/Module 使用Maven下载以下依赖 : jackson-annotations-2.9.9.jar、 jackson-core-2.9.9.jar、 jackson-databind-2.9.9.jar 放到 $HADOOP_HOME/share/hadoop/common下
