spark单机部署

https://spark.apache.org/docs/2.2.0/

 

wget https://archive.apache.org/dist/spark/spark-2.2.0/spark-2.2.0-bin-hadoop2.7.tgz

 

tar xf spark-2.2.0-bin-hadoop2.7.tgz

 

ln -s /home/centos/spark-2.2.0-bin-hadoop2.7 spark

 

修改conf/spark-env.sh

export JAVA_HOME=/usr/java/jdk1.8.0_221

export SPARK_LOCAL_IP=192.168.xxx.xxx

export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HADOOP_HOME/lib/native/

export SPARK_DIST_CLASSPATH=$(/home/centos/hadoop/bin/hadoop classpath)

export HADOOP_HOME=/home/centos/hadoop

export HADOOP_CONF=$HADOOP_HOME/etc

export YARN_CONF=$HADOOP_HOME/etc

 

启动spark

 

bin/run-example SparkPi 2>&1 |grep "Pi is roughly"

Pi is roughly 3.1406957034785172