Dream------spark--spark集群的环境搭建

1、下载安装scala

http://www.scala-lang.org/download/2.11.6.html

 

2、解压下载后的文件，配置环境变量：编辑/etc/profile文件，添加如下内容：

 

export SCALA_HOME=/root/scala-2.11.6

export PATH=$SCALA_HOME/bin:$PATH

 

运行source /etc/profile使环境变量的修改立即生效

 

3、配置conf/spark-env.sh环境变量

 

进入$SPARK_HOME/conf 目录， 将 spark-env.sh.template 重命名为 spark-env.sh，修改 spark-env.sh，添加如下内容

 

export JAVA_HOME=/usr/java/jdk1.8.0_45

     export SPARK_MASTER_IP=wls01

     #export SPARK_MASTER_IP=localhost

     export SPARK_MASTER_PORT=7077

     export SPARK_WORKER_CORES=1

     export SPARK_WORKER_INSTANCES=1

     export SPARK_WORKER_MEMORY=1g

　　export HADOOP_HOMEE=/root/hadoop-2.6.0

     #export SPARK_DAEMON_JAVA_OPTS="-Dspark.deploy.recoveryMode=FILESYSTEM -Dspark.deploy.recoveryDirectory=/nfs/spark/recovery"

   

     export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop

     export YARN_CONF_DIR=$HADOOP_HOME/etc/hadoop

     

     export SPARK_HOME=/root/spark-1.4.1-bin-hadoop2.6

     export SPARK_JAR=/root/spark-1.4.1-bin-hadoop2.6/lib/spark-assembly-1.4.1-hadoop2.6.0.jar

     export PATH=$SPARK_HOME/bin:$PATH

 

 

4、 修改/conf/slaves文件

wls02

wls03

 

5、拷贝到其他机器

 

scp scala-2.11.6.tgz root@wls02:/root

scp scala-2.11.6.tgz root@wls03:/root

scp spark-1.4.1-bin-hadoop2.6.zip root@wls03:/root

scp spark-1.4.1-bin-hadoop2.6.zip root@wls02:/root

scp spark-1.4.1-bin-hadoop2.6/conf/* root@wls02:/root/spark-1.4.1-bin-hadoop2.6/conf/

 

 

6、修改权限(不修改貌似也行，因为本身就是可执行文件)

bin目录下

chmod a+x spark-shell

chmod a+x spark-submit

chmod a+x spark-class

chmod a+x compute-classpath.sh

 

sbin目录下

chmod a+x *.sh

 

7、启动

cd到sbin目录下

 

./start-all.sh

 

 

运行（集群模式，其他模式官网写的很清楚，直接运行就可以）

重点就是class，是自己的类，jars有mysql的驱动和项目打的jar包

 

　　（官网的例子）

./bin/spark-submit --class org.apache.spark.examples.SparkPi \
    --master yarn-cluster \
    --num-executors 3 \
    --driver-memory 4g \
    --executor-memory 2g \
    --executor-cores 1 \
    --queue thequeue \
    lib/spark-examples*.jar \
    10

（自己写的推荐）

./bin/spark-submit --class \
com.tuijian.SparkToJDBC \
--master yarn-cluster \
--num-executors 3 \
--driver-memory 1g \
--executor-memory 1g \
--executor-cores 1 \
--jars /root/founderRecommend.jar \
lib/mysql*.jar

 

wls01:8080

 

这样也行

# /etc/profile

 

# System wide environment and startup programs, for login setup

# Functions and aliases go in /etc/bashrc

###########java

export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/protobuf/lib

export JAVA_HOME=/usr/soft/jdk1.7.0_71

PATH=$PATH:$JAVA_HOME/bin

#CLASSPATH=.:$JAVA_HOME/lib:$JAVA_HOME/jre/lib:/usr/soft/jdk1.7.0_71/lib/tools.jar

export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

#############hadoop 

export HADOOP_INSTALL=/usr/hadoopsoft/hadoop-2.5.2

export HADOOP_HOME=$HADOOP_INSTALL

 

export HADOOP_MAPRED_HOME=$HADOOP_INSTALL

export HADOOP_COMMON_HOME=$HADOOP_INSTALL

export HADOOP_HDFS_HOME=$HADOOP_INSTALL

export YARN_HOME=$HADOOP_INSTALL

export HADOOP_CONF_DIR=$HADOOP_INSTALL/etc/hadoop

export PATH=$PATH:$HADOOP_INSTALL/bin

export PATH=$PATH:$HADOOP_INSTALL/sbin

 

###########scala

SCALA_HOME=/usr/soft/scala-2.10.4

MVN_HOME=/usr/soft/apache-maven-3.1.1

ANT_HOME=/usr/soft/apache-ant-1.9.4

MAHOUT_HOME=/root/mahout-mahout-0.9

 

PATH=$PATH:$SCALA_HOME/bin:$MVN_HOME/bin:$ANT_HOME/bin:$MAHOUT_HOME/bin

export FINDBUGS_HOME=/usr/soft/findbugs-3.0.0

PATH=$PATH:$FINDBUGS_HOME/bin

 

export PATH

export MAVEN_OPTS="-Xmx2g -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=512m"

###############################

export SPARK_HOME=/usr/hadoopsoft/spark-1.3.1-bin-hadoop2.4

export MASTER=spark://spark001:7077

export SPARK_JAR=/usr/hadoopsoft/spark-1.3.1-bin-hadoop2.4/lib/spark-assembly-1.3.1-hadoop2.4.0.jar

export PATH=$SPARK_HOME/bin:$PATH

 

 

 

 #It's NOT a good idea to change this file unless you know what you

# are doing. It's much better to create a custom.sh shell script in

# /etc/profile.d/ to make custom changes to your environment, as this

# will prevent the need for merging in future updates.

 

pathmunge () {

    case ":${PATH}:" in

        *:"$1":*)

            ;;

        *)

            if [ "$2" = "after" ] ; then

                PATH=$PATH:$1

            else

                PATH=$1:$PATH

            fi

    esac

}

 

 

if [ -x /usr/bin/id ]; then

    if [ -z "$EUID" ]; then

        # ksh workaround

        EUID=`id -u`

        UID=`id -ru`

    fi

    USER="`id -un`"

    LOGNAME=$USER

    MAIL="/var/spool/mail/$USER"

fi

 

# Path manipulation

if [ "$EUID" = "0" ]; then

    pathmunge /sbin

    pathmunge /usr/sbin

    pathmunge /usr/local/sbin

else

    pathmunge /usr/local/sbin after

    pathmunge /usr/sbin after

    pathmunge /sbin after

fi

 

HOSTNAME=`/bin/hostname 2>/dev/null`

HISTSIZE=1000

if [ "$HISTCONTROL" = "ignorespace" ] ; then

    export HISTCONTROL=ignoreboth

else

    export HISTCONTROL=ignoredups

fi

 

export PATH USER LOGNAME MAIL HOSTNAME HISTSIZE HISTCONTROL

 

# By default, we want umask to get set. This sets it for login shell

# Current threshold for system reserved uid/gids is 200

# You could check uidgid reservation validity in

# /usr/share/doc/setup-*/uidgid file

if [ $UID -gt 199 ] && [ "`id -gn`" = "`id -un`" ]; then

    umask 002

else

    umask 022

fi

 

for i in /etc/profile.d/*.sh ; do

    if [ -r "$i" ]; then

        if [ "${-#*i}" != "$-" ]; then

            . "$i"

        else

            . "$i" >/dev/null 2>&1

        fi

    fi

done

 

unset i

unset -f pathmunge