test

export SPARK_CLASSPATH=$SPARK_CLASSPATH:/home/ocdc/app/spark-1.0.1-bin-2.0.0-cdh4.2.1/lib/mysql-connector-java-5.1.13.jar:/home/ocdc/app/hadoop-2.0.0-cdh4.2.1/lib/hadoop-lzo-0.4.15.jar
export JAVA_HOME=/home/ocdc/app/jdk1.7.0_55
export SPARK_MASTER_IP=oc-etl-name-02
export SPARK_MASTER_PORT=9020
export SPARK_MASTER_WEBUI_PORT=9021
export SPARK_WORKER_CORES=1
export SPARK_WORKER_INSTANCES=2
export SPARK_WORKER_MEMORY=6g
export HADOOP_CONF_DIR=/home/ocdc/app/hadoop-2.0.0-cdh4.2.1/etc/hadoop
export SPARK_JAVA_OPTS="$SPARK_JAVA_OPTS -Dspark.ui.port=8080 -Djava.library.path=/home/ocdc/app/hadoop-2.0.0-cdh4.2.1/lib/native/"
export MASTER=spark://oc-etl-name-02:9020

 

 

shark:

lzo mysql放到$SHARK_HOME/lib下

export SPARK_MEM=1g

HADOOP_HOME=/home/ocdc/app/hadoop-2.0.0-mr1-cdh4.2.1

export MASTER=spark://oc-etl-data-10:9020

export SPARK_HOME=/home/ocdc/app/spark-hadoop_2.0.0-mr1-cdh4.2.1

export SPARK_MEM=2g

export SPARK_JAVA_OPTS="$SPARK_JAVA_OPTS -Dspark.ui.port=8080"

 

sharkserver:  nohup ./shark --service sharkserver2 &

beeline :   ./beeline -u jdbc:hive2://oc-etl-data-10:10002 -n ocdc -p ocdc3.14

 

生产环境ssh使用方式：

alias  ss=ssh
ss oc-etl-data-10

 

./deploy.sh  ~/app/amplab-shark/ ~/app sparksql deploy.conf

 

./deploy.sh  ~/app/amplab-shark/conf/shark-env.sh  ~/app/amplab-shark/conf/ sparksql deploy.conf

./deploy.sh  ~/app/spark-hadoop_2.0.0-mr1-cdh4.2.1/ ~/app sparksql deploy.conf

select count(*) from vgop_st_wlan_hot_yyyymm