echo "export SPARK_HOME='/usr/local/Cellar/apache-spark/1.5.2/libexec/'" >> ~/.bashrc
ipython profile create pyspark
touch ~/.ipython/profile_pyspark/startup/00-spark-setup.py
1 import os
2 import sys
3
4 spark_home = os.environ.get('SPARK_HOME', None)
5 sys.path.insert(0, os.path.join(spark_home, 'python'))
6 sys.path.insert(0, os.path.join(spark_home, 'python/lib/py4j-0.8.2.1-src.zip'))
7 execfile(os.path.join(spark_home, 'python/pyspark/shell.py'))
ipython --profile=spark
#Java gateway process exited before sending the driver its port number
export PYSPARK_SUBMIT_ARGS="--master local[2] pyspark-shell"
