spark 提交

#!/bin/bash
num_executors=100
executor_memory=4g
executor_cores=20
principal=BotSpider
queue=DB_QUEUE
keytab=/opt/keys/user.keytab
python_path="hdfs://deng/python-3.7.3.tgz#python-3.7.3"
app_name=deng_app

spark-submit \
--master yarn \
--deploy-mode cluster \
--executor-memory $executor_memory \
--executor-cores $executor_cores \
--nums-executors $num_executors \
--driver-cores 4 \
--queue $queue \
--driver-memory 5g \
--keytab $keytab \
--principal $principal \
--name $app_name \
--conf spark.yarn.maxApppAttempts=1 \
--conf spark.speculation=true \
--conf.spark.speculation.interval=6000 \
--conf spark.speculation.quantile=0.85 \
--conf spark.speculaiton.multiplier=10 \
--conf spark.pyspark.driver.python="./python-3.7.3/bin/python" \
--conf spark.python="./python-3.7.3/bin/python" \
code.py