Spark hive to ES Elasticsearch

#!/bin/bash
date=`date  -d "-1 day"  +%F`
spark-submit \
--class ****** \
--name "*** ***" \
--master yarn \
--deploy-mode client \
--num-executors 30 \
--driver-memory 7g \
--executor-memory 2g \
--executor-cores 2 \
--conf spark.default.parallelism=200 \
--conf spark.sql.shuffle.partitions=200 \
--queue batch \
ossref://***.jar "select * from table where dt = '$date' " 'index name' $date

 

posted @ 2022-03-02 09:50  cup_leo  阅读(66)  评论(0编辑  收藏  举报