mysql分库分表数据导入hive的sqoop-shell脚本

sqoop常规导入脚本：

#!/bin/bash
#coding=UTF-8

sqoop=/data/cloudera/parcels/CDH-5.14.4-1.cdh5.14.4.p0.3/bin/sqoop

$sqoop import \
-Dhadoop.security.credential.provider.path=jceks://hdfs/nameservice1/conf/ps/dev181ps \
--connect jdbc:mysql://ip:3306/mall_order0 \
--table orders_0 \
--username root \
--password-alias dev181ps \
--delete-target-dir \
--target-dir /warehouse/malls/ods/ods_mall_order

 

sqoop导入按天分表数据：

#!/bin/bash
start=$1
end=$2
order_table=media_daily_enter_ 
startDate=`date -d "${start}" +%s`
endDate=`date -d "${end}" +%s`
##计算两个时间戳的差值除于每天86400s即为天数差
stampDiff=`expr $endDate - $startDate`
dayDiff=`expr $stampDiff / 86400`

##根据日期循环输出日期
for ((i=0;i<$dayDiff;i++)); do
process_date=`date -d "${start} $i day" +'%Y%m%d'`
echo ${order_table}$process_date

sqoop=/data/cloudera/parcels/CDH-5.14.4-1.cdh5.14.4.p0.3/bin/sqoop
$sqoop import \
-Dhadoop.security.credential.provider.path=jceks://hdfs/nameservice1/conf/ps/dev181ps \
--connect jdbc:mysql://ip:3306/mall_order0 \
--table ${order_table}$process_date \
--username root \
--password-alias dev181ps \
--delete-target-dir \
--target-dir /warehouse/malls/ods/ods_mall_order

done

 

echo结果示例：

[root@atone test_sh]# sh sub_tables.sh 20220501 20220601
media_daily_enter_20220501
media_daily_enter_20220502
media_daily_enter_20220503
media_daily_enter_20220504
media_daily_enter_20220505
media_daily_enter_20220506
media_daily_enter_20220507
media_daily_enter_20220508
media_daily_enter_20220509
media_daily_enter_20220510
media_daily_enter_20220511
media_daily_enter_20220512
media_daily_enter_20220513
media_daily_enter_20220514
media_daily_enter_20220515
media_daily_enter_20220516
media_daily_enter_20220517
media_daily_enter_20220518
media_daily_enter_20220519
media_daily_enter_20220520
media_daily_enter_20220521
media_daily_enter_20220522
media_daily_enter_20220523
media_daily_enter_20220524
media_daily_enter_20220525
media_daily_enter_20220526
media_daily_enter_20220527
media_daily_enter_20220528
media_daily_enter_20220529
media_daily_enter_20220530
media_daily_enter_20220531

　　

参考：

sqoop分库分表全量导入脚本