数据相关脚本
#!/bin/sh . /home/db2inst1/.profile #数据上报程序 #author huodaihao #获得系统 DATE=$(date +%Y%m%d) DATE_TIME=$(date +'%Y%m%d%H%M%S') ORG="Z2017112000011" DBNAME="pbctjuat" USERNAME="db2admin" PASSWORD="db2@admin" #shell文件目录 SHELL_PATH=/home/tianjing/sh #csv文件基本路径 CSV_PATH=/home/tianjing/csv #csv文件路径永久路径 CSV_SAVE_PATH=/home/tianjing/savecsv #日志存放路径 LOG_PATH=/home/tianjing/log #定义一个正在执行 TMPFILE=/home/tianjing/sh/shell_csv.tmp #检查是否有其他的实 if [ -e ${TMPFILE} ] then echo "Other instance is running!" exit 0 #退出本脚本的执行 else touch ${TMPFILE} #新建一个临时文件 chmod 600 ${TMPFILE} fi #设置一个 rap "rm -f ${TMPFILE}; exit" 0 1 2 3 9 15 echo "`date +%Y-%m-%d\ %T` ------------START----------">>${LOG_PATH}/load_data_${DATE}.log #连接数据库 输出日志信息 db2 connect to $DBNAME user $USERNAME using $PASSWORD echo "`date +%Y-%m-%d\ %T` -清空ods_temp下的T_SC_MERCHANT_INFO,T_SC_MERCHANT_IM_DETAIL,T_SC_MERCHANT_EX_DETAIL,T_RBYHZHXX表的数据">>${LOG_PATH}/load_data_${DATE}.log db2 "delete from ODS_TEMP.T_SC_MERCHANT_INFO " db2 "delete from ODS_TEMP.T_SC_MERCHANT_IM_DETAIL " db2 "delete from ODS_TEMP.T_SC_MERCHANT_EX_DETAIL " db2 "delete from ODS_TEMP.T_RBYHZHXX " db2 commit for file_all_name in ${CSV_PATH}/*.csv do echo "$file_all_name --------------" #获取csv文件名称 CSV_NAME=$(basename "$file_all_name") echo "$CSV_NAME----" echo "`date +%Y-%m-%d\ %T` csv文件的文件名称是:${CSV_NAME}">>${LOG_PATH}/load_data_${DATE}.log # gbk --> utf-8 if file "${file_all_name}"|grep -q UTF-8 >/dev/null ; then echo "csv is utf-8" >> ${LOG_PATH}/load_data_${DATE}.log else echo "csv is gbk to utf-8 success" >> ${LOG_PATH}/load_data_${DATE}.log iconv -c -f gbk -t utf-8 ${file_all_name}> ${LOG_PATH}/temp_${CSV_NAME} rm -f ${file_all_name} mv ${LOG_PATH}/temp_${CSV_NAME} ${CSV_PATH}/${CSV_NAME} fi #截取1个字 FLAG=${CSV_NAME:0:6} ORG_CODE=${CSV_NAME:9:14} LOAD_DATE=${CSV_NAME:24:8} #商户基本信息表 TEMPLATE_FILE_ID1='20160813173727372741975374175096280' #商户入金支付明细 TEMPLATE_FILE_ID2='16081318351835185705507814252728000' #商户出金支付明细 TEMPLATE_FILE_ID3='16081318360836823267029553581420000' #融宝商户对应账号的信息 TEMPLATE_FILE_ID4='16081318365236529830235585519220000' sed -i '1d' $file_all_name sed -i "s/\//-/g" $file_all_name if [ $FLAG = "SSJBXX" ] then db2 "update DPS.T_SYS_COLLECTION_LOG set VERIFY_STATUS=1 where FISCAL_TERM='${LOAD_DATE}' and ORG_ID='${ORG_CODE}' and TEMPLATE_FILE_ID='${TEMPLATE_FILE_ID1}'" echo "`date +%Y-%m-%d\ %T` load 商户基本信息数据">>${LOG_PATH}/load_data_${DATE}.log echo "1111111111111111" #商户基本信息 db2 "LOAD client FROM '${file_all_name}' OF DEL MODIFIED BY CODEPAGE=1208 COLDEL, DECPLUSBLANK DATESISO INSERT INTO ODS_TEMP.T_SC_MERCHANT_INFO NONRECOVERABLE " db2 commit fi if [ $FLAG = "SHRJMX" ] then db2 "update DPS.T_SYS_COLLECTION_LOG set VERIFY_STATUS=1 where FISCAL_TERM='${LOAD_DATE}' and ORG_ID='${ORG_CODE}' and TEMPLATE_FILE_ID='${TEMPLATE_FILE_ID2}'" echo "`date +%Y-%m-%d\ %T` load 入金支付明细">>${LOG_PATH}/load_data_${DATE}.log #入金支付明细 db2 "LOAD client FROM '${file_all_name}' OF DEL MODIFIED BY CODEPAGE=1208 COLDEL, DECPLUSBLANK DATESISO INSERT INTO ODS_TEMP.T_SC_MERCHANT_IM_DETAIL NONRECOVERABLE" db2 commit fi if [ $FLAG = "SSCJMX" ] then db2 "update DPS.T_SYS_COLLECTION_LOG set VERIFY_STATUS=1 where FISCAL_TERM='${LOAD_DATE}' and ORG_ID='${ORG_CODE}' and TEMPLATE_FILE_ID='${TEMPLATE_FILE_ID3}'" echo "`date +%Y-%m-%d\ %T` load 出金支付">>${LOG_PATH}/load_data_${DATE}.log #出金支付明细 db2 "LOAD client FROM '${file_all_name}' OF DEL MODIFIED BY CODEPAGE=1208 COLDEL, DECPLUSBLANK DATESISO INSERT INTO ODS_TEMP.T_SC_MERCHANT_EX_DETAIL NONRECOVERABLE" db2 commit fi if [ $FLAG = "RBYHXX" ] then db2 "update DPS.T_SYS_COLLECTION_LOG set VERIFY_STATUS=1 where FISCAL_TERM='${LOAD_DATE}' and ORG_ID='${ORG_CODE}' and TEMPLATE_FILE_ID='${TEMPLATE_FILE_ID4}'" echo "`date +%Y-%m-%d\ %T` load荣宝信息">>${LOG_PATH}/load_data_${DATE}.log #融宝银行账户信息 db2 "LOAD client FROM '${CSV_PATH}/${CSV_NAME}' OF DEL MODIFIED BY CODEPAGE=1208 COLDEL, DECPLUSBLANK DATESISO INSERT INTO ODS_TEMP.T_RBYHZHXX NONRECOVERABLE" db2 commit fi mv -f ${CSV_PATH}/${CSV_NAME} ${CSV_SAVE_PATH} done echo "`date +%Y-%m-%d\ %T` load ods_temp success">>${LOG_PATH}/load_data_${DATE}.log db2 "update DPS.T_SYS_COLLECTION_LOG set VERIFY_STATUS=1 where FISCAL_TERM='${LOAD_DATE}' and ORG_ID='${ORG_CODE}' and TEMPLATE_FILE_ID='${TEMPLATE_FILE_ID1}'" db2 "update DPS.T_SYS_COLLECTION_LOG set VERIFY_STATUS=1 where FISCAL_TERM='${LOAD_DATE}' and ORG_ID='${ORG_CODE}' and TEMPLATE_FILE_ID='${TEMPLATE_FILE_ID2}'" db2 "update DPS.T_SYS_COLLECTION_LOG set VERIFY_STATUS=1 where FISCAL_TERM='${LOAD_DATE}' and ORG_ID='${ORG_CODE}' and TEMPLATE_FILE_ID='${TEMPLATE_FILE_ID3}'" db2 "update DPS.T_SYS_COLLECTION_LOG set VERIFY_STATUS=1 where FISCAL_TERM='${LOAD_DATE}' and ORG_ID='${ORG_CODE}' and TEMPLATE_FILE_ID='${TEMPLATE_FILE_ID4}'" db2 "call ODS.P_PAY_TYSHXX_WLJYMX('${LOAD_DATE}','${ORG_CODE}')" echo "`date +%Y-%m-%d\ %T` load ods success">>${LOG_PATH}/load_data_${DATE}.log db2 "call DW.P_PAY_TYSHXX('${LOAD_DATE}','${ORG_CODE}','','')" db2 "call DW.P_PAY_WLJYMX('${LOAD_DATE}','${ORG_CODE}','','')" echo "`date +%Y-%m-%d\ %T` load DW success">>${LOG_PATH}/load_data_${DATE}.log db2 commit; db2 terminate; rm -f ${TMPFILE}
#!/bin/sh #---------------------------------------------------- #功能:将文件上传到FTP文件服务器 # #时间:2015-03-22 #---------------------------------------------------- #FTP地址 IP_ADDRESS="192.168.0.111" #FTP端存放目录 REMOTE_PATH="/load_data/files/test_data" #shell存放目录 BASE_PATH="/load_shell" #本地需要上传的文件存放目录 LOCAL_PATH="/home/pbc-file-maxfiles" #本地文件上传后备份存放目录 LOCAL_ZIP_PATH="/home/pbc-file-maxfiles/data_bak" #FTPP用户名 USERNAME="db2inst1" #FTP密码 PASSWORD="db2inst1" #获取系统时间到天 DATE=$(date +%Y%m%d) #echo $DATE echo "`date +%Y-%m-%d\ %T` : ---------------------------START----------------------------------" >>$BASE_PATH/log/ftp_up_$DATE.log echo "`date +%Y-%m-%d\ %T` : 开始上传文件" >>$BASE_PATH/log/ftp_up_$DATE.log ftp -in <<EOF open $IP_ADDRESS user $USERNAME $PASSWORD binary cd $REMOTE_PATH lcd $LOCAL_PATH espv4 off passive mput "*.csv" bye EOF echo "`date +%Y-%m-%d\ %T` : 上传文件结束" >>$BASE_PATH/log/ftp_up_$DATE.log #获取系统时间到天 #DATE=$(date +%Y%m) echo "`date +%Y-%m-%d\ %T` : 准备移动文件到 ${LOCAL_ZIP_PATH} 目录 " >>$BASE_PATH/log/ftp_up_$DATE.log for file_all_zip_name in ${LOCAL_PATH}/*;do ZIP_FILE_NAME=$(basename "$file_all_zip_name") # temp_zip_file=$file_all_zip_name #截取文件名 TABLE_NAME_STR=${ZIP_FILE_NAME:0:4} ORG=${ZIP_FILE_NAME:4:14} DAY=${ZIP_FILE_NAME:18:8} MONTH=${ZIP_FILE_NAME:18:6} FILE_NAME=${ZIP_FILE_NAME:0:26} if [ ! -d "$LOCAL_ZIP_PATH/$MONTH" ]; then mkdir $LOCAL_ZIP_PATH/$MONTH fi # echo "`date +%Y-%m-%d\ %T` : 移动文件 ${ZIP_FILE_NAME}" >>$BASE_PATH/log/ftp_up_$DATE.log #将解压完的文件从临时文件夹移到永久保存文件夹 mv -f $temp_zip_file $LOCAL_ZIP_PATH/$MONTH done echo "`date +%Y-%m-%d\ %T` : 文件移动完毕" >>$BASE_PATH/log/ftp_up_$DATE.log #cd $LOCAL_ZIP_PATH #if [ ! -d "$LOCAL_ZIP_PATH/$DATE" ]; # then # mkdir $LOCAL_ZIP_PATH/$DATE #fi #mkdir $DATE #将解压完的文件从临时文件夹移到永久保存文件夹 #mv -f $LOCAL_PATH/* $LOCAL_ZIP_PATH/$DATE echo "`date +%Y-%m-%d\ %T` : ---------------------------END----------------------------------" >>$BASE_PATH/log/ftp_up_$DATE.log