1 #! /bin/bash
2
3 #/python2292/CNEDMp/ETLscript/CNEDMp_etl_v1.0/testhis
4 LOAD_PATH=$(cd $(dirname $0); pwd)
5 #/python2292/CNEDMp/ETLscript/CNEDMp_etl_v1.0
6 DEPLOY_PATH=$(cd $(dirname $LOAD_PATH); pwd)
7 #/python2292/CNEDMp/ETLscript
8 BASE_PATH=$(cd "$(dirname "$DEPLOY_PATH")"; pwd)
9 #/python2292/CNEDMp
10 MAIN_PATH=$(cd "$(dirname "$BASE_PATH")"; pwd)
11
12 FILE_LANDING=$DEPLOY_PATH'/File_Landing.sh'
13 LOAD_HDFS=$DEPLOY_PATH'/Load_hdfs.sh'
14 LOAD_HIVE=$DEPLOY_PATH'/Load_hive.sh'
15 LANDING_PATH=$MAIN_PATH'/landing/'
16 JAR_FILE=$DEPLOY_PATH'/dailydata-1.0-SNAPSHOT.jar'
17 INI_FILE=$DEPLOY_PATH'/cnedmp_etl.ini'
18 LOG_FILE=$DEPLOY_PATH'/shellrun.log'
19
20 CODING='GB2312'
21
22 #Funtion for Log
23 function logging(){
24 logTime=$(date +"%Y-%m-%d %T")
25 echo "$logTime $1 $2"
26 }
27
28
29 source $INI_FILE
30
31 cd $LOAD_PATH
32 rm -r dataspace
33 mkdir dataspace
34 cd dataspace
35 ziplist=$(cat $LOAD_PATH/ziplist.txt)
36 logging INFO "Get zip file list $ziplist"
37 for zipname in $ziplist
38 do
39 logging INFO "-------------------Handling zipfile $zipname----------------------------"
40 cp /home/infra1/jason/DataFiles/$zipname ./
41 foldername=$(echo $zipname|awk -F . '{print $1}')
42 echo $foldername
43 mkdir $foldername
44 unzip -d ./$foldername $zipname
45 cd $foldername
46 partition=$foldername
47 filelist=$(cat $LOAD_PATH/filelist.txt)
48 logging INFO "Get file list $filelist"
49 for filename in $filelist
50 do
51 cp $filename $LANDING_PATH && logging INFO "$filename copied to $LANDING_PATH"
52 done
53 cd ..
54 sh $LOAD_PATH/Loading_Meta_Init_history.sh $partition
55 sh $LOAD_PATH/schedule_history.sh $partition
56 cd $LOAD_PATH
57 rm -r $LOAD_PATH/dataspace
58 logging INFO "-------------------zipfile $zipname finished----------------------------"
59 done
