shell diff + kettle = mail 预警
1 #!bin/bash 2 kettleConfigHome=/home/vmuser/linbo/kettleDemo/job/config 3 apiInfos=/home/vmuser/linbo/kettleDemo/job/data/apiInfo 4 api_csv_path_home=/home/vmuser/linbo/test_upload/data_file 5 sendMailData=/home/vmuser/linbo/kettleDemo/job/sendData 6 dt=`date +"%Y%m%d"` 7 lastDay=`date -d "1 day ago" +"%Y%m%d"` 8 dmt=`date -d "1 day ago" +"%Y-%m-%d"` 9 declare -A diffResult_eq 10 declare -A diffResult_nt 11 #先将pord 接口生成的文件load test的情况告知相关人 ; 12 sh /home/vmuser/linbo/kettleDemo/job/execSH/fileCpCase.sh 13 apiCnt=(`ls -rt $api_csv_path_home`) 14 #apiCnt=(unilever_three_shop_sales_retrun) 15 for((i=0;i<${#apiCnt[*]} ;i++)); 16 do 17 strInfo="" 18 #默认不发送邮件; 19 sendMail=false 20 apiName=${apiCnt[$i]} 21 #到prod 目录下查将今天生成的文件筛选出来 22 cd $api_csv_path_home/$apiName/prod 23 cmmd00="ls --full-time ./ | sed -n '/$(date +"%Y-%m-%d")/p' | awk '{print \$9}'" 24 # cmmd00="ls --full-time ./ | sed -n '/${dmt}/p' | awk '{print \$9}'" 25 prod_currentDateFile=(`eval $cmmd00`) 26 cd $api_csv_path_home/$apiName/test 27 cmmd02="ls --full-time ./ | sed -n '/$(date +"%Y-%m-%d")/p' | awk '{print \$9}'" 28 # cmmd02="ls --full-time ./ | sed -n '/${dmt}/p' | awk '{print \$9}'" 29 test_currentDateFile=(`eval $cmmd02`) 30 #echo "${i}_$apiName.prod.[${#prod_currentDateFile[*]}] ：${prod_currentDateFile[*]} " 31 #echo "${i}_$apiName.test.[${#test_currentDateFile[*]}] ：${test_currentDateFile[*]} " 32 if (( ${#prod_currentDateFile[*]} == 0 )); then 33 echo "**************** $apiName 今天没有生成文件 不需要对比 continue ! ***************" 34 continue 35 fi 36 for((j=0;j<${#prod_currentDateFile[*]};j++)); 37 do 38 apiCsvPath=${api_csv_path_home}/${apiName} 39 prod_csvName=${prod_currentDateFile[$j]} 40 #假如prod接口下今天只生成了1个文件 并且test环境也有文件生成 就直接使用test环境的文件进行比较,否则使用生产环境的文件与测试环境的文件进行绝对匹配 ; 41 if [[ ${#prod_currentDateFile[*]} = 1 && ${#test_currentDateFile[*]} = 1 ]]; then 42 uat_csvName=${test_currentDateFile[0]} 43 else 44 uat_csvName="uat_$prod_csvName" 45 fi 46 #获取prod 和 test 环境当天生成文件的大小和行数 ;注意是文件创建日期为今天并不是文件名称中的日期; 47 prod_fileSize=`ls -l ${apiCsvPath}/prod/${prod_csvName} | awk '{print \$5}'` 48 test_fileSize=`ls -l ${apiCsvPath}/test/${uat_csvName} | awk '{print \$5}'` 49 prod_fileRowCnt=`cat ${apiCsvPath}/prod/${prod_csvName} | wc -l` 50 test_fileRowCnt=`cat ${apiCsvPath}/test/${uat_csvName} | wc -l` 51 52 if (( $prod_fileSize != 0 )); then 53 # echo "删除上一天的发送容 rm -rf ${sendMailData}/${apiName}/${lastDay} " 54 rm -rf ${sendMailData}/${apiName}/${lastDay} 55 if [[ -d "${sendMailData}/${apiName}/${dt}/${prod_csvName%.*}" ]]; then 56 #echo "${prod_csvName%.*} 文件存在 则删除，保持每次都是最新需要发送的内容 ！ " 57 rm -rf ${sendMailData}/${apiName}/${dt}/${prod_csvName%.*}/* 58 fi 59 #"创建 ${sendMailData}/${apiName}/${dt}/${prod_csvName%.*} 目录" 60 mkdir -p ${sendMailData}/${apiName}/${dt}/${prod_csvName%.*} 61 #"将 ${prod_csvName} 和 ${uat_csvName} 文件cp 到 $sendMailData/${apiName}/${dt}/${prod_csvName%.*} 目录下 " 62 cp $apiCsvPath/prod/${prod_csvName} $sendMailData/${apiName}/$dt/${prod_csvName%.*} 63 cp $apiCsvPath/test/${uat_csvName} $sendMailData/${apiName}/$dt/${prod_csvName%.*} 64 cd $sendMailData/${apiName}/$dt/${prod_csvName%.*}/ 65 sort ${prod_csvName} -o ${prod_csvName} 66 sort ${uat_csvName} -o ${uat_csvName} 67 diff -iHB ${prod_csvName} ${uat_csvName} >./${prod_csvName%.*}_$(date +"%Y%m%d_%H%M%S")_diff.log 68 diffFlag=`cat ./${prod_csvName%.*}_$(date +"%Y%m%d_%H%M%S")_diff.log | wc -l` 69 if (( $diffFlag != 0 )); then 70 strInfo="<li>$prod_csvName 【 fileSize=${prod_fileSize}B , rowCnt=${prod_fileRowCnt}行 】; </li> <li>$uat_csvName 【 fileSize=${test_fileSize}B , rowCnt=${test_fileRowCnt}行 】;</li>$strInfo" 71 diffResult_nt["${apiName}_${j}"]="${strInfo}" 72 # echo "--------修改发送邮件标志--------" 73 sendMail=true 74 else 75 rm -rf ${prod_csvName} ${uat_csvName} ${prod_csvName%.*}_$(date +"%Y%m%d_%H%M%S")_diff.log 76 diffResult_eq["${apiName}_${j}"]="${apiName}.${prod_csvName}和${uat_csvName}[${prod_fileSize}B : ${prod_fileRowCnt}行]\n" 77 fi 78 : ' 79 else 80 echo " $prod_csvName 和 $uat_csvName 文件大小为 0 无需比较 ！" 81 ' 82 fi 83 done 84 if [[ $sendMail = true ]]; then 85 #发送邮件前需要修改的配置 ; 86 sed -i "s#^attachment_filePath=.*#attachment_filePath=//home//vmuser//linbo//kettleDemo//job//sendData//${apiName}//${dt}#g" $kettleConfigHome/sandMailVariate.properties 87 sed -i "s#^var1=.*#var1=<ul>${strInfo}</ul>#g" $kettleConfigHome/sandMailVariate.properties 88 sed -i "s#^var2=.*#var2=$apiName#g" $kettleConfigHome/sandMailVariate.properties 89 echo ">>>>>>>>>>>>>>>>>>>>>>>> 开始发送邮件 >>>>>>>>>>>>>>>>>>>>>" 90 sh /home/vmuser/linbo/kettleDemo/job/kjb/execSendMail.sh 91 fi 92 done 93 echo -e "${diffResult_eq[*]}" >/home/vmuser/linbo/kettleDemo/job/log/${dt}/diffResult_eq_$(date +"%Y%m%d_%H").log 94 echo "${diffResult_nt[*]}" >/home/vmuser/linbo/kettleDemo/job/log/${dt}/diffResult_nt_$(date +"%Y%m%d_%H").log
1 #!/bin/bash 2 kettleConfigHome=/home/vmuser/linbo/kettleDemo/job/config 3 api_csv_path_home=/home/vmuser/linbo/test_upload/data_file 4 prod_tableHead="<table border=\"1\"style=\"border-collapse:collapse;\"><tr><th>No</th><th>evn.prod</th><th>fileName</th><th>fileSize</th><th>fileRowCnt</th></tr>" 5 test_tableHead="<table border=\"1\"style=\"border-collapse:collapse;\"><tr><th>No</th><th>evn.test</th><th>fileName</th><th>fileSize</th><th>fileRowCnt</th></tr>" 6 evnArr=(prod test) 7 declare -A prod_tableRowInfo 8 declare -A test_tableRowInfo 9 #dtformat=`date -d "4 day ago" +"%Y-%m-%d"` 10 #lastday=`date -d "4 day ago" +"%Y-%m-%d"` 11 dtformat=`date +"%Y-%m-%d"` 12 index=0 13 k=0 14 apiNames=(`ls -rt $api_csv_path_home`) 15 #t_burberry_sales_daily bz_return_order bz_return_order_line unilever_three_shop_sales_retrun ca_order_return unilever_sales_and_retrun mk_refund nike_gc_dgtl_hk_demand nike_odi_hk nike_odi_tmall nike_gc_dgtl_gp_order nike_hk_china_demand mk_bzsales disney_sales_return 16 #apiNames=(t_burberry_sales_daily bz_return_order bz_return_order_line unilever_three_shop_sales_retrun ca_order_return unilever_sales_and_retrun mk_refund nike_gc_dgtl_hk_demand nike_odi_hk nike_odi_tmall nike_gc_dgtl_gp_order nike_hk_china_demand mk_bzsales disney_sales_return) 17 for apiName in ${apiNames[*]} 18 do 19 for ((i=0;i<${#evnArr[*]};i++)); 20 do 21 cd $api_csv_path_home/$apiName/${evnArr[i]} 22 cmmd0="ls --full-time ./ | sed -n '/${dtformat}/p' | awk '{print \$9}'" 23 cmmd1="ls --full-time ./ | sed -n '/${dtformat}/p' | awk '{print \$5\"B\"}'" 24 25 if [[ ${evnArr[i]} = prod ]]; then 26 prod_fileArr=(`eval $cmmd0`) 27 prod_fileSizeArr=(`eval $cmmd1`) 28 prod_EvnAndDataArr[k]="${apiName}.${evnArr[i]}.[${#prod_fileArr[*]}]" 29 #获取每个文件的行数 ; 30 prod_fizeRowCntArr=() 31 for (( a=0;a<${#prod_fileArr[*]};a++)); 32 do 33 rowCnt0=`cat ${prod_fileArr[$a]} | wc -l` 34 prod_fizeRowCntArr[$a]="${rowCnt0}行" 35 done 36 # 将数据按一定的格式放入prod_tableRowInfo 中 37 if [[ ${#prod_fileArr[*]} -le 1 ]]; then 38 # echo " ${#prod_fileArr[*]} -eq 1 " 39 row="<tr><td>${index}</td><td>${prod_EvnAndDataArr[0]}</td><td>${prod_fileArr[0]}</td><td>${prod_fileSizeArr[0]}</td><td>${prod_fizeRowCntArr[0]}</td></tr>" 40 elif [[ ${#prod_fileArr[*]} -gt 1 ]]; then 41 # echo " ${#prod_fileArr[*]} -gt 1 " 42 for ((b=0;b<${#prod_fileArr[*]};b++)); 43 do 44 if [ $b -eq 0 ]; then 45 row="<tr><td rowspan=\"${#prod_fileArr[*]}\">${index}</td> <td rowspan=\"${#prod_fileArr[*]}\">${prod_EvnAndDataArr[$b]}</td><td>${prod_fileArr[$b]}</td><td>${prod_fileSizeArr[$b]}</td><td>${prod_fizeRowCntArr[$b]}</td></tr>" 46 else 47 row="${row}<tr> <td colspan=\"1\">${prod_fileArr[$b]}</td> <td>${prod_fileSizeArr[$b]}</td><td>${prod_fizeRowCntArr[$b]}</td></tr>" 48 fi 49 done 50 fi 51 prod_tableRowInfo["$index"]="$row" 52 53 else 54 test_fileArr=(`eval $cmmd0`) 55 test_fileSizeArr=(`eval $cmmd1`) 56 test_EvnAndDataArr[k]="${apiName}.${evnArr[i]}.[${#test_fileArr[*]}]" 57 #获取每个文件的行数 ; 58 test_fizeRowCntArr=() 59 for (( a=0 ;a<${#test_fileArr[*]} ;a++ )); 60 do 61 rowCnt1=`cat ${test_fileArr[$a]} | wc -l` 62 test_fizeRowCntArr[$a]="${rowCnt1}行" 63 done 64 # ------------------------------------------------------------------------------------------------------------------------------------------- 65 if [[ ${#test_fileArr[*]} -le 1 ]]; then 66 # echo " ${#test_fileArr[*]} -eq 1 " 67 row="<tr><td>${index}</td><td>${test_EvnAndDataArr[0]}</td><td>${test_fileArr[0]}</td><td>${test_fileSizeArr[0]}</td><td>${test_fizeRowCntArr[0]}</td></tr>" 68 elif [[ ${#test_fileArr[*]} -gt 1 ]]; then 69 # echo " ${#test_fileArr[*]} -gt 1 " 70 for ((b=0;b<${#test_fileArr[*]};b++)); 71 do 72 if [ $b -eq 0 ]; then 73 row="<tr><td rowspan=\"${#test_fileArr[*]}\">${index}</td> <td rowspan=\"${#test_fileArr[*]}\">${test_EvnAndDataArr[$b]}</td><td>${test_fileArr[$b]}</td><td>${test_fileSizeArr[$b]}</td><td>${test_fizeRowCntArr[$b]}</td></tr>" 74 else 75 row="${row}<tr> <td colspan=\"1\">${test_fileArr[$b]}</td> <td>${test_fileSizeArr[$b]}</td><td>${test_fizeRowCntArr[$b]}</td></tr>" 76 fi 77 done 78 fi 79 test_tableRowInfo["$index"]="$row" 80 # ------------------------------------------------------------------------------------------------------------------------------------------- 81 82 fi 83 done 84 index=`expr $index + 1` 85 done 86 87 #保证顺序 88 for ((x=0;x<${#prod_tableRowInfo[*]};x++)); 89 do 90 str1="${str1}${prod_tableRowInfo[$x]}" 91 str2="${str2}${test_tableRowInfo[$x]}" 92 done 93 94 #var3="${prod_tableHead}${prod_tableRowInfo[*]}</table>" 95 #var4="${test_tableHead}${test_tableRowInfo[*]}</table>" 96 var3="${prod_tableHead}${str1}</table>" 97 var4="${test_tableHead}${str2}</table>" 98 99 #echo ${!prod_tableRowInfo[*]} 100 #echo "${var3}" 101 #echo "*************************************************************" 102 #echo "${var4}" 103 #echo ${!test_tableRowInfo[*]} 104 sed -i "s#^var3=.*#var3=${var3}#g" $kettleConfigHome/sandMailVariate.properties 105 sed -i "s#^var4=.*#var4=${var4}#g" $kettleConfigHome/sandMailVariate.properties 106 # 发送邮件 107 sh /home/vmuser/linbo/kettleDemo/job/kjb/fileCpEvnEmailSend.sh
