shell csv/txt文件對比

 

 

 1 #!bin/bash
 2 kettleConfigHome=/home/vmuser/linbo/kettleDemo/job/config
 3 apiInfos=/home/vmuser/linbo/kettleDemo/job/data/apiInfo
 4 api_csv_path_home=/home/vmuser/linbo/test_upload/data_file
 5 sendMailData=/home/vmuser/linbo/kettleDemo/job/sendData
 6 dtm=`date +"%Y-%m-%d %H:%M:%S"`
 7 dt=`date +"%Y%m%d"`
10 lastDay=`date -d "1 day ago" +"%Y%m%d"`
11 declare -A diffResult
12 #先將pord 接口生成的文件load test的情況告知相關人 ;
13 #sh /home/vmuser/linbo/kettleDemo/job/execSH/fileCpCase.sh
14 apiCnt=(`ls -rt $api_csv_path_home`)
15 #apiCnt=(unilever_three_shop_sales_retrun)
16 for((i=0;i<${#apiCnt[*]} ;i++)); 
17 do 
18     strInfo=""
19     #默認不發送郵件;
20     sendMail=false 
21     apiName=${apiCnt[$i]}        
22     #到prod 目錄下查將今天生成的文件篩選出來
23     cd $api_csv_path_home/$apiName/prod
24     cmmd00="ls --full-time ./ | sed -n '/$(date +"%Y-%m-%d")/p' | awk '{print \$9}' "
25     prod_currentDateFile=(`eval $cmmd00`)
26     #刪除prod 3天前接口生成的文件 
27     find ./ -mtime +3 -delete
28     cd $api_csv_path_home/$apiName/test
29     cmmd02="ls --full-time ./ | sed -n '/$(date +"%Y-%m-%d")/p' | awk '{print \$9}'"
30     test_currentDateFile=(`eval $cmmd02`)
31     #刪除test 3天前接口生成的文件 
32     find ./ -mtime +3 -delete
33     #echo "${i}_$apiName.prod.[${#prod_currentDateFile[*]}] ：${prod_currentDateFile[*]} "
34     #echo "${i}_$apiName.test.[${#test_currentDateFile[*]}] ：${test_currentDateFile[*]} "
35     if (( ${#prod_currentDateFile[*]} == 0 )); then 
36     echo "**************** $apiName 今天沒有生成文件 不需要對比 continue ! ***************"
37         continue 
38     fi         
39     for((j=0;j<${#prod_currentDateFile[*]};j++));
40     do 
41         apiCsvPath=${api_csv_path_home}/${apiName}
42         prod_csvName=${prod_currentDateFile[$j]}
43         #假如prod接口下今天只生成了1個文件 並且test環境也有文件生成 就直接使用test環境的文件進行比較,否則使用生產環境的文件與測試環境的文件進行絕對匹配 ;
44         if [[ ${#prod_currentDateFile[*]} = 1 &&  ${#test_currentDateFile[*]} = 1 ]]; then
45             uat_csvName=${test_currentDateFile[0]}
46         else 
47             uat_csvName="uat_$prod_csvName"
48         fi    
49         #獲取prod 和 test 環境當天生成文件的大小和行數 ;注意是文件創建日期爲今天並不是文件名稱中的日期;
50         prod_fileSize=`ls -l ${apiCsvPath}/prod/${prod_csvName} | awk '{print \$5}'`
51         test_fileSize=`ls -l ${apiCsvPath}/test/${uat_csvName} | awk '{print \$5}'`
52         prod_fileRowCnt=`cat  ${apiCsvPath}/prod/${prod_csvName} | wc -l`
53         test_fileRowCnt=`cat  ${apiCsvPath}/test/${uat_csvName} | wc -l`
54 
55         if (( $prod_fileSize != 0 )); then
56 #            echo "刪除上一天的發送容  rm -rf ${sendMailData}/${apiName}/${lastDay} "
57             rm -rf ${sendMailData}/${apiName}/${lastDay}
58             if [[ -d "${sendMailData}/${apiName}/${dt}/${prod_csvName%.*}" ]]; then
59                 #echo "${prod_csvName%.*} 文件存在 則刪除，保持每次都是最新需要發送的內容 ！ "
60                 rm -rf ${sendMailData}/${apiName}/${dt}/${prod_csvName%.*}/*
61             fi
62             #"創建 ${sendMailData}/${apiName}/${dt}/${prod_csvName%.*} 目錄"
63             mkdir -p ${sendMailData}/${apiName}/${dt}/${prod_csvName%.*}
64             #"將 ${prod_csvName} 和 ${uat_csvName} 文件cp 到 $sendMailData/${apiName}/${dt}/${prod_csvName%.*} 目錄下 "
65             cp $apiCsvPath/prod/${prod_csvName} $sendMailData/${apiName}/$dt/${prod_csvName%.*}
66             cp $apiCsvPath/test/${uat_csvName} $sendMailData/${apiName}/$dt/${prod_csvName%.*}
67             cd $sendMailData/${apiName}/$dt/${prod_csvName%.*}/
68             sort ${prod_csvName} -o ${prod_csvName}
69             sort ${uat_csvName} -o ${uat_csvName}
70             diff -iHB ${prod_csvName} ${uat_csvName} >./${prod_csvName%.*}_$(date +"%Y%m%d_%H%M%S")_diff.log
71             diffFlag=`cat ./${prod_csvName%.*}_$(date +"%Y%m%d_%H%M%S")_diff.log | wc -l`             
72             if (( $diffFlag != 0 )); then
73                 strInfo="<li>$prod_csvName    【 fileSize=${prod_fileSize}B ,  rowCnt=${prod_fileRowCnt}行 】; </li> <li>$uat_csvName  【 fileSize=${test_fileSize}B , rowCnt=${test_fileRowCnt}行 】;</li>$strInfo"
74             #    echo "--------修改發送郵件標誌--------"
75                 sendMail=true            
76             else                 
77                 rm -rf ${prod_csvName} ${uat_csvName} ${prod_csvName%.*}_$(date +"%Y%m%d_%H%M%S")_diff.log
78                 diffResult["${apiName}_${j}"]="${apiName}.${prod_csvName}和${uat_csvName}[$prod_fileSize : $prod_fileRowCnt]\n"
79             fi
80         : '
81         else 
82             echo " $prod_csvName 和 $uat_csvName  文件大小爲 0  無需比較 ！"
83             '
84         fi
85     done 
86 if [[ $sendMail = true ]]; then 
87     #發送郵件前需要修改的配置 ;
88     sed -i "s#^attachment_filePath=.*#attachment_filePath=//home//vmuser//linbo//kettleDemo//job//sendData//${apiName}//${dt}#g" $kettleConfigHome/sandMailVariate.properties
89     sed -i "s#^var1=.*#var1=<ul>${strInfo}</ul>#g" $kettleConfigHome/sandMailVariate.properties
90     sed -i "s#^var2=.*#var2=$apiName#g" $kettleConfigHome/sandMailVariate.properties
91     echo ">>>>>>>>>>>>>>>>>>>>>>>> 開始發送郵件 >>>>>>>>>>>>>>>>>>>>>"
92     #sh /home/vmuser/linbo/kettleDemo/job/kjb/execSendMail.sh
93 fi
94 
95 done
96 echo -e "${diffResult[*]}" >>/home/vmuser/linbo/kettleDemo/job/log/${dt}/$(date +"%Y%m%d_%H").log