Kylin中使用Api構建cube以及狀態監控腳本

Kylin中使用RESTful API 每日構建cube以及進行狀態監控

1.認證

• 目前Kylin使用basic authentication。
• 給第一個請求加上用於認證的 Authorization 頭部。

POST http://localhost:7070/kylin/api/user/authentication

Authorization:Basic xxxxJD124xxxGFxxxSDF
Content-Type: application/json;charset=UTF-8

2.跟蹤任務狀態

• GET http://localhost:7070/kylin/api/jobs/{job_uuid}
• 返回的 job_status 代表job的當前狀態。

####　3.如果構建任務出現錯誤，可以重新開始它

• PUT http://localhost:7070/kylin/api/jobs/{job_uuid}/resume

4.shell腳本編寫

#!/bin/bash
set -x
cubeName="$2"
kylin_user="$3"
kylin_pass="$4"
kylin_server="$5"
　
#加入Kylin增量build cube
#day 截止的時間
#cubeName  cube的名稱
#endTime  執行build cube的結束時間
#buildType  BUILD 構建cube操作(還有Refresh、Merge等操作，增量構建爲BUILD)
#today=`date -d now +%Y-%m-%d`

#執行cube構建
cube_rebuild(){     
        kylinMinusTime=$((8 * 60 * 60 * 1000)) #這裏是UTC時間，需要加8個小時
        #用戶名、密碼base64加密
        baseUsernamePasswd=`python -c "import base64; print base64.standard_b64encode('$kylin_user:$kylin_pass')"`

        day="$1"
        dayTimeStamp=`date -d "$day 00:00:00" +%s`
        errorTimeStamp=`date "+%N"`
        res=`echo $errorTimeStamp |grep '^0'`
        if [ -z $res ];then
			echo $res
        else
			errorTimeStamp=`echo $errorTimeStamp |cut -c2-10`
        fi

        dayTimeStampMs=$(($dayTimeStamp*1000 + $errorTimeStamp/1000000)) #將current轉換爲時間戳，精確到毫秒
        endTime=$(($dayTimeStampMs + $kylinMinusTime))

        #build
        response=`curl -X PUT -H "Authorization: Basic ${baseUsernamePasswd}" -H 'Content-Type: application/json' -d '{"endTime":'$endTime', "buildType":"BUILD"}' \
        http://${kylin_server}:7070/kylin/api/cubes/${cubeName}/rebuild`

        #獲取jobid
        jobid=`echo $response |awk -F',' '{print $1}' |grep 'uuid' |awk -F':' '{print $2}' |xargs`
           if [ x${jobid} != 'x' ];then	  
			  status_check
        else
              echo "$cubeName 沒有開始構建"
              exit
          fi      
}

#2，執行任務狀態檢測

status_check(){
        run_count=0
        fail_count=0
        while true
        do
                   sleep 1200     #默認爲s
                   let run_count++
                   #job_status=`echo $line |bash |awk -F',' '{print $(NF-2)}' |grep 'job_status' |awk -F':' '{print $2}' |xargs`
				
                   result=`curl -X PUT -H "Authorization: Basic ${baseUsernamePasswd}" http://${kylin_server}:7070/kylin/api/jobs/${jobid}`
                   
				   job_status=`echo $result |grep  -oE job_status.* |grep 'job_status' |awk -F':' '{print $2}' |awk -F',' '{print $1}'|xargs`
                   job_duration=`echo $result |grep  -oE duration.* |grep 'duration' |awk -F':' '{print $2}' |awk -F',' '{print $1}'|xargs`

                   #如果當前cube狀態爲FINISHED則直接退出監控
                   if [ $job_status == 'FINISHED' ] ;then
                       exit
                 elif [ $job_status == 'RUNNING' -o $job_status == 'PENDING' ] ;then
                       if [ $job_duration -ge 3600 ] ;then
                          result=`curl -X PUT -H "Authorization: Basic ${baseUsernamePasswd}" http://${kylin_server}:7070/kylin/api/jobs/${jobid}/pause`
                          echo "當前 $cubeName cube已構建時長${job_duration}s,已暫停構建,請及時查看原因" 
                          exit
                       fi
                 elif [ $job_status == 'ERROR' ] ;then
                       let error_count++
                       if [ $error_count -ge 2 ] ;then
                          echo "當前 $cubeName cube已構建失敗 ${error_count} 次，已終止重啓，請及時查看原因"
                          exit
                       else
                          cat $resumefile |grep $jobid |sh
                          echo "當前 $cubeName cube已構建失敗 ${error_count} 次，已重新啓動構建，請及時查看原因"
                       fi
                   fi
               #監控10次後停止監控
               [ $run_count -eq 10 ] && break
      done
}