Kylin中使用Api构建cube以及状态监控脚本

Kylin中使用RESTful API 每日构建cube以及进行状态监控

1.认证

• 目前Kylin使用basic authentication。
• 给第一个请求加上用于认证的 Authorization 头部。

POST http://localhost:7070/kylin/api/user/authentication

Authorization:Basic xxxxJD124xxxGFxxxSDF
Content-Type: application/json;charset=UTF-8

2.跟踪任务状态

• GET http://localhost:7070/kylin/api/jobs/{job_uuid}
• 返回的 job_status 代表job的当前状态。

####　3.如果构建任务出现错误，可以重新开始它

• PUT http://localhost:7070/kylin/api/jobs/{job_uuid}/resume

4.shell脚本编写

#!/bin/bash
set -x
cubeName="$2"
kylin_user="$3"
kylin_pass="$4"
kylin_server="$5"
　
#加入Kylin增量build cube
#day 截止的时间
#cubeName  cube的名称
#endTime  执行build cube的结束时间
#buildType  BUILD 构建cube操作(还有Refresh、Merge等操作，增量构建为BUILD)
#today=`date -d now +%Y-%m-%d`

#执行cube构建
cube_rebuild(){     
        kylinMinusTime=$((8 * 60 * 60 * 1000)) #这里是UTC时间，需要加8个小时
        #用户名、密码base64加密
        baseUsernamePasswd=`python -c "import base64; print base64.standard_b64encode('$kylin_user:$kylin_pass')"`

        day="$1"
        dayTimeStamp=`date -d "$day 00:00:00" +%s`
        errorTimeStamp=`date "+%N"`
        res=`echo $errorTimeStamp |grep '^0'`
        if [ -z $res ];then
			echo $res
        else
			errorTimeStamp=`echo $errorTimeStamp |cut -c2-10`
        fi

        dayTimeStampMs=$(($dayTimeStamp*1000 + $errorTimeStamp/1000000)) #将current转换为时间戳，精确到毫秒
        endTime=$(($dayTimeStampMs + $kylinMinusTime))

        #build
        response=`curl -X PUT -H "Authorization: Basic ${baseUsernamePasswd}" -H 'Content-Type: application/json' -d '{"endTime":'$endTime', "buildType":"BUILD"}' \
        http://${kylin_server}:7070/kylin/api/cubes/${cubeName}/rebuild`

        #获取jobid
        jobid=`echo $response |awk -F',' '{print $1}' |grep 'uuid' |awk -F':' '{print $2}' |xargs`
           if [ x${jobid} != 'x' ];then	  
			  status_check
        else
              echo "$cubeName 没有开始构建"
              exit
          fi      
}

#2，执行任务状态检测

status_check(){
        run_count=0
        fail_count=0
        while true
        do
                   sleep 1200     #默认为s
                   let run_count++
                   #job_status=`echo $line |bash |awk -F',' '{print $(NF-2)}' |grep 'job_status' |awk -F':' '{print $2}' |xargs`
				
                   result=`curl -X PUT -H "Authorization: Basic ${baseUsernamePasswd}" http://${kylin_server}:7070/kylin/api/jobs/${jobid}`
                   
				   job_status=`echo $result |grep  -oE job_status.* |grep 'job_status' |awk -F':' '{print $2}' |awk -F',' '{print $1}'|xargs`
                   job_duration=`echo $result |grep  -oE duration.* |grep 'duration' |awk -F':' '{print $2}' |awk -F',' '{print $1}'|xargs`

                   #如果当前cube状态为FINISHED则直接退出监控
                   if [ $job_status == 'FINISHED' ] ;then
                       exit
                 elif [ $job_status == 'RUNNING' -o $job_status == 'PENDING' ] ;then
                       if [ $job_duration -ge 3600 ] ;then
                          result=`curl -X PUT -H "Authorization: Basic ${baseUsernamePasswd}" http://${kylin_server}:7070/kylin/api/jobs/${jobid}/pause`
                          echo "当前 $cubeName cube已构建时长${job_duration}s,已暂停构建,请及时查看原因" 
                          exit
                       fi
                 elif [ $job_status == 'ERROR' ] ;then
                       let error_count++
                       if [ $error_count -ge 2 ] ;then
                          echo "当前 $cubeName cube已构建失败 ${error_count} 次，已终止重启，请及时查看原因"
                          exit
                       else
                          cat $resumefile |grep $jobid |sh
                          echo "当前 $cubeName cube已构建失败 ${error_count} 次，已重新启动构建，请及时查看原因"
                       fi
                   fi
               #监控10次后停止监控
               [ $run_count -eq 10 ] && break
      done
}