將數據從mysql導入到hive表

代碼腳本如下：

#!/bin/bash
# 連接mysql數據庫，並查詢結果，將結果輸出到文件中
# 1.定義連接變量
mysql_hostname='172.28.65.133'
mysql_port='3306'
mysql_username='root'
mysql_password='xxxxxx'
#
dbname='recom_video'
time=`date -d "1 day ago" +"%Y-%m-%d"`
save_f="t_st_video_info_${time}.txt"
save_p="/data/recommend/"
#
mysql_sq="select 
        IFNULL(id,''),
        IFNULL(video_id,''),
        IFNULL(video_name,''),
        IFNULL(video_desc,''),
        IFNULL(video_url,''),
        IFNULL(hits_num,''),
        IFNULL(praise_num,''),
        IFNULL(create_time,''),
        IFNULL(update_time,'')
from t_st_video_info 
where create_time <= '${time} 23:59:59'"

echo "開始鏈接數據庫..."

# 2.連接數據庫
result=`mysql -h${mysql_hostname} -P${mysql_port} -u${mysql_username} -p${mysql_password} << EOF
use recom_video;
$mysql_sq;
quit
EOF`

# 判斷是否連接成功
if [ $? = 0 ] 
then
 echo "連接成功。。。"
else
 echo "連接失敗。。。"
 exit
fi
echo "寫入查詢結果..."

# 將結果寫入文本
echo "$result" | tail -n +2 | sed 's/\t/,/g' >> $save_p$save_f
echo "寫入完成..."

#
target_table="t_di_video_info_ad"
op_time=`date -d "1 day ago" +"%Y%m%d"`

hive -e "alter table ${target_table} drop partition(dayid='$op_time');"
hive -e "use recommend_video;load data local inpath'$save_p$save_f' overwrite into table ${target_table} partition(dayid='${op_time}');"

 

hive表：

CREATE TABLE `t_di_video_info_ad`(
  `id` int, 
  `video_id` string, 
  `video_name` string, 
  `video_desc` string, 
  `video_url` string, 
  `hits_num` string, 
  `praise_num` string, 
  `create_time` string, 
  `update_time` string)
COMMENT '視頻信息表'
PARTITIONED BY ( `dayid` string)
ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' 
LINES TERMINATED BY '\n' STORED AS TEXTFILE

 

SQL：

load data inpath '/recommend/video/t_st_video_info.txt' into table video_info PARTITION (date = '20200608');