hive_sql優化

-- 優化時間計算導致的資源消耗 --

-- 優化hive性能:tez,spark,lmpala,mapreduce; 矢量化--
set hive.execution.engine = spark;

set hive.vectorized.execution.enabled = true;
set hive.vectorized.execution.reduce.enabled = true;

set hive.exec.parallel = true;
set hive.limit.optimize.enable = true;

-- mr 優化 --
set mapred.reduce.tasks = 300;
set hive.auto.convert.join = true;

-- 數據傾斜 --
set hive.optimize.skewjoin = true;

-- hive建表 經驗設置
set hive.exec.compress.output=true;
set mapred.output.compress=true;
set mapred.output.compression.codec=org.apache.hadoop.io.compress.GzipCodec;
set io.compression.codecs=org.apache.hadoop.io.compress.GzipCodec;
set mapred.reduce.tasks=200;
set mapreduce.reduce.shuffle.memory.limit.percent=0.10;
set yarn.nodemanager.pmem-check-enabled=false;
set yarn.nodemanager.vmem-check-enabled=false;
set mapreduce.map.memory.mb=3072;
set mapreduce.reduce.memory.mb=3072;

-- hive當前時間轉化爲：'20180425'
WHERE pt = from_unixtime(unix_timestamp(date_sub(from_unixtime(unix_timestamp(),'yyyy-MM-dd'),1),'yyyy-MM-dd'),'yyyyMMdd')

-- hive在shell上執行
-- hive_data.sh

table_name='t_news'
tt=$(date -d '-3 days' +%Y%m%d) --'20180422'

hive(gohive) -e "select * from ${table_name} limit 5;" -- method_1

hive(gohive) -hiveconf table_name='t_news' -f news.sql -- method_2
-- news.sql: select * from ${hiveconf: table_name} limit 5;

# linux_shell注意事項：
# 變量賦值‘等號’兩邊不能有空格，要緊挨；
# 變量存在的語句一定使用雙引號，而不能用單引號；