hive表數據去重

根據user name查重
SELECT test_user_name, count(*) c from default.test GROUP BY test_user_name HAVING c> 1;
根據id查重
SELECT id ,count(*) c FROM default.test GROUP BY id HAVING c >1;
去重：注意min用法，取相同數據的最小id，去重的重點
min:使用group by 分組後取分組內最小id，以便去掉其它重複數據
INSERT OVERWRITE table default.test partition(test_data_source) select * from default.test where id in ( select min(id) as id from default.test group by test_user_name) ;