hadoop 基本操作手冊

####hdfs
./bin/hdfs dfs -mkdir /user
./bin/hdfs dfs -mkdir /user/Administrator
./bin/hdfs dfs -chmod 777 /user/Administrator
 bin/hadoop dfs -put /home/logonuser/hadoop-2.6.0/aa.txt /lbda/
bin/hadoop dfs -ls /

查看hdfs 50070
    mapreduce  8088
		 
####mapreduce執行jar
hadoop jar wordcount.jar org.apache.hadoop.examples.WordCount /lbda/rtd/2015/04/DT_CDMA_20150408_1 /lbda/output

mapreduce設置輸出分隔符
job.getConfiguration().set("mapred.textoutputformat.separator", "\t");

####hive創建表
create table test(type1 string,type2 string,type3 string)row format delimited fields terminated by ','
####外部表
create EXTERNAL table IF NOT EXISTS testrtd 
(type1 string,type2 string, type3 string, type4 string, type5 string,type6 string, type7 string,type8 string,type9 string,type10 string,type11 string)  
ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' location '/lbda/rtd/2015/04';
####hive導出hdfs
insert overwrite  directory '/lbda/count' 
select count(*) from ( select type1,type2,type3,row_number() over (distribute by type1,type3) as rn  from test ) t where t.rn=1

####hdfs導入hive
load data inpath '/lbda/asd' into table test;

hive服務啓動
hive --service hiveserver2 &
啓動元數據存儲服務
hive --service metastore &
查看hive錯誤
hive -hiveconf hive.root.logger=DEBUG,console
修改mysql數據庫編碼
alter database hive character set latin1 
 
 
hive服務端啓動查看
./beeline
 !connect jdbc:hive2://172.17.5.120:10000