hadoop僞分佈式環境apache版本切換爲CDH

1.在/opt/modules下創建一個cdh空目錄

$ mkdir cdh       //cdh版本的hadoop， hive，sqoop安裝在此目錄下

2.解壓cdh版本的hadoop，hive，sqoop

$ tar -zxf hadoop-2.5.0-cdh5.3.6.tar.gz -C /opt/modules/cdh

$ tar -zxf hive-0.13.1-cdh5.3.6.tar.gz -C /opt/modules/cdh

$ tar -zxf sqoop-1.4.5-cdh5.3.6.tar.gz -C /opt/modules/cdh

3.配置hadoop

     1).${HADOOP_HOME}/etc/hadoop下

hadoop-env.sh    mared-env.sh   yarn-env.sh

添加

export JAVA_HOME=/opt/modules/jdk1.7.0_79

     2).${HADOOP_HOME}/etc/hadoop/core-site.xml(核心配置文件)

<!--NameNode地址，8020是指定進程8020，訪問入口 -->

    <property>

        <name>fs.defaultFS</name>

        <value>hdfs://[hostname]:8020</value>

    </property>

     <!--hadoop在運行時產生的文件 -->

     <property>

          <name>hadoop.tmp.dir</name>

          <value>/opt/modules/cdh/hadoop-2.5.0-cdh5.3.6/data</value>

     </property>

3).${HADOOP_HOME}/etc/hadoop/hdfs-site.xml （HDFS模塊配置信息）

     <!-- 存放在hdfs上的副本數，在僞分佈式設置爲1 -->

     <property>

        <name>dfs.replication</name>

        <value>1</value>

    </property>

    <!-- 關閉hdf權限控制 -->

    <property>

        <name>dfs.permissions</name>

        <value>false</value>          

     </property>

4).${HADOOP_HOME}/etc/hadoop/yarn-site.xml

  <!-- 指定ResorceManager所在服務器的主機名-->

<property>

     <name>yarn.resourcemanager.hostname</name>

     <value>[hostname]</value>

</property>

<!-- 指明在執行MapReduce的時候使用shuffle-->

<property>

     <name>yarn.nodemanager.aux-services</name>

     <value>mapreduce_shuffle</value>

</property>

<!--啓用日誌聚合功能-->

<property>

     <name>yarn.log-aggregation-enable</name>

     <value>true</value>

</property>

     <!--日誌保存時間-->

<property>

     <name>yarn.log-aggregation.retain-seconds</name>

     <value>86400</value>

</property>

5).${HADOOP_HOME}/etc/hadoop/複製並重命名模板文件

$ cp mapred-site.xml.template mapred-site.xml

<!-- 指定MapReduce基於Yarn來運行-->

<property>

     <name>mapreduce.framework.name</name>

     <value>yarn</value>

</property>

<property>

     <name>mapreduce.jobhistory.address</name>

     <!--配置實際的主機名和端口-->

     <value>[hostname]:10020</value>

</property>

<property>

     <name>mapreduce.jobhistory.webapp.address</name>

     <value>[hostname]:19888</value>

</property>

6).${HADOOP_HOME}/etc/hadoop/slaves

hadoop.beifeng.com

7).${HADOOP_HOME}/

     格式化hdfs

## 注意：是cdh版本的hadoop下的hdfs命令

$ bin/hdfs namenode -format

     8).${HADOOP_HOME}/啓動hadoop

     $ sbin/start-dfs.sh

     $ sbin/start-yarn.sh

     $ sbin/mr-jobhistory-daemon.sh start historyserver

     3.hive部署

     首先在${HIVE_HOME}/conf/重命名生成配置文件

     $ cp hive-env.sh.template hive-env.sh

     $ cp hive-default.xml.template  hive-site.xml

     $ cp  hive-log4j.properties.template hive-log4j.properties

1).${HIVE_HOME}/conf/hive-env.sh

export JAVA_HOME=/opt/modules/jdk1.7.0_79

export HADOOP_HOME=/opt/modules/cdh/hive-0.13.1-cdh5.3.6

export HIVE_CONF_DIR=/opt/modules/cdh/hive-0.13.1-cdh5.3.6/conf

2).${HIVE_HOME}/conf/hive-site.xml

     --JDBC 連接四要素

     <!--132行 -->

<property>

  <name>javax.jdo.option.ConnectionURL</name>

  <value>jdbc:mysql://[hostname]:3306/metastore?createDatabaseIfNotExist=true</value>

  <description>JDBC connect string for a JDBC metastore</description>

</property>

<!--138行 -->

<property>

  <name>javax.jdo.option.ConnectionDriverName</name>

  <value>com.mysql.jdbc.Driver</value>

  <description>Driver class name for a JDBC metastore</description>

</property>

<!--162行 -->

<property>

  <name>javax.jdo.option.ConnectionUserName</name>

  <value>root</value>

  <description>username to use against metastore database</description>

</property>

<property>

  <name>javax.jdo.option.ConnectionPassword</name>

  <value>root123</value>

  <description>password to use against metastore database</description>

</property>

===配置hiveserver2

<!--2183行 去掉5000L後面的“L”-->

<property>

  <name>hive.server2.long.polling.timeout</name>

  <value>5000L</value>

  <description>Time in milliseconds that HiveServer2 will wait, before responding to asynchronous calls that use long polling</description>

</property>

<!--2196行 -->

<property>

  <name>hive.server2.thrift.port</name>

  <value>10000</value>

  <description>Port number of HiveServer2 Thrift interface.

  Can be overridden by setting $HIVE_SERVER2_THRIFT_PORT</description>

</property>

<property>

  <name>hive.server2.thrift.bind.host</name>

  <value>[hostname]</value>

  <description>Bind host on which to run the HiveServer2 Thrift interface.

  Can be overridden by setting $HIVE_SERVER2_THRIFT_BIND_HOST</description>

</property>

***<!--2786行少一個<property>標籤 -->

     3).${HIVE_HOME}/conf/hive-log4j.properties

hive.log.dir=${HIVE_HOME}/logs

     4).## 拷貝驅動包到${HIVE_HOME}/lib

$ cp mysql-connector-java-5.1.34-bin.jar /opt/modules/cdh/hive-0.13.1-cdh5.3.6/lib/

     5).在HDFS上創建相關目錄並修改權限

$ bin/hdfs dfs -mkdir /tmp

$ bin/hdfs dfs -mkdir -p  /user/hive/warehouse

$ bin/hdfs dfs -chmod g+w /tmp

$ bin/hdfs dfs -chmod g+w  /user/hive/warehouse

     4.Zookeeper安裝

  1）${ZOOKEEPER_HOME}/conf

  $ cp zoo_sample.cfg zoo.cfg

  2）${ZOOKEEPER_HOME}/conf

  $ vi  zoo.cfg

dataDir=/opt/modules/zookeeper-3.4.5/zkData

啓動zookeeper

bin/zkServer.sh start

8967 QuorumPeerMain

##查看zookeeper的狀態

bin/zkServer.sh status