setup cluster -- configure hadoop

after above two step now we can configure our hadoop
at first we must install the "jdk" and then we can install hadoop
1. install jdk
    i use the command "sudo apt-get install openjdk-7-jdk"
    and you can down the new jdk package and install it
2. down the hadoop package  at below url
    http://www.apache.org/dyn/closer.cgi/hadoop/common/hadoop-2.6.0/hadoop-2.6.0.tar.gz
    and you can search "hadoop releases" by google to find
3. install and configure the file
    (1 install it
    uncompress the package
    at the "Downloads" directory use the below commands
    "tar-vxzf hadoop-2.6.0.tar.gz -C /home/warrior/bigData"
    "cd /home/warrior/bigData"
    "mv hadoop-2.6.0 hadoop"
    (2 configure it 
    at first set the "/etc/profile"
    add the content 

點擊(此處)摺疊或打開

export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64/
export HADOOP_INSTALL=/home/warrior/bigData/hadoop
export PATH=$PATH:$HADOOP_INSTALL/bin
export PATH=$PATH:$HADOOP_INSTALL/sbin
export HADOOP_MAPRED_HOME=$HADOOP_INSTALL
export HADOOP_COMMON_HOME=$HADOOP_INSTALL
export HADOOP_HDFS_HOME=$HADOOP_INSTALL
export YARN_HOME=$HADOOP_INSTALL

    (3 set the /home/warrior/bigData/hadoop/etc/hadoop/core-site.xml

點擊(此處)摺疊或打開

<?xml
version="1.0"
encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, 
Version 2.0 
(the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an 
"AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
    /* path of HDFS
*/
    <property>
        <name>fs.defaultFS</name>
    </property>
    /* buffer size
*/
    <property>
        <name>fs.default.name</name>
        <value>hdfs://warrior:9000</value>
    </property>
    /* temp folder
*/
    <property>
        <name>hadoop.tmp.dir</name>
        <value>file:/home/warrior/tmp</value>
        <description>Abase for other temporary directories.</description>
    </property>


</configuration>

    (4 set the /home/warrior/bigData/hadoop/etc/hadoop/yarn-site.xml

點擊(此處)摺疊或打開

<?xml
version="1.0"?>
<!--
  Licensed under the Apache License, 
Version 2.0 
(the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an 
"AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<configuration>

<!-- Site specific YARN configuration properties -->
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>

    <property>
        <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
        <value>org.apache.hadoop.mapred.ShuffleHandler</value>
    </property>

    <property>
        <name>yarn.resourcemanager.address</name>
        <value>warrior:8032</value>
    </property>
    <property>
        <name>yarn.resourcemanager.scheduler.address</name>
        <value>warrior:8030</value>
    </property>

    <property>
        <name>yarn.resourcemanager.resource-tracker.address</name>
        <value>warrior:8031</value>
    </property>

    <property>
        <name>yarn.resourcemanager.admin.address</name>
        <value>warrior:8033</value>
    </property>

    <property>
        <name>yarn.resourcemanager.webapp.address</name>
        <value>warrior:8088</value>
    </property>
</configuration>

    (5 set the /home/warrior/bigData/hadoop/etc/hadoop/mapres-site.xml

點擊(此處)摺疊或打開

<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>

    <property>
        <name>mapreduce.jobhistory.address</name>
        <value>warrior:10020</value>
    </property>

    <property>
        <name>mapred.job.tracker</name>
        <value>warrior:10001</value>
    </property>

    <property>
        <name>mapreduce.jobhistory.webapp.address</name>
        <value>warrior:19888</value>
    </property>
</configuration>

    (6 make directory "namenode" "datanode" at "/home/warrior/bigData/"
    and set the /home/warrior/bigData/hadoop/etc/hadoop/hdfs-site.xml

點擊(此處)摺疊或打開

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet
type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0
(the "License");
  you may not use 
this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to
in writing, software
  distributed under the License is distributed on an 
"AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS 
OF ANY KIND, either express 
or implied.
  See the License for the specific language governing permissions
and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides
in this file.
-->

<configuration>
    <property>
        <name>dfs.namenode.secondary.http-address</name>
        <value>warrior:9001</value>
    </property>

    <property>
        <name>dfs.namenode.name.dir</name>
        <value>file:/home/warrior/bigData/hdfs//namenode</value>
    </property>

    <property>
        <name>dfs.datanode.data.dir</name>
        <value>file:/home/warrior/bigData/hdfs//datanode</value>
    </property>

    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>

    <property>
        <name>dfs.webdfs.enabled</name>
        <value>true</value>
    </property>
</configuration>

    (7 set the "master" and "slaves" 
    below is my set 
    set the /home/warrior/bigData/hadoop/etc/hadoop/master
    [email protected]
    set the /home/warrior/bigData/hadoop/etc/hadoop/slaves
    [email protected]
    [email protected]

4. scp the directory to all the other machines
    like at 10.0.2.10 to 10.0.2.20
    "scp /home/warrior/bigData/hadoop/  [email protected]:/home/warrior/bigData/hadoop/"
    to other like above
5. use below command test the hadoop cluster
    "./bin/hadoop namenode -format"
    "./sbin/start-all.sh"

閱讀(11) | 評論(0) | 轉發(0) |

1

上一篇：setup cluster -- ssh to each other with no key ssh免密鑰登陸配置

下一篇：搭建spark cluster過程中遇到的網絡問題

相關熱門文章

• [官版翻譯ing]OpenStack雲計算...
• hive的安裝和配置
• [官版翻譯ing]OpenStack雲計算...
• OpenStack安裝記
• Hadoop運行class類出現Excepti...

• linux dhcp peizhi roc
• 關於Unix文件的軟鏈接
• 求教這個命令什麼意思，我是新...
• sed -e "/grep/d" 是什麼意思...
• 誰能夠幫我解決LINUX 2.6 10...

評論熱議