[hadoop] 搭建自己的hadoop集羣

1>  準備工作
       a>  五臺centos6.2虛擬機,配置主機名、IP地址、yum源、
              192.168.68.201  master01
              192.168.68.202  master02
              192.168.68.203  slave01
              192.168.68.204  slvae02
              192.168.68.205  slave03   
       b>  準備所需要的軟件包
              jdk-6u26-linux-x64-rpm.bin 
              hadoop-0.20.2.tar.gz
2> 配置我自己的hadoop 集羣
      a>  修改5臺機器的hosts文件

            ＃vim /etc/hosts           

        192.168.68.201  master01
        192.168.68.202  master02
        192.168.68.203  slave01
        192.168.68.204  slvae02
        192.168.68.205  slave03

      b>  配置master無密碼登錄slave,在master01和master02上執行以下命令:   

             #ssh-keygen
             #ssh-copy-id -i .ssh/id_rsa.pub root@master01
             #ssh-copy-id -i .ssh/id_rsa.pub root@master02
             #ssh-copy-id -i .ssh/id_rsa.pub root@slave01
             #ssh-copy-id -i .ssh/id_rsa.pub root@slave02
             #ssh-copy-id -i .ssh/id_rsa.pub root@slave03

      c>  5臺機器分別安裝java虛擬機，設置java環境變量

            #./jdk-6u26-linux-x64-rpm.bin      
            #cat >>/etc/profile <<EOF
              export JAVA_HOME=/usr/java/jdk1.6.0_26
              export PATH=\$JAVA_HOME/bin:\$PATH
              EOF 
      d>  在master01上解壓hadoop-0.20.2.tar.gz 配置hadoop集羣
             #tar -zxvf hadoop-0.20.2.tar.gz     
            #vim hadoop-0.20.2/conf/hadoop-env.sh 

       export JAVA_HOME=/usr/java/jdk1.6.0_26
       export HADOOP_NAMENODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_NAMENODE_OPTS"
       export HADOOP_SECONDARYNAMENODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_SECONDARYNAMENODE_OPTS"
       export HADOOP_DATANODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_DATANODE_OPTS"
       export HADOOP_BALANCER_OPTS="-Dcom.sun.management.jmxremote $HADOOP_BALANCER_OPTS"
       export HADOOP_JOBTRACKER_OPTS="-Dcom.sun.management.jmxremote $HADOOP_JOBTRACKER_OPTS"

           #vim hadoop-0.20.2/conf/hdfs-site.xml

<configuration>
    <property>
        <name>dfs.http.address</name>
        <value>192.168.68.201:50070</value>
    </property>
    <property>
        <name>dfs.name.dir</name>
        <value>${hadoop.tmp.dir}/dfs/name,/data/name</value>
    </property>
    <property>
        <name>dfs.data.dir</name>
        <value>/data/data</value>
    </property>
    <property> 
        <name>dfs.replication</name> 
        <value>2</value> 
    </property>
</configuration>

         #vim hadoop-0.20.2/conf/mapred-site.xml 

<configuration>
    <property> 
        <name>mapred.job.tracker</name> 
        <value>master02:8021</value> 
    </property>
    <property> 
        <name>mapred.local.dir</name> 
        <value>/data/mapred/local</value> 
    </property> 
    <property> 
        <name>mapred.system.dir</name> 
        <value>/data/mapred/system</value> 
    </property>
    <property> 
        <name>mapred.job.tracker.http.address</name> 
        <value>192.168.68.202:50030</value> 
    </property>
</configuration>

           #vim hadoop-0.20.2/conf/core-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property> 
        <name>fs.default.name</name> 
        <value>hdfs://master01:8020</value> 
    </property>
    <property> 
        <name>hadoop.tmp.dir</name> 
        <value>/data/hadoop/tmp</value> 
    </property>
</configuration>

           #vim hadoop-0.20.2/conf/masters

     master02

          #vim hadoop-0.20.2/conf/slaves           

     slave01
     slave02
     slave03

      e>  將master01上的hadoop-0.20.2 複製到其他機器    

      #scp -r hadoop-0.20.2 root@master02:/root/
      #scp -r hadoop-0.20.2 root@slave01:/root/
      #scp -r hadoop-0.20.2 root@slave02:/root/
      #scp -r hadoop-0.20.2 root@slave03:/root/

     
3>  啓動hadoop集羣
       a>  在master01上格式化hdfs系統
              #./hadoop-0.20.2/bin/hadoop namenode -format  
       b>  在master01上啓動hdfs系統  ,執行jps查看啓動的進程 
              #./hadoop-0.20.2/bin/start-dfs.sh

              #jps

                1872 Jps

                1654 NameNode

      c>  在master02上啓動MapReduce，執行jps查看啓動的進程
             #./hadoop-0.20.2/bin/start-mapred.sh

             #jps   

               1956 Jps
               1737 SecondaryNameNode
               1895 JobTracker

      d>  查看slave01上啓動的進程

             #jps

               2418 Jps
               1758 TaskTracker
               1827 DataNode