Linux下安裝Hadoop集羣（2.部署Hadoop）

1.下載解壓hadoop2.7.1

cd /usr/local
wget https://archive.apache.org/dist/hadoop/common/hadoop-2.7.1/hadoop-2.7.1.tar.gz
tar -zxvf hadoop-2.7.1.tar.gz

嫌速度慢的話可以下載這個（不要臉.jpg）
hadoop2.7.1下載

2.配置環境變量

vi /etc/profile
#添加如下配置
export HADOOP_HOME=/usr/local/hadoop-2.7.1
export PATH=$PATH:$HADOOP_HOME/bin
#使配置生效
source /etc/profile
#測試（輸入hadoop有提示則表示成功）
hadoop
#將profile發送到其他節點
scp /etc/profile hadoop@dn2:/etc
scp /etc/profile hadoop@dn3:/etc
#記得在其他節點下source
source /etc/profile

3.修改core-site.xml(配置Service的URL地址、Hadoop集羣的臨時目錄等信息)

#在usr/local下創建hdfs目錄
cd /usr/local 
mkdir hdfs
cd hdfs
mkdir name
mkdir data
mkdir tmp
#將hdfs文件夾傳到其他兩個節點，注意權限問題
chmod 777 usr/local #其他節點也要
chmod 777 usr/local/hdfs
scp -r /usr/local/hdfs hadoop@dn2:/usr/local
scp -r /usr/local/hdfs hadoop@dn3:/usr/local
#進入配置文件路徑
cd /usr/local/hadoop-2.7.1/etc/hadoop
vi core-site.xml

core-site.xml的配置如下

<configuration>
 <property>
   <name>hadoop.tmp.dir</name>
   <value>file:/usr/local/hdfs/tmp</value> 
  </property>
 <property>
   <name>io.file.buffer.size</name>
   <value>131072</value>
 </property>
 <property>
   <name>fs.default.name</name>
   <value>hdfs://dn1:9000</value>
 </property>
 <property>
   <name>hadoop.proxyuser.root.hosts</name>
   <value>*</value>
 </property>
 <property>
   <name>hadoop.proxyuser.root.groups</name>
   <value>*</value>
 </property>
</configuration>

4.修改hdfs-site.xml

vi hdfs-site.xml

hdfs-site.xml的配置如下

<configuration>
 <property>
   <name>dfs.replication</name>
   <value>2</value>
 </property>
 <property>
   <name>dfs.namenode.name.dir</name>
   <value>file:/usr/local/hdfs/name</value>
   <final>true</final>
 </property>
 <property>
   <name>dfs.datanode.data.dir</name>
   <value>file:/usr/local/hdfs/data</value>
   <final>true</final>
 </property>
 <property>
   <name>dfs.namenode.secondary.http-address</name>
   <value>dn1:9001</value>
 </property>
 <property>
   <name>dfs.webhdfs.enabled</name>
   <value>true</value>
 </property>
 <property>
   <name>dfs.permissions</name>
   <value>false</value>
 </property>
</configuration>

5.添加修改mapred-site.xml

#將mapred-site.xml.template複製爲mapred-site.xml
cp mapred-site.xml.template  mapred-site.xml

mapred-site.xml的配置如下

<!--計算任務託管的資源框架名稱-->
 <property>
   <name>mapreduce.framework.name</name>
   <value>yarn</value>
 </property>

6. 修改vim yarn-site.xml

vi yarn-site.xml

yarn-site.xml的配置如下

<configuration>

<!-- Site specific YARN configuration properties -->
 <property>
   <name>yarn.resourcemanager.address</name>
   <value>dn1:8132</value>
 </property>
 <property>
   <name>yarn.resourcemanager.scheduler.address</name>
   <value>dn1:8130</value>
 </property>
 <property>
   <name>yarn.resourcemanager.webapp.address</name>
   <value>dn1:8188</value>
 </property>
 <property>
   <name>yarn.resourcemanager.resource-tracker.address</name>
   <value>dn1:8131</value>
 </property>
 <property>
   <name>yarn.resourcemanager.admin.address</name>
   <value>dn1:8033</value>
 </property>
 <property>
   <name>yarn.nodemanager.aux-services</name>
   <value>mapreduce_shuffle</value>
 </property>

 <property>
   <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
   <value>org.apache.hadoop.mapred.ShuffleHandler</value>
 </property>
  <property>
   <name>yarn.nodemanager.resource.memory-mb</name>
   <value>2048</value>
 </property>
 <property>
   <name>yarn.nodemanager.resource.cpu-vcores</name>
   <value>1</value>
 </property>
</configuration>

7.修改slaves和hadoop-env.sh文件

#修改slaves文件
vi /usr/local/hadoop-2.7.1/etc/hadoop/slaves
#將localhost改成下方
dn2
dn3
#修改hadoop-env.sh文件(註釋掉之前的，添加完整路徑)
vi /usr/local/hadoop-2.7.1/etc/hadoop/hadoop-env.sh 
#export JAVA_HOME=${JAVA_HOME}
export JAVA_HOME=/usr/local/jdk1.8.0_231

8.將hadoop2.7.1的文件夾複製到dn2和dn3

scp -r /usr/local/hadoop-2.7.1 hadoop@dn2:/usr/local
scp -r /usr/local/hadoop-2.7.1 hadoop@dn3:/usr/local

運行Hadoop

#格式化
hadoop namenode -format
cd /usr/local/hadoop-2.7.1/sbin/
#啓動hadoop
sh ./start-all.sh
#查看集羣狀態
/usr/local/hadoop-2.7.1/bin/hdfs dfsadmin -report

#測試yarm，登錄 地址是（yarn.resourcemanager.webapp.address）的value
http://192.168.65.77:8188/cluster/cluster

#測試HDFS
http://192.168.65.77:50070/dfshealth.html