需準備東西:
1、3個或3個以上Linux服務器;
Linux安裝安裝教程見:https://blog.csdn.net/Hjchidaozhe/article/details/103434690
2、Linux jdk 8 ;
官網下載方式見:https://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html
3、ftp傳輸工具;
下載Filezilla 百度網盤:https://pan.baidu.com/s/149i0H94bVuC0n9QbphFegw (fsvb) (使用時,Linux默認端口22)
4、Hadoop安裝包;
coluder 版本鏡像地址:http://archive.cloudera.com/cdh5/cdh/5/
現在默認跳轉官網,可在最後一步跳轉的網址後添加 .tar.gz 例如:http://archive.cloudera.com/cdh5/cdh/5/hadoop-2.6.0-cdh5.9.3.tar.gz
Apache Hadoop下載地址:http://mirror.bit.edu.cn/apache/hadoop/common/
方式採用新建hadoop用戶,將集羣整個權限賦給hadoop操作,安裝時,在主節點安裝,拷貝至從節點
在無說明的情況下,均在root權限下操作,本文中所有密碼均設置爲123456
本文默認下載文件通過ftp傳輸至虛擬機:/tmp 下面,且所有文件安裝在 /usr/local/
下面開始準備系統環境:
# 修改主機名(3臺)
# 臨時修改主機名
hostname centos-a
# 永久修改主機名,將文件中的名字替換
vi /etc/hostname
centos-a
# 修改hosts文件(3臺)
# 編輯文件,在裏面添加以下內容
vi /etc/hosts
192.168.190.128 centos-a
192.168.190.129 centos-b
192.168.190.130 centos-c
# 新建hadoop用戶,並設置密碼(3臺)
useradd hadoop
passwd hadoop
# 輸入兩遍密碼
# 卸載centos自帶jdk,並重新安裝下載版本(3臺)
# 查看已存在的jdk信息:(3臺)
java -version
# 查看jdk詳細信息:(找出jdk文件,並將其卸載)(3臺)
rpm -qa|grep jdk 或者 rpm -qa|grep java
# 執行卸載命令(3臺)
rpm -e --nodeps java-1.7.0-openjdk-1.7.0.191-2.6.15.5.el7.x86_64
rpm -e --nodeps java-1.7.0-openjdk-headless-1.7.0.191-2.6.15.5.el7.x86_64
rpm -e --nodeps java-1.8.0-openjdk-headless-1.8.0.181-7.b13.el7.x86_64
rpm -e --nodeps java-1.8.0-openjdk-1.8.0.181-7.b13.el7.x86_64
# 再次查看是否卸載完成(3臺)
java -version
# 移動下載好的jdk壓縮包到指定目錄(3臺)
mv jdk-8u191-linux-x64.tar.gz /usr/local/
# 解壓安裝(3臺)
tar -zxvf jdk-8u191-linux-x64.tar.gz
# 修改環境變量,添加以下內容(3臺)
export JAVA_HOME=/usr/local/jdk1.8.0_191
export JRE_HOME=${JAVA_HOME}/jre
export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib
export PATH=${JAVA_HOME}/bin:$PATH
# 使環境變量立即生效(3臺)
source /etc/profile
# 再次查看Java版本,並確認是否安裝完成(3臺)
java -version
# 創建3臺服務器間訪問密鑰,免密登陸,切換到hadoop用戶下,且3臺都需要操作(3臺)
su hadoop
# 創建密鑰(hadoop用戶下)(3臺)
ssh-keygen -t rsa (這裏一直回車默認即可)
# 複製密鑰到3臺,自己也需要(hadoop用戶下)(3臺)
ssh-copy-id -i /home/hadoop/.ssh/id_rsa.pub centos-a (輸入yes,然後輸入密碼:123456)
ssh-copy-id -i /home/hadoop/.ssh/id_rsa.pub centos-b (輸入yes,然後輸入密碼:123456)
ssh-copy-id -i /home/hadoop/.ssh/id_rsa.pub centos-c (輸入yes,然後輸入密碼:123456)
# 下面開始安裝hadoop,只需在主節點配置
# 切換到安裝目錄,並移動下載好的hadoop壓縮包
mv /tmp/hadoop-2.6.0-cdh5.9.3.tar.gz -/usr/local/
cd /usr/local/
# 解壓安裝
tar -zxvf hadoop-2.6.0-cdh5.9.3.tar.gz
# 將整個目錄權限授給hadoop用戶
chown -R hadoop.hadoop /usr/local/hadoop-2.6.0-cdh5.9.3
# 下面配置hadoop相關文件,切換到配置文件目錄(hadoop用戶下)
cd /usr/local/hadoop-2.6.0-cdh5.9.3/etc/hadoop/
# 創建文件夾,供後續使用(hadoop用戶下)
mkdir /usr/local/hadoop-2.6.0-cdh5.9.3/tmp
mkdir /usr/local/hadoop-2.6.0-cdh5.9.3/var
mkdir /usr/local/hadoop-2.6.0-cdh5.9.3/dfs
mkdir /usr/local/hadoop-2.6.0-cdh5.9.3/dfs/name
mkdir /usr/local/hadoop-2.6.0-cdh5.9.3/dfs/data
# 修改文件 slaves ,裏面只放如下信息(從節點信息)(hadoop用戶下)
vi slaves
centos-b
centos-c
# 修改文件 hadoop-env.sh ,找到並修改指定內容(hadoop用戶下)
vi hadoop-env.sh
export JAVA_HOME=/usr/local/jdk1.8.0_191
# 修改文件 core-site.xml ,在 <configuration> </configuration>文件之間添加以下內容,注意不要重複 <configuration> </configuration>,這裏只是說明添加位置(hadoop用戶下)
vi core-site.xml
<configuration>
<property>
<name>hadoop.tmp.dir</name>
<value>/usr/local/hadoop-2.6.0-cdh5.9.3/tmp</value>
</property>
<property>
<name>fs.default.name</name>
<value>hdfs://centos-a:9000</value>
</property>
</configuration>
# 修改文件 hdfs-site.xml ,添加以下內容(hadoop用戶下)
vi hdfs-site.xml
<configuration>
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>centos-a:9091</value>
<description>The secondary namenode http server address andport.</description>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:///usr/local/hadoop-2.6.0-cdh5.9.3/dfs/name</value>
<description>Path on the local filesystem where the NameNodestores the namespace and transactions logs persistently.</description>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:///usr/local/hadoop-2.6.0-cdh5.9.3/dfs/data</value>
<description>Comma separated list of paths on the local filesystemof a DataNode where it should store its blocks.</description>
</property>
<property>
<name>dfs.replication</name>
<value>3</value>
</property>
</configuration>
# 拷貝指定文件,並修改拷貝之後的文件(hadoop用戶下)
cp mapred-site.xml.template mapred-site.xml
# 修改文件 mapred-site.xml ,添加以下內容(hadoop用戶下)
vim mapred-site.xml
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
<description>Theruntime framework for executing MapReduce jobs. Can be one of local, classic or yarn.</description>
</property>
</configuration>
# 修改文件 yarn-site.xml ,添加以下內容(hadoop用戶下)
vi yarn-site.xml
<configuration>
<property>
<name>yarn.resourcemanager.hostname</name>
<value>centos-a</value>
<description>The hostname of theRM.</description>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
<description>Shuffle service that needs to be set for Map Reduceapplications.</description>
</property>
<property>
<description>The address of the applications manager interface in the RM.</description>
<name>yarn.resourcemanager.address</name>
<value>centos-a:8032</value>
</property>
<property>
<description>The address of the scheduler interface.</description>
<name>yarn.resourcemanager.scheduler.address</name>
<value>centos-a:8030</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>centos-a:8031</value>
</property>
<property>
<description>The address of the RM admin interface.</description>
<name>yarn.resourcemanager.admin.address</name>
<value>centos-a:8033</value>
</property>
<property>
<description>The http address of the RM web application.</description>
<name>yarn.resourcemanager.webapp.address</name>
<value>centos-a:8088</value>
</property>
<property>
<description>The https adddress of the RM web application.</description>
<name>yarn.resourcemanager.webapp.https.address</name>
<value>centos-a:8090</value>
</property>
<property>
<name>yarn.scheduler.maximum-allocation-mb</name>
<value>2048</value>
<discription>每個節點可用內存,單位MB,默認8182MB</discription>
</property>
</configuration>
# hadoop文件配置完畢,下面在主節點拷貝至另外兩個節點,root用戶下
scp -r /usr/local/hadoop-2.6.0-cdh5.9.3/ root@centos-b:/usr/local/
scp -r /usr/local/hadoop-2.6.0-cdh5.9.3/ root@centos-c:/usr/local/
# 在另外兩個節點分別授權hadoop目錄權限(另外兩臺)
chown -R hadoop.hadoop /usr/local/hadoop-2.6.0-cdh5.9.3
# 下面再聲明hadoop命令的環境變量,編輯文件,添加以下內容(3臺)
vi /etc/profile
export HADOOP_HOME=/usr/local/hadoop-2.6.0-cdh5.9.3/
export PATH=${HADOOP_HOME}/bin:$PATH
# 使環境變量立即生效(3臺)
source /etc/profile
# 爲防止意外,建議重啓三臺機器(3臺)
reboot
# 重啓完成後,在主節點,切換至hadoop用戶下
su hadoop
# 切換目錄,初始化化hadoop(hadoop用戶下)
cd /usr/local/hadoop-2.6.0-cdh5.9.3/
./bin/hadoop namenode -format
# 下面開啓集羣,如果有需要確認,則輸入yes再回車(一般第一次開啓需要)(hadoop用戶下)
./sbin/start-all.sh
# 然後3臺服務器查看後臺進程,主節點顯示入下1,從節點顯示如下2,則安裝完成(hadoop用戶下)(3臺)
jps
12628 NameNode
13158 ResourceManager
16648 Jps
13274 NodeManager
12750 DataNode
12974 SecondaryNameNode
8976 NodeManager
8858 DataNode
11627 Jps
# 下面可以用文件再次測試確認安裝完成(hadoop用戶下)
cd /home/hadoop/
vi test.txt
hello world
hadoop fs -put test.txt /
hadoop fs -ls /
hadoop fs -cat /test.txt
# 關閉集羣(hadoop用戶下)
./sbin/stop-all.sh
錯誤提示:
一般開啓時會報如下警告:WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
解決方案參考:https://blog.csdn.net/Hjchidaozhe/article/details/103430157