準備
下載包
http://archive.cloudera.com/cm5/cm/5/
cloudera-manager-centos7-cm5.9.0_x86_64.tar.gz
http://archive.cloudera.com/cdh5/parcels/latest/
CDH-5.9.0-1.cdh5.9.0.p0.23-el7.parcel
CDH-5.9.0-1.cdh5.9.0.p0.23-el7.parcel.sha1
manifest.json
依賴包
yum install -y python-lxml httpd mod_ssl
yum -y install cyrus-sasl-plain cyrus-sasl-devel cyrus-sasl-gssapi
rpm jdk包
http://archive.cloudera.com/cm5/redhat/7/x86_64/cm/5.15.1/RPMS/x86_64/
rpm -vih jdk-6u31-linux-amd64.rpm
rpm -vih oracle-j2sdk1.7-1.7.0+update67-1.x86_64.rpm
掛載nfs
mkdir /opt/nfs
mount -t nfs 10.0.1.19:/opt/software /opt/nfs
hosts文件
10.0.1.120 cdh1.ops.net
10.0.1.121 cdh2.ops.net
10.0.1.122 cdh3.ops.net
10.0.1.123 cdh4.ops.net
推送ssh key
## 生成
mkdir /opt/nfs/software/hadoop/key -p
cd /opt/nfs/software/hadoop/key && ssh-keygen -t rsa -P '' -f hadoop #生成key
## 所有主機的操作
mkdir /root/.ssh
cp /opt/nfs/software/hadoop/key/hadoop.pub /root/.ssh/authorized_keys
chmod 600 /root/.ssh/authorized_keys
cp /opt/nfs/software/hadoop/key/hadoop /root/.ssh/id_rsa
master1 ntp(確保時間一致性)
which ntpd|xargs rpm -qf ||yum install ntp -y
ntp配置文件
restrict 0.0.0.0 mask 0.0.0.0 nomodify notrap noquery
restrict 127.0.0.1
restrict default nomodify notrap
server ntp1.aliyun.com iburst
server ntp2.aliyun.com iburst
# 阿里雲ntp
server 127.127.1.0
fudge 127.127.1.0 stratum 10
# 定義的server都不可用時，將使用local時間作爲ntp服務提供給ntp客戶端
includefile /etc/ntp/crypto/pw
driftfile /var/lib/ntp/drift
keys /etc/ntp/keys
啓用服務
systemctl enable ntpd && systemctl start ntpd
測試命令
ntpstat
synchronised to NTP server (120.25.115.20) at stratum 3 time correct to within 27 ms polling server every 64 s
其他主機同步時間
grep 'cdh-1' /var/spool/cron/root &>/dev/null ||echo '*/5 * * * * ntpdate cdh-1' > /var/spool/cron/root
禁用Transparent Hugepage
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
cat /sys/kernel/mm/transparent_hugepage/enabled #查看狀態
調整vm.swappiness Linux內核參數
grep vm.swappiness /etc/sysctl.conf || echo "vm.swappiness = 1" >> /etc/sysctl.conf
sysctl -p
安裝mysql
master1 節點安裝mysql
安裝jdk
java -version
java version "1.8.0_102" Java(TM) SE Runtime Environment (build 1.8.0_102-b14) Java HotSpot(TM) 64-Bit Server VM (build 25.102-b14, mixed mode)
安裝MySQL JDBC驅動程序
wget https://cdn.mysql.com//Downloads/Connector-J/mysql-connector-java-5.1.46.tar.gz
tar zxvf mysql-connector-java-5.1.46.tar.gz
mkdir -p /usr/share/java/
cd mysql-connector-java-5.1.46
cp mysql-connector-java-5.1.46-bin.jar /usr/share/java/mysql-connector-java.jar
#http://download.softagency.net/MySQL/Downloads/Connector-J/mysql-connector-java-5.1.42.tar.gz
#JDBC目前已經有6.x，但與CDH 5.x存在不兼容
Cloudera Manager 安裝與集羣配置
...
安裝CDH5
...
清理
mysql
drop database oozie;
drop database hive;
drop database hue;
drop database scm;
刪除文件
rm -Rf /var/lib/flume-ng/ /var/lib/hadoop* /var/lib/navigator /var/lib/hue /var/lib/oozie /var/lib/solr /var/lib/sqoop* /var/lib/zookeeper /etc/hadoop/ /etc/hbase /etc/hive /dfs /var/lib/hive /usr/lib/hue /var/lib/cloudera* /var/cache/yum/cloudera* /var/log/cloudera* /var/run/cloudera*
rm /opt/cm -rf #軟連接
rm -rf /tmp/.scm_prepare_node.lock
rm /opt/cm-5.15.1/lib/cloudera-scm-agent/cm_guid -f
rm /opt/cm/lib/cloudera-scm-agent/cm_guid -f