Cloudera(CDH) 部署在centos7系統教程

系統

[root@i-138dr3jn ~]# cat /etc/redhat-release 
CentOS Linux release 7.2.1511 (Core)

主機節點

主機	ip
cdh.master.aniu.so master	10.39.47.63
cdh.node1.aniu.so node1	10.39.47.64
cdh.node2.aniu.so node2	10.39.47.65
cdh.node3.aniu.so node3	10.39.47.66

設置主機名

[root@master ~]#   hostnamectl set-hostname master

主機更新

yum update -y

關閉防火牆

sudo chkconfig iptables off
sudo service iptables stop
systemctl stop firewalld
systemctl disable firewalld

設置selinux

參考https://www.cloudera.com/documentation/enterprise/6/6.0/topics/install_cdh_disable_selinux.html

sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config && setenforce 0

設置每臺服務器之間設置免密認證

[root@node3 ~]# cat /etc/hosts
10.39.47.63 cdh.master.aniu.so master
10.39.47.64 cdh.node1.aniu.so node1
10.39.47.65 cdh.node2.aniu.so node2
10.39.47.66 cdh.node3.aniu.so node3

# 設置swap參數
echo never > /sys/kernel/mm/transparent_hugepage/defrag  #建議寫到開啓啓動新裏
sysctl -w vm.swappiness=0  # 建議寫進sysctl.conf

同步主機時間

參考 https://www.cloudera.com/documentation/enterprise/6/6.0/topics/install_cdh_enable_ntp.html
yum install ntp -y

啓動服務：systemctl start ntpd.service 
開機自啓：systemctl enable ntpd.service

hwclock --systohc

yum源

wget  https://archive.cloudera.com/cm6/6.0.0/redhat7/yum/cloudera-manager.repo  -P /etc/yum.repos.d/ 

rpm --import https://archive.cloudera.com/cm6/6.0.0/redhat7/yum/RPM-GPG-KEY-cloudera

安裝 oracle jdk

yum install oracle-j2sdk1.8 -y

安裝 Cloudera Manager Packages

yum install cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server -y

安裝數據庫並創建數據庫以及安裝數據庫驅動
參考
https://www.cloudera.com/documentation/enterprise/6/6.0/topics/cm_ig_mysql.html#cmig_topic_5_5

create database cmf DEFAULT CHARACTER SET utf8;
GRANT ALL PRIVILEGES ON `cmf`.* TO 'cmf'@'%' IDENTIFIED BY 'enN#$12345';
flush privileges;

[root@master ~]# wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.46.tar.gz
--2018-11-17 11:30:21--  https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.46.tar.gz
Resolving dev.mysql.com (dev.mysql.com)... 137.254.60.11
Connecting to dev.mysql.com (dev.mysql.com)|137.254.60.11|:443... connected.
HTTP request sent, awaiting response... 302 Found
Location: https://cdn.mysql.com//Downloads/Connector-J/mysql-connector-java-5.1.46.tar.gz [following]
--2018-11-17 11:30:24--  https://cdn.mysql.com//Downloads/Connector-J/mysql-connector-java-5.1.46.tar.gz
Resolving cdn.mysql.com (cdn.mysql.com)... 2.19.60.91
Connecting to cdn.mysql.com (cdn.mysql.com)|2.19.60.91|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 4434926 (4.2M) [application/x-tar-gz]
Saving to: ‘mysql-connector-java-5.1.46.tar.gz’

100%[====================================================================================================================>] 4,434,926   1.03MB/s   in 4.1s   

2018-11-17 11:30:29 (1.03 MB/s) - ‘mysql-connector-java-5.1.46.tar.gz’ saved [4434926/4434926]

[root@master ~]# tar zxvf mysql-connector-java-5.1.46.tar.gz
[root@master mysql-connector-java-5.1.46]# cp mysql-connector-java-5.1.46-bin.jar /usr/share/java/mysql-connector-java.jar
[root@master mysql-connector-java-5.1.46]# ls -l /usr/share/java/mysql-connector-java.jar

數據庫表

https://www.cloudera.com/documentation/enterprise/6/6.0/topics/cm_ig_mysql.html#cmig_topic_5_5

啓動

systemctl start cloudera-scm-server
systemctl status cloudera-scm-server

查看是否啓動

[root@master ~]# netstat -tlnp 
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name    
tcp        0      0 0.0.0.0:7180            0.0.0.0:*               LISTEN      26110/java          
tcp        0      0 0.0.0.0:7182            0.0.0.0:*               LISTEN      26110/java          
tcp        0      0 0.0.0.0:22              0.0.0.0:*               LISTEN      16396/sshd          
tcp6       0      0 :::3306                 :::*                    LISTEN      27191/mysqld        
tcp6       0      0 :::22                   :::*                    LISTEN      16396/sshd

相關配置文件

[root@master ~]# cat /etc/cloudera-scm-server/db.properties 
# Copyright (c) 2012 Cloudera, Inc. All rights reserved.
#
# This file describes the database connection.
#

# The database type
# Currently 'mysql', 'postgresql' and 'oracle' are valid databases.
com.cloudera.cmf.db.type=mysql

# The database host
# If a non standard port is needed, use 'hostname:port'
com.cloudera.cmf.db.host=master

# The database name
com.cloudera.cmf.db.name=cmf

# The database user
com.cloudera.cmf.db.user=cmf

# The database user's password
com.cloudera.cmf.db.password=enN#$12345

# The db setup type
# After fresh install it is set to INIT
# and will be changed post config.
# If scm-server uses Embedded DB then it is set to EMBEDDED
# If scm-server uses External DB then it is set to EXTERNAL
com.cloudera.cmf.db.setupType=EXTERNAL

[root@master ~]# cat /etc/default/cloudera-scm-server
#
# Specify any command line arguments for the Cloudera SCM Server here.
#

CMF_SERVER_ARGS=""

#
# Locate the JDBC driver jar file.
#
# The default value is the default system mysql driver on RHEL/CentOS/Ubuntu
# and the standard, documented location for where to put the oracle jar in CM
# deployments.
#

export CMF_JDBC_DRIVER_JAR="/usr/share/java/mysql-connector-java.jar:/usr/share/java/oracle-connector-java.jar:/usr/share/java/postgresql-connector-java.jar"

#
# Java Options.
#
# Default value sets Java maximum heap size to 2GB, and Java maximum permanent
# generation size to 256MB.
#

export CMF_JAVA_OPTS="-Xmx2G -XX:MaxPermSize=512m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp"

訪問 http://10.39.47.63:7180