1:MySQL離線部署 元數據
2:CM離線部署
3:Parcel文件離線源部署 hfs yarn hive base zk
節點初始化
配置hosts
內網ip
172.31.232.209 hadoop001
172.31.232.208 hadoop002
172.31.232.207 hadoop003
注意三臺機器都要配置hosts
配置防火牆開放外部可以訪問的網站
關閉防火牆
[root@hadoop001 ~]# systemctl stop firewalld
[root@hadoop001 ~]# systemctl disable firewalld
重啓服務器
配置時區(ntp)
[root@hadoop001 ~]# yum -y install ntp
[root@hadoop002 ~]# yum -y install ntp
[root@hadoop003 ~]# yum -y install ntp
配置主
[root@hadoop001 ~]# vi /etc/ntp.conf
server 0.asia.pool.ntp.org
server 1.asia.pool.ntp.org
server 2.asia.pool.ntp.org
server 3.asia.pool.ntp.org
server 127.0.0.1 iburst local clock
restrict 172.31.232.0 mask 255.255.255.0 nomodify
[root@hadoop001 ~]# systemctl start ntpd
[root@hadoop001 ~]# systemctl status ntpd
[root@hadoop001 ~]# ntp -p
客戶端與主同步(有多少個客戶端就執行幾次)
[root@hadoop002 ~]# systemctl stop ntpd
[root@hadoop002 ~]# systemctl disable ntpd
Removed symlink /etc/systemd/system/multi-user.target.wants/ntpd.service.
[root@hadoop002 ~]# which ntpdate
/usr/sbin/ntpdate
[root@hadoop002 ~]# /usr/sbin/ntpdate hadoop001
23 May 16:17:50 ntpdate[2354]: step time server 172.31.232.209 offset -2.166869 sec
修正時間
[root@hadoop002 ~]# crontab -e
00 00 * * * /usr/sbin/ntpdate hadoop001
安裝jdk
[root@hadoop001 ~]# scp cdh5.16.1/jdk-8u45-linux-x64.gz hadoop002:/root/software/
[root@hadoop001 ~]# scp cdh5.16.1/jdk-8u45-linux-x64.gz hadoop003:/root/software/
三臺機器配置jdk
[root@hadoop001 ~]# mkdir /usr/java
[root@hadoop001 cdh5.16.1]# tar -zxvf jdk-8u45-linux-x64.gz -C /usr/java/
修改jdk權限
[root@hadoop001 java]# chown -R root:root jdk1.8.0_45/
[root@hadoop001 java]# chown -R root:root jdk1.8.0_45/*
[root@hadoop001 jdk1.8.0_45]# vi /etc/profile
export JAVA_HOME=/usr/java/jdk1.8.0_45
export PATH=$JAVA_HOME/bin:$PATH
[root@hadoop001 jdk1.8.0_45]# source /etc/profile
離線部署mysql
[root@hadoop001 cdh5.16.1]# tar -zxvf mysql-5.7.11-linux-glibc2.5-x86_64.tar.gz -C /usr/local/
[root@hadoop001 local]# chown -R root:root mysql-5.7.11-linux-glibc2.5-x86_64/
[root@hadoop001 local]# chown -R root:root mysql-5.7.11-linux-glibc2.5-x86_64/*
[root@hadoop001 local]# mv mysql-5.7.11-linux-glibc2.5-x86_64/ mysql
[root@hadoop001 local]# mkdir mysql/arch mysql/data mysql/tmp
[root@hadoop001 local]# vi /etc/my.cnf
[client]
port = 3306
socket = /usr/local/mysql/data/mysql.sock
default-character-set=utf8mb4
[mysqld]
port = 3306
socket = /usr/local/mysql/data/mysql.sock
skip-slave-start
skip-external-locking
key_buffer_size = 256M
sort_buffer_size = 2M
read_buffer_size = 2M
read_rnd_buffer_size = 4M
query_cache_size= 32M
max_allowed_packet = 16M
myisam_sort_buffer_size=128M
tmp_table_size=32M
table_open_cache = 512
thread_cache_size = 8
wait_timeout = 86400
interactive_timeout = 86400
max_connections = 600
# Try number of CPU's*2 for thread_concurrency
#thread_concurrency = 32
#isolation level and default engine
default-storage-engine = INNODB
transaction-isolation = READ-COMMITTED
server-id = 1739
basedir = /usr/local/mysql
datadir = /usr/local/mysql/data
pid-file = /usr/local/mysql/data/hostname.pid
#open performance schema
log-warnings
sysdate-is-now
binlog_format = ROW
log_bin_trust_function_creators=1
log-error = /usr/local/mysql/data/hostname.err
log-bin = /usr/local/mysql/arch/mysql-bin
expire_logs_days = 7
innodb_write_io_threads=16
relay-log = /usr/local/mysql/relay_log/relay-log
relay-log-index = /usr/local/mysql/relay_log/relay-log.index
relay_log_info_file= /usr/local/mysql/relay_log/relay-log.info
log_slave_updates=1
gtid_mode=OFF
enforce_gtid_consistency=OFF
# slave
slave-parallel-type=LOGICAL_CLOCK
slave-parallel-workers=4
master_info_repository=TABLE
relay_log_info_repository=TABLE
relay_log_recovery=ON
#other logs
#general_log =1
#general_log_file = /usr/local/mysql/data/general_log.err
#slow_query_log=1
#slow_query_log_file=/usr/local/mysql/data/slow_log.err
#for replication slave
sync_binlog = 500
#for innodb options
innodb_data_home_dir = /usr/local/mysql/data/
innodb_data_file_path = ibdata1:1G;ibdata2:1G:autoextend
innodb_log_group_home_dir = /usr/local/mysql/arch
innodb_log_files_in_group = 4
innodb_log_file_size = 1G
innodb_log_buffer_size = 200M
#根據生產需要，調整pool size
innodb_buffer_pool_size = 2G
#innodb_additional_mem_pool_size = 50M #deprecated in 5.6
tmpdir = /usr/local/mysql/tmp
innodb_lock_wait_timeout = 1000
#innodb_thread_concurrency = 0
innodb_flush_log_at_trx_commit = 2
innodb_locks_unsafe_for_binlog=1
#innodb io features: add for mysql5.5.8
performance_schema
innodb_read_io_threads=4
innodb-write-io-threads=4
innodb-io-capacity=200
#purge threads change default(0) to 1 for purge
innodb_purge_threads=1
innodb_use_native_aio=on
#case-sensitive file names and separate tablespace
innodb_file_per_table = 1
lower_case_table_names=1
[mysqldump]
quick
max_allowed_packet = 128M
[mysql]
no-auto-rehash
default-character-set=utf8mb4
[mysqlhotcopy]
interactive-timeout
[myisamchk]
key_buffer_size = 256M
sort_buffer_size = 256M
read_buffer = 2M
write_buffer = 2M
注意：把配置文件的內容全清除
創建用戶用戶組
[root@hadoop001 local]# groupadd -g 101 dba
useradd -u 514 -g dba -G root -d /usr/local/mysql mysqladmin
創建用戶有問題，需要執行下面一條命令操作
[root@hadoop001 local]# cp /etc/skel/.* /usr/local/mysql/
配置環境變量
[root@hadoop001 local]# vi mysql/.bash_profile
把原有的export刪除
export MYSQL_BASE=/usr/local/mysql
export PATH=${MYSQL_BASE}/bin:$PATH
unset USERNAME
#stty erase ^H
set umask to 022
umask 022
PS1=`uname -n`":"'$USER'":"'$PWD'":>"; export PS1
可以看一下效果
[root@hadoop001 local]# su - mysqladmin
上一次登錄：六 5月 23 20:53:53 CST 2020pts/0 上
hadoop001:mysqladmin:/usr/local/mysql:>exit
logout
賦予權限
[root@hadoop001 local]# chown mysqladmin:dba /etc/my.cnf
[root@hadoop001 local]# chmod 640 /etc/my.cnf
[root@hadoop001 local]# chown -R mysqladmin:dba /usr/local/mysql/
[root@hadoop001 local]# chmod -R 755 /usr/local/mysql/
設置開機自啓動
[root@hadoop001 local]# cd /usr/local/mysql/
[root@hadoop001 mysql]# cp support-files/mysql.server /etc/rc.d/init.d/mysql
[root@hadoop001 mysql]# chmod +x /etc/rc.d/init.d/mysql
[root@hadoop001 mysql]# chkconfig --del mysql
[root@hadoop001 mysql]# chkconfig --add mysql
[root@hadoop001 mysql]# chkconfig --level 345 mysql on
[root@hadoop001 mysql]# chkconfig --list
注意：該輸出結果只顯示 SysV 服務，並不包含原生 systemd 服務。SysV 配置數據可能被原生 systemd 配置覆蓋。
如果您想列出 systemd 服務,請執行 'systemctl list-unit-files'。
欲查看對特定 target 啓用的服務請執行
'systemctl list-dependencies [target]'。
aegis 0:關 1:關 2:開 3:開 4:開 5:開 6:關
mysql 0:關 1:關 2:開 3:開 4:開 5:開 6:關
netconsole 0:關 1:關 2:關 3:關 4:關 5:關 6:關
network 0:關 1:關 2:開 3:開 4:開 5:開 6:關
[root@hadoop001 mysql]# yum -y install libaio
[root@hadoop001 mysql]# su - mysqladmin
進行mysql初始化
hadoop001:mysqladmin:/usr/local/mysql:>bin/mysqld \
> --defaults-file=/etc/my.cnf \
> --user=mysqladmin \
> --basedir=/usr/local/mysql/ \
> --datadir=/usr/local/mysql/data/ \
> --initialize
等待....
獲取初始密碼
hadoop001:mysqladmin:/usr/local/mysql:>cd data/
hadoop001:mysqladmin:/usr/local/mysql/data:>ll
total 2097196
-rw-r----- 1 mysqladmin dba 56 May 23 21:11 auto.cnf
-rw-r----- 1 mysqladmin dba 7727 May 23 21:12 hostname.err
-rw-r----- 1 mysqladmin dba 407 May 23 21:12 ib_buffer_pool
-rw-r----- 1 mysqladmin dba 1073741824 May 23 21:12 ibdata1
-rw-r----- 1 mysqladmin dba 1073741824 May 23 21:11 ibdata2
drwxr-x--- 2 mysqladmin dba 4096 May 23 21:11 mysql
drwxr-x--- 2 mysqladmin dba 4096 May 23 21:11 performance_schema
drwxr-x--- 2 mysqladmin dba 12288 May 23 21:12 sys
hadoop001:mysqladmin:/usr/local/mysql/data:>cat hostname.err | grep password
2020-05-23T13:11:59.402441Z 1 [Note] A temporary password is generated for root@localhost: _e;N9JPoIEjc
hadoop001:mysqladmin:/usr/local/mysql/data:>cd ..
啓動
hadoop001:mysqladmin:/usr/local/mysql:>/usr/local/mysql/bin/mysqld_safe --defaults-file=/etc/my.cnf &
查看mysql 端口
hadoop001:mysqladmin:/usr/local/mysql:>ps -ef | grep mysql
root 12115 11811 0 21:08 pts/0 00:00:00 su - mysqladmin
mysqlad+ 12116 12115 0 21:08 pts/0 00:00:00 -bash
mysqlad+ 12174 12116 0 21:15 pts/0 00:00:00 /bin/sh /usr/local/mysql/bin/mysqld_safe --defaults-file=/etc/my.cnf
mysqlad+ 12991 12174 0 21:15 pts/0 00:00:00 /usr/local/mysql/bin/mysqld --defaults-file=/etc/my.cnf --basedir=/usr/local/mysql --datadir=/usr/local/mysql/data --plugin-dir=/usr/local/mysql/lib/plugin --log-error=/usr/local/mysql/data/hostname.err --pid-file=/usr/local/mysql/data/hostname.pid --socket=/usr/local/mysql/data/mysql.sock --port=3306
mysqlad+ 13023 12116 0 21:16 pts/0 00:00:00 ps -ef
mysqlad+ 13024 12116 0 21:16 pts/0 00:00:00 grep --color=auto mysql
hadoop001:mysqladmin:/usr/local/mysql:>exit
logout
[root@hadoop001 mysql]# ps -ef | grep mysql
mysqlad+ 12174 1 0 21:15 pts/0 00:00:00 /bin/sh /usr/local/mysql/bin/mysqld_safe --defaults-file=/etc/my.cnf
mysqlad+ 12991 12174 0 21:15 pts/0 00:00:00 /usr/local/mysql/bin/mysqld --defaults-file=/etc/my.cnf --basedir=/usr/local/mysql --datadir=/usr/local/mysql/data --plugin-dir=/usr/local/mysql/lib/plugin --log-error=/usr/local/mysql/data/hostname.err --pid-file=/usr/local/mysql/data/hostname.pid --socket=/usr/local/mysql/data/mysql.sock --port=3306
root 13026 11811 0 21:18 pts/0 00:00:00 grep --color=auto mysql
[root@hadoop001 mysql]# netstat -nlp|grep 12991
tcp6 0 0 :::3306 :::* LISTEN 12991/mysqld
unix 2 [ ACC ] STREAM LISTENING 64182 12991/mysqld /usr/local/mysql/data/mysql.sock
[root@hadoop001 mysql]# su - mysqladmin
hadoop001:mysqladmin:/usr/local/mysql:>service mysql status
嘗試登錄
hadoop001:mysqladmin:/usr/local/mysql:>mysql -uroot -p
Enter password:
Welcome to the MySQL monitor. Commands end with ; or \g.
Your MySQL connection id is 2
Server version: 5.7.11-log
Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved.
Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
mysql>
重置密碼
mysql> alter user root@localhost identified by 'Hadoop12345678';
Query OK, 0 rows affected (0.00 sec)
mysql> GRANT ALL PRIVILEGES ON *.* TO ‘root'@'%' IDENTIFIED BY 'Hadoop12345678';
Query OK, 0 rows affected, 1 warning (0.00 sec)
mysql> flush privileges;
Query OK, 0 rows affected (0.00 sec)
創建db用戶
mysql> create database cmf default character set utf8;
Query OK, 1 row affected (0.01 sec)
mysql> GRANT ALL PRIVILEGES ON cmf.* TO 'cmf'@'%' IDENTIFIED BY 'Hadoop12345678';
Query OK, 0 rows affected, 1 warning (0.00 sec)
mysql> GRANT ALL PRIVILEGES ON amon.* TO 'amon'@'%' IDENTIFIED BY 'Hadoop12345678';
Query OK, 0 rows affected, 1 warning (0.00 sec)
mysql> create database amon default character set utf8;
mysql> flush privileges;
Query OK, 0 rows affected (0.00 sec)
部署mysql jdbc jar
[root@hadoop001 ~]# mkdir -p /usr/share/java
[root@hadoop001 ~]# cp cdh5.16.1/mysql-connector-java-5.1.47.jar /usr/share/java/mysql-connector-java.jar
部署cm
[root@hadoop001 ~]# scp cdh5.16.1/cloudera-manager-centos7-cm5.16.1_x86_64.tar.gz hadoop002:/root/software/
[root@hadoop001 ~]# scp cdh5.16.1/cloudera-manager-centos7-cm5.16.1_x86_64.tar.gz hadoop003:/root/software/
[root@hadoop001 sofware]# mkdir /opt/cloudera-manager
[root@hadoop002 sofware]# mkdir /opt/cloudera-manager
[root@hadoop003 sofware]# mkdir /opt/cloudera-manager
[root@hadoop001 sofware]# tar -zxvf cloudera-manager-centos7-cm5.16.1_x86_64.tar.gz -C /opt/cloudera-manager/
[root@hadoop002 sofware]# tar -zxvf cloudera-manager-centos7-cm5.16.1_x86_64.tar.gz -C /opt/cloudera-manager/
[root@hadoop003 sofware]# tar -zxvf cloudera-manager-centos7-cm5.16.1_x86_64.tar.gz -C /opt/cloudera-manager/
修改agent端
[root@hadoop001 sofware]# cd /opt/cloudera-manager/cm-5.16.1/
[root@hadoop001 cm-5.16.1]# cd etc/cloudera-scm-agent/
[root@hadoop001 cloudera-scm-agent]# vi config.ini
server_host=hadoop001
[root@hadoop002 software]# cd /opt/cloudera-manager/cm-5.16.1/etc/cloudera-scm-agent
[root@hadoop002 cloudera-scm-agent]# vi config.ini
server_host=hadoop001
[root@hadoop003 software]# cd /opt/cloudera-manager/cm-5.16.1/etc/cloudera-scm-agent
[root@hadoop003 cloudera-scm-agent]# vi config.ini
server_host=hadoop001
[root@hadoop001 etc]# cd cloudera-scm-server/
[root@hadoop001 cloudera-scm-server]# vi db.properties
# Copyright (c) 2012 Cloudera, Inc. All rights reserved.
#
# This file describes the database connection.
#
# The database type
# Currently 'mysql', 'postgresql' and 'oracle' are valid databases.
com.cloudera.cmf.db.type=mysql
# The database host
# If a non standard port is needed, use 'hostname:port'
com.cloudera.cmf.db.host=hadoop001
# The database name
com.cloudera.cmf.db.name=cmf
# The database user
com.cloudera.cmf.db.user=cmf
# The database user's password
com.cloudera.cmf.db.password=Hadoop12345678
# The db setup type
# By default, it is set to INIT
# If scm-server uses Embedded DB then it is set to EMBEDDED
# If scm-server uses External DB then it is set to EXTERNAL
com.cloudera.cmf.db.setupType=EXTERNAL
[root@hadoop001 cloudera-scm-server]# useradd --system --home=/opt/cloudera-manager/cm-5.16.1/run/cloudera-scm-server/ --no-create-home --shell=/bin/false cloudera-scm
[root@hadoop002 cloudera-scm-server]# useradd --system --home=/opt/cloudera-manager/cm-5.16.1/run/cloudera-scm-server/ --no-create-home --shell=/bin/false cloudera-scm
[root@hadoop003 cloudera-scm-server]# useradd --system --home=/opt/cloudera-manager/cm-5.16.1/run/cloudera-scm-server/ --no-create-home --shell=/bin/false cloudera-scm
[root@hadoop001 cloudera-scm-server]# chown -R cloudera-scm:cloudera-scm /opt/cloudera-manager/
[root@hadoop002 cloudera-scm-server]# chown -R cloudera-scm:cloudera-scm /opt/cloudera-manager/
[root@hadoop003 cloudera-scm-server]# chown -R cloudera-scm:cloudera-scm /opt/cloudera-manager/
部署離線源
[root@hadoop001 cdh5.16.1]# mkdir -p /opt/cloudera/parcel-repo
[root@hadoop001 cdh5.16.1]# mv CDH-5.16.1-1.cdh5.16.1.p0.3-el7.parcel /opt/cloudera/parcel-repo/
[root@hadoop001 cdh5.16.1]# mv CDH-5.16.1-1.cdh5.16.1.p0.3-el7.parcel.sha1 /opt/cloudera/parcel-repo/CDH-5.16.1-1.cdh5.16.1.p0.3-el7.parcel.sha
[root@hadoop001 cdh5.16.1]# mv manifest.json /opt/cloudera/parcel-repo/
[root@hadoop001 cdh5.16.1]# cd /opt/cloudera/parcel-repo/
[root@hadoop001 parcel-repo]# chown -R cloudera-scm:cloudera-scm /opt/cloudera
[root@hadoop001 parcel-repo]# mkdir -p /opt/cloudera/parcels
[root@hadoop002 parcel-repo]# mkdir -p /opt/cloudera/parcels
[root@hadoop003 parcel-repo]# mkdir -p /opt/cloudera/parcels
[root@hadoop001 parcel-repo]# chown -R cloudera-scm:cloudera-scm /opt/cloudera
[root@hadoop002 parcel-repo]# chown -R cloudera-scm:cloudera-scm /opt/cloudera
[root@hadoop003 parcel-repo]# chown -R cloudera-scm:cloudera-scm /opt/cloudera
[root@hadoop001 parcel-repo]# cd /opt/cloudera-manager/cm-5.16.1/etc/
[root@hadoop001 etc]# cd init.d/
[root@hadoop001 init.d]# ./cloudera-scm-server start
[root@hadoop001 init.d]# ./cloudera-scm-server status
cloudera-scm-server (pid 4383) 正在運行…
[root@hadoop001 init.d]# ./cloudera-scm-agent start
[root@hadoop002 init.d]# ./cloudera-scm-agent start
[root@hadoop003 init.d]# ./cloudera-scm-agent start