HBase搭建 之 分佈式模式

目錄

• Vagrant

• Prepare

• Hadoop

  • 下載配置

  • 啓動服務

  • 相關工具

• ZooKeeper

  • 下載配置

  • 啓動服務

  • 相關工具

• HBase

  • 下載配置

  • 啓動服務

  • 相關工具

Vagrant

vagrant -v
# Vagrant 2.2.6

vagrant box list | grep ubuntu/xenial64
# ubuntu/xenial64    (virtualbox, 20200407.0.0)

vim sources.list

deb-src http://archive.ubuntu.com/ubuntu xenial main restricted
deb http://mirrors.aliyun.com/ubuntu/ xenial main restricted
deb-src http://mirrors.aliyun.com/ubuntu/ xenial main restricted multiverse universe
deb http://mirrors.aliyun.com/ubuntu/ xenial-updates main restricted
deb-src http://mirrors.aliyun.com/ubuntu/ xenial-updates main restricted multiverse universe
deb http://mirrors.aliyun.com/ubuntu/ xenial universe
deb http://mirrors.aliyun.com/ubuntu/ xenial-updates universe
deb http://mirrors.aliyun.com/ubuntu/ xenial multiverse
deb http://mirrors.aliyun.com/ubuntu/ xenial-updates multiverse
deb http://mirrors.aliyun.com/ubuntu/ xenial-backports main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ xenial-backports main restricted universe multiverse
deb http://archive.canonical.com/ubuntu xenial partner
deb-src http://archive.canonical.com/ubuntu xenial partner
deb http://mirrors.aliyun.com/ubuntu/ xenial-security main restricted
deb-src http://mirrors.aliyun.com/ubuntu/ xenial-security main restricted multiverse universe
deb http://mirrors.aliyun.com/ubuntu/ xenial-security universe
deb http://mirrors.aliyun.com/ubuntu/ xenial-security multiverse

vim Vagrantfile

cluster = {
    "node01" => { :ip => "192.168.56.161", :vb => "node01" },
    "node02" => { :ip => "192.168.56.162", :vb => "node02" },
    "node03" => { :ip => "192.168.56.163", :vb => "node03" },
    "node04" => { :ip => "192.168.56.164", :vb => "node04" }
}

Vagrant.configure("2") do |config|
    cluster.each_with_index do |(hostname, info), index|
        config.vm.define hostname do |cfg|
            cfg.vm.provider "virtualbox" do |vb, override|
                cfg.vm.box = "ubuntu/xenial64"
                override.vm.host_name = hostname
                override.vm.network "private_network", ip: "#{info[:ip]}"
                vb.name = "#{info[:vb]}"
                vb.cpus = "1"
                vb.memory = "1500"
                cfg.vm.provision "file", source: "~/.ssh/id_rsa", destination: "~/.ssh/id_rsa"
                cfg.vm.provision "file", source: "~/.ssh/id_rsa.pub", destination: "~/.ssh/id_rsa.pub"
                cfg.vm.provision "file", source: "sources.list", destination: "/tmp/sources.list"
                cfg.vm.provision "shell", inline: <<-SHELL
                    cat /home/vagrant/.ssh/id_rsa.pub >> /home/vagrant/.ssh/authorized_keys
                SHELL
                cfg.vm.provision "shell", inline: <<-SHELL
                    sudo mv /tmp/sources.list /etc/apt/sources.list && sudo apt update
                SHELL
            end
        end
    end
end

vagrant up

ssh [email protected]

vagrant@node01:~$ ssh [email protected]

vagrant@node02:~$ ssh [email protected]

vagrant@node03:~$ ssh [email protected]

Prepare

vagrant ssh node01

sudo vim /etc/hosts
# 要刪除其他node0&解析!!! 
# 192.168.56.161 node01
# 192.168.56.162 node02
# 192.168.56.163 node03
# 192.168.56.164 node04

sudo apt install -y openjdk-8-jdk

java -version
# openjdk version "1.8.0_275"

• node02~04都完成上述操作

Hadoop

下載配置

vagrant ssh node01

wget https://mirror.tuna.tsinghua.edu.cn/apache/hadoop/common/hadoop-3.2.1/hadoop-3.2.1.tar.gz

tar xf hadoop-3.2.1.tar.gz

sudo mv hadoop-3.2.1 /opt/hadoop

mkdir -p /opt/hadoop/hdfs/tmp /opt/hadoop/hdfs/name /opt/hadoop/hdfs/data

vim /opt/hadoop/etc/hadoop/hadoop-env.sh
# export JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::")

vim /opt/hadoop/etc/hadoop/core-site.xml
# <configuration>
#     <property>
#         <name>fs.defaultFS</name>
#         <value>hdfs://node01:9010</value>
#     </property>
#     <property>
#         <name>hadoop.tmp.dir</name>
#         <value>file:/opt/hadoop/hdfs/tmp</value>
#     </property>
# </configuration>

vim /opt/hadoop/etc/hadoop/hdfs-site.xml
# <configuration>
#     <property>
#         <name>dfs.namenode.http-address</name>
#         <value>node01:51070</value>
#     </property>
#     <property>
#         <name>dfs.replication</name>
#         <value>3</value>
#     </property>
#     <property>
#         <name>dfs.namenode.name.dir</name>
#         <value>file:/opt/hadoop/hdfs/name</value>
#     </property>
#     <property>
#         <name>dfs.datanode.data.dir</name>
#         <value>file:/opt/hadoop/hdfs/data</value>
#     </property>
# </configuration>

上述配置可以參考core-default.xml & hdfs-default.xml

scp -r /opt/hadoop/ vagrant@node02:/home/vagrant/

ssh vagrant@node02

sudo mv hadoop /opt/

scp -r /opt/hadoop/ vagrant@node03:/home/vagrant/

ssh vagrant@node03

sudo mv hadoop /opt/

scp -r /opt/hadoop/ vagrant@node04:/home/vagrant/

ssh vagrant@node04

sudo mv hadoop /opt/

啓動服務

vagrant ssh node01

/opt/hadoop/bin/hdfs namenode -format

/opt/hadoop/bin/hdfs --daemon start namenode

jps
# 11756 NameNode

ssh vagrant@node02

/opt/hadoop/bin/hdfs --daemon start datanode

jps
# 11909 DataNode

ssh vagrant@node03

/opt/hadoop/bin/hdfs --daemon start datanode

jps
# 11913 DataNode

ssh vagrant@node04

/opt/hadoop/bin/hdfs --daemon start datanode

jps
# 11907 DataNode

相關工具

vagrant ssh node01

/opt/hadoop/bin/hdfs dfs -mkdir /hello

/opt/hadoop/bin/hdfs dfs -ls /

vagrant ssh node01

sudo apt install -y nginx

sudo vim /etc/nginx/sites-enabled/hadoop.conf
# server {
#     listen 80;
#     server_name hadoop.com;
#
#     location / {
#         proxy_pass http://192.168.56.161:51070;
#     }
# }

sudo nginx -t

sudo nginx -s reload

• 主機添加域名解析/etc/hosts

192.168.56.161 hadoop.com

• 瀏覽器訪問http://hadoop.com

ZooKeeper

下載配置

vagrant ssh node01

wget https://mirror.tuna.tsinghua.edu.cn/apache/zookeeper/stable/apache-zookeeper-3.5.8-bin.tar.gz

tar xf apache-zookeeper-3.5.8-bin.tar.gz

sudo mv apache-zookeeper-3.5.8-bin /opt/zookeeper

mkdir -p /opt/zookeeper/data

vim /opt/zookeeper/conf/zoo.cfg
# tickTime=2000
# initLimit=10
# syncLimit=5
# dataDir=/opt/zookeeper/data
# clientPort=2181
# server.1=node01:2888:3888
# server.2=node02:2888:3888
# server.3=node03:2888:3888
# server.4=node04:2888:3888

echo 1 > /opt/zookeeper/data/myid

scp -r /opt/zookeeper/ vagrant@node02:/home/vagrant

ssh vagrant@node02

sudo mv zookeeper /opt/

echo 2 > /opt/zookeeper/data/myid

scp -r /opt/zookeeper/ vagrant@node03:/home/vagrant

ssh vagrant@node03

sudo mv zookeeper /opt/

echo 3 > /opt/zookeeper/data/myid

scp -r /opt/zookeeper/ vagrant@node04:/home/vagrant

ssh vagrant@node04

sudo mv zookeeper /opt/

echo 4 > /opt/zookeeper/data/myid

啓動服務

vagrant ssh node01

/opt/zookeeper/bin/zkServer.sh start

jps
# 12302 QuorumPeerMain

ssh vagrant@node02

/opt/zookeeper/bin/zkServer.sh start

jps
# 12171 QuorumPeerMain

ssh vagrant@node03

/opt/zookeeper/bin/zkServer.sh start

jps
# 12174 QuorumPeerMain

ssh vagrant@node04

/opt/zookeeper/bin/zkServer.sh start

jps
# 12164 QuorumPeerMain

相關工具

/opt/zookeeper/bin/zkServer.sh status

/opt/zookeeper/bin/zkCli.sh
# ls /

HBase

下載配置

vagrant ssh node01

wget https://mirror.tuna.tsinghua.edu.cn/apache/hbase/2.2.6/hbase-2.2.6-bin.tar.gz

tar xf hbase-2.2.6-bin.tar.gz

sudo mv hbase-2.2.6 /opt/hbase

vim /opt/hbase/conf/hbase-env.sh
# export JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::")
# export HBASE_MANAGES_ZK=false

vim /opt/hbase/conf/hbase-site.xml
# <configuration>
#     <property>
#         <name>hbase.rootdir</name>
#         <value>hdfs://node01:9010/hbase</value>
#     </property>
#     <property>
#         <name>hbase.cluster.distributed</name>
#         <value>true</value>
#     </property>
#     <property>
#         <name>hbase.zookeeper.quorum</name>
#         <value>node01,node02,node03,node04</value>
#     </property>
# </configuration>

scp -r /opt/hbase/ vagrant@node02:/home/vagrant/

ssh vagrant@node02

sudo mv hbase /opt/

scp -r /opt/hbase/ vagrant@node03:/home/vagrant/

ssh vagrant@node03

sudo mv hbase /opt/

scp -r /opt/hbase/ vagrant@node04:/home/vagrant/

ssh vagrant@node04

sudo mv hbase /opt/

啓動服務

vagrant ssh node01

/opt/hbase/bin/hbase-daemon.sh start master

jps
# 12764 HMaster

ssh vagrant@node02

/opt/hbase/bin/hbase-daemon.sh start regionserver

jps
# 12530 HRegionServer

ssh vagrant@node03

/opt/hbase/bin/hbase-daemon.sh start regionserver

jps
# 12567 HRegionServer

ssh vagrant@node04

/opt/hbase/bin/hbase-daemon.sh start regionserver

jps
# 12999 HRegionServer

相關工具

vagrant ssh node01

/opt/hbase/bin/hbase shell

create 'test', 'cf'

list

describe 'test'

put 'test', 'row1', 'cf:a', 'value1'

scan 'test'

get 'test', 'row1'

vagrant ssh node01

sudo vim /etc/nginx/sites-enabled/hbase.conf
# server {
#     listen 80;
#     server_name hbase.com;
#
#     location / {
#         proxy_pass http://192.168.56.161:16010;
#     }
# }

sudo nginx -t

sudo nginx -s reload

• 主機添加域名解析/etc/hosts

192.168.56.161 hbase.com

• 瀏覽器訪問http://hbase.com

參考

• Hadoop Cluster Setup

• 2.4. Fully Distributed for Production