大數據組件kafka部署安裝與測試練習

環境說明

10.176.2.101 master
10.176.2.103 zjx03
10.176.2.105 zjx05

cent-os6.5
zookeeper cdh 3.4.5
hadoop apache 2.7.7
mysql 5.17
jdk 1.8.191
kafka 2.11_2.01

kafka官網

http://kafka.apache.org/

kafka安裝

#master
[root@master softwares]# cd /opt/softwares
[root@master softwares]# wget https://mirrors.tuna.tsinghua.edu.cn/apache/kafka/2.0.1/kafka_2.11-2.0.1.tgz -c /opt/softwares/
[root@master softwares]# tar -zxvf kafka_2.11-2.0.1.tgz
[root@master softwares]# cd ./kafka_2.11-2.0.1/config
#10.176.2.101 master節點修改broker.id、listeners、zookeeper.connect
[root@master config]# vim server.properties
broker.id=0
#listeners=PLAINTEXT://:9092
listeners=PLAINTEXT://master:9092
#advertised.listeners=PLAINTEXT://your.host.name:9092
advertised.listeners=PLAINTEXT://master:9092
zookeeper.connect=master:2181,zjx03:2181,zjx05:2181
#kafaka複製至其它機器
[root@master softwares]# scp -r ./kafka_2.11-2.0.1 zjx03:/opt/softwares/
[root@master softwares]# scp -r ./kafka_2.11-2.0.1 zjx05:/opt/softwares/
#10.176.2.103 zjx03節點修改broker.id、listeners、zookeeper.connect
[root@zjx03 config]# vim server.properties
broker.id=1
#listeners=PLAINTEXT://:9092
listeners=PLAINTEXT://zjx03:9092
#advertised.listeners=PLAINTEXT://your.host.name:9092
advertised.listeners=PLAINTEXT://zjx03:9092
zookeeper.connect=master:2181,zjx03:2181,zjx05:2181
#10.176.2.105 zjx05節點修改broker.id、listeners、zookeeper.connect
[root@zjx05 config]# vim server.properties
broker.id=2
#listeners=PLAINTEXT://:9092
listeners=PLAINTEXT://zjx05:9092
#advertised.listeners=PLAINTEXT://your.host.name:9092
advertised.listeners=PLAINTEXT://zjx05:9092
zookeeper.connect=master:2181,zjx03:2181,zjx05:2181
#配置環境變量
[root@master config]# vim /etc/profile
export KAFKA_HOME=/opt/softwares/kafka_2.11-2.0.1
export PATH=$KAFKA_HOME/bin:$PATH
[root@master config]# source /etc/profile
#另兩個節點同樣進行環境變量配置

後臺開啓3臺虛擬機kafka程序

cd /opt/softwares/kafka_2.11-2.0.1
./bin/kafka-server-start.sh -daemon config/server.properties
#或者
./bin/kafka-server-start.sh  config/server.properties &

創建topic

可指定Partition、Replication數量
建議--replication-factor > 1，否則Broker宕掉後將無法寫入消息

#3個分區3個副本
[root@master bin]# ./kafka-topics.sh --create --topic topic_zjx_test --zookeeper master:2181,zjx03:2181,zjx05:2181 --partitions 3 --replication-factor 3
#3個分區1個副本
[root@master bin]# ./kafka-topics.sh --create --topic topic_zjx  --zookeeper master:2181,zjx03:2181,zjx05:2181 --partitions 3 --replication-factor 1
#查看topic列表
[root@master bin]# ./kafka-topics.sh --zookeeper master:2181,zjx03:2181,zjx05:2181 --list
topic_zjx
topic_zjx_test
#查看topic詳情 
[root@master bin]# ./kafka-topics.sh  --describe --topic topic_zjx  --zookeeper master:2181,zjx03:2181,zjx05:2181
Topic:topic_zjx PartitionCount:3    ReplicationFactor:1 Configs:
    Topic: topic_zjx    Partition: 0    Leader: 0   Replicas: 0 Isr: 0
    Topic: topic_zjx    Partition: 1    Leader: 0   Replicas: 0 Isr: 0
    Topic: topic_zjx    Partition: 2    Leader: 0   Replicas: 0 Isr: 0
[root@master bin]# ./kafka-topics.sh  --describe --topic topic_zjx_test  --zookeeper master:2181,zjx03:2181,zjx05:2181
Topic:topic_zjx_test    PartitionCount:3    ReplicationFactor:3 Configs:
    Topic: topic_zjx_test   Partition: 0    Leader: 2   Replicas: 2,0,1 Isr: 2,0,1
    Topic: topic_zjx_test   Partition: 1    Leader: 0   Replicas: 0,1,2 Isr: 0,1,2
    Topic: topic_zjx_test   Partition: 2    Leader: 1   Replicas: 1,2,0 Isr: 1,2,0

創建producer

[root@master config]# kafka-console-producer.sh --broker-list master:9092, zjx03:9092, zjx05:9092 --topic topic_zjx_test

創建consumer

新開一個session

[root@master ~]# cd /opt/softwares/kafka_2.11-2.0.1/bin
[root@master bin]# ./kafka-console-consumer.sh --bootstrap-server master:9092, zjx03:9092, zjx05:9092 --topic topic_zjx_test

在producer session中輸入，consumer session中會自動接收消息

[root@master bin]# ./kafka-console-consumer.sh --bootstrap-server master:9092, zjx03:9092, zjx05:9092 --topic topic_zjx_test
>zhoujixiang
>hello kafka
>yudd
[root@master bin]# ./kafka-console-consumer.sh --bootstrap-server master:9092, zjx03:9092, zjx05:9092 --topic topic_zjx_test
zhoujixiang
hello kafka
yudd

停止kafka

kafka-server-stop.sh

參考鏈接：
https://blog.csdn.net/qq_38270106/article/details/83715910
https://www.cnblogs.com/xyj179/p/10020865.html
https://blog.csdn.net/weixin_38120374/article/details/80608496
kafaka常用命令：
https://www.cnblogs.com/dragkiss/p/5668019.html