一、環境準備
| 主機名 | IP | 運行服務 |
|---|---|---|
| kafka1 | 192.168.171.131 | kafka+zookeeper |
| kafka2 | 192.168.171.134 | kafka+zookeeper |
| kafka3 | 192.168.171.135 | kafka+zookeeper |
二、部署zookeeper服務
源碼包（提取碼：6q58）
1、kafka1配置如下
#部署zookeeper
[root@kafka1 ~]# tar zxf zookeeper-3.4.9.tar.gz
[root@kafka1 ~]# mv zookeeper-3.4.9 /usr/local/zookeeper
[root@kafka1 ~]# cd /usr/local/zookeeper/conf/
[root@kafka1 conf]# cp zoo_sample.cfg zoo.cfg
[root@kafka1 conf]# sed -i 's/dataDir=\/tmp\/zookeeper/dataDir=\/usr\/local\/zookeeper\/data/g' zoo.cfg
#聲明羣集節點信息，2888和3888端口用於羣集內部通信
[root@kafka1 conf]# echo "server.1 192.168.171.131:2888:3888" >> zoo.cfg
[root@kafka1 conf]# echo "server.2 192.168.171.134:2888:3888" >> zoo.cfg
[root@kafka1 conf]# echo "server.3 192.168.171.135:2888:3888" >> zoo.cfg
tickTime=2000 #節點之間的心跳檢測時間單位爲毫秒
initLimit=10 #達到5個訪問進行同步數據
syncLimit=5 #節點之間檢查失敗次數超過後斷開相應的節點
dataDir=/usr/local/zookeeper/data #日誌文件存放路徑
clientPort=2181
#聲明參與集羣的主機
server.1 192.168.171.131:2888:3888
server.2 192.168.171.134:2888:3888
server.3 192.168.171.135:2888:3888
#創建所需目錄及設置節點的ID號
[root@kafka1 conf]# mkdir /usr/local/zookeeper/data
[root@kafka1 conf]# echo 1 > /usr/local/zookeeper/data/myid
#將配置好的zookeeper目錄複製到羣集內的其他節點
[root@kafka1 conf]# scp -r /usr/local/zookeeper/ [email protected]:/usr/local/
[root@kafka1 conf]# scp -r /usr/local/zookeeper/ [email protected]:/usr/local/
#啓動zookeeper服務
[root@kafka1 conf]# /usr/local/zookeeper/bin/zkServer.sh start
[root@kafka1 conf]# netstat -antp | egrep '2181|2888|3888' 2、kafka2配置如下
#修改ID號爲2
[root@kafka2 ~]# echo 2 > /usr/local/zookeeper/data/myid
[root@kafka2 ~]# /usr/local/zookeeper/bin/zkServer.sh start 3、kafka3配置如下
#修改ID號爲3
[root@kafka3 ~]# echo 3 > /usr/local/zookeeper/data/myid
[root@kafka3 ~]# /usr/local/zookeeper/bin/zkServer.sh start 4、查看zookeeper羣集內節點的角色
#kafka1
[root@kafka1 conf]# /usr/local/zookeeper/bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
Mode: follower #角色爲follower
#kafka2
[root@kafka2 ~]# /usr/local/zookeeper/bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
Mode: leader #角色爲leader
#kafka3
[root@kafka3 ~]# /usr/local/zookeeper/bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
Mode: follower #角色爲follower三、部署kafka集羣
1、kafka1配置如下
[root@kafka1 ~]# wget https://archive.apache.org/dist/kafka/2.2.1/kafka_2.11-2.2.1.tgz
[root@kafka1 ~]# tar zxf kafka_2.11-2.2.1.tgz
[root@kafka1 ~]# mv kafka_2.11-2.2.1/ /usr/local/kafka
#修改配置文件
[root@kafka1 ~]# cd /usr/local/kafka/config/
[root@kafka1 config]# sed -i 's/broker.id=0/broker.id=1/g' server.properties
[root@kafka1 config]# sed -i 's/#listeners=PLAINTEXT:\/\/:9092/listeners=PLAINTEXT:\/\/192.168.171.131:9092/g' server.properties
[root@kafka1 config]# sed -i 's/#advertised.listeners=PLAINTEXT:\/\/your.host.name:9092/advertised.listeners=PLAINTEXT:\/\/192.168.171.131:9092/g' server.properties
[root@kafka1 config]# sed -i 's/log.dirs=\/tmp\/kafka-logs/log.dirs=\/usr\/local\/zookeeper\/data/g' server.properties
[root@kafka1 config]# sed -i 's/zookeeper.connect=localhost:2181/zookeeper.connect=192.168.171.131:2181,192.168.171.134:2181,192.168.171.135:2181/g' server.properties
[root@kafka1 config]# sed -i 's/zookeeper.connection.timeout.ms=6000/zookeeper.connection.timeout.ms=600000/g' server.properties
[root@kafka1 config]# egrep -v '^$|^#' server.properties
broker.id=1 #kafka的ID號，這裏爲1，其他節點依次是2、3
listeners=PLAINTEXT://192.168.171.131:9092 #節點監聽地址，填寫每個節點自己的IP地址
advertised.listeners=PLAINTEXT://192.168.171.131:9092 #集羣中節點內部交流使用的端口，填寫每個節點自己的IP地址
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/usr/local/zookeeper/data
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=192.168.171.131:2181,192.168.171.134:2181,192.168.171.135:2181 #聲明鏈接zookeeper節點的地址
zookeeper.connection.timeout.ms=600000 #修改這的時間，單位是毫秒，爲了防止連接zookeeper超時
group.initial.rebalance.delay.ms=0
[root@kafka1 config]# scp -r /usr/local/kafka/ [email protected]:/usr/local/
[root@kafka1 config]# scp -r /usr/local/kafka/ [email protected]:/usr/local/
#啓動kafka
[root@kafka1 bin]# ./kafka-server-start.sh ../config/server.properties &2、kafka2配置如下：
#修改與kafka01衝突之處
[root@kafka2 ~]# cd /usr/local/kafka/
[root@kafka2 kafka]# sed -i 's/192.168.171.131/192.168.171.134/g' config/server.properties
[root@kafka2 kafka]# sed -i 's/broker.id=1/broker.id=2/g' config/server.properties
#啓動kafka服務
[root@kafka2 kafka]# cd bin/
[root@kafka2 bin]# ./kafka-server-start.sh ../config/server.properties &
[root@kafka2 bin]# netstat -anupt | grep 9092
3、kafka3配置如下：
[root@kafka3 ~]# cd /usr/local/kafka/
[root@kafka3 kafka]# sed -i 's/192.168.171.131/192.168.171.135/g' config/server.properties
[root@kafka3 kafka]# sed -i 's/broker.id=1/broker.id=3/g' config/server.properties
#啓動kafka服務
[root@kafka3 kafka]# cd bin/
[root@kafka3 bin]# ./kafka-server-start.sh ../config/server.properties &
[root@kafka3 bin]# netstat -anupt | grep 90924、發佈與訂閱消息測試
#創建名爲my-replicated-topic的topic
[root@kafka1 bin]# ./kafka-topics.sh --create --bootstrap-server 192.168.171.131:9092 --replication-factor 3 --partitions 1 --topic my-replicated-topic
#查看topic的狀態和leader
[root@kafka1 bin]# ./kafka-topics.sh --describe --bootstrap-server 192.168.171.131:9092 --topic my-replicated-topic
Topic:my-replicated-topic PartitionCount:1 ReplicationFactor:3 Configs:segment.bytes=1073741824
Topic: my-replicated-topic Partition: 0 Leader: 2 Replicas: 2,3,1 Isr: 2,3,1
#返回的信息表示partition數量爲1，副本數量爲3，segment字節數爲1073741824
#名稱爲“my-replicated-topic”，ID爲2的節點爲leader
[root@kafka1 bin]# ./kafka-console-producer.sh --broker-list 192.168.171.131:9092 --topic my-replicated-topic
>abc
>123
#在其他節點上訂閱消息
[root@kafka2 bin]# ./kafka-console-consumer.sh --bootstrap-server 192.168.171.134:9092 --from-beginning --topic my-replicated-topic
abc
1235、模擬leader宕機，查看topic的狀態及新的leader
#可以看到當前leader是ID爲2的節點
[root@kafka1 bin]# ./kafka-topics.sh --describe --bootstrap-server 192.168.171.131:9092 --topic my-repated-topic
Topic:my-replicated-topic PartitionCount:1 ReplicationFactor:3 Configs:segment.bytes=1073741824
Topic: my-replicated-topic Partition: 0 Leader: 2 Replicas: 2,3,1 Isr: 2,3,1
#到kafka02主機上停止kafka服務
[root@kafka2 bin]# ./kafka-server-stop.sh
#再次查看leader是哪個節點？（可以發現leader換成了ID爲3的節點）
[root@kafka1 bin]# ./kafka-topics.sh --describe --bootstrap-server 192.168.171.131:9092 --topic my-replicated-topic
Topic:my-replicated-topic PartitionCount:1 ReplicationFactor:3 Configs:segment.bytes=1073741824
Topic: my-replicated-topic Partition: 0 Leader: 3 Replicas: 2,3,1 Isr: 3,1