一.需求背景
隨着公司業務發展，rabbitMQ 吞吐量滿足不了當前業務，將rabbitMQ 替換爲kafka.
二.實驗環境 & 機器
1.版本
$ /etc/redhat-release
CentOS Linux release 7.9.2009 (Core)
2.機器列表
192.168.82.252 centos-1
192.168.82.253 centos-2
192.168.82.130 centos-3
三.安裝 kafka zookeeper jkd
$ wget https://mirror-hk.koddos.net/apache/kafka/2.6.2/kafka_2.13-2.6.2.tgz
$ wget https://downloads.apache.org/zookeeper/zookeeper-3.6.3/apache-zookeeper-3.6.3-bin.tar.gz
$ tar -xvf kafka_2.13-2.6.2.tgz && mv kafka_2.13-2.6.2 /usr/local/kafka
$ tar -xvf apache-zookeeper-3.6.3-bin.tar.gz && mv apache-zookeeper-3.6.3-bin /usr/local/zookeeper
$ tar -xvf jdk-8u301-linux-x64.tar.gz && mv jdk1.8.0_301 /usr/local/jdk1.8Step 1. 設置jdk 環境變量
$ vim /etc/profile
*******************
export JAVA_HOME=/usr/java/jdk1.8
export PATH=$PATH:$JAVA_HOME/bin
JAVA_HOME=/usr/local/jdk1.8/
JAVA_BIN=/usr/local/jdk1.8/bin
JRE_HOME=/usr/local/jdk1.8/jre
PATH=$PATH:/usr/local/jdk1.8/bin:/usr/local/jdk1.8/jre/bin
CLASSPATH=/usr/local/jdk1.8/jre/lib:/usr/local/jdk1.8/lib:/usr/local/jdk1.8/jre/lib/charsets.jar
#加載配置
$ source /etc/profile
Step 2.配置zookeeper
1.centos-1配置文件
$ vim /usr/local/zookeeper/conf/zoo.cfg
dataDir=/data/zookeeper
#集羣選舉配置
server.1=centos-1:2888:3888
server.2=centos-2:2888:3888
server.3=centos-3:2888:3888
1.創建data 目錄
$ mkdir -p /data/zookeeper
2.創建 & 設置zookeeper id 文件
$ touch /data/zookeeper/myid
$ echo '1' >> /data/zookeeper/myid
2.centos-2配置文件
$ vim /usr/local/zookeeper/conf/zoo.cfg
dataDir=/data/zookeeper
#集羣選舉配置
server.1=centos-1:2888:3888
server.2=centos-2:2888:3888
server.3=centos-3:2888:3888
1.創建data 目錄
$ mkdir -p /data/zookeeper
2.創建 & 設置zookeeper id 文件
$ touch /data/zookeeper/myid
$ echo '2' >> /data/zookeeper/myid
3.centos-3配置文件
$ vim /usr/local/zookeeper/conf/zoo.cfg
dataDir=/data/zookeeper
#集羣選舉配置
server.1=centos-1:2888:3888
server.2=centos-2:2888:3888
server.3=centos-3:2888:3888
1.創建data 目錄
$ mkdir -p /data/zookeeper
2.創建 & 設置zookeeper id 文件
$ touch /data/zookeeper/myid
$ echo '3' >> /data/zookeeper/myid
Step 3.kafka 配置
1.centos-1 kafka配置文件
$ vim /usr/local/kafka/config/server.properties
#每一個broker在集羣中的唯一標示，要求是正數。在改變IP地址，不改變broker.id的話不會影響consumers
broker.id=1
#服務器IP與監聽端口
listeners=PLAINTEXT://centos-1:9092
kafka數據的存放地址，多個地址的話用逗號分割
log.dirs=/var/log/kafka/kafka-logs
#日誌保留時長
log.retention.hours=2
#日誌數據存儲的最大字節數。超過這個時間會根據policy處理數據
log.retention.bytes=107374182$
#指定zookeeper集羣
zookeeper.connect=centos-1:2181,centos-2:2181,centos-3:2181
2.centos-2 kafka配置文件
$ vim /usr/local/kafka/config/server.properties
#每一個broker在集羣的唯一標示，要求是正數。在改變IP地址，不改變broker.id的話不會影響consumers
broker.id=2
#服務器IP與監聽端口
listeners=PLAINTEXT://centos-2:9092
kafka數據的存放地址，多個地址的話用逗號分割
log.dirs=/var/log/kafka/kafka-logs
#日誌保留時長
log.retention.hours=2
#日誌數據存儲的最大字節數。超過這個時間會根據policy處理數據
log.retention.bytes=107374182$
#指定zookeeper集羣
zookeeper.connect=centos-1:2181,centos-2:2181,centos-3:2181
3.centos-3 kafka配置文件
$ vim /usr/local/kafka/config/server.properties
#每一個broker在集羣中的唯一標示，要求是正數。在改變IP地址，不改變broker.id的話不會影響consumers
broker.id=3
#服務器IP與監聽端口
listeners=PLAINTEXT://centos-3:9092
kafka數據的存放地址，多個地址的話用逗號分割
log.dirs=/var/log/kafka/kafka-logs
#日誌保留時長
log.retention.hours=2
#日誌數據存儲的最大字節數。超過這個時間會根據policy處理數據
log.retention.bytes=107374182$
#指定zookeeper集羣
zookeeper.connect=centos-1:2181,centos-2:2181,centos-3:2181
4.開啓了jmx監控(centos-1~centos-3 配置)
$ vim /usr/local/bin/kafka/bin/kafka-server-start.sh
if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
##添加這兩條配置
export KAFKA_HEAP_OPTS="-server -Xms2G -Xmx2G -XX:PermSize=128m -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:ParallelGCThreads=8 -XX:ConcGCThreads=5 -XX:InitiatingHeapOccupancyPercent=70"
export JMX_PORT="9999"
#export KAFKA_HEAP_OPTS="-Xmx1G -Xms1G -Dcom.sun.management.jmxremote.port=$JMX_PORT"
fi
四.啓動zookeeper kafka
#zookeeper啓動
$ /usr/local/zookeeper/bin/zkServer.sh start
#kafka 啓動
$ nohup /usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties &
五.kafka 運維
1.創建topic
$ /usr/local/kafka/bin/kafka-topics.sh --create --zookeeper 192.168.82.130:2181 --replication-factor 1 --partitions 1 --topic my_test
Created topic my_test.
2.創建group
$ /usr/local/kafka/bin/kafka-console-consumer.sh --bootstrap-server 192.168.82.252:9092 --topic my_test --consumer-property group.id=1
3.添加partition
$ /usr/local/kafka/bin/kafka-topics.sh --alter --zookeeper 192.168.82.252:2181 --partitions 6 --topic my_test
Adding partitions succeeded!
#可見有6個partition
$ /usr/local/kafka/bin/kafka-topics.sh --zookeeper 192.168.82.252:2181 --describe --topic my_test
Topic: my_test PartitionCount: 6 ReplicationFactor: 1 Configs:
Topic: my_test Partition: 0 Leader: 3 Replicas: 3 Isr: 3
Topic: my_test Partition: 1 Leader: 1 Replicas: 1 Isr: 1
Topic: my_test Partition: 2 Leader: 2 Replicas: 2 Isr: 2
Topic: my_test Partition: 3 Leader: 3 Replicas: 3 Isr: 3
Topic: my_test Partition: 4 Leader: 1 Replicas: 1 Isr: 1
Topic: my_test Partition: 5 Leader: 2 Replicas: 2 Isr: 2
4.delete topic
$ /usr/local/kafka/bin/kafka-topics.sh --delete --zookeeper 192.168.82.252:2181 --topic my_test
5.查看消費位移
$ /usr/local/kafka/bin/kafka-consumer-groups.sh --bootstrap-server 192.168.82.252:9092 --describe --group 1
GROUP TOPIC PARTITION CURRENT-OFFSET LOG-END-OFFSET LAG CONSUMER-ID HOST CLIENT-ID
1 my_test 0 0 0 0 consumer-1-1-d2fbd6eb-19c7-4817-a2e3-a360e7078d6d /192.168.82.252 consumer-1-1
6.查看指定 Topic 明細
$ /usr/local/kafka/bin/kafka-topics.sh --zookeeper 192.168.82.252:2181 --describe --topic my_test
Topic: my_test PartitionCount: 1 ReplicationFactor: 1 Configs:
Topic: my_test Partition: 0 Leader: 3 Replicas: 3 Isr: 3
7.查看topic 列表
$ /usr/local/kafka/bin/kafka-topics.sh --list --bootstrap-server centos-1:9092,centos-2:9092,centos-3:9092
__consumer_offsets
click_log
my_test
8.創建生產者
$ /usr/local/kafka/bin/kafka-console-producer.sh --broker-list centos-1:9092 --topic my_test
>5
>6
>7
>8
>9
>10
9.創建消費者
$ /usr/local/kafka/bin/kafka-console-consumer.sh --bootstrap-server centos-1:9092 --topic my_test --consumer-property group.id=1
5
6
7
8
9
10
10.通過位移查看消費情況
$ /usr/local/kafka/bin/kafka-consumer-groups.sh --bootstrap-server 192.168.82.252:9092 --describe --group 1
GROUP TOPIC PARTITION CURRENT-OFFSET LOG-END-OFFSET LAG CONSUMER-ID HOST CLIENT-ID
1 my_test 4 2 2 0 consumer-1-1-ebc4b05c-b601-45fc-ad03-c4b6916205cc /192.168.82.252 consumer-1-1
1 my_test 3 4 4 0 consumer-1-1-ebc4b05c-b601-45fc-ad03-c4b6916205cc /192.168.82.252 consumer-1-1
1 my_test 0 5 5 0 consumer-1-1-ebc4b05c-b601-45fc-ad03-c4b6916205cc /192.168.82.252 consumer-1-1
1 my_test 5 4 4 0 consumer-1-1-ebc4b05c-b601-45fc-ad03-c4b6916205cc /192.168.82.252 consumer-1-1
1 my_test 1 4 4 0 consumer-1-1-ebc4b05c-b601-45fc-ad03-c4b6916205cc /192.168.82.252 consumer-1-1
1 my_test 2 5 5 0 consumer-1-1-ebc4b05c-b601-45fc-ad03-c4b6916205cc /192.168.82.252 consumer-1-1