CentOS 7 下單節點 zookeeper + kafka 環境搭建

安裝包準備

• zookeeper 官網

• kafka 官網

服務器準備

操作系統：CentOS 7
CPU：2核
內存：4GB
磁盤：30GB
安裝包目錄：/soft
安裝目錄：/home/soft
JDK 版本：jdk-8u211
Zookeeper 版本：apache-zookeeper-3.6.1
Kafka 版本：kafka_2.12-2.5.0

服務安裝

JDK 安裝

• jdk 安裝

[root@bogon soft]# rpm -ivh jdk-8u211-linux-x64.rpm 
warning: jdk-8u211-linux-x64.rpm: Header V3 RSA/SHA256 Signature, key ID ec551f03: NOKEY
Preparing...                          ################################# [100%]
Updating / installing...
   1:jdk1.8-2000:1.8.0_211-fcs        ################################# [100%]
Unpacking JAR files...
	tools.jar...
	plugin.jar...
	javaws.jar...
	deploy.jar...
	rt.jar...
	jsse.jar...
	charsets.jar...
	localedata.jar...

• jdk 環境變量配置

[root@bogon soft]# vim /etc/profile     # 最後追加

# Set Java Env
export JAVA_HOME=/usr/java/jdk1.8.0_211-amd64/
export CLASSPATH=.:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export PATH=$PATH:$JAVA_HOME/bin

• jdk 驗證

[root@bogon soft]# source /etc/profile
[root@bogon soft]# echo $JAVA_HOME
/usr/java/jdk1.8.0_211-amd64/

zookeeper 安裝配置

• 解壓安裝包

[root@bogon soft]# tar -zxf apache-zookeeper-3.6.1-bin.tar.gz -C /home/soft/
[root@bogon soft]# cd /home/soft
[root@bogon soft]# mv apache-zookeeper-3.6.1-bin zookeeper

• 編輯配置文件

[root@bogon soft]# cd zookeeper/conf/
[root@bogon conf]# cp zoo_sample.cfg zoo.cfg

[root@bogon conf]# cat zoo.cfg | grep -v '^#' | grep -v '^$'
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/home/soft/zookeeper/data
dataLogdir=/home/soft/zookeeper/logs
clientPort=2181
extendedTypesEnabled=true
emulate353TTLNodes=true
admin.enableServer=false
4lw.commands.whitelist=*
server.1=192.168.1.127:2888:3888

• 創建對應目錄及文件

[root@bogon conf]# cd /home/soft/zookeeper
[root@bogon zookeeper]# mkdir {data,logs}
[root@bogon zookeeper]# echo "1" > data/myid

• 添加環境變量

[root@bogon ~]# vim /etc/profile     # 最後追加

# Set Zookeeper Env
export ZOOKEEPER_HOME=/home/soft/zookeeper
export PATH=$PATH:$ZOOKEEPER_HOME/bin
[root@bogon ~]# source /etc/profile

• 啓動 Zookeeper 服務

[root@bogon ~]# zkServer.sh start
ZooKeeper JMX enabled by default
Using config: /home/soft/zookeeper/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED

• 查看 Zookeeper 服務

[root@bogon ~]# zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /home/soft/zookeeper/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost.
Mode: standalone

Kafka 安裝

• 解壓安裝包

[root@bogon soft]# tar -zxf kafka_2.12-2.5.0.tgz -C /home/soft/
[root@bogon soft]# cd /home/soft
[root@bogon soft]# mv kafka_2.12-2.5.0 kafka

• 編輯配置文件

[root@bogon ~]# cd /home/soft/kafka/config/
[root@bogon config]# vim server.properties

[root@bogon config]# cat server.properties | grep -v '^#' | grep -v '^$'
broker.id=1
listeners=PLAINTEXT://192.168.1.127:9092
host.name=192.168.1.127
port=9092
advertised.listeners=PLAINTEXT://192.168.1.127:9092
advertised.host.name=192.168.1.127
advertised.port=9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/home/soft/kafka/logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=192.168.1.127:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0

• 添加環境變量

[root@bogon ~]# vim /etc/profile     # 最後追加

# Set Kafka Env
export KAFKA_HOME=/home/soft/kafka
export PATH=$PATH:$KAFKA_HOME/bin
[root@bogon ~]# source /etc/profile

• 啓動 Kafka 服務

[root@bogon config]# cd /home/soft/kafka/
[root@bogon kafka]# bin/kafka-server-start.sh config/server.properties

• 查看 zookeeper 中 broker 信息

[zk: localhost:2181(CONNECTED) 4] get /brokers/ids/1
{"listener_security_protocol_map":{"PLAINTEXT":"PLAINTEXT"},"endpoints":["PLAINTEXT://192.168.1.127:9092"],"jmx_port":-1,"host":"192.168.1.127","timestamp":"1591718323700","port":9092,"version":4}

• 新建 topic

[root@bogon ~]# cd /home/soft/kafka/bin/
[root@bogon bin]# ./kafka-topics.sh --create --zookeeper 192.168.1.127:2181 --replication-factor 1 --partitions 1 --topic test
Created topic test.

### 新建 topic 後 kafka 日誌
[2020-06-10 00:10:42,001] INFO [Log partition=test-0, dir=/home/soft/kafka/logs] Completed load of log with 1 segments, log start offset 0 and lo end offset 0 in 42 ms (kafka.log.Log)
[2020-06-10 00:10:42,003] INFO Created log for partition test-0 in /home/soft/kafka/logs/test-0 with properties {compression.type -> producer, mesage.downconversion.enable -> true, min.insync.replicas -> 1, segment.jitter.ms -> 0, cleanup.policy -> [delete], flush.ms -> 9223372036854775807 segment.bytes -> 1073741824, retention.ms -> 604800000, flush.messages -> 9223372036854775807, message.format.version -> 2.5-IV0, file.delete.deay.ms -> 60000, max.compaction.lag.ms -> 9223372036854775807, max.message.bytes -> 1048588, min.compaction.lag.ms -> 0, message.timestamp.type ->CreateTime, preallocate -> false, min.cleanable.dirty.ratio -> 0.5, index.interval.bytes -> 4096, unclean.leader.election.enable -> false, retenton.bytes -> -1, delete.retention.ms -> 86400000, segment.ms -> 604800000, message.timestamp.difference.max.ms -> 9223372036854775807, segment.indx.bytes -> 10485760}. (kafka.log.LogManager)
[2020-06-10 00:10:42,004] INFO [Partition test-0 broker=1] No checkpointed highwatermark is found for partition test-0 (kafka.cluster.Partition)
[2020-06-10 00:10:42,007] INFO [Partition test-0 broker=1] Log loaded for partition test-0 with initial high watermark 0 (kafka.cluster.Partition
[2020-06-10 00:10:42,008] INFO [Partition test-0 broker=1] test-0 starts at leader epoch 0 from offset 0 with high watermark 0. Previous leader eoch was -1. (kafka.cluster.Partition)

• 查看 topic

[root@bogon bin]# ./kafka-topics.sh --list --zookeeper 192.168.1.127:2181
test

[root@bogon bin]# ./kafka-topics.sh --describe --zookeeper 192.168.1.127:2181 --topic test
Topic: test	PartitionCount: 1	ReplicationFactor: 1	Configs: 
	Topic: test	Partition: 0	Leader: 1	Replicas: 1	Isr: 1

• 啓動消息生產者

[root@bogon bin]# ./kafka-console-producer.sh --broker-list 192.168.1.127:9092 --topic test

• 啓動消息消費者

[root@bogon bin]# ./kafka-console-consumer.sh --bootstrap-server 192.168.1.127:9092 --from-beginning --topic test

• 驗證消息創建 && 消費

參考

• Zookeeper+Kafka的單節點配置