安裝Zookeeper和Kafka集羣

配置/etc/hosts文件

vi /etc/hosts

# 添加
192.168.200.110 master
192.168.200.111 slave1
192.168.200.112 slave2

安裝Zookeeper集羣

下載Zookeeper安裝包

下載地址：https://www.apache.org/dyn/closer.lua/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz

創建對應的ZK數據和日誌目錄

# 創建ZK的數據目錄，同時需要創建myid指定這個節點的ID
mkdir -p /software/zookeeper/zkdata/
vi /software/zookeeper/zkdata/myid

# 創建ZK的日誌目錄
mkdir /software/zookeeper/zklogs/

修改zoo.cfg文件

先複製一份：

cp zoo_sample.cfg zoo.cfg

開始修改：

cat zoo.cfg

# The number of milliseconds of each tick
tickTime=2000
# The number of ticks that the initial 
# synchronization phase can take
initLimit=10
# The number of ticks that can pass between 
# sending a request and getting an acknowledgement
syncLimit=5
# the directory where the snapshot is stored.
# do not use /tmp for storage, /tmp here is just 
# example sakes.
#dataDir=/tmp/zookeeper
# the port at which the clients will connect
clientPort=2181
# the maximum number of client connections.
# increase this if you need to handle more clients
#maxClientCnxns=60
#
# Be sure to read the maintenance section of the 
# administrator guide before turning on autopurge.
#
# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
#
# The number of snapshots to retain in dataDir
#autopurge.snapRetainCount=3
# Purge task interval in hours
# Set to "0" to disable auto purge feature
#autopurge.purgeInterval=1

# ZK的數據目錄，myid也在裏面
dataDir=/software/zookeeper/zkdata
# ZK的日誌目錄
dataLogDir=/software/zookeeper/zklogs
# 集羣模式的各個節點，如果是自身的話，需要配置爲0.0.0.0，而不是master/IP地址，不然可能會出現zk之間無法連接通信的情況。(如果您提供了公共IP，則偵聽器將無法連接到端口，您必須爲當前節點指定0.0.0.0)
server.1=0.0.0.0:2888:3888
server.2=slave1:2888:3888
server.3=slave2:2888:3888

啓動/停止ZK/查看狀態/重啓

zkServer.sh start

zkServer.sh stop

zkServer.sh status

zkServer.sh restart

ZK的相關命令操作

# 連接ZK
[root@master conf]# zkCli.sh -server 127.0.0.1:2181

# 顯示根目錄下文件
[zk: 127.0.0.1:2181(CONNECTED) 0] ls /
[admin, brokers, cluster, config, consumers, controller_epoch, isr_change_notification, latest_producer_id_block, log_dir_event_notification, zookeeper]

# 顯示根目錄下文件，並能看到更新次數等數據
[zk: 127.0.0.1:2181(CONNECTED) 1] ls2 /
'ls2' has been deprecated. Please use 'ls [-s] path' instead.
[cluster, controller_epoch, brokers, zookeeper, admin, isr_change_notification, consumers, log_dir_event_notification, latest_producer_id_block, config]
cZxid = 0x0
ctime = Thu Jan 01 08:00:00 CST 1970
mZxid = 0x0
mtime = Thu Jan 01 08:00:00 CST 1970
pZxid = 0x400000003
cversion = 10
dataVersion = 0
aclVersion = 0
ephemeralOwner = 0x0
dataLength = 0
numChildren = 10

# 創建文件，並設置初始化內容
[zk: 127.0.0.1:2181(CONNECTED) 3] create /lzhpo-test1 "hello,lzhpo-test1~"
Created /lzhpo-test1

# 查看文件內容
[zk: 127.0.0.1:2181(CONNECTED) 6] get /lzhpo-test1
hello,lzhpo-test1~

# 修改文件內容
[zk: 127.0.0.1:2181(CONNECTED) 7] set /lzhpo-test1 "test1"

[zk: 127.0.0.1:2181(CONNECTED) 8] get /lzhpo-test1
test1

# 刪除文件
[zk: 127.0.0.1:2181(CONNECTED) 9] delete /lzhpo-test1

[zk: 127.0.0.1:2181(CONNECTED) 10] ls /
[admin, brokers, cluster, config, consumers, controller_epoch, isr_change_notification, latest_producer_id_block, log_dir_event_notification, zookeeper]

剩下的，請看官方文檔：https://zookeeper.apache.org/doc/r3.5.8/zookeeperStarted.html

安裝Kafka集羣

下載Kafka安裝包

下載地址：https://www.apache.org/dyn/closer.cgi?path=/kafka/2.2.2/kafka_2.12-2.2.2.tgz

開啓JMX監控(可選)

方法1

如果不想做kafka的監控的話，可以選擇不開啓JMX監控。

就像下圖這樣子的監控，如果是自己寫了一個Kafka的監控平臺的話，不開啓JMX監控就無法獲取Kafka的一些信息。

修改KAFKA_HEAP_OPTS爲主機運行內存的一半；

vi /software/kafka/kafka_2.12-2.2.0/bin/kafka-server-start.sh

添加export JMX_PORT="9999"，也就是添加JMX監控，用來監控Kafka集羣。

if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
    export KAFKA_HEAP_OPTS="-Xmx2G -Xms2G"
    export JMX_PORT="9999"
fi

方法2(廢棄)

這種方法在創建topic的時候，會報錯：

Error: Exception thrown by the agent : java.rmi.server.ExportException: Port already in use: 9988; nested exception is: 
        java.net.BindException: Address in use (Bind failed)
sun.management.AgentConfigurationError: java.rmi.server.ExportException: Port already in use: 9988; nested exception is: 

# 修改kafka-run-class.sh
vi /software/kafka/kafka_2.12-2.2.0/bin/kafka-run-class.sh

# 第一行增加(也就是開啓JMX監控，制定JMX監控端口)
JMX_PORT=9988

# 某些服務器可能無法正確綁定ip，這時候我們需要顯示指定綁定的host
# 在參數KAFKA_JMX_OPTS，增加一個，也就是制定服務IP
# 不同的主機的需要修改爲對應的IP地址
-Djava.rmi.server.hostname=192.168.10.110

修改bin/kafka-server-start.sh

這是我優化的地方，優化JVM。

vi  /software/kafka/kafka_2.12-2.2.0/bin/kafka-server-start.sh

# 調整KAFKA_HEAP_OPTS="-Xmx16G -Xms16G”的值
# 推薦配置：一般HEAP SIZE的大小不超過主機內存的50%。

修改config/server.properties

vi /software/kafka/kafka_2.12-2.2.0/bin/server.properties

# 修改處1：broker.id，集羣中唯一的ID，不同主機的不能相同。
broker.id=0

# 修改處2：對外網開放地址。不同主機需要修改爲對應的地址
listeners=PLAINTEXT://192.168.200.110:9092

# 修改處3：監聽地址。不同主機需要修改爲對應的地址
advertised.listeners=PLAINTEXT://192.168.200.110:9092

# 修改處4：socket最大請求，int類型的，不能超過int類型的最大值
socket.request.max.bytes=2147483600

# 修改處5：partition的個數
num.partitions=6

# 修改處6：是否啓用log壓縮，一般不用啓用，啓用的話可以提高性能
log.cleaner.enable=false

# 修改處7：zookeeper集羣地址
zookeeper.connect=192.168.200.110:2181,192.168.200.111:2181,192.168.200.112:2181

# 修改處8：是否允許刪除
# 是否允許刪除topic
delete.topic.enable=true

# 修改處9：消息存放的目錄，這個目錄可以配置爲“，”逗號分割的表達式，上面的num.io.threads要大於這個目錄的個數這個目錄，如果配置多個目錄，新創建的topic他把消息持久化的地方是，當前以逗號分割的目錄中，那個分區數最少就放那一個
log.dirs=/software/kafka/kafka-logs

# 創建消息存放的目錄
mkdir -p /software/kafka/kafka-logs/

Kafka調優請看我的文章：https://www.lzhpo.com/article/126

配置文件調優詳細說明(說明示例)

# 當前機器在集羣中的唯一標識，和zookeeper的myid性質一樣  建議用自己主機的後三位  每臺（主機）broker不一致
broker.id

# 當前kafka對外提供服務的端口默認是9092  生產者（producer）要以這個端口爲準[kafka-0.1.x之前]
#port

# 這個參數默認是關閉的，在0.8.1有個bug，DNS解析問題，失敗率的問題。  （填寫本機地址即可）[kafka-0.1.x之前]
#host.name

# Kafka啓動的端口[kafka-0.1.x之前]
#advertised.port=9092
# kafka的本機IP地址[kafka-0.1.x之前]
#advertised.host.name=192.168.10.130

# 監聽
listeners=PLAINTEXT://192.168.10.130:9092

# 對外開放。kafka-0.1.x之後的就是這樣子修改的了，啓用了上面的開放端口和地址方式
advertised.listeners=PLAINTEXT://192.168.10.130:9092

# 這個是borker進行網絡處理的線程數
# 【優化：num.network.threads主要處理網絡io，讀寫緩衝區數據，基本沒有io等待，配置線程數量爲cpu核數加1。】
num.network.threads=3

# 這個是borker進行I/O處理的線程數
# 【優化：num.io.threads主要進行磁盤io操作，高峯期可能有些io等待，因此配置需要大些。配置線程數量爲cpu核數2倍，最大不超過3倍。】
num.io.threads=8 

# 消息存放的目錄，這個目錄可以配置爲“，”逗號分割的表達式，上面的num.io.threads要大於這個目錄的個數這個目錄，如果配置多個目錄，新創建的topic他把消息持久化的地方是，當前以逗號分割的目錄中，那個分區數最少就放那一個
log.dirs=/software/kafka/kafka-logs

# 是否允許刪除topic
delete.topic.enable=true

# 發送緩衝區buffer大小，數據不是一下子就發送的，先回存儲到緩衝區了到達一定的大小後在發送，能提高性能
socket.send.buffer.bytes=102400

# kafka接收緩衝區大小，當數據到達一定大小後在序列化到磁盤
socket.receive.buffer.bytes=102400

# 這個參數是向kafka請求消息或者向kafka發送消息的請請求的最大數，這個值不能超過java的堆棧大小
# 【優化：這個是int類型的取值，int類型範圍是-2147483648~2147483647。不能超出，超出之後報錯：org.apache.kafka.common.config.ConfigException: Invalid value 8589934592 for configuration socket.request.max.bytes: Not a number of type INT。如果很糾結的話，那就按我推薦設置爲2147483600】
socket.request.max.bytes=104857600

# 默認的分區數，一個topic默認1個分區數
# 【優化：默認partition數量1，如果topic在創建時沒有指定partition數量，默認使用此值。Partition的數量選取也會直接影響到Kafka集羣的吞吐性能，配置過小會影響消費性能，建議改爲6。】
num.partitions=1

# 默認消息的最大持久化時間，168小時，7天
log.retention.hours=168

# 消息保存的最大值5M
message.max.byte=5242880

# kafka保存消息的副本數，如果一個副本失效了，另一個還可以繼續提供服務
default.replication.factor=2

# 取消息的最大直接數
replica.fetch.max.bytes=5242880

# 這個參數是：因爲kafka的消息是以追加的形式落地到文件，當超過這個值的時候，kafka會新起一個文件
log.segment.bytes=1073741824

# 每隔300000毫秒去檢查上面配置的log失效時間（log.retention.hours=168 ），到目錄查看是否有過期的消息如果有，刪除
log.retention.check.interval.ms=300000

# 是否啓用log壓縮，一般不用啓用，啓用的話可以提高性能
log.cleaner.enable=false

# 設置zookeeper的連接端口 消費的時候要以這個端口消費
zookeeper.connect=192.168.10.130:2181,192.168.10.128:2181,192.168.10.129:2181
# zookeeper連接超時時間
zookeeper.connection.timeout.ms=6000

# 【優化：爲了大幅度提高producer寫入吞吐量，需要定期批量寫文件。一般無需改動，如果topic的數據量較小可以考慮減少log.flush.interval.ms和log.flush.interval.messages來強制刷寫數據，減少可能由於緩存數據未寫盤帶來的不一致。推薦配置分別message 10000，間隔1s。】
# 每當producer寫入10000條消息時，刷數據到磁盤
log.flush.interval.messages=10000
# 每間隔1秒鐘時間，刷數據到磁盤
log.flush.interval.ms=1000

默認的config/server.properties配置文件：

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# see kafka.server.KafkaConfig for additional details and defaults

############################# Server Basics #############################

# The id of the broker. This must be set to a unique integer for each broker.
broker.id=0

############################# Socket Server Settings #############################

# The address the socket server listens on. It will get the value returned from 
# java.net.InetAddress.getCanonicalHostName() if not configured.
#   FORMAT:
#     listeners = listener_name://host_name:port
#   EXAMPLE:
#     listeners = PLAINTEXT://your.host.name:9092
#listeners=PLAINTEXT://:9092

# Hostname and port the broker will advertise to producers and consumers. If not set, 
# it uses the value for "listeners" if configured.  Otherwise, it will use the value
# returned from java.net.InetAddress.getCanonicalHostName().
#advertised.listeners=PLAINTEXT://your.host.name:9092

# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details
#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL

# The number of threads that the server uses for receiving requests from the network and sending responses to the network
num.network.threads=3

# The number of threads that the server uses for processing requests, which may include disk I/O
num.io.threads=8

# The send buffer (SO_SNDBUF) used by the socket server
socket.send.buffer.bytes=102400

# The receive buffer (SO_RCVBUF) used by the socket server
socket.receive.buffer.bytes=102400

# The maximum size of a request that the socket server will accept (protection against OOM)
socket.request.max.bytes=104857600


############################# Log Basics #############################

# A comma separated list of directories under which to store log files
log.dirs=/tmp/kafka-logs

# The default number of log partitions per topic. More partitions allow greater
# parallelism for consumption, but this will also result in more files across
# the brokers.
num.partitions=1

# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown.
# This value is recommended to be increased for installations with data dirs located in RAID array.
num.recovery.threads.per.data.dir=1

############################# Internal Topic Settings  #############################
# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state"
# For anything other than development testing, a value greater than 1 is recommended for to ensure availability such as 3.
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1

############################# Log Flush Policy #############################

# Messages are immediately written to the filesystem but by default we only fsync() to sync
# the OS cache lazily. The following configurations control the flush of data to disk.
# There are a few important trade-offs here:
#    1. Durability: Unflushed data may be lost if you are not using replication.
#    2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush.
#    3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks.
# The settings below allow one to configure the flush policy to flush data after a period of time or
# every N messages (or both). This can be done globally and overridden on a per-topic basis.

# The number of messages to accept before forcing a flush of data to disk
#log.flush.interval.messages=10000

# The maximum amount of time a message can sit in a log before we force a flush
#log.flush.interval.ms=1000

############################# Log Retention Policy #############################

# The following configurations control the disposal of log segments. The policy can
# be set to delete segments after a period of time, or after a given size has accumulated.
# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens
# from the end of the log.

# The minimum age of a log file to be eligible for deletion due to age
log.retention.hours=168

# A size-based retention policy for logs. Segments are pruned from the log unless the remaining
# segments drop below log.retention.bytes. Functions independently of log.retention.hours.
#log.retention.bytes=1073741824

# The maximum size of a log segment file. When this size is reached a new log segment will be created.
log.segment.bytes=1073741824

# The interval at which log segments are checked to see if they can be deleted according
# to the retention policies
log.retention.check.interval.ms=300000

############################# Zookeeper #############################

# Zookeeper connection string (see zookeeper docs for details).
# This is a comma separated host:port pairs, each corresponding to a zk
# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
# You can also append an optional chroot string to the urls to specify the
# root directory for all kafka znodes.
zookeeper.connect=localhost:2181

# Timeout in ms for connecting to zookeeper
zookeeper.connection.timeout.ms=6000


############################# Group Coordinator Settings #############################

# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance.
# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms.
# The default value for this is 3 seconds.
# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing.
# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup.
group.initial.rebalance.delay.ms=0

修改完，分發到其它機器

scp -r /software/kafka/ slave1:/software/kafka/

scp -r /software/kafka/ slave2:/software/kafka/

修改其它機器的參數

config/server.properties

需要修改的參數，修改爲自己主機的實際地址：

broker.id、listeners、advertised.listeners

bin/kafka-run-class.sh

需要修改的：

-Djava.rmi.server.hostname

kafka相關操作命令

# 啓動kafka
[root@master /]# kafka-server-start.sh -daemon /software/kafka/kafka_2.12-2.2.0/config/server.properties

# 停止kafka
[root@master /]# kafka-server-stop.sh

# 創建topic，設置分區數爲3，副本數爲6
[root@master kafka-logs]# kafka-topics.sh --create --zookeeper 192.168.200.110:2181,192.168.200.111:2181,192.168.200.112:2181 --replication-factor 3 --partitions 6 --topic lzhpo-topic-test06
Created topic lzhpo-topic-test06.

# 查看集羣所有topic
[root@master kafka-logs]# kafka-topics.sh --list --zookeeper 192.168.200.110:2181,192.168.200.111:2181,192.168.200.112:2181
__consumer_offsets
lzhpo-topic-test01
lzhpo-topic-test02
lzhpo-topic-test03
lzhpo-topic-test04
lzhpo-topic-test05
lzhpo-topic-test06
lzhpo-topic-test3
lzhpo-topic-test4
topic-test-01

# 在slave2節點啓動消費者consumer，監聽lzhpo-topic-test3的topic
[root@slave2 ~]# kafka-console-consumer.sh --bootstrap-server 192.168.200.112:9092 --topic lzhpo-topic-test05 --from-beginning


# 在slave1啓動一個生產者發送消息
[root@slave1 ~]# kafka-console-producer.sh --broker-list 192.168.200.111:9092 --topic lzhpo-topic-test05

slave2節點啓動一個消費者：

slave1節點啓動一個生產者發送消息：

# 查看topic的信息
[root@master kafka-logs]# kafka-topics.sh --describe --zookeeper master:2181,slave1:2181,slave2:2181 --topic lzhpo-topic-test05
Topic:lzhpo-topic-test05	PartitionCount:6	ReplicationFactor:3	Configs:
	Topic: lzhpo-topic-test05	Partition: 0	Leader: 2	Replicas: 2,0,1	Isr: 2,0,1
	Topic: lzhpo-topic-test05	Partition: 1	Leader: 0	Replicas: 0,1,2	Isr: 0,1,2
	Topic: lzhpo-topic-test05	Partition: 2	Leader: 1	Replicas: 1,2,0	Isr: 1,2,0
	Topic: lzhpo-topic-test05	Partition: 3	Leader: 2	Replicas: 2,1,0	Isr: 2,1,0
	Topic: lzhpo-topic-test05	Partition: 4	Leader: 0	Replicas: 0,2,1	Isr: 0,2,1
	Topic: lzhpo-topic-test05	Partition: 5	Leader: 1	Replicas: 1,0,2	Isr: 1,0,2

kafka調優

調優參數：

• replication.factor：這是給topic設置的參數，也就是副本數，這個參數值必須大於1，默認是1，要求每個partition必須至少有2個副本。kafka集羣雖然是高可用的，但是該topic在有broker宕機時，可能發生無法使用的情況。

  • 方法1：建立topic的時候就設置。

    bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 3 --partitions 6 --topic lzhpo01
    

    查看修改情況：

    bin/kafka-topics.sh --describe --zookeeper localhost:2181 --topic lzhpo01
    

  • 方法2：動態的給已經創建的topic添加replication-factor

    a. 首先我們配置topic的副本，保存爲json文件（）
    例如， 我們想把lzhpo01的部分設置爲3，（我的kafka集羣有3個broker，id分別爲0,1,2）， json文件名稱爲increase-replication-factor.json：

    {"version":1,
    "partitions":[
    {"topic":"lzhpo01","partition":0,"replicas":[0,1,2]},
    {"topic":"lzhpo01","partition":1,"replicas":[0,1,2]},
    {"topic":"lzhpo01","partition":2,"replicas":[0,1,2]}
    ]}
    

    b. 執行腳本：

    bin/kafka-reassign-partitions.sh -zookeeper 127.0.0.1:2181 --reassignment-json-file increase-replication-factor.json --execute
    

    c.查看修改之後的topic副本因子：

    bin/kafka-topics.sh --describe --zookeeper localhost:2181 --topic lzhpo01
    

    參考官方文檔：https://kafka.apache.org/documentation/#basic_ops_increase_replication_factor

• min.insync.replicas：這是給Kafka服務端設置的參數，這個參數值也必須大於1，意思是要求一個Leader至少有一個Follower跟自己保持聯繫，也就是數據和Leader一直是保持同步的，這樣子就確保了Leader掛掉了，至少還有一個Follower，不至於丟失數據。

  設置min.insync.replicas：

  bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 3 --partitions 6 --min.insync.replicas 2 --topic lzhpo01
  

• acks=all：這是在生產者端設置的，這是要求每條數據必須是寫入所有的replica之後，才能認爲是寫入成功了。

  就舉個例子，SpringBoot集成Kafka的話，可以直接在application.yml中配置：

[外鏈圖片轉存失敗,源站可能有防盜鏈機制,建議將圖片保存下來直接上傳(img-fh1k0mw1-1590571812227)(http://cdn.lzhpo.com/Kafka/images/%E7%94%9F%E4%BA%A7%E8%80%85%E7%9A%84acks%E8%AE%BE%E7%BD%AE%E4%B8%BAall.png)]

• retries=MAX：這個是要求一旦寫入失敗，就無限充值，卡在這裏。

  這個我沒有研究過，略…

歡迎關注我

一個熱愛技術、熱愛籃球的程序猿。

我的博客：https://www.lzhpo.com

我的微信公衆號：會打籃球的程序猿