kafka安裝
機器初始化
參考ES安裝，安裝java，關閉防火牆，selinux等
安裝zookeeper
yum install -y wget
wget https://downloads.apache.org/kafka/2.6.0/kafka_2.13-2.6.0.tgz #下載zookeeper
mkdir /usr/share/kafka
tar zxvf kafka_2.13-2.6.0.tgz -C /usr/share/kafka/ #解壓至/usr/share/kafka/下
mkdir -p /data/zookeeper/{data,logs} #創建zookeeper數據及日誌存放路徑
mkdir -p /data/kafka/{data,logs} #創建kafka數據及日誌存放路徑配置zookeeper
vim /usr/share/kafka/kafka_2.13-2.6.0/config/zookeeper.properties #修改zookeeper配置文件
-----------------------------------------------------------------
dataDir=/data/zookeeper/data
dataLogDir=/data/zookeeper/logs
clientPort=2181
maxClientCnxns=0
admin.enableServer=false
tickTime=2000
initLimit=20
syncLimit=10
server.1=192.168.110.64:2888:3888
server.2=192.168.110.65:2888:3888
server.3=192.168.110.66:2888:3888配置kafka
-----------------------------------------------------------------
#每臺生成一個Myid 後期要用，我有三臺kafka，第一臺設置爲1，第二臺設置爲2，第三臺設置爲3
echo 2 > /data/zookeeper/data/myid #創建Myid後面啓動kafka時使用
vim /usr/share/kafka/kafka_2.13-2.6.0/config/server.properties
-----------------------------------------------------------------
broker.id=2 #修改爲自己創建的myid
listeners=PLAINTEXT://192.168.110.66:9092 #修改爲kafka本機地址
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/data/kafka/logs
num.partitions=6
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=2
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
zookeeper.connect=192.168.110.64:2181,192.168.110.65:2181,192.168.110.66:2181
zookeeper.connection.timeout.ms=6000
group.initial.rebalance.delay.ms=0啓動kafka
cd /usr/share/kafka/kafka_2.13-2.6.0/
nohup bin/zookeeper-server-start.sh config/zookeeper.properties & #啓動zookeeper
yum install -y net-tools
netstat -nlpt | grep -E "2181|2888|3888" #驗證zookeeper羣集，羣集擁有2888端口即爲leader
nohup bin/kafka-server-start.sh config/server.properties & #啓動kafka
#創建一個topic驗證kafka羣集
bin/kafka-topics.sh --create --zookeeper 192.168.110.61:2181 --replication-factor 1 --partitions 1 --topic testtopic
#查看topic
bin/kafka-topics.sh --zookeeper 192.168.110.64 --list
#測試生產消息
bin/kafka-console-producer.sh --broker-list 192.168.110.64:9092 --topic testtopic
#測試消費消息
bin/kafka-console-consumer.sh --bootstrap-server 192.168.110.64:9092 --topic testtopic --from-beginning
#查看topic中的內容
bin/kafka-console-consumer.sh --bootstrap-server 192.168.36.194:9092,192.168.36.195:9092,192.168.36.196:9092 --topic TestEx01_Transport_Logs