cd /opt
wget http://apache-mirror.rbc.ru/pub/apache/kafka/0.10.1.0/kafka_2.11-0.10.1.0.tgz
tar xvzf kafka_2.11-0.10.1.0.tgz
ln -s kafka_2.11-0.10.1.0/ kafka
vi /etc/systemd/system/kafka-zookeeper.service
[Unit]
Description=Apache Zookeeper server (Kafka)
Documentation=http://zookeeper.apache.org
Requires=network.target remote-fs.target
After=network.target remote-fs.target
[Service]
Type=simple
User=nano
Group=nano
Environment=JAVA_HOME=/usr/java/jdk1.8.0_102
ExecStart=/opt/kafka/bin/zookeeper-server-start.sh /opt/kafka/config/zookeeper.properties
ExecStop=/opt/kafka/bin/zookeeper-server-stop.sh
[Install]
WantedBy=multi-user.target
vi /etc/systemd/system/kafka.service
[Unit]
Description=Apache Kafka server (broker)
Documentation=http://kafka.apache.org/documentation.html
Requires=network.target remote-fs.target
After=network.target remote-fs.target kafka-zookeeper.service
[Service]
Type=simple
User=nano
Group=nano
Environment=JAVA_HOME=/usr/java/jdk1.8.0_102
ExecStart=/opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties
ExecStop=/opt/kafka/bin/kafka-server-stop.sh
[Install]
WantedBy=multi-user.target
vi kafka/config/server.properties
edit listeners propertie
listeners=PLAINTEXT://192.168.13.133:9092
systemctl daemon-reload
systemctl start kafka-zookeeper.service
systemctl start kafka.service
-
Install Kafka
cd /opt curl -O http://www.gtlib.gatech.edu/pub/apache/kafka/0.11.0.0/kafka_2.11-0.11.0.0.tgz tar xvzf kafka_2.11-0.11.0.0.tgz ln -s kafka_2.11-0.11.0.0/ kafka -
Edit
/usr/lib/systemd/system/kafka-zookeeper.service[Unit] Description=Apache Zookeeper server (Kafka) Documentation=http://zookeeper.apache.org Requires=network.target remote-fs.target After=network.target remote-fs.target [Service] Type=simple User=kafka Group=kafka Environment=JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk ExecStart=/opt/kafka/bin/zookeeper-server-start.sh /opt/kafka/config/zookeeper.properties ExecStop=/opt/kafka/bin/zookeeper-server-stop.sh [Install] WantedBy=multi-user.target -
Edit
/usr/lib/systemd/system/kafka.service[Unit] Description=Apache Kafka server (broker) Documentation=http://kafka.apache.org/documentation.html Requires=network.target remote-fs.target After=network.target remote-fs.target kafka-zookeeper.service [Service] Type=simple User=kafka Group=kafka Environment=JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk ExecStart=/opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties ExecStop=/opt/kafka/bin/kafka-server-stop.sh [Install] WantedBy=multi-user.target -
Edit
/opt/kafka/config/server.propertiesand change thelistenerspropertylisteners=PLAINTEXT://0.0.0.0:9092 -
Reload and start the systemd services
systemctl daemon-reload systemctl start kafka-zookeeper.service systemctl start kafka.service
kafka 是一個分佈式的基於 push-subscribe 的消息系統，快速、可擴展、可持久化， 可以實時的處理大量數據以滿足各種需求場景，具有以下特點：
- 高吞吐量、低延遲：每秒可以處理幾十萬條消息，它的延遲最低只有幾毫秒
- 可擴展性：集羣支持熱擴展
- 持久性、可靠性：消息被持久化到本地磁盤，並且支持數據備份防止數據丟失
- 容錯性：允許集羣中節點失敗（若副本數量爲 n,則允許 n-1 個節點失敗）
- 高併發：支持數千個客戶端同時讀寫
本博客講解 debian9.4 系統下 kafka 集羣的搭建。
集羣節點配置
10.0.0.2: zookeeper, kafka10.0.0.3: zookeeper, kafka10.0.0.4: zookeeper, kafka
安裝 zookeeper
kafka 使用 zookeeper 來維護集羣的狀態，因此應該首先搭建一個 zookeeper 集羣， 具體步驟請見博文：搭建zookeeper集羣。
安裝 kafka
下載 kafka
wget http://mirrors.tuna.tsinghua.edu.cn/apache/kafka/1.1.0/kafka_2.12-1.1.0.tgz
安裝 kafka
分別在三臺節點機器上運行解壓命令，解壓到 /opt 目錄：
tar -xzvf kafka_2.12-1.1.0.tgz -C /opt
創建 kafka 用戶、組
groupadd kafka
useradd -g kafka -M -s /usr/sbin/nologin kafka
配置 kafka
創建 kafka 數據目錄：
mkdir -p /var/lib/kafka
配置 /opt/kafka_2.12-1.1.0/config/server.properties 文件(10.0.0.2 節點)如下：
broker.id=2
listeners=PLAINTEXT://10.0.0.2:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/var/lib/kafka
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=10.0.0.2:2181,10.0.0.3:2181,10.0.0.4:2181/kafka
zookeeper.connection.timeout.ms=6000
group.initial.rebalance.delay.ms=0
其中 broker.id 同 zookeeper 的 myid 類似，是集羣中的唯一標識，在此設爲 ip 地址尾數。
配置 systemd service
文件 /etc/systemd/system/kafka.service 文件如下：
[Unit]
Description=Kafka
After=zookeeper.service
[Service]
Type=simple
User=kafka
Group=kafka
ExecStart=/opt/kafka_2.12-1.1.0/bin/kafka-server-start.sh /opt/kafka_2.12-1.1.0/config/server.properties
Restart=always
[Install]
WantedBy=multi-user.target
在每一臺機器上運行開啓 kafka
systemctl start kafka
可以將 kafka 設爲開機自啓動：
systemctl enable kafka
測試 kafka 集羣
創建 topic
/opt/kafka_2.12-1.1.0/bin/kafka-topics.sh \
--zookeeper 10.0.0.2:2181,10.0.0.3:2181,10.0.0.4:2181/kafka \
--create \
--replication-factor 1 \
--partitions 1 \
--topic test
查看已經存在的 topic
/opt/kafka_2.12-1.1.0/bin/kafka-topics.sh \
--zookeeper 10.0.0.2:2181,10.0.0.3:2181,10.0.0.4:2181/kafka \
--list
接收消息
/opt/kafka_2.12-1.1.0/bin/kafka-console-consumer.sh \
--bootstrap-server 10.0.0.2:9092,10.0.0.3:9092,10.0.0.4:9092 \
--topic test \
--from-beginning
發送消息
Kafka and Zookeeper as service
|
# Modify systemd script
modify _kafka.service_ by applying [this template](./kafka.service) to /etc/systemd/system/kafka.service followed by executing: ``` sudo systemctl daemon-reload ``` FOR KAFKA
If Zookeeper is added as service, then add the 'Requires' parameter to the unit section. So when kafka is started it will start zookeeper service automatically.
FOR ZOOKEEPER
### Activating systemd scripts ``` sudo systemctl enable zookeeper sudo systemctl enable kafka ``` ### service management via systemd ``` sudo systemctl start zookeeper sudo systemctl start kafka |
其他
https://www.digitalocean.com/community/tutorials/how-to-install-apache-kafka-on-centos-7