Docker學習筆記-Kafka集羣搭建

Linux版本：Ubuntu 16.04.2 LTS

Docker版本：Docker version 17.09.0-ce, build afdb6d4

Docker Compose版本：docker-compose version 1.22.0, build f46880fe

JDK版本：java version "1.8.0_102"

Zookeeper版本：zookeeper-3.4.14

Kafka版本：kafka_2.12-2.3.0

相關構建配置文件如下：

Kafka Dockerfile

From jdk:1.8.0.102
MAINTAINER xxxxx

ADD kafka_2.12-2.3.0.tgz /usr/local/
RUN mv /usr/local/kafka_2.12-2.3.0 /usr/local/kafka

ENV KAFKA_HOME=/usr/local/kafka \
    KAFKA_CONF_DIR=/usr/local/kafka/config \
    KAFKA_LOG_DIR=/usr/local/kafka/logs

RUN mkdir -p "$KAFKA_LOG_DIR" 

RUN sed -i 's/log.dirs=\/tmp\/kafka-logs/log.dirs=\/usr\/local\/kafka\/logs/g' $KAFKA_CONF_DIR/server.properties

WORKDIR $KAFKA_HOME
VOLUME ["$KAFKA_LOG_DIR"]

EXPOSE 9092

ENV PATH=$PATH:$KAFKA_HOME/bin

COPY kafka-entrypoint.sh /
RUN chmod +x /kafka-entrypoint.sh
ENTRYPOINT ["/kafka-entrypoint.sh"]

kafka-entrypoint.sh

#!/bin/bash

set -e

CONFIG="$KAFKA_CONF_DIR/server.properties"
if [[ -f "$CONFIG" ]]; then
    sed -i "s/broker.id=0/broker.id=$KAFKA_BROKER_ID/g" $CONFIG   
    sed -i "s/zookeeper.connect=localhost:2181/zookeeper.connect=$KAFKA_ZOOKEEPER_CONNECT/g" $CONFIG
    echo "" >> "$CONFIG"
    echo "listeners=PLAINTEXT://$KAFKA_LISTENERS_HOSTNAME:9092" >> "$CONFIG"   
fi

$KAFKA_HOME/bin/kafka-server-start.sh $CONFIG

exec "$@"

docker build --no-cache -t kafka:2.3.0 images/kafka

docker-compose.yml

version: '3.4'

services:
  zookeeper1:
    image: zookeeper:3.4.14
    restart: always
    hostname: zookeeper1
    container_name: zookeeper1
    ports:
    - 12181:2181
    volumes:
    - "/home/docker/zookeeper-3.4.14/cluster/zookeeper1/data:/usr/local/zookeeper/data"
    - "/home/docker/zookeeper-3.4.14/cluster/zookeeper1/datalog:/usr/local/zookeeper/datalog"
    - "/home/docker/zookeeper-3.4.14/cluster/zookeeper1/logs:/usr/local/zookeeper/logs"
    environment:
      ZOOKEEPER_MY_ID: 1
      ZOOKEEPER_SERVERS: server.1=0.0.0.0:2888:3888 server.2=zookeeper2:2888:3888 server.3=zookeeper3:2888:3888

  zookeeper2:
    image: zookeeper:3.4.14
    restart: always
    hostname: zookeeper2
    container_name: zookeeper2
    ports:
    - 22181:2181
    volumes:
    - "/home/docker/zookeeper-3.4.14/cluster/zookeeper2/data:/usr/local/zookeeper/data"
    - "/home/docker/zookeeper-3.4.14/cluster/zookeeper2/datalog:/usr/local/zookeeper/datalog"
    - "/home/docker/zookeeper-3.4.14/cluster/zookeeper2/logs:/usr/local/zookeeper/logs"
    environment:
      ZOOKEEPER_MY_ID: 2
      ZOOKEEPER_SERVERS: server.1=zookeeper1:2888:3888 server.2=0.0.0.0:2888:3888 server.3=zookeeper3:2888:3888

  zookeeper3:
    image: zookeeper:3.4.14
    restart: always
    hostname: zookeeper3
    container_name: zookeeper3
    ports:
    - 32181:2181
    volumes:
    - "/home/docker/zookeeper-3.4.14/cluster/zookeeper3/data:/usr/local/zookeeper/data"
    - "/home/docker/zookeeper-3.4.14/cluster/zookeeper3/datalog:/usr/local/zookeeper/datalog"
    - "/home/docker/zookeeper-3.4.14/cluster/zookeeper3/logs:/usr/local/zookeeper/logs"
    environment:
      ZOOKEEPER_MY_ID: 3
      ZOOKEEPER_SERVERS: server.1=zookeeper1:2888:3888 server.2=zookeeper2:2888:3888 server.3=0.0.0.0:2888:3888

  kafka1:
    image: kafka:2.3.0
    restart: always
    hostname: kafka1
    container_name: kafka1
    ports:
    - 19092:9092
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181,zookeeper2:2181,zookeeper3:2181
      KAFKA_LISTENERS_HOSTNAME: kafka1
    volumes:
    - "/home/docker/kafka-2.3.0/cluster/kafka1/logs:/usr/local/kafka/logs"
    links:
    - zookeeper1
    - zookeeper2
    - zookeeper3
    tty: true

  kafka2:
    image: kafka:2.3.0
    restart: always
    hostname: kafka2
    container_name: kafka2
    ports:
    - 29092:9092
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181,zookeeper2:2181,zookeeper3:2181
      KAFKA_LISTENERS_HOSTNAME: kafka2
    volumes:
    - "/home/docker/kafka-2.3.0/cluster/kafka2/logs:/usr/local/kafka/logs"
    links:
    - zookeeper1
    - zookeeper2
    - zookeeper3
    tty: true

  kafka3:
    image: kafka:2.3.0
    restart: always
    hostname: kafka3
    container_name: kafka3
    ports:
    - 39092:9092
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181,zookeeper2:2181,zookeeper3:2181
      KAFKA_LISTENERS_HOSTNAME: kafka3
    volumes:
    - "/home/docker/kafka-2.3.0/cluster/kafka3/logs:/usr/local/kafka/logs"
    links:
    - zookeeper1
    - zookeeper2
    - zookeeper3
    tty: true

docker-compose up -d

進入容器kafka1測試驗證Kafka集羣

# docker exec -it kafka1 /bin/bash

# kafka-topics.sh --create --zookeeper zookeeper1:2181,zookeeper2:2181,zookeeper3:2181 --replication-factor 1 --partitions 3 --topic test

# kafka-topics.sh --list --zookeeper zookeeper1:2181,zookeeper2:2181,zookeeper3:2181