Docker搭建Zookeeper和Kafka集羣

安裝Docker-compose

創建一個帶子網的專有網絡

創建一個叫zoo_kafka的網絡,子網網段172.23.0.10/20

root@localhost ]# docker network create --subnet=172.23.0.10/20 zoo_kafka
root@localhost ]# docker network ls
NETWORK ID          NAME                DRIVER              SCOPE
4fad358ef523        bridge              bridge              local
2ea1ad136fa2        host                host                local
a66aec056df4        kafka_default       bridge              local
9a4fbf8d993d        none                null                local
d584dd2084c7        zoo_kafka           bridge              local

創建Zookeeper集羣

創建文件夾

root@localhost ]# mkdir zookeeper
root@localhost ]# cd zookeeper

編寫docker-compose.yml文件

root@localhost ]# touch docker-compose.yml
root@localhost ]# vim docker-compose.yml

內容如下:

version: '2'
services:
  zoo1:
    image: zookeeper:3.4 # 鏡像名稱
    restart: always # 當發生錯誤時自動重啓
    hostname: zoo1
    container_name: zoo1
    privileged: true
    ports: # 端口
      - 2181:2181
    volumes: # 掛載數據卷
      - ./zoo1/data:/data
      - ./zoo1/datalog:/datalog 
    environment:
      TZ: Asia/Shanghai
      ZOO_MY_ID: 1 # 節點ID
      ZOO_PORT: 2181 # zookeeper端口號
      ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888 # zookeeper節點列表
    networks:
      default:
        ipv4_address: 172.23.0.11

  zoo2:
    image: zookeeper:3.4
    restart: always
    hostname: zoo2
    container_name: zoo2
    privileged: true
    ports:
      - 2182:2181
    volumes:
      - ./zoo2/data:/data
      - ./zoo2/datalog:/datalog
    environment:
      TZ: Asia/Shanghai
      ZOO_MY_ID: 2
      ZOO_PORT: 2181
      ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
    networks:
      default:
        ipv4_address: 172.23.0.12

  zoo3:
    image: zookeeper:3.4
    restart: always
    hostname: zoo3
    container_name: zoo3
    privileged: true
    ports:
      - 2183:2181
    volumes:
      - ./zoo3/data:/data
      - ./zoo3/datalog:/datalog
    environment:
      TZ: Asia/Shanghai
      ZOO_MY_ID: 3
      ZOO_PORT: 2181
      ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
    networks:
      default:
        ipv4_address: 172.23.0.13

networks:
  default:
    external:
      name: zoo_kafka

啓動並驗證

• 後臺啓動,並查看容器

root@localhost zookeeper]# docker-compose up -d
Starting zoo3 ... done
Starting zoo2 ... done
Starting zoo1 ... done
[root@localhost zookeeper]# docker ps
CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS                                        NAMES
125a78286d6b        zookeeper:3.4       "/docker-entrypoint.…"   22 minutes ago      Up 12 minutes       2888/tcp, 0.0.0.0:2181->2181/tcp, 3888/tcp   zoo1
4998e54b7140        zookeeper:3.4       "/docker-entrypoint.…"   22 minutes ago      Up 12 minutes       2888/tcp, 3888/tcp, 0.0.0.0:2182->2181/tcp   zoo2
5eec0034d73b        zookeeper:3.4       "/docker-entrypoint.…"   22 minutes ago      Up 12 minutes       2888/tcp, 3888/tcp, 0.0.0.0:2183->2181/tcp   zoo3

• 驗證zookeeper集羣
  這裏看到zoo3爲leader,zoo1和zoo2爲follower

[root@localhost zookeeper]# docker exec zoo1 zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /conf/zoo.cfg
Mode: follower
[root@localhost zookeeper]# docker exec zoo2 zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /conf/zoo.cfg
Mode: follower
[root@localhost zookeeper]# docker exec zoo3 zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /conf/zoo.cfg
Mode: leader

創建Kafka集羣

創建文件夾

創建文件夾

root@localhost ]# mkdir kafka
root@localhost ]# cd kafka

編寫docker-compose.yml文件

root@localhost ]# touch docker-compose.yml
root@localhost ]# vim docker-compose.yml

內容如下:

version: '2'

services:
  broker1:
    image: wurstmeister/kafka:2.11-0.11.0.3
    restart: always
    hostname: broker1
    container_name: broker1
    privileged: true
    ports:
      - "9091:9092"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_LISTENERS: PLAINTEXT://broker1:9092
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker1:9092
      KAFKA_ADVERTISED_HOST_NAME: broker1
      KAFKA_ADVERTISED_PORT: 9092
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181/kafka1,zoo2:2181/kafka1,zoo3:2181/kafka1
      JMX_PORT: 9988
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ./broker1:/kafka/kafka\-logs\-broker1
    external_links:
    - zoo1
    - zoo2
    - zoo3
    networks:
      default:
        ipv4_address: 172.23.0.14

  broker2:
    image: wurstmeister/kafka:2.11-0.11.0.3
    restart: always
    hostname: broker2
    container_name: broker2
    privileged: true
    ports:
      - "9092:9092"
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_LISTENERS: PLAINTEXT://broker2:9092
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker2:9092
      KAFKA_ADVERTISED_HOST_NAME: broker2
      KAFKA_ADVERTISED_PORT: 9092
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181/kafka1,zoo2:2181/kafka1,zoo3:2181/kafka1
      JMX_PORT: 9988
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ./broker2:/kafka/kafka\-logs\-broker2
    external_links:  # 連接本compose文件以外的container
    - zoo1
    - zoo2
    - zoo3
    networks:
      default:
        ipv4_address: 172.23.0.15

  broker3:
    image: wurstmeister/kafka:2.11-0.11.0.3
    restart: always
    hostname: broker3
    container_name: broker3
    privileged: true
    ports:
      - "9093:9092"
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_LISTENERS: PLAINTEXT://broker3:9092
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker3:9092
      KAFKA_ADVERTISED_HOST_NAME: broker3
      KAFKA_ADVERTISED_PORT: 9092
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181/kafka1,zoo2:2181/kafka1,zoo3:2181/kafka1
      JMX_PORT: 9988
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ./broker3:/kafka/kafka\-logs\-broker3
    external_links:  # 連接本compose文件以外的container
    - zoo1
    - zoo2
    - zoo3
    networks:
      default:
        ipv4_address: 172.23.0.16

  kafka-manager:
    image: sheepkiller/kafka-manager:latest
    restart: always
    container_name: kafka-manager
    hostname: kafka-manager
    ports:
      - "9000:9000"
    links:            # 連接本compose文件創建的container
      - broker1
      - broker2
      - broker3
    external_links:   # 連接本compose文件以外的container
      - zoo1
      - zoo2
      - zoo3
    environment:
      ZK_HOSTS: zoo1:2181/kafka1,zoo2:2181/kafka1,zoo3:2181/kafka1
      KAFKA_BROKERS: broker1:9092,broker2:9092,broker3:9092
      APPLICATION_SECRET: letmein
      KM_ARGS: -Djava.net.preferIPv4Stack=true
    networks:
      default:
        ipv4_address: 172.23.0.10

networks:
  default:
    external:   # 使用已創建的網絡
      name: zoo_kafka

啓動並驗證

[root@localhost kafka]# docker-compose up -d
Creating broker1 ... done
Creating broker3 ... done
Creating broker2 ... done
Creating kafka-manager ... done
[root@localhost kafka]# docker ps
CONTAINER ID        IMAGE                              COMMAND                  CREATED             STATUS              PORTS                                        NAMES
8140b9c9eb07        sheepkiller/kafka-manager:latest   "./start-kafka-manag…"   4 minutes ago       Up 4 minutes        0.0.0.0:9000->9000/tcp                       kafka-manager
5df60f08f5aa        wurstmeister/kafka:2.11-0.11.0.3   "start-kafka.sh"         4 minutes ago       Up 4 minutes        0.0.0.0:9093->9092/tcp                       broker3
4d17d490faa7        wurstmeister/kafka:2.11-0.11.0.3   "start-kafka.sh"         4 minutes ago       Up 4 minutes        0.0.0.0:9092->9092/tcp                       broker2
07206f603d20        wurstmeister/kafka:2.11-0.11.0.3   "start-kafka.sh"         4 minutes ago       Up 4 minutes        0.0.0.0:9091->9092/tcp                       broker1
125a78286d6b        zookeeper:3.4                      "/docker-entrypoint.…"   37 minutes ago      Up 27 minutes       2888/tcp, 0.0.0.0:2181->2181/tcp, 3888/tcp   zoo1
4998e54b7140        zookeeper:3.4                      "/docker-entrypoint.…"   37 minutes ago      Up 27 minutes       2888/tcp, 3888/tcp, 0.0.0.0:2182->2181/tcp   zoo2
5eec0034d73b        zookeeper:3.4                      "/docker-entrypoint.…"   37 minutes ago      Up 27 minutes       2888/tcp, 3888/tcp, 0.0.0.0:2183->2181/tcp   zoo3