docker-compose安裝kafka集羣及管理監控工具

編寫docker-compose文件配置zk 和kafka集羣

#vim kafka.yml

version: '2'

services:

    zoo1:

        image: zookeeper

        restart: always

        container_name: zoo1

        ports:

            - "2181:2181"

        volumes:

            - "/data/zookeeper/zookeeper1/data:/data"

            - "/data/zookeeper/zookeeper1/datalog:/datalog"

        environment:

            ZOO_MY_ID: 1

            ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888


    zoo2:

        image: zookeeper

        restart: always

        container_name: zoo2

        ports:

            - "2182:2181"

        volumes:

            - "/data/zookeeper/zookeeper2/data:/data"

            - "/data/zookeeper/zookeeper2/datalog:/datalog"

        environment:

            ZOO_MY_ID: 2

            ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888


    zoo3:

        image: zookeeper

        restart: always

        container_name: zoo3

        ports:

            - "2183:2181"

        volumes:

            - "/data/zookeeper/zookeeper3/data:/data"

            - "/data/zookeeper/zookeeper3/datalog:/datalog"

        environment:

            ZOO_MY_ID: 3

            ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888


    broker1:

        image: wurstmeister/kafka

        restart: always

        container_name: broker1

        ports:

          - "9091:9092"

        depends_on:

          - zoo1

          - zoo2

          - zoo3

        environment:

          KAFKA_BROKER_ID: 1

          KAFKA_ADVERTISED_HOST_NAME: broker1

          KAFKA_ADVERTISED_PORT: 9092

          KAFKA_HOST_NAME: broker1

          KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181

          KAFKA_LISTENERS: PLAINTEXT://broker1:9092

          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker1:9092

        volumes:

          - /var/run/docker.sock:/var/run/docker.sock

          - "/data/kafka/kafka1/:/kafka"


    broker2:

        image: wurstmeister/kafka

        restart: always

        container_name: broker2

        ports:

          - "9092:9092"

        depends_on:

          - zoo1

          - zoo2

          - zoo3

        environment:

          KAFKA_BROKER_ID: 2

          KAFKA_ADVERTISED_HOST_NAME: broker2

          KAFKA_ADVERTISED_PORT: 9092

          KAFKA_HOST_NAME: broker2

          KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181

          KAFKA_LISTENERS: PLAINTEXT://broker2:9092

          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker2:9092

        volumes:

          - /var/run/docker.sock:/var/run/docker.sock

          - "/data/kafka/kafka2/:/kafka"


    broker3:

        image: wurstmeister/kafka

        restart: always

        container_name: broker3

        ports:

          - "9093:9092"

        depends_on:

          - zoo1

          - zoo2

          - zoo3

        environment:

          KAFKA_BROKER_ID: 3

          KAFKA_ADVERTISED_HOST_NAME: broker3

          KAFKA_ADVERTISED_PORT: 9092

          KAFKA_HOST_NAME: broker3

          KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181

          KAFKA_LISTENERS: PLAINTEXT://broker3:9092

          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker3:9092

        volumes:

          - /var/run/docker.sock:/var/run/docker.sock

          - "/data/kafka/kafka3/:/kafka"

運行compose文件

#COMPOSE_PROJECT_NAME=kafkatest docker-compose -f kafka.yml up -d

查看docker啓動進程

[root@localhost ~]# docker ps

zk 和kafka已啓動image.png


kafka-manager管理工具安裝

#vim kafka_manager.yml

version: '3.1'

services:

  kafka-manager:

    container_name: kafka-manager

    image: sheepkiller/kafka-manager

    restart: always

    ports:

      - 9000:9000

    environment:

      KM_VERSION: 1.3.3.18

      ZK_HOSTS: zoo1:2181,zoo2:2182,zoo3:2183

啓動docker容器

docker-compsoe -f kafka-manager.yml up -d

docker啓動後,登陸到docker容器裏

  #vim conf/application.conf


    kafka-manager.zkhosts="zoo1:2181,zoo2:2182,zoo3:2183"

    #kafka-manager.zkhosts=${?ZK_HOSTS}

將配置文件注掉一條,然後將zk的地址寫入,不然界面是出不來的

以上是出現如下錯誤的解決方法

Yikes! Ask timed out on [ActorSelection[Anchor(akka://kafka-manager-system/), Path(/user/kafka-manager)]] after [5000 ms]

kafka-manager不支持硬盤掛載,我也被坑了好久,


kafkaoffsetmonitor監控工具安裝配置

#vim kafkaoffsetmonitor.yml

vim kafkaoffsetmonitor.yml

version: '2'

services:

  kafka-offset-monitor:

    image: 564239555/kafkaoffsetmonitor

    volumes:

      - /data/kafkaoffsetmonitor/conf:/kafkaoffsetmonitor

    ports:

      - "8089:8089"

    environment:

     ZK_HOSTS: zoo1:2181,zoo2:2182,zoo3:2183

     KAFKA_BROKERS: 172.16.70.51:9091,172.16.70.51:9092,172.16.70.51:90093

     REFRESH_SECENDS: 10

     RETAIN_DAYS: 2

啓動鏡像

docker-compose -f kafkaoffsetmonitor.yml up -d 

kafkaoffsetmonitor需要一個文件來啓動,KafkaOffsetMonitor-assembly-0.2.1.jar,我已將些文件放到鏡像中

或者自行下載這個軟件,將KafkaOffsetMonitor-assembly-0.2.1.jar 拷貝到docker 容器裏,

docker cp /data/kafkaoffsetmonitor/conf/KafkaOffsetMonitor-assembly-0.2.1.jar a45b7987f267:/kafkaoffsetmonitor/

寫個啓動腳本

#vim /kafkaoffsetmonitor/start.sh

#!/bin/bash

nohup java -cp KafkaOffsetMonitor-assembly-0.2.1.jar com.quantifind.kafka.offsetapp.OffsetGetterWeb --zk zoo1:2181,zoo2:2182,zoo3:2183 --port 8089 --refresh 10.seconds --retain 2.days &

如果啓動報錯,將--zk 改成宿主機IP 

給執行權限然後啓動腳本 

#chmod +x /kafkaoffsetmonitor/start.sh

#sh /kafkaoffsetmonitor/start.sh

看到8089的端口啓動,就可以通過瀏覽器http:IP:8089去訪問

網上很多鏡像沒有這個文件,啓動就報錯

或者直接下載我的鏡像文件

https://hub.docker.com/r/564239555/kafkaoffsetmonitor/

docker pull 564239555/kafkaoffsetmonitor

docker啓動後,登陸到docker容器裏,執行腳本,看到8089的端口啓動就可以通過瀏覽器去訪問了http://宿主機IP:8089

#docker exec -it $dockerCONTAINER ID bash

修改docker啓動所需要的腳本文件

#cd /kafkaoffsetmonitor/conf/

需要將zk更改成你自已集羣的IP 地址

#./kafkaoffsetmonitor.sh

#ss -tunl





發表評論
所有評論
還沒有人評論,想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.
相關文章