Docker搭建ELK,並將MySql數據同步到ES中

創建Docker自定義網絡

# 查看一下 docker network 用法
[root@daigd dgd]# docker network --help
Usage:  docker network COMMAND
Manage networks
Commands:
  connect     Connect a container to a network
  create      Create a network
  disconnect  Disconnect a container from a network
  inspect     Display detailed information on one or more networks
  ls          List networks
  prune       Remove all unused networks
  rm          Remove one or more networks
# 查看一下 docker network create 用法
[root@daigd dgd]# docker network create --help
Usage:  docker network create [OPTIONS] NETWORK
Create a network
Options:
      --attachable           Enable manual container attachment
      --aux-address map      Auxiliary IPv4 or IPv6 addresses used by Network driver (default map[])
      --config-from string   The network from which copying the configuration
      --config-only          Create a configuration only network
  -d, --driver string        Driver to manage the Network (default "bridge")
      --gateway strings      IPv4 or IPv6 Gateway for the master subnet
      --ingress              Create swarm routing-mesh network
      --internal             Restrict external access to the network
      --ip-range strings     Allocate container ip from a sub-range
      --ipam-driver string   IP Address Management Driver (default "default")
      --ipam-opt map         Set IPAM driver specific options (default map[])
      --ipv6                 Enable IPv6 networking
      --label list           Set metadata on a network
  -o, --opt map              Set driver specific options (default map[])
      --scope string         Control the network's scope
      --subnet strings       Subnet in CIDR format that represents a network segment

結合幫助手冊，創建一個自定義網絡：

# -d bridge 指定網絡驅動方式爲橋接模式（默認也是這個）
# --subnet 指定子網掩碼
# --gateway 指定網關
[root@daigd dgd]# docker network create -d bridge --subnet 192.162.0.0/16 --gateway 192.162.0.1 mynet

用Docker搭建ElasticSearch

# --name 指定容器名字
# --net 指定網絡
# -e "discovery.type=single-node" 使用開發模式，即只有一個節點
# -e ES_JAVA_OPTS="-Xms128m -Xmx512m" 限制JVM內存最多使用512M，因爲ES非常佔內存，默認啓動會佔去1G多
docker run -d --name es --net mynet -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e ES_JAVA_OPTS="-Xms128m -Xmx512m" elasticsearch:7.7.1

# 測試是否啓動成功
# 有如下信息出來表示ES已啓動成功
[root@daigd dgd]# curl localhost:9200
{
  "name" : "fffa6a086b64",
  "cluster_name" : "docker-cluster",
  "cluster_uuid" : "ypkwhS_TSQm7IG1LqHNWTA",
  "version" : {
    "number" : "7.7.1",
    "build_flavor" : "default",
    "build_type" : "docker",
    "build_hash" : "ad56dce891c901a492bb1ee393f12dfff473a423",
    "build_date" : "2020-05-28T16:30:01.040088Z",
    "build_snapshot" : false,
    "lucene_version" : "8.5.1",
    "minimum_wire_compatibility_version" : "6.8.0",
    "minimum_index_compatibility_version" : "6.0.0-beta1"
  },
  "tagline" : "You Know, for Search"
}

用Dockers搭建Kibana

docker run -d --name kibana --net mynet  -p 5601:5601 kibana:7.7.1
# 測試一下Kibana容器是否啓動成功
[root@daigd dgd]# curl localhost:5601
Kibana server is not ready yet

# 提示Kibana server is not ready yet，啓動失敗了，我們看下容器啓動日誌
# --tail 10 查看日誌的最後10條記錄

[root@daigd dgd]# docker logs --tail 10 kibana 
....
{"type":"log","@timestamp":"2020-06-06T10:40:11Z","tags":["warning","elasticsearch","admin"],"pid":6,"message":"Unable to revive connection: http://elasticsearch:9200/"}
{"type":"log","@timestamp":"2020-06-06T10:40:11Z","tags":["warning","elasticsearch","admin"],"pid":6,"message":"No living connections"}

通過察看Kibana容器的啓動日誌，我們看到一行關鍵日誌：message":"Unable to revive connection: http://elasticsearch:9200/。

通過http://elasticsearch:9200/無法訪問ES，進容器修改下Kibana配置。

# 以交互模式進入到kibana容器
[root@daigd dgd]# docker exec -it kibana /bin/bash
bash-4.2$ ls
LICENSE.txt  NOTICE.txt  README.txt  bin  built_assets  config  data  node  node_modules  optimize  package.json  plugins  src  webpackShims  x-pack
bash-4.2$ cd config/
bash-4.2$ ls
kibana.yml
# 修改配置文件
bash-4.2$ vi kibana.yml 
#
# ** THIS IS AN AUTO-GENERATED FILE **
#

# Default Kibana configuration for docker target
server.name: kibana
server.host: "0"
# 將http://elasticsearch:9200 修改成 http://es:9200 es是我們ES容器取的名字，因爲都指定了自定義網絡，故容器之間可通過容器名訪問
elasticsearch.hosts: [ "http://elasticsearch:9200" ]
monitoring.ui.container.elasticsearch.enabled: true

# 修改後保存退出,查看一下是否修改正確
bash-4.2$ cat kibana.yml 
#
# ** THIS IS AN AUTO-GENERATED FILE **
#

# Default Kibana configuration for docker target
server.name: kibana
server.host: "0"
elasticsearch.hosts: [ "http://es:9200" ]
monitoring.ui.container.elasticsearch.enabled: true

# 重啓kibana 容器
[root@daigd dgd]# docker restart kibana
kibana

# 再測試一下Kibana容器是否啓動成功，無任何內容表示，表示沒問題，這時我們再在瀏覽器上輸入ip(docker所在主機IP地址):5601，來訪問kibana，可以成功訪問了
[root@daigd dgd]# curl localhost:5601

用Docker搭建Mysql

# --name 指定容器名
# --p 暴露端口
# --net 指定自定義網絡
# -e MYSQL_ROOT_PASSWORD=123456 指定數據庫密碼
# -v "$PWD/mysql/data":/var/lib/mysql 將mysql的/var/lib/myql 目錄掛載到當前路徑下的/mysql/data目錄
# -v "$PWD/mysql/conf":/etc/mysql/conf.d 將mysql的配置掛載到當前路徑下的/mysql/conf目錄
docker run -d --name mysql8 -p 3306:3306 --net mynet \
 -e MYSQL_ROOT_PASSWORD=123456 \
 -v "$PWD/mysql/data":/var/lib/mysql \
 -v "$PWD/mysql/conf":/etc/mysql/conf.d \
 mysql:8

docker 啓動mysql後遠程無法訪問：

# 以交互模式進入容器
[root@daigd dgd]# docker exec -it mysql8 /bin/bash
# 連接數據庫
root@c038801dbb6c:/# cd /var/lib/mysql/
root@c038801dbb6c:/var/lib/mysql# mysql -uroot -p123456root@c038801dbb6c:/# cd /var/lib/mysql/

mysql> show databases;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| mysql              |
| performance_schema |
| sys                |
+--------------------+
4 rows in set (0.00 sec)

# 切換到mysql庫
mysql> use mysql;

mysql> select host,user from user where user='root';
+-----------+------+
| host      | user |
+-----------+------+
| %         | root |
| localhost | root |
+-----------+------+
2 rows in set (0.00 sec)

# 修改user表
mysql> ALTER USER 'root'@'%' IDENTIFIED WITH mysql_native_password BY '123456';
Query OK, 0 rows affected (0.00 sec)

創建數據庫及相關表

# 創建一個表
DROP TABLE IF EXISTS `article`;
CREATE TABLE `article`(
id BIGINT NOT NULL PRIMARY KEY auto_increment COMMENT '文章ID',
title VARCHAR(16) NOT NULL COMMENT '標題',
content text NOT NULL COMMENT '內容',
author VARCHAR(16) COMMENT '作者',
created_at datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '創建時間',
created_by VARCHAR(16) NOT NULL COMMENT '創建人',
updated_by VARCHAR(16) NOT NULL COMMENT '更新人',
updated_at datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新時間'
);

INSERT INTO `article`(`title`,`content`,`author`,`created_by`,`updated_by`) VALUES ('Go編程學習','第一章節:Go歷史；第二章節：Hello Go!','daigd','sys','sys');

用Docker搭建Logstash

# 使用具名掛載logstash的配置目錄和jar包目錄
docker run -d --name logstash --net mynet \
-v logstash-config:/usr/share/logstash/config \
-v logstash-lib:/usr/share/logstash/lib \
docker.elastic.co/logstash/logstash:7.7.1

# 容器成功啓動後，進入容器
docker exec -it logstash /bin/bash
# 查看當前目錄
bash-4.2$ pwd
/usr/share/logstash
bash-4.2$ 
# 進入config 目錄下，創建一個 jdbc.conf 文件
bash-4.2$ cd config/
bash-4.2$ pwd
/usr/share/logstash/config

jdbc.conf 文件內容：

mysql-connector-java-8.0.20.jar爲Myslq驅動包，在掛載目錄 logstash-lib上傳該文件，就會自動同步到容器內。

input {
  jdbc {
    jdbc_driver_library => "../lib/mysql-connector-java-8.0.20.jar"
    jdbc_driver_class => "Java::com.mysql.jdbc.Driver"
    jdbc_connection_string => "jdbc:mysql://mysql8:3306/forum"
    jdbc_user => "root"
    jdbc_password => "123456"
    statement => "select `id`,`title`,`content`,`author`,`created_at`,`created_by` FROM forum.article"
    jdbc_paging_enabled => "true"
    jdbc_page_size => "50000"
  }
}

filter {
}

output {
  stdout {
    codec => rubydebug
  }

  elasticsearch {
    hosts => ["es:9200"]
    index => "forum-mysql"
  }
}

# 執行腳本
../bin/logstash -f jdbc.conf 
# 發現如下輸出如下日誌
......
[2020-06-06T14:54:57,659][FATAL][logstash.runner          ] Logstash could not be started because there is already another instance using the configured data directory.  If you wish to run multiple instances, you must change the "path.data" setting.
[2020-06-06T14:54:57,661][ERROR][org.logstash.Logstash    ] java.lang.IllegalStateException: Logstash stopped processing because of an error: (SystemExit) exit

在網上找到該錯誤的解決方法：

If you want to run multiple logstash instances, you need to define the path.data either by command,make sure the directory is writable

bin/logstash -f <config_file.conf> --path.data PATH

創建一個目錄mydata,重新執行以下命令：

 ../bin/logstash -f jdbc.conf --path.data ../mydata/

稍等片刻之後，看到輸出以下日誌：

......
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/awesome_print-1.7.0/lib/awesome_print/formatters/base_formatter.rb:31: warning: constant ::Fixnum is deprecated
{
    "created_by" => "sys",
            "id" => 1,
       "content" => "第一章節:Go歷史；第二章節：Hello Go!",
         "title" => "Go編程學習",
    "@timestamp" => 2020-06-06T15:07:16.364Z,
        "author" => "daigd",
    "created_at" => 2020-06-06T09:46:50.000Z,
      "@version" => "1"
}
[2020-06-06T15:07:18,077][INFO ][logstash.javapipeline    ] Pipeline terminated {"pipeline.id"=>".monitoring-logstash"}
[2020-06-06T15:07:18,854][INFO ][logstash.runner          ] Logstash shut down.

程序正常退出，去Kibana上查看索引情況，在Dev Tools上執行以下命令：

GET forum-mysql/_search

有如下結果輸出，表明Logstash已正常將Mysql數據到ES中。

{
  "took" : 1,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 1,
      "relation" : "eq"
    },
    "max_score" : 1.0,
    "hits" : [
      {
        "_index" : "forum-mysql",
        "_type" : "_doc",
        "_id" : "mOorinIBhQ3tXszZ-CUZ",
        "_score" : 1.0,
        "_source" : {
          "created_by" : "sys",
          "id" : 1,
          "content" : "第一章節:Go歷史；第二章節：Hello Go!",
          "title" : "Go編程學習",
          "@timestamp" : "2020-06-06T15:07:16.364Z",
          "author" : "daigd",
          "created_at" : "2020-06-06T09:46:50.000Z",
          "@version" : "1"
        }
      }
    ]
  }
}
        "_source" : {
          "created_by" : "sys",
          "id" : 1,
          "content" : "第一章節:Go歷史；第二章節：Hello Go!",
          "title" : "Go編程學習",
          "@timestamp" : "2020-06-06T15:07:16.364Z",
          "author" : "daigd",
          "created_at" : "2020-06-06T09:46:50.000Z",
          "@version" : "1"
        }
      }
    ]
  }
}