Logstatsh數據同步-Mysql同步ES

1.獲取Logstash模板

GET    http://10.0.0.219:9200/_template/logstash

2.在sync目錄下創建logstash-ik.json，將logstash模板修改後粘貼其中。
修改前：

{
    "logstash": {
        "order": 0,
        "version": 60001,
        "index_patterns": [
            "logstash-*"
        ],
        "settings": {
            "index": {
                "refresh_interval": "5s"
            }
        },
        "mappings": {
            "_default_": {
                "dynamic_templates": [
                    {
                        "message_field": {
                            "path_match": "message",
                            "match_mapping_type": "string",
                            "mapping": {
                                "type": "text",
                                "norms": false
                            }
                        }
                    },
                    {
                        "string_fields": {
                            "match": "*",
                            "match_mapping_type": "string",
                            "mapping": {
                                "type": "text",
                                "norms": false,
                                "fields": {
                                    "keyword": {
                                        "type": "keyword",
                                        "ignore_above": 256
                                    }
                                }
                            }
                        }
                    }
                ],
                "properties": {
                    "@timestamp": {
                        "type": "date"
                    },
                    "@version": {
                        "type": "keyword"
                    },
                    "geoip": {
                        "dynamic": true,
                        "properties": {
                            "ip": {
                                "type": "ip"
                            },
                            "location": {
                                "type": "geo_point"
                            },
                            "latitude": {
                                "type": "half_float"
                            },
                            "longitude": {
                                "type": "half_float"
                            }
                        }
                    }
                }
            }
        },
        "aliases": {}
    }
}

修改後：

{
  "order": 1,
  "version": 1,
  "index_patterns": [
    "*"
  ],
  "settings": {
    "index": {
      "refresh_interval": "60s",
      "number_of_shards": 3,
      "number_of_replicas": 0
    }
  },
  "mappings": {
    "_default_": {
      "dynamic_templates": [
        {
          "message_field": {
            "path_match": "message",
            "match_mapping_type": "string",
            "mapping": {
              "type": "text",
              "norms": false
            }
          }
        },
        {
          "string_fields": {
            "match": "*",
            "match_mapping_type": "string",
            "mapping": {
              "type": "text",
              "norms": false,
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            }
          }
        }
      ],
      "properties": {
        "@timestamp": {
          "type": "date"
        },
        "@version": {
          "type": "keyword"
        },
        "compname": {
          "type": "text",
          "analyzer": "ik_smart",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "compnewname": {
          "type": "text",
          "analyzer": "ik_smart",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "id": {
          "type": "long"
        },
        "legalpersonname": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "regcapital": {
          "type": "long"
        }
      }
    }
  },
  "aliases": {}
}

 

3.修改配置文件logstash-db-sync.conf 

input {
    stdin{
    }
    jdbc {
      # 數據庫
      jdbc_connection_string => "jdbc:mysql://XXXX:XXXX/XXXX"
      jdbc_user => "XXXXX"
      jdbc_password => "XXXXXX"
      # mysql驅動解壓的位置
      jdbc_driver_library => "/database/chentong/opt/logstash-6.5.3/config/mysql-connector-java-8.0.12.jar"
       # mysql驅動類
      jdbc_driver_class => "com.mysql.jdbc.Driver"
      #分頁查詢
	  jdbc_paging_enabled => true
      jdbc_page_size => 50000
      #可以使用sql文件的方式
      #statement_filepath => ""
      #要同步的表
      statement => "SQL語句"
      schedule => "* * * * *"
      #索引設置時區
      jdbc_default_timezone => "Asia/Shanghai"
    }
}

filter {
    ruby {
        code => "
            event.set('regcapital', event.get('regcapital').to_f);
        "
    }
}


output {
    elasticsearch {
        hosts => ["es節點"]
        index => "索引名稱"
        template_name => "sy_comp_basic"
        template_overwrite => true
        manage_template => true
        template => "/database/chentong/opt/logstash-6.5.3/templates/sy_comp_basic_info.json"
		document_id => "%{id}"
    }
    stdout { codec => rubydebug }
}