Logstash7.3基礎配置

1.根據系統選擇自己需要的版本下載（https://www.elastic.co/cn/downloads/logstash）

2.配置文件（新建logstash-7.3.1/config/logstash-test.conf）

3.配置文件如下

input{

     kafka {
        bootstrap_servers => "127.0.0.1:9092"
        topics => "ip"
		client_id => "ip_b"
        group_id => "ip_a"
        auto_offset_reset => "earliest"
        consumer_threads => 2
        # 從kafka獲取json數據解析
        codec => "json"
      }

      kafka {
        bootstrap_servers => "127.0.0.1:9092"
        topics => "ips"
		client_id => "ips_b"
        group_id => "ips_a"
        auto_offset_reset => "earliest"
        consumer_threads => 2
        codec => "json"
      }

  }
  
filter{

  grok {
      # 移除不需要的字段
      remove_field => ["@timestamp","@version"]
  }

 mutate {
 	# 增加一個request_time_format用於標識時間
     add_field => {
        "request_time_format" => ""
     }
 }

 json {
   source => "message"
 }
# 根據標識字段判斷使用不同的過濾規則
 if[fields][log_source] == "ip" {
    date {
    # 這裏是格式化時間2019-09-26 19:17:56
      match =>  ["get_time" ,"yyyy-MM-dd HH:mm:ss"]
      target => "request_time_format"
      locale => "cn"
    }
    ruby {   
    	# 真實的時間是比中國時間慢八個小時，這裏把時間加上去
        code => "event.set('request_time_format', event.get('request_time_format').time.localtime + 8*60*60)"
    } 
  }
	 if[fields][log_source] == "ips" {
    date {
    # 格式化微秒時間UNIX_MS，具體請參看文檔
      match =>  [ "request_time" ,"yyyy-MM-dd HH:mm:ss,SSS" , "UNIX_MS" ]
     # 過濾的時間格式賦值到request_time_format上
      target => "request_time_format"
      locale => "cn"
    }
    ruby {   
        code => "event.set('request_time_format', event.get('request_time_format').time.localtime + 8*60*60)"
    }
  }
}

output{
    # elast參數配置
    # 輸出信息
     if [action_method] == "ip" {
          elasticsearch {
             hosts => ["127.0.0.1:9200"]
            # 索引的名稱
             index => "ip"
             codec => line { format => "%{message}"}
         }
      }
    # if[fields][log_source] == "ips" {
      elasticsearch {
    		hosts => ["127.0.0.1:9200"]
         # 索引的名稱
        index => "ips"
         codec => line { format => "%{message}"}
       }
     }
     # 打印輸出傳輸到ElasticSearch
    stdout{
        codec => rubydebug
    }

}

3.檢查logstash的配置文件是否配置正確（logstash根目錄下運行）

./bin/logstash -f ./config/logstash-test.conf --config.test_and_exit

4. 運行logstash根目錄下運行）

./bin/logstash -f config/logstash-test.conf

5.Logstash配置含義