ELK日誌平臺----解耦配置文件

本文記錄了三個配置文件：

第一個：all.conf 通過一個配置文件，配置輸入輸出，實例；

第二個：shipper.conf配置logstash收集日誌內容到redis裏；

第三個：indexer.conf配置logstash從redis裏讀取日誌內容輸出到Elasticsearch裏。

第二個跟第三個配置解耦收集日誌

ELK 解耦

logstash ---------->redis ---------->logstash -------->elasticsearch----------->kibana

收集                     消息隊列                 分析                       存儲                                 顯示

192.168.1.87       192.168.1.87         192.168.1.88        192.168.1.87                192.168.1.87

all.conf 通過一個配置文件，配置輸入輸出，實例；
[root@y0 ~]# cat all.conf
input {
    file {
       	path => "/var/log/messages"
    	type => "system"	
    	start_position => "beginning"
    	}
    file {
        path => "/var/log/nginx/access_json.log"
        codec => "json"
        type => "nginx_log"
    	start_position => "beginning"
    }
    file {
        path => "/var/log/elasticsearch/Mint.log"
        type => "es-error"
        start_position => "beginning"
        codec => multiline {
            pattern => "^\["
            negate => true
            what => "previous"
        }
    }
    syslog {
        type => "system-syslog"
        host => "192.168.1.87"
        port => "514"
    }
}
output {
    if [type] == "system" {
        elasticsearch {
            hosts => ["192.168.1.87:9200"]
            index => "system-%{+YYYY.MM.dd}"
        }
    }
    if [type] == "es-error" {
        elasticsearch {
            hosts => ["192.168.1.87:9200"]
            index => "es-error-%{+YYYY.MM.dd}"
        }
    }
    if [type] == "nginx_log" {
        elasticsearch {
            hosts => ["192.168.1.87:9200"]
            index => "nginx_log-%{+YYYY.MM.dd}"
        }
    }
    if [type] == "system-syslog" {
        elasticsearch {
            hosts => ["192.168.1.87:9200"]
            index => "system-syslog-%{+YYYY.MM.dd}"
        }
    }
}

shipper.conf配置logstash收集日誌內容到redis裏；
[root@y0 ~]# cat shipper.conf
input {
    file {
   	path => "/var/log/messages"
	type => "system"	
	start_position => "beginning"
	}
    file {
        path => "/var/log/nginx/access_json.log"
        codec => "json"
        type => "nginx_log"
	start_position => "beginning"
}
    file {
        path => "/var/log/elasticsearch/Mint.log"
        type => "es-error"
	start_position => "beginning"
        codec => multiline {
            pattern => "^\["
            negate => true
            what => "previous"
        }
    }
    
    syslog {
        type => "system-syslog"
        host => "192.168.1.87"
        port => "514"
        }
}
output {
    if [type] == "system" {
        redis {
        host => "192.168.1.87 "
        port => "6379"
        db => "6"
        data_type => "list"
        key => "system"
        }
    }
    if [type] == "es-error" {
        redis {
        host => "192.168.1.87 "
        port => "6379"
        db => "6"
        data_type => "list"
        key => "es-error"
        }
    }
    if [type] == "nginx_log" {
    redis {
        host => "192.168.1.87 "
        port => "6379"
        db => "6"
        data_type => "list"
        key => "nginx_log"
       }
    }
    if [type] == "system-syslog" {
        redis {
            host => "192.168.1.87 "
            port => "6379"
            db => "6"
            data_type => "list"
            key => "system-syslog"
        }
    }
}

indexer.conf 配置logstash從redis裏讀取日誌內容輸出到Elasticsearch裏。
[root@test01 ~]# cat indexer.conf
input {
    redis {
        type => "system"
        host => "192.168.1.87 "
        port => "6379"
        db => "6"
        data_type => "list"
        key => "system"
    }
    redis {
        type => "es-error"
        host => "192.168.1.87 "
        port => "6379"
        db => "6"
        data_type => "list"
        key => "es-error"
    }
    redis {
        type => "nginx_log"
        host => "192.168.1.87 "
        port => "6379"
        db => "6"
        data_type => "list"
        key => "nginx_log"
    }
    redis {
        type => "system-syslog"
        host => "192.168.1.87 "
        port => "6379"
        db => "6"
        data_type => "list"
        key => "system-syslog"
    }
}
output {
    if [type] == "system" {
        elasticsearch {
            hosts => ["192.168.1.87:9200"]
            index => "system-%{+YYYY.MM.dd}"
        }
    }
    if [type] == "es-error" {
        elasticsearch {
            hosts => ["192.168.1.87:9200"]
            index => "es-error-%{+YYYY.MM.dd}"
        }
    }
    if [type] == "nginx_log" {
        elasticsearch {
            hosts => ["192.168.1.87:9200"]
            index => "nginx_log-%{+YYYY.MM.dd}"
        }
    }
    if [type] == "system-syslog" {
        elasticsearch {
            hosts => ["192.168.1.87:9200"]
            index => "system-syslog-%{+YYYY.MM.dd}"
        }
    }
}

歡迎打算使用阿里雲服務器的小夥伴，加我私聊！！QQ:3533470970  或直接領取阿里雲幸運券：https://promotion.aliyun.com/ntms/act/ambassador/sharetouser.html?userCode=b6h928qb&utm_source=b6h928qb