logstash信息采集

信息采集

采集之前完成elasticsearch的搭建。
elasticsearch的搭建

server1：

rpm -ivh logstash-2.3.3-1.noarch.rpm
cd /opt/logstash/bin/
[root@server1 bin]# /opt/logstash/bin/logstash -e 'input { stdin {} } output { stdout {} }'  #输出终端
[root@server1 bin]# /opt/logstash/bin/logstash -e 'input { stdin {} } output { stdout { codec => rubydebug } }' # 更改输出格式
[root@server1 bin]# /opt/logstash/bin/logstash -e 'input { stdin {} } output { elasticsearch { hosts => ["172.25.11.1"] index => "logstash-%{+YYYY.MM.dd}" }  stdout { codec => rubydebug } }'  # 输出到es

编写文件，采集信息（文件的名字是*.conf）

vim /etc/lagstash/es.conf
input {
        stdin {}
}
output {
        elasticsearch {
                hosts => ["172.25.11.1"]
                index => "logstash-%{+YYYY.MM.dd}"
        }
        stdout {
                codec => rubydebug
        }
        file {
                path => "/tmp/demo.file"
                codec => line { format => "custom format: %{message}" }
        }
}

/opt/logstash/bin/logstash -f /etc/logstash/es.conf

终端输出：

elasticsearch输出：


文件输出：

将系统日志文件输出到elasticsearch中

vim /etc/lagstash/message.conf
input {
        file {
                path => "/var/log/messages"
                start_position => beginning
        }
}
output {
        elasticsearch {
                hosts => ["172.25.11.1"]
                index => "message-%{+YYYY.MM.dd}"
        }
}

/opt/logstash/bin/logstash -f /etc/logstash/es.conf

[root@server1 ~]# cat .sincedb_452905a167cf4509fd08acb964fdb20c  # elasticsearch 的读取记录，记录文件读取的内容地址
1044500 0 64768 32556

rsyslog 远程控制

server1:
vim /etc/logstash/message.conf
input {
        syslog {
                port => 514
        }
}
output {
#       elasticsearch {
#               hosts => ["172.25.11.1"]
#               index => "message-%{+YYYY.MM.dd}"
#       }
        stdout {
                codec => rubydebug
        }
}

server1上没有514端口

/opt/logstash/bin/logstash -f /etc/logstash/es.conf
netstat -antulp |grep :514

server2:
vim /etc/rsyslog.conf
*.*     @@172.25.11.1:514
/etc/init.d/rsyslog restart

多行日志采集

input {
        file {
                path => "/var/log/elasticsearch/my-es.log"
                start_position => "beginning"
        }
}

filter {
        multiline {
#               type => "type"
                pattern => "^\["
                negate => true
                what => "previous"
        }
}

output {
        elasticsearch {
                hosts => ["172.25.11.1"]
                index => "message-%{+YYYY.MM.dd}"
        }
        stdout {
                codec => rubydebug
        }
}

多行成为一行输出

apache信息采集

vim /etc/logstash/conf.d/test.conf
input {
        file {
                path => ["/var/log/httpd/access_log","/var/log/httpd/error_log"]
                start_position => "beginning"
        }
}

filter {
        grok {
                match => { "message" => "%{COMBINEDAPACHELOG}" }
        }
}
output {
        elasticsearch {
                hosts => ["172.25.11.1"]
                index => "apache-%{+YYYY.MM.dd}"
        }
        stdout {
                codec => rubydebug
        }
}