注：本系統使用的是Logstash+elasticsearch+elastic+nignx 進行日誌分析、展示
3 Kibana與elasticsearch的啓動... 6
1環境版本：
操作系統：CentOS 7.2.1511
內核：Linux Logs3.10.0-123.9.3.el7.x86_64
JDK: 1.8.0_74
logstash-2.2.2
下載地址(github)：https://github.com/elastic/logstash/tree/2.2
功能：對輸入日誌進行收集、分析，並將其存儲供以後使用（如，搜索）。
elasticsearch-2.2.0
功能：對logstash分析結果的輸入提供進行自定義搜索
下載地址(github)：https://github.com/elastic/elasticsearch/tree/2.2
kibana-4.4.1
功能：連接elasticsearch-2.2.0,提供web界面
下載地址(github)：https://github.com/elastic/kibana/tree/4.4
nginx: 1.9.12
將 kibana 的端口轉發到 80，並定義好訪問用的域名。
1.1主機：
web1: 10.46.90.80(內網)，xx.xx.xx.xx(外網)
logs: 10.46.90.147(內網)，xx.xx.xx.xx(外網)
1.2前提：
Nfs
在logs搭建好nfs，共享 /opt/logs，掛載到 web1 的 /home/wwwlogs，web1 的 php 日誌直接輸出到 /home/wwwlogs/*/
logstash 、kibana 、elasticsearch都下載到 /opt/
JDK已經安裝
安裝好nginx
2 Logstash配置
Logstash 可以git下載到本地直接使用，其配置是最主要的，它會對日誌進行收集、分析，並將其存儲供以後使用（如，搜索）。
logstash 的 shipper.conf 配置文件 grok 篩選都使用 ruby 正則表達式，在此推薦一個guby 正則表達式模擬器：http://www.rubular.com/
新建配置文件並配置：
[root@Logs ~]# mkdir /opt/logstash/conf.d
[root@Logs ~]# vi /opt/logstash/conf.d/shipper.conf
input {
#stdin {
#}
#file {
#path =>"/opt/logs/*/*_nginx.log"
#type => "access"
#codec => json
#}
file {
path => "/opt/logs/php/admin.etcchebao.com/*.log"
#path =>"/opt/logs/php/admin.etcchebao.com/admin.log"
type => "admin"
codec => multiline {
# Grok pattern names are valid!:)
pattern => "^\[\d{4}" #開頭匹配[+4個年份字符
#pattern =>"^%{TIMESTAMP_ISO8601} "
negate => true
what => previous
}
}
file {
path => "/opt/logs/php/passport.etcchebao.com/*.log"
#path =>"/opt/logs/php/passport.etcchebao.com/passport.log"
type => "passport"
codec => multiline {
# Grok pattern names are valid!:)
pattern => "^\[\d{4}" #開頭匹配[+4個年份字符
#pattern =>"^%{TIMESTAMP_ISO8601} "
negate => true
what => previous
}
}
file {
path => "/opt/logs/php/push.etcchebao.com/*.log"
#path =>"/opt/logs/php/push.etcchebao.com/push.log"
type => "push"
codec => multiline {
# Grok pattern names are valid!:)
pattern =>"^\[\d{4}" #開頭匹配[+4個年份字符
#pattern =>"^%{TIMESTAMP_ISO8601} "
negate => true
what => previous
}
}
file {
path => "/opt/logs/php/seller.etcchebao.com/*.log"
#path =>"/opt/logs/php/seller.etcchebao.com/seller.log"
type => "seller"
codec => multiline {
# Grok pattern names are valid!:)
pattern =>"^\[\d{4}" #開頭匹配[+4個年份字符
#pattern =>"^%{TIMESTAMP_ISO8601} "
negate => true
what => previous
}
}
file {
path => "/opt/logs/php/m.etcchebao.com/*.log"
#path =>"/opt/logs/php/m.etcchebao.com/m.log"
type => "m"
codec => multiline {
# Grok pattern names are valid!:)
pattern =>"^\[\d{4}" #開頭匹配[+4個年份字符
#pattern =>"^%{TIMESTAMP_ISO8601} "
negate => true
what => previous
}
}
file {
path => "/opt/logs/php/pay.etcchebao.com/*.log"
#path =>"/opt/logs/php/pay.etcchebao.com/pay.log"
type => "pay"
codec => multiline {
# Grok pattern names are valid!:)
pattern =>"^\[\d{4}" #開頭匹配[+4個年份字符
#pattern =>"^%{TIMESTAMP_ISO8601} "
negate => true
what => previous
}
}
}
filter {
# if [type] == "access" {
# grok {
# match => {"message" => "%{COMBINEDAPACHELOG}" }
# }
# date {
# match => ["timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
# }
# }
grok {
match => [
#404錯誤
"message","\:(?<Error_class>\d{3}?)\]",
#Error錯誤
"message","\[(?<Error_class>\Error?)\]",
#500錯誤
"message","系統(?<Error_class>\d{3}?)錯誤\.*ERROR_NO:(?<err_no>[0-9]*$?).*ERROR_STR:(?<err_str>.*$?)\\.*ERROR_LINE:(?<err_line>[0-9]*$?).*ERROR_FILE:(?<err_file>\\.*$?)\\n"
]
}
}
#輸出到redis
#output {
# redis {
# host => "127.0.0.1"
# port => "6379"
# type => "nginx-log"
# data_type => "list"
# key => "logstash"
# }
#}
#輸出到elasticsearch
output {
elasticsearch {
#hosts => ["127.0.0.1:9300"]
hosts => "127.0.0.1"
index => "logstash-%{type}-%{+YYYY.MM.dd}"
document_type => "%{type}"
#workers => 1
#flush_size => 20000
#idle_flush_time => 10
#template_overwrite => true
}
# if [Error_class] != "404" {
# exec {
# #command => "echo'%{timestamp}:%{message}' | mail -s 'Log_error: HttpException error'[email protected]"
# command => "echo'%{timestamp}:%{message}' | mail -s 'Log_error: HttpException [HttpException]'[email protected]"
# }
# }
}
output{
if[Error_class] != "404" {
exec {
#command =>"echo '%{timestamp}:%{message}' | mail -s 'Log_error: HttpException error'[email protected]"
command =>"echo '%{timestamp}:%{message}' | mail -s 'Log_error: HttpException[HttpException]' [email protected]"
}
}
}
#屏幕輸出-test
output {
stdout {
codec => rubydebug
}
}
Logstash的啓動：
[root@Logs ~]# nohup/opt/logstash/bin/logstash -f /opt/logstash/conf.d/shipper.conf > /dev/null2>&1 &
檢查啓動情況：
3 Kibana與elasticsearch 的啓動
kibana 與 elasticsearch 都無需安裝，只要下載到本地即可直接使用，最好先啓動logstash。要注意的是，但默認不允許使用 root帳號啓動，所以使用專門運行nginx的www用戶啓動。
3.1 elasticsearch
[www@Logs ~]$ nohup/opt/elasticsearch-2.2.0/bin/elasticsearch > /dev/null 2>&1 &
[www@Logs ~]$ ps -elf|grep elasticsearch
檢查進程：
檢查端口：
3.2 kibana
[www@Logs ~]$ nohup /opt/kibana/bin/kibana> /dev/null 2>&1 &
[www@Logs ~]$ ps -elf|grep kibana
檢查進程：
檢查端口：
4 Nginx的配置：
[www@Logs ~]$ vi/usr/local/nginx/conf/vhost/logs.etcchebao.cn.conf
server {
listen 80;
server_name logs.etcchebao.cn;
location/ {
auth_basic "secret";
auth_basic_user_file /usr/local/nginx/logs_etcchebao.passwd;
proxy_pass http://127.0.0.1:5601;
}
}