ELK日誌分析系統

       注意事項:
   我的軟件包都是在ftp://192.168.10.250/pub/package/elk軟件包/ 這個服務器當中

實驗環境:
   兩臺內存爲4G 的Linux服務器,一臺http Apache服務器
   192.168.1.101 node1           + kibana
   192.168.1.102 node2             #elasticsearch的從服務器
   192.168.1.103 httpd-server  + logstash

#添加解析
vim /etc/hosts
192.168.1.101 node1
192.168.1.102 node2

yum -y install java 安裝java環境
首先獲取elasticsearch 的RPM包

rpm -ivh elasticsearch-5.5.0.rpm    #安裝elasticsearch 的RPM包
systemctl daemon-reload                    
systemctl enable elasticsearch.service

#備份配置文件
cp /etc/elasticsearch/elasticsearch.yml /etc/elasticsearch/elasticsearch.yml.bak

vim /etc/elasticsearch/elasticsearch.yml

17/ cluster.name: my-elk-cluster
23/ node.name: node1
33/ path.data: /data/elk_data
37/ path.logs: /var/log/elasticsearch/
43/ bootstrap.memory_lock: false
55/ network.host: 0.0.0.0
59/ http.port: 9200
68/ discovery.zen.ping.unicast.hosts: ["node1", "node2"]

grep -v "^#" /etc/elasticsearch/elasticsearch.yml

mkdir -p /data/elk_data #創建elk的data目錄
chown -R elasticsearch:elasticsearch /data/elk_data/   #更換data目錄的屬主跟屬組

systemctl start elasticsearch.service
netstat -ntap | grep 9200       #啓動並查看端口是否啓動成功

使用瀏覽器打開   http://192.168.1.101:9200  下面是節點信息

http://192.168.1.101:9200/_cluster/health?pretty    測試節點是否健康
http://192.168.1.102:9200/_cluster/health?pretty    測試節點是否健康

#獲取node的源碼包

tar zxvf node-v8.2.1.tar.gz     #解壓縮軟件包
./configure
make -j 2 && make install      #編譯加安裝

####安裝phantomjs###前端框架
獲取phantomjs的前段框架的軟件包

tar xjvf phantomjs-2.1.1-linux-x86_64.tar.bz2   #解壓該壓縮包
cp phantomjs-2.1.1-linux-x86_64/bin/phantomjs /usr/local/bin/   #複製框架到/usr/local/bin下面

###安裝elasticsearch-head###數據可視化工具
獲取軟件包
tar zxvf elasticsearch-head.tar.gz -C /usr/local/src/
cd /usr/local/src/elasticsearch-head/
npm install

###修改主配置文件###
vim /etc/elasticsearch/elasticsearch.yml
末行添加以下兩行
http.cors.enabled: true
http.cors.allow-origin: "*"

systemctl restart elasticsearch.service

###啓動elasticsearch-head 啓動服務器###
cd /usr/local/src/elasticsearch-head/
npm run start &




##########以上爲elasticsearch羣集的配置##########
##########從下面開始搭建logstash+httpd服務的#####
!!!!!創建索引


Apache服務器
yum -y install httpd   #安裝Apache
關閉防火牆 跟 安全功能

###獲取logstashRPM包然後安裝
yum -y instsall logstash-5.5.1.rpm
systemctl start logstash.service
systemctl enable logstash.service


ln -s /usr/share/logstash/bin/logstash /usr/local/bin/


#普通輸出
logstash -e 'input { stdin{} } output { stdout{} }'

logstash -e 'input { stdin{} } output { stdout { codec=>rubydebug } }'  #格式化輸出

logstash -e 'input { stdin{} } output { elasticsearch { hosts=>["192.168.1.101:9100"] } }'


vim /etc/logstash/conf.d/system.conf    #編寫系統日誌蒐集配置文件
input {
       file{
       path => "/var/log/messages"
       type => "system"
       start_position => "beginning"
       }
}
output {
       elasticsearch {
       hosts => ["192.168.1.101:9200"]
       index => "system-%{+YYYY.MM.dd}"
       }
}

systemctl restart logstash.service          #啓動logstash服務

rpm -ivh kibana-5.5.1-x86_64.rpm                               #安裝kibana軟件    
cp /etc/kibana/kibana.yml /etc/kibana/kibana.yml.bak

vim /etc/kibana/kibana.yml
 2 server.port: 5601
 7 server.host: "0.0.0.0"
 21 elasticsearch.url: "<a href="http://192.168.1.101:9200" "="">http://192.168.1.101:9200"
 30 kibana.index: ".kibana"
 
systemctl start kibana.service
systemctl enable kibana.service      #開啓kibana服務

http://192.168.1.101:5601/


chmod o+r /var/log/messages #添加可讀權限

###對接Apache主機的Apache日誌文件(訪問的錯誤的)
vim /etc/logstash/conf.d/apache_log.conf

input {
       file{
       path => "/etc/httpd/logs/access_log"
       type => "access"
       start_position => "beginning"
       }
       file{
       path => "/etc/httpd/logs/error_log"
       type => "error"
       start_position => "beginning"
       }
}
output {
       if [type] == "access" {
       elasticsearch {
       hosts => ["192.168.1.101:9200"]
       index => "apache_access-%{+YYYY.MM.dd}"
               }
       }
       if [type] == "error" {
       elasticsearch {
       hosts => ["192.168.1.101:9200"]
       index => "apache_error-%{+YYYY.MM.dd}"
           }
       }
}

chmod -R o+r /etc/httpd/logs/       #給log可讀的權限
logstash -f /etc/logstash/conf.d/apache_log.conf        #讀取Apache_log