ELSA(全稱：Enterprise Log Search and Archive)是一款基於syslog-ng（新一代日誌收集器，但目前多數Linux發現版都不帶此工具）、MySQL的開源級企業日誌歸檔查詢工具，由於它和Sphinx的完美搭配，支持全文索引可以像搜索Web一樣輕鬆地搜索上億個日誌中的任意字符串（前提是你的服務器配置足夠高）。單節點ELSA日誌採集系統的工作原理圖如下所示：
上面這張架構圖可以看出ELSA從架構上分爲三層：
日誌接收器，由syslog-ng完成負責接收來自本地、網絡以及導入的日誌文件
日誌存儲索引，存儲由MySQL數據庫完成，索引由sphinx完成。
Web前端 。
ELSA利用syslog-ng的pattern-db解析器進行有效的日誌規範化，並利用Sphinx全文索引進行日誌搜索。系統內部API將查詢結果彙總後，發送給客戶端，整個系統是異步執行，可以跑多個查詢。接收器syslog-ng在接收日誌時並沒有進行歸一化處理（類比OSSIM-Agent插件），所以對日誌的正則表達式計算量不大，可以在syslog-ng中保持高效的日誌接收率，系統大部分有Perl腳本組成，MySQL每秒可插入100K行數據。Sphinx在索引中爲新插入的行建立索引，每個2小時會重新建立一次永久索引。整個系統最大效率發揮時每秒鐘可以處理100K條日誌。
如果你具備ELK實戰經驗的話，可以把ELSA理解爲簡版的ELK系統，結構簡單，速度快。安裝（感興趣的朋友可以在基於Debian（包括Ubuntu）的OS上測試，在ELSA Google Code主頁上獲取安裝tar包）比較簡單就不介紹了，下面直接切入正題。
1.採集Windows服務器日誌
我們可以採用Eventlog-to-Syslog工具將Windows平臺的日誌發送到ELSA服務器
方法：
將evtsys.exe和evtsys.dll複製到系統目錄下輸入下面命令
evtsys.exe -i -h ELSA服務器的IP
志將使用syslog協議發送到您的ELSA服務器，在該服務器中，日誌將被解析爲“ WINDOWS”類
2.採集Linux系統及相關服務的日誌
Linux/Unix系統都有rsyslog 或 Syslogd進程，在其配置文件中加入下面的配置即可
. @ELSA服務器IP
3.配置文件
ELSA的主要配置文件是/etc/elsa_node.conf
{
本地數據庫連接信息
"database" : {
"db": "syslog",
"data_db": "syslog_data",
"dsn" : "dbi:mysql:database=syslog",
"username" : "elsa",
"password" : "biglog"
},// 系統協調鎖的目錄
"lockfile_dir": "/opt/elsa/node/tmp/locks",
"num_indexes": 200,//如果要歸檔日誌，請保留此項
"archive": {
Uncomment to establish a retention period in days for archive logs
#"days": 90,
"percentage": 33,
"table_size": 10000000
},
//日誌大小限制+索引大小。設置爲磁盤總空間的95-90%。
"log_size_limit" : 8000000000,
"sphinx" : {
"indexer": "/usr/bin/indexer",
"allowed_temp_percent" : 40,
"allowed_mem_percent": 25
"host" : "127.0.0.1",
"port" : 9312,
"mysql_port" : 9306,
"config_file" : "/etc/sphinxsearch/sphinx.conf",
"index_path" : "/nsm/elsa/data/sphinx",
"index_interval" : 60,
"perm_index_size" : 10000000,
# Where the optional stopwords file is
"stopwords": {
"file": "/etc/sphinxsearch/sphinx_stopwords.txt",
"top_n": 0,
"interval": 0,
"whitelist": []
},
"pid_file": "/var/run/sphinxsearch/searchd.pid"
},
"logdir" : "/nsm/elsa/data/elsa/log",
"mysql_dir": "/nsm/elsa/data/elsa/mysql",
"num_log_readers" : 1,
#調試跟蹤級別
"debug_level" : "TRACE",
"buffer_dir" : "/nsm/elsa/data/elsa/tmp/buffers/",
"log_parse_errors": 1,
"stats" : {
"retention_days": 365
},
"min_expected_hosts": 2}
ELSA的Web配置文件 /etc/elsa_web.conf
{
#定義API密鑰
"apikeys": {
"elsa": "b7292980d34c99e2581d36681831667b"
},
"version": {
"Author": "mcholste",
"Date": "2014-07-17 15:12:58 -0700 (Thu, 17 Jul 2014)",
"Rev": "1205",
"Sphinx": "Sphinx 2.1.9"
},
"peers": {
"127.0.0.1": {
"url": "http://127.0.0.1:3154/",
"username": "elsa",
"apikey": "b7292980d34c99e2581d36681831667b"
}
},
"admin_email_address": "root@localhost",
"connectors": {
},
"dashboards": {
},
"datasources": {
},
"transforms": {
"whois": {
"known_subnets": {
"10.0.0.0": {
"end": "10.255.255.255",
"org": "MyOrg"
},
"192.168.0.0": {
"end": "192.168.255.255",
"org": "MyOrg"
},
"172.16.0.0": {
"end": "172.31.255.255",
"org": "MyOrg"
}
},
"known_orgs": {
"MyOrg": {
"name": "MyOrg",
"org": "MyOrg",
"descr": "MyOrg",
"cc": "US",
"country": "United States",
"city": "Anytown",
"state": "Somestate"
}
}
},
"parse": {
"tld": [
{
"field": "domain",
"pattern": "\.([a-zA-Z]+)$",
"extractions": [
"tld"
]
},
{
"field": "site",
"pattern": "\.([a-zA-Z]+)$",
"extractions": [
"tld"
]
},
{
"field": "uri",
"pattern": "\.([a-zA-Z]+)(:|/|$)",
"extractions": [
"tld"
]
}
],
"url": [
{
"field": "uri",
"pattern": "(?:(?<proto>[a-zA-Z]+)://)?(?:(?<username>[^/]+):(?<password>[^/]+)@)?(?<domain>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|[^/]+\.(?<tld>[a-zA-Z]+))(?::(?<port>\d+))?(?<resource>/[^?])?(?:\?(?<query_string>.))?$",
"extractions": [
"proto",
"username",
"password",
"domain",
"tld",
"port",
"resource",
"querystring"
]
}
],
"mimetype": [
{
"field": "msg",
"pattern": "[\"'\(\[\s\|;:](?<mime>(?<type>application|audio|chemical|image|message|model|multipart|text|video)/(?<subtype>[\w-
]+))[\"'\)\]\s\|;:]",
"extractions": [
"mime",
"type",
"subtype"
]
}
]
}
},
"plugins": {
"SNORT": "Info::Snort",
"WINDOWS": "Info::Windows",
"URL": "Info::Url",
"BRO_NOTICE": "Info::Bro"
},
"info": {
"snort": {
"url_templates": [
"http://doc.emergingthreats.net/bin/view/Main/%d"
]
},
"url": {
"url_templates": [
"http://whois.domaintools.com/%s"
]
},
"windows": {
"url_templates": [
"http://www.ultimatewindowssecurity.com/securitylog/encyclopedia/event.aspx?eventid=%d"
]
}
},
"max_concurrent_archive_queries": 4,
"schedule_interval": 60,
"node_info_cache_timeout": 60,
"email": {
"display_address": "[email protected]",
"base_url": "http://elsa/",
"subject": "ELSA Alert"
},
"link_key": "secret",
"yui": {
"local": "inc"
},
"data_db": {
"db": "syslog",
"username": "elsa",
"password": "biglog"
},
"meta_db": {
"dsn": "dbi:mysql:database=elsa_web",
"username": "elsa",
"password": "biglog"
},
"auth": {
"method": "security_onion"
},
"admin_groups": [
"system",
"admin"
],
"auth_db": {
"dsn": "dbi:mysql:database=securityonion_db",
"username": "root",
"password": "",
"auth_statement": "SELECT PASSWORD(password) FROM user_info WHERE username=?",
"email_statement": "SELECT email FROM user_info WHERE username=?"
},
"peer_id_multiplier": 1000000000000,
"query_timeout": 55,
"pcap_url": "/capme",
"logdir": "/nsm/elsa/data/elsa/log",
"buffer_dir": "/nsm/elsa/data/elsa/tmp/buffers",
"debug_level": "TRACE",
"default_start_time_offset": 2,
"livetail": {
"poll_interval": 5,
"time_limit": 3600
}
}
4.典型應用場景（截圖）
着重對ELSA軟件的幾個重點功能進行展示。
1.連接數 Top N
2.動態儀表盤展示
動態展示單位時間內處理日誌的數量、查詢量、採集主機的地址以及日誌類型等參數。
3.查詢日誌詳細信息
我們在Field Summary（字段摘要）中發現這些日誌有15個字段（主機IP、進程名稱、源地址、源端口、目的地址、目的端口、協議類型、輸入字節數量、服務類型、持續時間、輸出字節、輸入數據包數量、輸出數據包數量、國家代碼等），每個字段後面是出現的次數，各個字段之間通過“|”符號分割。
4.查詢ossec日誌信息
5.偵測到針對MySQL 3306端口掃描報警日誌信息
6.端口掃描報警日誌信息
7.Ping報警日誌信息
有關日誌分析的相關話題大家可以閱讀暢銷書《Unix/Linux網絡日誌分析與流量監控》。