graylog日誌分析系統上手教程

{"type":"doc","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"日誌分析系統可以實時收集、分析、監控日誌並報警，當然也可以非實時的分析日誌。splunk是功能強大且用起來最省心的，但是要收費，免費版有每天500M的限制，超過500M的日誌就沒法處理了。ELK系統是最常見的，缺點是配置麻煩一些，比較重量級。graylog是開源免費的，配置上要比ELK系統簡單。綜上，本文嘗試容器方式搭建一套graylog系統，不做實時收集日誌和報警的配置，只完成非實時被動接收網站日誌，分析日誌各項指標的功能。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"docker官方鏡像國內速度我覺得慢，改成國內鏡像。新建文件daemon.json如下"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"vi /etc/docker/daemon.json\n{\n\"registry-mirrors\": [\"https://registry.docker-cn.com\"]\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"也可以用網易鏡像http://hub-mirror.c.163.com"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"配置完重啓docker才能生效 "}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"#service docker restart"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"拉取如下三個鏡像"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"docker pull mongo:3\ndocker pull docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.10\ndocker pull graylog/graylog:3.3"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"不要急着按照網上的方法啓動鏡像，我開始docker啓動elasticsearch，雖然顯示啓動成功，但過半分鐘後偷偷退出，這導致graylog在瀏覽器打不開。最後通過查看容器啓動時的日誌，發現elasticsearch對於系統參數是有要求的，按如下修改。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在 /etc/sysctl.conf文件最後添加一行 "}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"vm.max_map_count=262144"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"vi /etc/security/limits.conf "}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"* - nofile 102400 "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"修改完成後重啓系統使變量生效。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"docker啓動elasticsearch時要加上參數"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"--ulimit nofile=65536:65536 --ulimit nproc=4096:4096"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"，確保容器內環境滿足要求，否則在docker pa -a命令下會看到exit(78)或exit(1)的容器異常退出錯誤。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"查看容器啓動報錯最準確的方法是“docker logs -f 容器ID”這個命令，我們不加--ulimit 參數試試"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"[root@bogon ~]# docker ps\nCONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES\n7e4a811093d9 docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.10 \"/usr/local/bin/dock 6 seconds ago Up 4 seconds 0.0.0.0:9200->9200/tcp, 0.0.0.0:9300->9300/tcp elasticsearch"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"用上面的CONTAINER ID產看啓動時的日誌"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"[root@bogon ~]# docker logs -f 7e4a811093d9\n最後會打印出\n[1]: max file descriptors [4096] for elasticsearch process is too low, increase to at least [65535]\n[2]: max number of threads [3869] for user [elasticsearch] is too low, increase to at least [4096]\n[2020-08-27T06:10:25,888][INFO ][o.e.n.Node ] [WG6mVz4] stopping ...\n[2020-08-27T06:10:25,903][INFO ][o.e.n.Node ] [WG6mVz4] stopped\n[2020-08-27T06:10:25,903][INFO ][o.e.n.Node ] [WG6mVz4] closing ...\n[2020-08-27T06:10:25,928][INFO ][o.e.n.Node ] [WG6mVz4] closed"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"兩行too low的提示就是容器退出的原因。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"三個容器正確的啓動命令如下"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"docker run --name mongo -d mongo:3\n\ndocker run --name elasticsearch \\\n -e \"http.host=0.0.0.0\" \\\n -e \"ES_JAVA_OPTS=-Xms512m -Xmx512m\" \\\n --ulimit nofile=65536:65536 --ulimit nproc=4096:4096 \\\n -p 9200:9200 -p 9300:9300 \\\n -d docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.10\n\ndocker run --name graylog --link mongo --link elasticsearch \\\n -p 9000:9000 -p 12201:12201 -p 1514:1514 -p 5555:5555 \\\n -v /home/graylog/geodata:/usr/share/graylog/log \\\n -e GRAYLOG_HTTP_EXTERNAL_URI=\"http://192.168.56.106:9000/\" \\\n -d graylog/graylog:3.3"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"mongo的啓動沒什麼可說的。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"elasticsearch的--ulimit必須加否則啓動後退出，-p 9200:9200是管理端口，將來刪除數據需要訪問這個端口。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"graylog 9000端口是系統界面，5555是開的tcp端口，用於被動接收日誌數據的。"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"-v /home/graylog/geodata:/usr/share/graylog/log"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"是把本地/home/graylog/geodata掛載到容器的/usr/share/graylog/log目錄，我這麼配置是爲了讓graylog能讀到GeoLite2-City.mmdb地理信息數據庫，這個庫是把ip和地理位置對應起來了。本來想把它拷貝到容器裏，但報錯"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"[root@localhost graylog]# docker cp ./GeoLite2-City.mmdb 151960c2f33b:/usr/share/graylog/data/\nError: Path not specified"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"說是要升級docker1.7到更高版本，不想升級，改成掛載方法了。如果不想掛載什麼文件，-v這行參數可以去掉。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"我是用命令“#docker exec -it graylog容器ID bash\" 先進入容器，看到容器內/usr/share/graylog/log目錄沒什麼東西，所以選擇掛載到這個目錄的。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"地理數據用於顯示訪問網站的ip分佈在哪個城市國家，還有世界地圖的顯示。需要在https://dev.maxmind.com/zh-hans/geoip/geoip2/geolite2/上下載，麻煩的是這裏需要註冊。我下載的是GeoLite2-City_20200825.tar.gz，解壓後有GeoLite2-City.mmdb，上傳這個文件到Linux的/home/graylog/geodata目錄，這個文件是需要掛載到容器，給graylog使用的。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"不想註冊請從下面鏈接下載"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"鏈接：https://pan.baidu.com/s/1LovroJyodJml4"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"niI66CkmA "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"提取碼：bsmm"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"GRAYLOG"},{"type":"text","marks":[{"type":"italic"}],"text":"HTTP"},{"type":"text","text":"EXTERNAL_URI的地址不要寫127.0.0.1，這樣如果在Linux的外部訪問，雖然能通，但是網頁是空白一片，要寫Linux對外的ip地址，這樣在外部瀏覽器打開才正常。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"另外graylog的啓動是依賴於mongo和elasticsearch的，等其它兩個都成功啓動，再啓動graylog。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下面開始演示如果配置graylog系統，並且分析網站的Apache標準格式的日誌。大概步驟如下"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"配置input->給input配置extractor->配置地理信息數據庫->手動輸入日誌->分析日誌。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"瀏覽器輸入http://192.168.56.106:9000/ 用戶名和密碼都是admin，登陸進graylog系統。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"system->input,"}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/eb/eb296e154e6c8e6b7ffed7463d651479.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"點擊select input右側的下拉箭頭，出現下拉列表，選擇raw/plaintext TCP"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/f5/f5ec8a3b0cebd6ee27cacbf7eee9880c.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"然後點擊Lanch new input，Node下拉唯一選擇給選上，Title隨意起名，Port寫5555，因爲我們docker啓動參數寫的-p 5555:5555 這兩個必須保持一致。"}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/ad/adc7b9d134e1bb3b6dd712c9be936bd5.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"其它不用填點擊下方SAVE按鈕，會自動啓動該input，可以看到local inputs下方增加了剛纔的配置。其實現在用cat access.log | nc localhost 5555等命令給5555端口發送日誌數據，數據就可以進入到graylog系統，並且可以進行簡單的搜索了。但這種搜索是最基礎的字符串匹配，價值不大。我們要分析日誌的各項指標，並且生成圖表，必須讓系統能解析每條日誌的各個field（字段或域值），例如clientip就是一個field，request也是一個field。要解析出field要給input配置extractor，點擊Manager exactor。"}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/ef/ef70579e0b062327e495f411bb695e05.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/5e/5e31c93ef93fe79654ecd7df6fe88c2a.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Extractors JSON裏貼入下面內容"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"{\n\"extractors\": [\n {\n\"title\": \"commonapache\",\n\"extractor_type\": \"grok\",\n\"converters\": [],\n\"order\": 0,\n\"cursor_strategy\": \"copy\",\n\"source_field\": \"message\",\n\"target_field\": \"\",\n\"extractor_config\": {\n\"grok_pattern\": \"%{COMMONAPACHELOG}\"\n },\n\"condition_type\": \"none\",\n\"condition_value\": \"\"\n }\n ],\n\"version\": \"3.3.5\"\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/35/35fcf0fa5437ae73590f4527c000b46a.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"最後點擊 add extrators to input，顯示successful即可。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"到這裏已經可以正確解析日誌得field了。但是如果我們想分析和地理位置相關的信息，還必須配置地理信息數據庫，上文下載的mmdb文件。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"system->configurations，最右下方有一項Geo-Location Processor，點擊改項目下方的update按鈕"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/a3/a3266936c69198e1937271f1e4324ad0.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"配置完成，點擊save。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Configurations最上方Message Processors Configuration下方表格裏要把GeoIP Resolver放在表格的最下方。點擊表格下方的update "}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/e9/e9b806973e7daa5bf88e096ec6f8e307.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"鼠標按住GeoIP Resolver往下方拖， "}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/66/6646a27b6d346368d28702e87cf1d866.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/d9/d97dd4994a370c239882937de982e798.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"完成後點擊save。Message Processors Configuration表格的GeoIP Resolver到了最下方。 "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下面是手動輸入日誌到input了，我將access2020-07-06.log放到了Linux目錄下，在目錄下執行 "}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"# cat access2020-07-06.log | head -n 10000 | nc localhost 5555"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"命令是將log從頭開始的10000行日誌發送到本機的5555端口，由於graylog的input配置的也是5555端口，docker run graylog時命令參數也是-p 5555:5555，只要這三處保持一致，這個命令是一定能成功的。這裏命令nc、ncat、netcat三個都能到達同樣的效果。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"導入完成後，選擇graylog最上方的search選項 "}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/69/695973046bf6fa29c3f856f3c903c362.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"上方的按鈕是查詢時間範圍，這個時間是日誌導入的時間，不是日誌本身記錄請求的時間，如果要查全部直接選擇search in all messages "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下方放大鏡按鈕就是搜索，後方可以添加搜索關鍵字，或者某個field的限制，有很多搜索語法非常方便，點擊搜索後，不符合條件的日誌記錄會被去除。 "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下方All Messages就是符合條件的原始的日誌結果。 "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如果想統計訪問來源於哪些城市，點擊左側邊欄最下的X（field）形按鈕。選擇clientip_cityname->show top values "}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/4c/4c582573163bb51a15103722df0f62dc.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"鼠標點擊右側灰色區域，回到主界面，訪問來源的城市信息已經在列表裏了。 "}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/a7/a7323767f1ba8415dd530885f32637d7.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"N/A表示的是有大量請求識別不出ip所在的城市，這有可能是我們的地理信息數據庫不全不新，或者有些192 172這種內網地址的訪問無法識別地區，這裏不重點討論了。如果要剔除N/A數據，只看可識別城市的分佈，鼠標放到N/A右側，會出現下拉菜單的箭頭，點擊箭頭，選擇exclude from results，N/A的數據就會去除，上面的搜索欄內也會自動增加這個篩選條件， "}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/9d/9d958db6e9b3e5f3fcebd3fe73d27a24.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/32/32dc57bfb9d7fec462c0a635bc1143dc.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"注意現在的統計是剔除了N/A的數據，數據範圍實際是比全部日誌範圍縮小了的，這在實際應用中很有價值，很多情況下我們統計某些指標，就是要看某個局部範圍的。下面我們看看訪問來源城市的統計圖，點擊右上角下拉箭頭，選擇Edit "}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/bb/bb322678db507082cb93d5c379f7fb7f.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"點擊左側Date Table處下拉菜單，可以看到柱狀圖、餅狀圖、散點圖等都列在裏面，選擇哪個右側就會出現那種統計的圖表。 "}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/c1/c10ab883fba898fcf08d328a86e46fcd.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/b6/b66f72fd9a3917c97153f589d5aee42b.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/e5/e50c2afe1a140ed9ac003c480cf8f609.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如果要展示訪問來源在世界地圖的分佈，field菜單選擇clientip_geolocation->show top values，"}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/1c/1cdd7121c0a8a9a4b91d6de5f2de1aef.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"彈出的統計表格是經緯度座標的訪問次數。和上面圖標一樣，進入Date Tabel下拉菜單，最下方有world map "}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/cb/cb8b292573784e1f47073e495d482181.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"選擇會顯示地圖統計結果，放大調整位置如下圖 "}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/50/504cff43d25fe8f440930e70526ff398.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"其它指標的統計如request分佈，訪問時間分佈，在field下列表裏都有，根據需要按上面同樣操作。地理信息數據和標準的Apache日誌可以結合生效，但一些自定義的extractor是否生效是不一定的。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"番外篇"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"給input配置extractor，上面配置的是標準的Apache格式日誌，如果日誌格式是nginx或者自定義的怎麼辦呢？"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"graylog提供了給日誌配置extractor的功能，假設我們配置完input，沒有給input配置extractor，直接導入日誌，按如下步驟配置extractor"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"input界面選擇manager extractor "}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/bd/bdb8c6db332b813a86772d77c2ea21e9.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"getstarted "}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/3b/3b71521c6254047f76fe2d7e6d402fc1.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"load message會將剛進入的日誌中的一條顯示出來，點擊message位置的select extractor type，表示我們要對message也就是整條信息配置extractor，下拉菜單選擇Grok Pattern。如果日誌進入時間比較久，load message無法展示日誌，需要通過旁邊message ID的標籤來搜索日誌，需要提供message ID和index，這兩個參數在搜索界面下方all message裏，隨便點擊一條日誌數據，展開就可以看到。message ID形如4b282600-e8d2-11ea-b962-0242ac110008，index形如graylog_0。 "}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/e2/e26ac7a944f556d950b35576b1045216.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"進入Extractor configuration，裏面的pattern要自己填寫，可以在右側已有的pattern選擇若干個組合，也可以自己定義，這裏需要對grok和正則語法熟練了。我這裏填寫的是解析nginx原生日誌的pattern格式，也是網上搜索的。填寫完點擊try against example，如果解析成功，下方會表格形式列出各個field對應該條日誌的值。不成功就會報錯，需要修改pattern直到不報錯。 "}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/6b/6b0fe4aad6844d29aa050394239a1279.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"我的pattern如下"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"^%{IPORHOST:clientip} (?:-|%{USER:ident}) (?:-|%{USER:auth}) \\[%{HTTPDATE:timestamp}\\] \\\"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|-)\\\" %{NUMBER:response} (?:-|%{NUMBER:bytes})([\\s\\S]{1})\\\"(?\\S+)\\\"([\\s\\S]{1})\\\"(?(\\S+\\s+)*\\S+)\\\".*%{BASE16FLOAT:request_time}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"解析成功，Extractor title隨便起個名，點擊最下方create extractor "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/32/32f04d77568de8d98eb688a3261d27cc.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"extractor已經成功添加給input了，上面的action有export extractor，點擊可以以json格式展示剛纔配置的extractor。 "}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/92/92fd87c6a634349621705565729548de.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"拷貝這個json文本，保存在本地，以後遇到nginx原生格式日誌，直接通過上面的import extractor就直接使用，不用在配置grok pattern測試了。 "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"需要說明的是一條日誌記錄有沒有被解析爲各個field，取決於日誌進入系統時，有沒有配置extractor。後配置的extractor對在之前的日誌是起不到解析作用的。 "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如果配置完extractor，相同格式日誌只進入系統一小部分，不要找其它原因了，原因就是pattern不對，雖然測試通過了，也配上了，但是還需要你重新修改pattern，如果pattern正確，符合格式的日誌應該全都進入系統。 "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"對於有些日誌格式，配置grok pattern需要大量調試，graylog調試並不方便，官方grok調試器網站國內都打不開了。下面提供一個工具，可以直接粘貼日誌到頁面調試 "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"鏈接：https://pan.baidu.com/s/1gWX4ZcAzh-zn5hSahdOMig "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"提取碼：t6q6 "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"windows的cmd直接java -jar GrokConstructor-0.1.0-SNAPSHOT-standalone.jar"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"然後瀏覽器訪問127.0.0.1:8080，點擊matcher，上面填寫日誌，下方填寫grok pattern， "}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/e2/e276c234e8817151224a340f1eb0f30e.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"點擊go，如果成功解析，會表格形式展示解析各個field結果。 "}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/11/1143df83fa73e63d60a370bcf15a4c68.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"ramdom example給出了一些常見日誌的例子和對應的pattern格式。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如果要重新配置graylog和輸入數據，先"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"docker stop $(docker ps -a -q)"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"停止所有容器，然後"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"docker rm $(docker ps -a -q)"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"刪除所有容器，然後docker run 按順序啓動三個容器，這樣啓動的容器是全新的，之前的配置和數據都會丟失。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"上面對容器的操作麻煩，可以使用"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"curl -L https://github.com/docker/compose/releases/download/1.25.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"安裝docker-compose，將啓動參數等要求寫到docker-compose.yml文件裏，這種方式操作命令會很簡單。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}}]}