搭建prometheus監控系統

搭建prometheus監控系統

安裝prometheus

sudo mkdir /docker/compose/prometheus -p
cd /docker/compose/prometheus
touch docker-compose.yml
#如果複製換行錯誤
:set paste

version: '3.7'
services:
   prometheus:
      image: prom/prometheus:latest
      container_name: prometheus
      restart: on-failure:1
      ports:
         - "9090:9090"
      volumes:
         - /docker/config/prometheus/:/etc/prometheus/
      command:
            - '--config.file=/etc/prometheus/prometheus.yml'
            - '--web.enable-admin-api'
            - '--web.enable-lifecycle'

創建配置文件

cd /docker/config/prometheus
touch prometheus.yml

# my global config
global:
  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
  alertmanagers:
    - static_configs:
        - targets:
          - 127.0.0.1:9093

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
    - "rules/*.yml"
  # - "first_rules.yml"
  # - "second_rules.yml"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: "prometheus"

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    static_configs:
      - targets: ["localhost:9090"]
      
  - job_name: "service_status_http"

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    http_sd_configs:
      - url: http://127.0.0.1:5220/api/Monitor/GetMonitorServiceTargets
        
  - job_name: "service_status_https"
    scheme: https
    http_sd_configs:
      - url: http://127.0.0.1:5220/api/Monitor/GetMonitorServiceTargets?scheme=https     
     
     
  - job_name: 'http_status'  # 配置job名
    metrics_path: /probe     # 定義metric獲取的路徑
    params:
      module: [http_2xx]     # 這裏就是我們在black_exporter中定義的模塊名
    #file_sd_configs:         # 因需要監控的地址很多，我們這裏將所有地址獨立出來，後面會介紹該文件
    #  - files: 
    #    - '/etc/prometheus/job_web.yaml'
    #    refresh_interval: 30s # 30秒刷新一次，當有新的監控地址時，會自動加載進來不需要重啓
    http_sd_configs:
      - url: http://127.0.0.1:5220/api/Monitor/GetMonitorSiteTargets
    relabel_configs:
      - source_labels: [__address__]  # 當前target的訪問地址，比如監控百度則爲 https://baidu.com
        target_label: __param_target  # __param是默認參數前綴，target爲參數，這裏可以理解爲把__address__ 的值賦給__param_target，若監控百度，則target=https://baidu.com
      - source_labels: [__param_target]
        target_label: instance        # 可以理解爲把__param_target的值賦給instance標籤
      - target_label: __address__
        replacement: 127.0.0.1:9115 # web監控原本的target爲站點的地址，但Prometheus不是直接去請求該地址，而是去請求black_exporter，故需要把目標地址替換爲black_exporter的地址

docker-compose

安裝docker-compose參考

# 2. 安裝docker-compose
pip install docker-compose

docker-compose up -d

安裝grafana

#先安裝原版,拿配置文件
docker run -d --name grafana \
-p 3000:3000 \
grafana/grafana:latest


docker cp grafana:/etc/grafana/ /docker/config/
docker cp grafana:/var/lib/grafana/ /docker/volumes/

docker run -d --name grafana \
-p 3000:3000 \
-v /docker/config/grafana/:/etc/grafana/ \
-v /docker/volumes/grafana/:/var/lib/grafana/ \
grafana/grafana:latest


#如果權限不足
# chmod -R 777 . 


#如果登錄報錯 /docker/config/grafana/grafana.ini
[log]
# Either "console", "file", "syslog". Default is console and  file
# Use space to separate multiple modes, e.g. "console file"
mode = console file

# Either "debug", "info", "warn", "error", "critical", default is "info"
level = debug

添加datasource

安裝alertManager

docker run --name alertmanager -d -p 9093:9093 --restart=always \
prom/alertmanager


docker cp alertmanager:/etc/alertmanager /docker/config/alertmanager
docker rm -f alertmanager

docker run --name alertmanager -d -p 9093:9093 --restart=always \
-v /docker/config/alertmanager/:/etc/alertmanager/ \
prom/alertmanager

http://10.0.17.178:9090/   #prometheus
http://10.0.17.178:3000/   #grafana
http://10.0.17.178:9093/   #altermessage

配置站點監控

參考

安裝配置black_expoter

mkdir -p /docker/compose/prometheus/black_expoter
touch docker-compose.yml

#重啓普羅米修斯
curl -XPOST http://127.0.0.1:9090/-/reload

#docker-compose.yml
version: '3.7'
services:
  blackbox_exporter:
    container_name: blackbox_exporter
    image: prom/blackbox-exporter:master
    volumes:
      - /docker/config/blackbox_exporter/config.yml:/etc/blackbox_exporter/config.yml
    ports:
      - 9115:9115

#config.yml
modules:
  http_2xx:      # 給模塊取名，後面在Prometheus的配置文件中會用到該名稱
    prober: http # 探針類型，探針有多種類型如http、tcp、icmp、dns，不同的探針具有不同的功能
    timeout: 5s  # 探針檢測超時時間
    http:
      valid_status_codes: [] # 有效的狀態碼，默認爲200，也可以自己定義，比如你的站點304也可能是正常的
      method: GET            # http使用get請求
      fail_if_body_not_matches_regexp: [] # 對返回結果進行正則匹配，如果未匹配成功則認爲失敗
      tls_config:
        insecure_skip_verify: true        # 不安全的https跳過確認，如某些證書不合法或者過期，如果你在瀏覽器訪問，那瀏覽器會讓你確認是否繼續，這裏也是類似的意思。

dashbord

https://grafana.com/grafana/dashboards/10915-asp-net-core-controller-summary-prometheus/

https://grafana.com/grafana/dashboards/10427-prometheus-net/

https://grafana.com/grafana/dashboards/12526-asp-net-core-services/

https://grafana.com/grafana/dashboards/17039-prometheus-net-process-metrics/