用ECharts繪製Prometheus圖表，實現類似Grafana的自定義Dashboard

  大家一般都是用Grafana自定義Dashboard來監控Prometheus數據的，作者這次嘗試用ECharts來繪製Prometheus數據圖表，一方面可以減少依賴，另一方面可以將監控界面靈活的集成進應用系統。至於如何在被監測機器上安裝NodeExporter以及如何部署Prometheus作者就不描述了，園子裏有很多文章介紹。

一、數據查詢及轉換

  Prometheus提供了Http Api來執行promql查詢，但需要將返回的數據格式轉換爲ECharts的格式，好在EChars的xAxis.type可以設置爲'time'類型，與Prometheus返回的格式接近。作者寫了個簡單的服務來執行查詢及轉換數據，詳見以下代碼：

public class MetricService
{
    private static readonly HttpClient http = new HttpClient()
    {
        //請修改指向Prometheus地址
        BaseAddress = new Uri("http://10.211.55.2:9090/api/v1/"),
        Timeout = TimeSpan.FromSeconds(2)
    };

    public async Task<object> GetCpuUsages(string node, DateTime start, DateTime end)
    {
        var promql = $"100-irate(node_cpu{{instance='{node}:9100',mode='idle'}}[5m])*100";
        return await QueryRange(promql, start, end, 20, 2);
    }

    public async Task<object> GetMemUsages(string node, DateTime start, DateTime end)
    {
        var promql = $"(1-(node_memory_MemAvailable{{instance='{node}:9100'}}/(node_memory_MemTotal{{instance='{node}:9100'}})))*100";
        return await QueryRange(promql, start, end, 20, 2);
    }

    public async Task<object> GetNetTraffic(string node, DateTime start, DateTime end)
    {
        var downql = $"irate(node_network_receive_bytes{{instance='{node}:9100',device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*'}}[5m])";
        var ls = await QueryRange(downql, start, end, 15/*4*/, 0);
        var upql = $"irate(node_network_transmit_bytes{{instance='{node}:9100',device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*'}}[5m])";
        ls.Add(await QueryRange(upql, start, end, 15/*4*/, 0));
        return ls;
    }

    public async Task<object> GetDiskIO(string node, DateTime start, DateTime end)
    {
        var readql = $"irate(node_disk_bytes_read{{instance='{node}:9100'}}[1m])";
        var ls = await QueryRange(readql, start, end, 15/*10*/, 0);
        var writeql = $"irate(node_disk_bytes_written{{instance='{node}:9100'}}[1m])";
        ls.Add(await QueryRange(writeql, start, end, 15/*10*/, 0));
        return ls;
    }

    #region ====Parse PromQL====
    private static async Task<List<object>> QueryRange(string promql, DateTime start, DateTime end, int step, int round)
    {
        if (start >= end) throw new ArgumentOutOfRangeException();
        var ts1 = (int)(start.ToUniversalTime() - DateTime.UnixEpoch).TotalSeconds;
        var ts2 = (int)(end.ToUniversalTime() - DateTime.UnixEpoch).TotalSeconds;
        var res = await http.GetAsync($"query_range?query={promql}&start={ts1}&end={ts2}&step={step}s");
        var stream = await res.Content.ReadAsStreamAsync();
        using (var sr = new System.IO.StreamReader(stream))
        using (var jr = new JsonTextReader(sr))
        {
            return ParseToSeries(jr, round);
        }
    }

    private static List<object> ParseToSeries(JsonTextReader jr, int round)
    {
        if (!jr.Read() || jr.TokenType != JsonToken.StartObject) throw new Exception();
        if (!jr.Read() || jr.TokenType != JsonToken.PropertyName || (string)jr.Value != "status")
            throw new Exception();
        var status = jr.ReadAsString();
        if (status != "success") throw new Exception();
        if (!jr.Read() || jr.TokenType != JsonToken.PropertyName || (string)jr.Value != "data")
            throw new Exception();

        if (!jr.Read() || jr.TokenType != JsonToken.StartObject) throw new Exception();
        if (!jr.Read() || jr.TokenType != JsonToken.PropertyName || (string)jr.Value != "resultType")
            throw new Exception();
        var resultType = jr.ReadAsString();
        if (!jr.Read() || jr.TokenType != JsonToken.PropertyName || (string)jr.Value != "result")
            throw new Exception();

        return ReadResultArray(jr, round);
        //No need read others
    }

    private static List<object> ReadResultArray(JsonTextReader jr, int round)
    {
        if (!jr.Read() || jr.TokenType != JsonToken.StartArray) throw new Exception();

        var ls = new List<object>();
        do
        {
            if (!jr.Read()) throw new Exception();
            if (jr.TokenType == JsonToken.EndArray) break;
            if (jr.TokenType != JsonToken.StartObject) throw new Exception();
            ls.Add(ReadResultItem(jr, round));
        } while (true);
        return ls;
    }

    private static List<double[]> ReadResultItem(JsonTextReader jr, int round)
    {
        //已讀取StartObject標記
        if (!jr.Read() || jr.TokenType != JsonToken.PropertyName || (string)jr.Value != "metric")
            throw new Exception();
        ReadMetric(jr);

        if (!jr.Read() || jr.TokenType != JsonToken.PropertyName || (string)jr.Value != "values")
            throw new Exception();
        var values = ReadValues(jr, round);
        if (!jr.Read() || jr.TokenType != JsonToken.EndObject) throw new Exception();
        return values;
    }

    private static void ReadMetric(JsonTextReader jr)
    {
        if (!jr.Read() || jr.TokenType != JsonToken.StartObject) throw new Exception();
        do
        {
            //PropertyName or EndObject
            if (!jr.Read()) throw new Exception();
            if (jr.TokenType == JsonToken.EndObject) return;
            //PropertyValue
            jr.Read();
        } while (true);
    }

    private static List<double[]> ReadValues(JsonTextReader jr, int round)
    {
        if (!jr.Read() || jr.TokenType != JsonToken.StartArray) throw new Exception();

        var ls = new List<double[]>();
        do
        {
            if (!jr.Read()) throw new Exception();
            if (jr.TokenType == JsonToken.EndArray) break;
            if (jr.TokenType != JsonToken.StartArray) throw new Exception();
            var ts = jr.ReadAsDouble().Value * 1000; //PromQL時間*1000
            var value = Math.Round(double.Parse(jr.ReadAsString()), round, MidpointRounding.ToEven); //PromQL值爲字符串
            ls.Add(new double[] { ts, value });
            if (!jr.Read() || jr.TokenType != JsonToken.EndArray) throw new Exception();
        } while (true);
        return ls;
    }
    #endregion

}

Tip: promql的寫法可參考grafana網站相關Dashboard。

二、單指標Vue組件

  作者使用Vue-ECharts作爲ECharts的包裝，以CPU使用率Vue組件爲例：

<v-chart theme="dark" autoresize :options="chartOptions" style="height:250px">
</v-chart>

@Component
export default class CpuUsages extends Vue {
    /** 目標實例IP */
    @Prop({ type: String, default: '10.211.55.3' }) node
    /** 開始時間 */
    @Prop({ type: Date, default: () => { var now = new Date(); return new Date(now.getFullYear(), now.getMonth(), now.getDate()) } }) start
    /** 結束時間 */
    @Prop({ type: Date, default: () => { return new Date() } }) end

    chartOptions = {
        title: { text: 'Cpu Usages', x: 'center' },
        tooltip: { trigger: 'axis' },
        xAxis: { type: 'time' },
        yAxis: { min: 0, max: 100 },
        series: []
    }

    refresh() {
        sys.Services.MetricService.GetCpuUsages(this.node, this.start, this.end).then(res => {
                this.chartOptions.series.splice(0)
                for (var i = 0; i < res.length; ++i) {
                    var seria = { type: 'line', name: 'cpu' + i, data: res[i], showSymbol: false }
                    this.chartOptions.series.push(seria)
                }
            }).catch(err => {
                this.$message(err)
            })
    }

    mounted() {
        this.refresh()
    }
}

三、組合多個組件形成Dashboard

  根據需要可以靈活組合多個指標組件，形成相應的Dashboard界面（如下圖所示）。

四、小結

  感謝Vue、ECharts、Vue-ECharts、Prometheus等項目，使得開發並集成監控Dashboard如此簡單。另碼文不易，碼技術文更不易，所以請您多多推薦！