Golang實戰之海量日誌收集系統（四）etcd介紹與使用etcd獲取配置信息

目錄:

GitHub項目地址：https://github.com/PlutoaCharon/Golang_logCollect⭐

Golang實戰之海量日誌收集系統（一）項目背景介紹

Golang實戰之海量日誌收集系統（二）收集應用程序日誌到Kafka中

Golang實戰之海量日誌收集系統（三）簡單版本logAgent的實現

Golang實戰之海量日誌收集系統（四）etcd介紹與使用etcd獲取配置信息

Golang實戰之海量日誌收集系統（五）根據etcd配置項創建多個tailTask

Golang實戰之海量日誌收集系統（六）監視etcd配置項的變更

Golang實戰之海量日誌收集系統（七）logTransfer之從kafka中獲取日誌信息

Golang實戰之海量日誌收集系統（八）logTransfer之將日誌入庫到Elasticsearch並通過Kibana進行展示

etcd介紹

高可用的分佈式key-value存儲，可以用於配置共享和服務發現

• 類似的項目：Zookeeper和consul
• 開發語言：go
• 接口：提供Restful的接口，使用簡單
• 實現算法：基於raft算法的強一致性，高可用的服務存儲目錄

etcd的應用場景：

• 服務發現和服務註冊
• 配置中心(我們實現的日誌收集客戶端需要用到)
• 分佈式鎖
• master選舉

etcd的命令驗證

PS E:\Study\etcd-v3.4.5-windows-amd64> .\etcdctl.exe put name xu
OK
PS E:\Study\etcd-v3.4.5-windows-amd64> .\etcdctl.exe get name
name
xu
PS E:\Study\etcd-v3.4.5-windows-amd64>

context 介紹和使用

Context即爲上下文管理，那麼context的作用是做什麼，主要有如下兩個作用：

• 控制goroutine的超時
• 保存上下文數據

context的超時控制：

package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"time"
)

type Result struct {
	r   *http.Response
	err error
}

func process() {
	// context的超時控制
	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
	defer cancel()
	tr := &http.Transport{}
	client := &http.Client{Transport: tr}
	c := make(chan Result, 1)
	// 如果請求成功了會將數據存入到管道中
	req, err := http.NewRequest("GET", "http://www.baidu.com", nil)
	//req, err := http.NewRequest("GET", "https://www.google.com", nil)

	if err != nil {
		fmt.Println("http request failed, err:", err)
		return
	}
	go func() {
		resp, err := client.Do(req)
		pack := Result{r: resp, err: err}
		c <- pack
	}()
	select {
	case <-ctx.Done(): // 如果超時, ctx.Done()返回一個管道,當管道里有數據即可說明超時
		//tr.CancelRequest(req)
		tr.CloseIdleConnections()
		res := <-c
		fmt.Println("Timeout! err:", res.err)
	case res := <-c: // c管道里的數據傳給res, 如果res裏有數據則證明請求成功
		defer res.r.Body.Close()
		out, _ := ioutil.ReadAll(res.r.Body)
		fmt.Printf("Server Response: %s", out)
	}
	return
}
func main() {
	process()
}

req, err := http.NewRequest("GET", "http://www.baidu.com", nil)
正常返回百度網站的網頁html
req, err := http.NewRequest("GET", "https://www.google.com", nil)
返回失敗
Timeout! err: Get "https://www.google.com": dial tcp 205.186.152.122:443: connectex: A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond.

context保存上下文

package main

import (
	"context"
	"fmt"
)

func process(ctx context.Context) {
	ret, ok := ctx.Value("trace_id").(int)
	if !ok {
		ret = 789
	}
	fmt.Printf("ret:%d\n", ret)

	s, _ := ctx.Value("session").(string)
	fmt.Printf("seesion:%s\n", s)
}
func main() {
	ctx := context.WithValue(context.Background(), "trace_id", 123)
	ctx = context.WithValue(ctx, "session", "This is a session")
	process(ctx)
}

ret:123
seesion:This is a session

結合etcd和context使用

我這裏使用的是Go1.14, 安裝github.com/coreos/etcd/clientv3時報錯etcd undefined: resolver.BuildOption
原因: grpc版本過高, 將grpc版本替換成v1.26.0版本
詳細參考這篇博客: 解決Golang1.14 etcd/clientv3報錯：etcd undefined: resolver.BuildOption

連接etcd

連接前要先啓動etcd

package main

import (
	"fmt"
	"github.com/coreos/etcd/clientv3"
	"time"
)

func main() {

	cli, err := clientv3.New(clientv3.Config{
		Endpoints:   []string{"localhost:2379", "localhost:22379", "localhost:32379"},
		DialTimeout: 5 * time.Second,})

	if err != nil {
		fmt.Println("connect failed, err:", err)
		return
	}

	fmt.Println("connect succ")
	defer cli.Close()
}

通過連接etcd，存值並取值

package main

import (
	"context"
	"fmt"
	"time"

	"github.com/coreos/etcd/clientv3"
)


func main() {
	EtcdExmaple()
}

func EtcdExmaple() {
	cli, err := clientv3.New(clientv3.Config{
		Endpoints:   []string{"localhost:2379", "localhost:22379", "localhost:32379"},
		DialTimeout: 5 * time.Second,
	})
	if err != nil {
		fmt.Println("connect failed, err:", err)
		return
	}

	fmt.Println("connect succ")
	defer cli.Close()
	// put操作
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	_, err = cli.Put(ctx, "/logagent/conf/", "sample_value")
	cancel()
	if err != nil {
		fmt.Println("put failed, err:", err)
		return
	}
	// get操作
	ctx, cancel = context.WithTimeout(context.Background(), time.Second)
	resp, err := cli.Get(ctx, "/logagent/conf/")
	cancel()
	if err != nil {
		fmt.Println("get failed, err:", err)
		return
	}
	for _, ev := range resp.Kvs {
		fmt.Printf("%s : %s\n", ev.Key, ev.Value)
	}
}

Watch操作
通過watch監控配置更改

package main

import (
	"context"
	"fmt"
	"github.com/coreos/etcd/clientv3"
	"time"
)

func main() {
	cli, err := clientv3.New(clientv3.Config{
		Endpoints:   []string{"localhost:2379"},
		DialTimeout: time.Second,
	})
	if err != nil {
		fmt.Printf("connect to etcd failed, err: %v\n", err)
		return
	}
	fmt.Println("connect etcd success.")

	defer cli.Close()

	// Watch操作
	wch := cli.Watch(context.Background(), "/logagent/conf/")
	for resp := range wch {
		for _, ev := range resp.Events {
			fmt.Printf("Type: %v, Key:%v, Value:%v\n", ev.Type, string(ev.Kv.Key), string(ev.Kv.Value))
		}
	}
}

構建運行，然後嘗試通過etcdctl向etcd指定的key /logagent/conf/發送數據測試

PS E:\Study\etcd-v3.4.5-windows-amd64> .\etcdctl.exe put /logagent/conf/ 1
OK
PS E:\Study\etcd-v3.4.5-windows-amd64> .\etcdctl.exe put /logagent/conf/ 2
OK
PS E:\Study\etcd-v3.4.5-windows-amd64>

終端查看

實現一個kafka的消費者代碼的簡單例子：

package main

import (
	"fmt"
	"strings"
	"sync"
	"time"
	"github.com/Shopify/sarama"
)

func main() {

	consumer, err := sarama.NewConsumer(strings.Split("localhost:9092",","), nil)
	if err != nil {
		fmt.Println("Failed to start consumer: %s", err)
		return
	}
	partitionList, err := consumer.Partitions("nginx_log")
	if err != nil {
		fmt.Println("Failed to get the list of partitions: ", err)
		return
	}
	fmt.Println(partitionList)

	// 按照分區來消費
	for partition := range partitionList {
		pc, err := consumer.ConsumePartition("nginx_log", int32(partition), sarama.OffsetNewest)
		if err != nil {
			fmt.Printf("Failed to start consumer for partition %d: %s\n", partition, err)
			return
		}
		defer pc.AsyncClose()
		go func(pc sarama.PartitionConsumer) {
			for msg := range pc.Messages() {
				fmt.Printf("Partition:%d, Offset:%d, Key:%s, Value:%s", msg.Partition, msg.Offset, string(msg.Key), string(msg.Value))
				fmt.Println()
			}
		}(pc)
	}
	time.Sleep(time.Hour)
	consumer.Close()
}

但是上面的代碼並不是最佳代碼，因爲這是通過time.sleep等待goroutine的執行，我們可以更改爲通過sync.WaitGroup方式實現

使用sync.WaitGroup優化

• 等待一組goroutine結束

• 使用Add方法設置等待的數量加1

• 使用Done方法設置等待的數量減1

• 當等待的數量等於0時,Wait函數返回

package main

import (
	"fmt"
	"github.com/Shopify/sarama"
	"strings"
	"sync"
)

var (
	wg sync.WaitGroup
)

func main() {

	consumer, err := sarama.NewConsumer(strings.Split("localhost:9092",","), nil)
	if err != nil {
		fmt.Println("Failed to start consumer: %s", err)
		return
	}
	partitionList, err := consumer.Partitions("nginx_log")
	if err != nil {
		fmt.Println("Failed to get the list of partitions: ", err)
		return
	}
	fmt.Println(partitionList)

	// 按照分區來消費
	for partition := range partitionList {
		pc, err := consumer.ConsumePartition("nginx_log", int32(partition), sarama.OffsetNewest)
		if err != nil {
			fmt.Printf("Failed to start consumer for partition %d: %s\n", partition, err)
			return
		}
		defer pc.AsyncClose()
		go func(pc sarama.PartitionConsumer) {
			wg.Add(1) // 增加一個goroutine
			for msg := range pc.Messages() {
				fmt.Printf("Partition:%d, Offset:%d, Key:%s, Value:%s", msg.Partition, msg.Offset, string(msg.Key), string(msg.Value))
				fmt.Println()
			}
			wg.Done() // 說明一個goroutine結束
		}(pc)
	}
	//time.Sleep(time.Hour)
	wg.Wait() // 當wg的內置計數爲0時返回, 即所有goroutine運行結束
	_ = consumer.Close()
}

從etcd中獲取配置信息

根據key從etcd中獲取配置項

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"time"

	"github.com/coreos/etcd/clientv3"
)

// 定義etcd的前綴key
const (
	EtcdKey = "/backend/logagent/config/192.168.0.11"
)

// 需要收集的日誌的配置信息
type LogConf struct {
	Path  string `json:"path"`  // 日誌存放的路徑
	Topic string `json:"topic"` // 日誌要發往Kafka中的哪個Topic
}

func SetLogConfToEtcd() {
	cli, err := clientv3.New(clientv3.Config{
		Endpoints:   []string{"localhost:2379", "localhost:22379", "localhost:32379"},
		DialTimeout: 5 * time.Second,
	})
	if err != nil {
		fmt.Println("connect failed, err:", err)
		return
	}

	fmt.Println("connect succ")
	defer cli.Close()

	// 日誌配置
	var logConfArr []LogConf
	logConfArr = append(
		logConfArr,
		LogConf{
			Path:  "E:/nginx/logs/access.log",
			Topic: "nginx_log",
		},
	)
	logConfArr = append(
		logConfArr,
		LogConf{
			Path:  "E:/nginx/logs/error.log",
			Topic: "nginx_log_err",
		},
	)

	// Json打包
	data, err := json.Marshal(logConfArr)
	if err != nil {
		fmt.Println("json failed, ", err)
		return
	}

	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	_, err = cli.Put(ctx, EtcdKey, string(data))
	cancel()
	if err != nil {
		fmt.Println("put failed, err:", err)
		return
	}

	ctx, cancel = context.WithTimeout(context.Background(), time.Second)
	resp, err := cli.Get(ctx, EtcdKey)
	cancel()
	if err != nil {
		fmt.Println("get failed, err:", err)
		return
	}
	for _, ev := range resp.Kvs {
		fmt.Printf("%s : %s\n", ev.Key, ev.Value)
	}
}

func main() {
	SetLogConfToEtcd()
}

測試能否正常拿到值

connect succ
/backend/logagent/config/192.168.0.11 : [{"path":"E:/nginx/logs/access.log","topic":"nginx_log"},{"path":"E:/nginx/logs/error.log","topic":"nginx_log_err"}]

現在我們可以通過操作etcd拿到配置信息，下一步就是拿着這些配置項進行日誌收集