Go日誌滾動模塊logrotator的使用方法及性能

logrotator是一款非常小巧的Go語言日誌滾動輸出模塊，而日誌寫入速度較go-file-rotatelogs顯著提升。模塊本身只實現日誌滾動輸出和舊日誌刪除兩個功能；由於模塊實現了io.Writer接口，所以可以結合其它日誌框架如logrus來實現更復雜的功能。

獲取： go get "github.com/pochard/[email protected]"

引用：import "github.com/pochard/logrotator"

示例1：按指定時間間隔滾動

NewTimeBasedRotator(pattern string, period time.Duration) 函數接受time.Duration作爲滾動間隔，如按天滾動 24*time.Hour，按小時滾動1* time.Hour,按20分鐘滾動20*time.Minute. 模塊使用服務器本地時間。

package main

import (
	"fmt"
	"github.com/pochard/logrotator"
	"io"
	"time"
)

func main() {
	writer, err := logrotator.NewTimeBasedRotator("/data/web_log/click-%Y%m%d-%H%M.log", 1*time.Hour)
	if err != nil {
		fmt.Printf("fail to create a NewTimeBasedRotator: %v\n", err)
	}
	defer writer.Close()

	s := "120024,1583893679,183.199.195.151,C03FD5AAB4CA,158379896859008,-,36781,pc,192_168_1_23\n"
	test(writer, s)
}

func test(writer io.Writer, s string) {
	_, err := writer.Write([]byte(s))
	if err != nil {
		fmt.Printf("Failed to write to log, %v\n", err)
	}
}

輸出日誌形如：
/data/web_log/click-20200317-1400.log
/data/web_log/click-20200317-1500.log
/data/web_log/click-20200317-1600.log

示例2：在指定時間（凌晨1:05）刪除修改日期在7天之前的舊日誌

通常負載很大的web應用需要把舊日誌刪除放在凌晨請求量較小的時候來進行。如下示例結合cron模塊來實現前述功能。

package main

import (
	"fmt"
	"github.com/pochard/logrotator"
	"github.com/robfig/cron/v3"
	"net/http"
	"time"
)

func main() {
	cleaner, err := logrotator.NewTimeBasedCleaner("/data/web_log/*.log", 7*24*time.Hour)
	if err != nil {
		fmt.Printf("%v\n", err)
		return
	}

	c := cron.New()
	c.AddFunc("5 1 * * *", func() {
		deleted, err := cleaner.Clean()
		if err != nil {
			fmt.Printf("%v\n", err)
			return
		}

		for _, d := range deleted {
			fmt.Printf("%s deleted\n", d)
		}
	})
	c.Start()

	http.ListenAndServe(":8080", nil)
}

輸出：
/data/web_log/click-20200317-1400.log deleted
/data/web_log/click-20200317-1500.log deleted
/data/web_log/click-20200317-1600.log deleted

性能對比：

logrotator和rotatelogs各開20萬個goroutine,每個goroutine寫入一條相同的日誌，測試總耗時。從測試結果來看logrotator顯著快於rotatelogs。

測試代碼 logrotate_performance.go

package main

import (
	"fmt"
	"github.com/lestrrat/go-file-rotatelogs"
	"github.com/pochard/logrotator"
	"io"
	"sync"
	"time"
)

var wg sync.WaitGroup

func main() {
	s := "120024,1583893679,183.199.195.151,C03FD5AAB4CA,158379896859008,b_ntms_1,335\n"
	writer, err := logrotator.NewTimeBasedRotator("/data/web_log/click1-%Y%m%d-%H%M.log", 1*time.Hour)
	if err != nil {
		fmt.Printf("%v\n", err)
	}
	defer writer.Close()
	elapse1 := test(writer, s)
	fmt.Printf("logrotator elapse: %v\n", elapse1)

	writerBenchmark, err := rotatelogs.New(
		"/data/web_log/click2-%Y%m%d-%H%M.log",
		rotatelogs.WithMaxAge(7*24*time.Hour),    // 文件最大保存時間
		rotatelogs.WithRotationTime(1*time.Hour), // 日誌切割時間間隔
	)
	if err != nil {
		fmt.Printf("%v", err)
	}

	elapse2 := test(writerBenchmark, s)
	fmt.Printf("rotatelogs elapse: %v\n", elapse2)
}

func test(writer io.Writer, s string) time.Duration{
	tSaved := time.Now()
	for i := 0; i != 200000; i++ {
		wg.Add(1)
		go func() {
			_, err := writer.Write([]byte(s))
			if err != nil {
				fmt.Printf("Failed to write to log, %v\n", err)
			}
			wg.Add(-1)
		}()
	}
	wg.Wait()
	return time.Now().Sub(tSaved)
}

[root@dev logrotate]# go run logrotate_performance.go
logrotator elapse: 941.173063ms
rotatelogs elapse: 1.85519449s
[root@dev logrotate]# go run logrotate_performance.go
logrotator elapse: 1.010939178s
rotatelogs elapse: 1.709002296s
[root@dev logrotate]# go run logrotate_performance.go
logrotator elapse: 940.848579ms
rotatelogs elapse: 1.778796964s

和日誌框架logrus結合使用：

package main

import (
	"fmt"
	"github.com/pochard/logrotator"
	logurs "github.com/sirupsen/logrus"
	"time"
)

func main() {
	writer, err := logrotator.NewTimeBasedRotator("/data/web_log/click1-%Y%m%d-%H%M.log", 1*time.Minute)
	if err != nil {
		fmt.Printf("%v\n", err)
	}
	defer writer.Close()
	logurs.SetOutput(writer)

	entry := logurs.WithFields(logurs.Fields{"request_id": 1, "user_ip": "127.0.0.1"})
	for j := 0; j < 2000; j++ {
		entry.Infof("%3d,%s ", 200, "A group of walrus emerges from the ocean")
		time.Sleep(3 * time.Second)
	}
}

日誌文件輸出：
[root@dev Development] ls -l /data/web_log/
total 8
-rw-r--r--. 1 root root  889 Mar 17 15:47 click1-20200317-1547.log
-rw-r--r--. 1 root root 1651 Mar 17 15:48 click1-20200317-1548.log

[root@dev Development] tail /data/web_log/click1-20200317-1547.log
time="2020-03-17T15:47:40+08:00" level=info msg="200,A group of walrus emerges from the ocean " request_id=1 user_ip=127.0.0.1
time="2020-03-17T15:47:43+08:00" level=info msg="200,A group of walrus emerges from the ocean " request_id=1 user_ip=127.0.0.1
time="2020-03-17T15:47:46+08:00" level=info msg="200,A group of walrus emerges from the ocean " request_id=1 user_ip=127.0.0.1
time="2020-03-17T15:47:49+08:00" level=info msg="200,A group of walrus emerges from the ocean " request_id=1 user_ip=127.0.0.1
time="2020-03-17T15:47:52+08:00" level=info msg="200,A group of walrus emerges from the ocean " request_id=1 user_ip=127.0.0.1
time="2020-03-17T15:47:55+08:00" level=info msg="200,A group of walrus emerges from the ocean " request_id=1 user_ip=127.0.0.1
time="2020-03-17T15:47:58+08:00" level=info msg="200,A group of walrus emerges from the ocean " request_id=1 user_ip=127.0.0.1