Go日志滚动模块logrotator的使用方法及性能

logrotator是一款非常小巧的Go语言日志滚动输出模块，而日志写入速度较go-file-rotatelogs显著提升。模块本身只实现日志滚动输出和旧日志删除两个功能；由于模块实现了io.Writer接口，所以可以结合其它日志框架如logrus来实现更复杂的功能。

获取： go get "github.com/pochard/[email protected]"

引用：import "github.com/pochard/logrotator"

示例1：按指定时间间隔滚动

NewTimeBasedRotator(pattern string, period time.Duration) 函数接受time.Duration作为滚动间隔，如按天滚动 24*time.Hour，按小时滚动1* time.Hour,按20分钟滚动20*time.Minute. 模块使用服务器本地时间。

package main

import (
	"fmt"
	"github.com/pochard/logrotator"
	"io"
	"time"
)

func main() {
	writer, err := logrotator.NewTimeBasedRotator("/data/web_log/click-%Y%m%d-%H%M.log", 1*time.Hour)
	if err != nil {
		fmt.Printf("fail to create a NewTimeBasedRotator: %v\n", err)
	}
	defer writer.Close()

	s := "120024,1583893679,183.199.195.151,C03FD5AAB4CA,158379896859008,-,36781,pc,192_168_1_23\n"
	test(writer, s)
}

func test(writer io.Writer, s string) {
	_, err := writer.Write([]byte(s))
	if err != nil {
		fmt.Printf("Failed to write to log, %v\n", err)
	}
}

输出日志形如：
/data/web_log/click-20200317-1400.log
/data/web_log/click-20200317-1500.log
/data/web_log/click-20200317-1600.log

示例2：在指定时间（凌晨1:05）删除修改日期在7天之前的旧日志

通常负载很大的web应用需要把旧日志删除放在凌晨请求量较小的时候来进行。如下示例结合cron模块来实现前述功能。

package main

import (
	"fmt"
	"github.com/pochard/logrotator"
	"github.com/robfig/cron/v3"
	"net/http"
	"time"
)

func main() {
	cleaner, err := logrotator.NewTimeBasedCleaner("/data/web_log/*.log", 7*24*time.Hour)
	if err != nil {
		fmt.Printf("%v\n", err)
		return
	}

	c := cron.New()
	c.AddFunc("5 1 * * *", func() {
		deleted, err := cleaner.Clean()
		if err != nil {
			fmt.Printf("%v\n", err)
			return
		}

		for _, d := range deleted {
			fmt.Printf("%s deleted\n", d)
		}
	})
	c.Start()

	http.ListenAndServe(":8080", nil)
}

输出：
/data/web_log/click-20200317-1400.log deleted
/data/web_log/click-20200317-1500.log deleted
/data/web_log/click-20200317-1600.log deleted

性能对比：

logrotator和rotatelogs各开20万个goroutine,每个goroutine写入一条相同的日志，测试总耗时。从测试结果来看logrotator显著快于rotatelogs。

测试代码 logrotate_performance.go

package main

import (
	"fmt"
	"github.com/lestrrat/go-file-rotatelogs"
	"github.com/pochard/logrotator"
	"io"
	"sync"
	"time"
)

var wg sync.WaitGroup

func main() {
	s := "120024,1583893679,183.199.195.151,C03FD5AAB4CA,158379896859008,b_ntms_1,335\n"
	writer, err := logrotator.NewTimeBasedRotator("/data/web_log/click1-%Y%m%d-%H%M.log", 1*time.Hour)
	if err != nil {
		fmt.Printf("%v\n", err)
	}
	defer writer.Close()
	elapse1 := test(writer, s)
	fmt.Printf("logrotator elapse: %v\n", elapse1)

	writerBenchmark, err := rotatelogs.New(
		"/data/web_log/click2-%Y%m%d-%H%M.log",
		rotatelogs.WithMaxAge(7*24*time.Hour),    // 文件最大保存时间
		rotatelogs.WithRotationTime(1*time.Hour), // 日志切割时间间隔
	)
	if err != nil {
		fmt.Printf("%v", err)
	}

	elapse2 := test(writerBenchmark, s)
	fmt.Printf("rotatelogs elapse: %v\n", elapse2)
}

func test(writer io.Writer, s string) time.Duration{
	tSaved := time.Now()
	for i := 0; i != 200000; i++ {
		wg.Add(1)
		go func() {
			_, err := writer.Write([]byte(s))
			if err != nil {
				fmt.Printf("Failed to write to log, %v\n", err)
			}
			wg.Add(-1)
		}()
	}
	wg.Wait()
	return time.Now().Sub(tSaved)
}

[root@dev logrotate]# go run logrotate_performance.go
logrotator elapse: 941.173063ms
rotatelogs elapse: 1.85519449s
[root@dev logrotate]# go run logrotate_performance.go
logrotator elapse: 1.010939178s
rotatelogs elapse: 1.709002296s
[root@dev logrotate]# go run logrotate_performance.go
logrotator elapse: 940.848579ms
rotatelogs elapse: 1.778796964s

和日志框架logrus结合使用：

package main

import (
	"fmt"
	"github.com/pochard/logrotator"
	logurs "github.com/sirupsen/logrus"
	"time"
)

func main() {
	writer, err := logrotator.NewTimeBasedRotator("/data/web_log/click1-%Y%m%d-%H%M.log", 1*time.Minute)
	if err != nil {
		fmt.Printf("%v\n", err)
	}
	defer writer.Close()
	logurs.SetOutput(writer)

	entry := logurs.WithFields(logurs.Fields{"request_id": 1, "user_ip": "127.0.0.1"})
	for j := 0; j < 2000; j++ {
		entry.Infof("%3d,%s ", 200, "A group of walrus emerges from the ocean")
		time.Sleep(3 * time.Second)
	}
}

日志文件输出：
[root@dev Development] ls -l /data/web_log/
total 8
-rw-r--r--. 1 root root  889 Mar 17 15:47 click1-20200317-1547.log
-rw-r--r--. 1 root root 1651 Mar 17 15:48 click1-20200317-1548.log

[root@dev Development] tail /data/web_log/click1-20200317-1547.log
time="2020-03-17T15:47:40+08:00" level=info msg="200,A group of walrus emerges from the ocean " request_id=1 user_ip=127.0.0.1
time="2020-03-17T15:47:43+08:00" level=info msg="200,A group of walrus emerges from the ocean " request_id=1 user_ip=127.0.0.1
time="2020-03-17T15:47:46+08:00" level=info msg="200,A group of walrus emerges from the ocean " request_id=1 user_ip=127.0.0.1
time="2020-03-17T15:47:49+08:00" level=info msg="200,A group of walrus emerges from the ocean " request_id=1 user_ip=127.0.0.1
time="2020-03-17T15:47:52+08:00" level=info msg="200,A group of walrus emerges from the ocean " request_id=1 user_ip=127.0.0.1
time="2020-03-17T15:47:55+08:00" level=info msg="200,A group of walrus emerges from the ocean " request_id=1 user_ip=127.0.0.1
time="2020-03-17T15:47:58+08:00" level=info msg="200,A group of walrus emerges from the ocean " request_id=1 user_ip=127.0.0.1