慶軍之簡單wal系統的實現記錄

要解決的問題:

我有一個系統，但是數據庫表示它不穩定了，我需要對外高可用。數據庫不能用的時候，總不能不用吧。所以，只能最終一致性。

臨時的數據存在文件wal裏面，由另一個線程單獨處理。

原來是打算用 faster來實現。但是我表示說，文檔很好。但是我不會用。我太菜。

最後某網友建議用日誌來處理。我寫了兩天。終於是簡單的湊出來了。

我用到了Serilog來寫wal.因爲不會寫它的過濾條件。搞不來輪子。所以指定了 Fatal級別，也就是枚舉裏面爲6的等級來記錄到另一個目錄下。

具體的配置如下。

//appsettings.json 中 

 1 "WriteTo": [
 2       //{ "Name": "Console" },
 3       {
 4         "Name": "File",
 5         "Args": {
 6           "path": "Log2s\\log.txt",
 7           "rollingInterval": "Day",
 8           "fileSizeLimitBytes": "10485760",
 9           "retainedFileCountLimit": 5,
10           "rollOnFileSizeLimit": true
11         }
12       },
13       {
14         "Name": "Logger",
15         "Args": {
16           "configureLogger": {
17             "Filter": [
18               {
19                 "Name": "ByIncludingOnly",
20                 "Args": {
21                   "expression": "@l = 'Fatal'"
22                 }
23               }
24             ],
25             "WriteTo": [
26               {
27                 "Name": "File",
28                 "Args": {
29                   "outputTemplate": "[{Timestamp:yyyy-MM-dd HH:mm:ss.fff}] {Message:lj}{NewLine}",
30                   //"outputTemplate": "[{Timestamp:g} {Level:u3}] [{SourceContext}] {Message:lj}{NewLine}",
31                   "path": "wal\\wal.log",
32                   "rollingInterval": "Day",
33                   "fileSizeLimitBytes": "10485760",
34                   "retainedFileCountLimit": 200,
35                   "rollOnFileSizeLimit": true,
36                   "formatter": "Serilog.Formatting.Json.JsonFormatter",
37                   "flushToDiskInterval": "00:00:01"//刷盤時間
38                 }
39               }
40             ]
41           }
42         }
43       }
44     ]

 

代碼調用

_logger?.LogCritical( "{@model}", model);

 

解析類:

 public class ReWALService : BackgroundService
    {
        public class ReWALDefine
        {    
            public long lastWriteTime { get; set; }//文件最後寫入的時間,解析的時候
            public string logfile { get; set; }//文件名稱
            public long streamPoint { get; set; }//在文件內容的位置
        }

        private readonly ILogger<ReWALService> _logger;
        
        private  string currentFile = "";
        private readonly DirectoryInfo _directoryInfo;
        public ReWALService(ILoggerFactory loggerFactory) 
        {
            this._logger = loggerFactory.CreateLogger<ReWALService>();
            var path = Path.Combine(Directory.GetCurrentDirectory(), "wal");
            _directoryInfo = new DirectoryInfo(path);
            //wallog
            currentFile = Path.Combine( path,"wallog.txt");
        }
        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
        {
            try
            {
                if (!File.Exists(currentFile))
                {
                    File.WriteAllText(currentFile, string.Empty);
                }

                while(!stoppingToken.IsCancellationRequested)
                {
                    var wallogContent = File.ReadAllText(currentFile);
                    ReWALDefine _walloginfo = fetchOrCreateDefine(wallogContent);
                    await Task.Delay(100);
                    DateTime oldDateTime = new DateTime(_walloginfo.lastWriteTime);
                    var walrecords = _directoryInfo.GetFiles("*.log").Where(q => q.LastWriteTime >= oldDateTime).Take(1).OrderBy(q => q.LastWriteTime);
                    var firstwalrecord = walrecords.FirstOrDefault(q => string.Compare(q.Name, currentFile, true) == 0);

                    foreach (var item in walrecords)
                    {   
                        _walloginfo.lastWriteTime = item.LastWriteTime.Ticks;
                        if (string.Compare(item.Name,_walloginfo.logfile,true) != 0)
                        {
                            _walloginfo.logfile = item.Name;
                            _walloginfo.streamPoint = 0;
                        }
                        _walloginfo.lastWriteTime += 1;

                        ReadFile(item.Open(FileMode.Open, FileAccess.Read, FileShare.ReadWrite), _walloginfo);
                        File.WriteAllText(currentFile, JsonConvert.SerializeObject(_walloginfo));
                    }
                }
            }
            catch (Exception ex)
            {
                _logger?.LogError(ex.ToString());
            }
        }

        private static ReWALDefine fetchOrCreateDefine(string wallogContent)
        {
            ReWALDefine _walloginfo;
            if (string.IsNullOrWhiteSpace(wallogContent))
            {
                _walloginfo = new ReWALDefine();
            }
            else
            {
                _walloginfo = JsonConvert.DeserializeObject<ReWALDefine>(wallogContent);
            }

            return _walloginfo;
        }private void ReadFile(FileStream fs, ReWALDefine reWALDefine) {
            try
            {
                fs.Position = reWALDefine.streamPoint;
                using (StreamReader sr = new StreamReader(fs, System.Text.Encoding.UTF8))
                {
                    while (sr.Peek() > -1)
                    {
                        Console.WriteLine("T:" + sr.ReadLine());//工作開始
                    }
                    Console.WriteLine(fs.Position);
                    Console.WriteLine(fs.Length);
                    reWALDefine.streamPoint = fs.Length;
                }
            }
            catch (Exception ex)
            {
                _logger?.LogError(ex.ToString());
            }
        }
    }

 後面看到微軟管道，決定來實現一個同樣的。

copy微軟的代碼，修改之後，如下:

static async Task ProcessMessagesAsync(
        PipeReader reader)
        {
            try
            {
                while (true)
                {
                    ReadResult readResult = await reader.ReadAsync();
                    ReadOnlySequence<byte> buffer = readResult.Buffer;

                    try
                    {
                        if (readResult.IsCanceled)
                        {
                            break;
                        }

                        if (TryParseLines(ref buffer, out string message))
                        {
                            Console.WriteLine(buffer.Start.GetInteger().ToString()+" "+buffer.End.GetInteger().ToString());
                            Console.WriteLine($"{message}");
                        }

                        if (readResult.IsCompleted)
                        {
                            if (!buffer.IsEmpty)
                            {
                                throw new InvalidDataException("Incomplete message.");
                            }
                            break;
                        }
                    }
                    finally
                    {
                        reader.AdvanceTo(buffer.Start, buffer.End);
                    }
                }
            }
            catch (Exception ex)
            {
                Console.Error.WriteLine(ex);
            }
            finally
            {
                await reader.CompleteAsync();
            }
        }

        static bool TryParseLines(
            ref ReadOnlySequence<byte> buffer,
            out string message)
        {
            SequencePosition? position;
            StringBuilder outputMessage = new StringBuilder();

            while (true)
            {
                position = buffer.PositionOf((byte)'\n');

                if (!position.HasValue)
                    break;

                outputMessage.Append(Encoding.UTF8.GetString(buffer.Slice(buffer.Start, position.Value).ToArray()))
                            .AppendLine();

                buffer = buffer.Slice(buffer.GetPosition(1, position.Value));
            };

            message = outputMessage.ToString();
            return message.Length != 0;
        }

        PipeReader reader = null;
        private async Task ReadFile2Async(FileStream fs, ReWALDefine reWALDefine)
        {
            try
            {

                fs.Position = reWALDefine.streamPoint;
                reader = PipeReader.Create(fs,new StreamPipeReaderOptions(leaveOpen:true));
                await ProcessMessagesAsync(reader);
                Console.WriteLine(fs.Position);
                Console.WriteLine(fs.Length);
                reWALDefine.streamPoint = fs.Position;
            }
            catch (Exception ex)
            {
                _logger?.LogError(ex.ToString());
            }
        }

相應調用的代碼如下:

 protected override async Task ExecuteAsync(CancellationToken stoppingToken)
        {
            try
            {
                if (!File.Exists(currentFile))
                {
                    File.WriteAllText(currentFile, string.Empty);
                }

                while(!stoppingToken.IsCancellationRequested)
                {
                    var wallogContent = File.ReadAllText(currentFile);
                    ReWALDefine _walloginfo = fetchOrCreateDefine(wallogContent);
                    await Task.Delay(100);
                    DateTime oldDateTime = new DateTime(_walloginfo.lastWriteTime);
                    var walrecords = _directoryInfo.GetFiles("*.log").Where(q => q.LastWriteTime >= oldDateTime).Take(1).OrderBy(q => q.LastWriteTime);
                    var firstwalrecord = walrecords.FirstOrDefault(q => string.Compare(q.Name, currentFile, true) == 0);

                    foreach (var item in walrecords)
                    {   
                        _walloginfo.lastWriteTime = item.LastWriteTime.Ticks;
                        if (string.Compare(item.Name,_walloginfo.logfile,true) != 0)
                        {
                            _walloginfo.logfile = item.Name;
                            _walloginfo.streamPoint = 0;
                        }
                        _walloginfo.lastWriteTime += 1;

                        await ReadFile2Async(item.Open(FileMode.Open, FileAccess.Read, FileShare.ReadWrite), _walloginfo);
                        File.WriteAllText(currentFile, JsonConvert.SerializeObject(_walloginfo));
                    }
                }

                reader?.CancelPendingRead();


            }
            catch (Exception ex)
            {
                _logger?.LogError(ex.ToString());
            }
        }