庆军之简单wal系统的实现记录
要解决的问题:
我有一个系统，但是数据库表示它不稳定了，我需要对外高可用。数据库不能用的时候，总不能不用吧。所以，只能最终一致性。
临时的数据存在文件wal里面，由另一个线程单独处理。
原来是打算用 faster来实现。但是我表示说，文档很好。但是我不会用。我太菜。
最后某网友建议用日志来处理。我写了两天。终于是简单的凑出来了。
我用到了Serilog来写wal.因为不会写它的过滤条件。搞不来轮子。所以指定了 Fatal级别，也就是枚举里面为6的等级来记录到另一个目录下。
具体的配置如下。
//appsettings.json 中
1 "WriteTo": [ 2 //{ "Name": "Console" }, 3 { 4 "Name": "File", 5 "Args": { 6 "path": "Log2s\\log.txt", 7 "rollingInterval": "Day", 8 "fileSizeLimitBytes": "10485760", 9 "retainedFileCountLimit": 5, 10 "rollOnFileSizeLimit": true 11 } 12 }, 13 { 14 "Name": "Logger", 15 "Args": { 16 "configureLogger": { 17 "Filter": [ 18 { 19 "Name": "ByIncludingOnly", 20 "Args": { 21 "expression": "@l = 'Fatal'" 22 } 23 } 24 ], 25 "WriteTo": [ 26 { 27 "Name": "File", 28 "Args": { 29 "outputTemplate": "[{Timestamp:yyyy-MM-dd HH:mm:ss.fff}] {Message:lj}{NewLine}", 30 //"outputTemplate": "[{Timestamp:g} {Level:u3}] [{SourceContext}] {Message:lj}{NewLine}", 31 "path": "wal\\wal.log", 32 "rollingInterval": "Day", 33 "fileSizeLimitBytes": "10485760", 34 "retainedFileCountLimit": 200, 35 "rollOnFileSizeLimit": true, 36 "formatter": "Serilog.Formatting.Json.JsonFormatter", 37 "flushToDiskInterval": "00:00:01"//刷盘时间 38 } 39 } 40 ] 41 } 42 } 43 } 44 ]
代码调用
_logger?.LogCritical( "{@model}", model);
解析类:
public class ReWALService : BackgroundService { public class ReWALDefine { public long lastWriteTime { get; set; }//文件最后写入的时间,解析的时候 public string logfile { get; set; }//文件名称 public long streamPoint { get; set; }//在文件内容的位置 } private readonly ILogger<ReWALService> _logger; private string currentFile = ""; private readonly DirectoryInfo _directoryInfo; public ReWALService(ILoggerFactory loggerFactory) { this._logger = loggerFactory.CreateLogger<ReWALService>(); var path = Path.Combine(Directory.GetCurrentDirectory(), "wal"); _directoryInfo = new DirectoryInfo(path); //wallog currentFile = Path.Combine( path,"wallog.txt"); } protected override async Task ExecuteAsync(CancellationToken stoppingToken) { try { if (!File.Exists(currentFile)) { File.WriteAllText(currentFile, string.Empty); } while(!stoppingToken.IsCancellationRequested) { var wallogContent = File.ReadAllText(currentFile); ReWALDefine _walloginfo = fetchOrCreateDefine(wallogContent); await Task.Delay(100); DateTime oldDateTime = new DateTime(_walloginfo.lastWriteTime); var walrecords = _directoryInfo.GetFiles("*.log").Where(q => q.LastWriteTime >= oldDateTime).Take(1).OrderBy(q => q.LastWriteTime); var firstwalrecord = walrecords.FirstOrDefault(q => string.Compare(q.Name, currentFile, true) == 0); foreach (var item in walrecords) { _walloginfo.lastWriteTime = item.LastWriteTime.Ticks; if (string.Compare(item.Name,_walloginfo.logfile,true) != 0) { _walloginfo.logfile = item.Name; _walloginfo.streamPoint = 0; } _walloginfo.lastWriteTime += 1; ReadFile(item.Open(FileMode.Open, FileAccess.Read, FileShare.ReadWrite), _walloginfo); File.WriteAllText(currentFile, JsonConvert.SerializeObject(_walloginfo)); } } } catch (Exception ex) { _logger?.LogError(ex.ToString()); } } private static ReWALDefine fetchOrCreateDefine(string wallogContent) { ReWALDefine _walloginfo; if (string.IsNullOrWhiteSpace(wallogContent)) { _walloginfo = new ReWALDefine(); } else { _walloginfo = JsonConvert.DeserializeObject<ReWALDefine>(wallogContent); } return _walloginfo; }private void ReadFile(FileStream fs, ReWALDefine reWALDefine) { try { fs.Position = reWALDefine.streamPoint; using (StreamReader sr = new StreamReader(fs, System.Text.Encoding.UTF8)) { while (sr.Peek() > -1) { Console.WriteLine("T:" + sr.ReadLine());//工作开始 } Console.WriteLine(fs.Position); Console.WriteLine(fs.Length); reWALDefine.streamPoint = fs.Length; } } catch (Exception ex) { _logger?.LogError(ex.ToString()); } } }
后面看到微软管道，决定来实现一个同样的。
copy微软的代码，修改之后，如下:
static async Task ProcessMessagesAsync( PipeReader reader) { try { while (true) { ReadResult readResult = await reader.ReadAsync(); ReadOnlySequence<byte> buffer = readResult.Buffer; try { if (readResult.IsCanceled) { break; } if (TryParseLines(ref buffer, out string message)) { Console.WriteLine(buffer.Start.GetInteger().ToString()+" "+buffer.End.GetInteger().ToString()); Console.WriteLine($"{message}"); } if (readResult.IsCompleted) { if (!buffer.IsEmpty) { throw new InvalidDataException("Incomplete message."); } break; } } finally { reader.AdvanceTo(buffer.Start, buffer.End); } } } catch (Exception ex) { Console.Error.WriteLine(ex); } finally { await reader.CompleteAsync(); } } static bool TryParseLines( ref ReadOnlySequence<byte> buffer, out string message) { SequencePosition? position; StringBuilder outputMessage = new StringBuilder(); while (true) { position = buffer.PositionOf((byte)'\n'); if (!position.HasValue) break; outputMessage.Append(Encoding.UTF8.GetString(buffer.Slice(buffer.Start, position.Value).ToArray())) .AppendLine(); buffer = buffer.Slice(buffer.GetPosition(1, position.Value)); }; message = outputMessage.ToString(); return message.Length != 0; } PipeReader reader = null; private async Task ReadFile2Async(FileStream fs, ReWALDefine reWALDefine) { try { fs.Position = reWALDefine.streamPoint; reader = PipeReader.Create(fs,new StreamPipeReaderOptions(leaveOpen:true)); await ProcessMessagesAsync(reader); Console.WriteLine(fs.Position); Console.WriteLine(fs.Length); reWALDefine.streamPoint = fs.Position; } catch (Exception ex) { _logger?.LogError(ex.ToString()); } }
相应调用的代码如下:
protected override async Task ExecuteAsync(CancellationToken stoppingToken) { try { if (!File.Exists(currentFile)) { File.WriteAllText(currentFile, string.Empty); } while(!stoppingToken.IsCancellationRequested) { var wallogContent = File.ReadAllText(currentFile); ReWALDefine _walloginfo = fetchOrCreateDefine(wallogContent); await Task.Delay(100); DateTime oldDateTime = new DateTime(_walloginfo.lastWriteTime); var walrecords = _directoryInfo.GetFiles("*.log").Where(q => q.LastWriteTime >= oldDateTime).Take(1).OrderBy(q => q.LastWriteTime); var firstwalrecord = walrecords.FirstOrDefault(q => string.Compare(q.Name, currentFile, true) == 0); foreach (var item in walrecords) { _walloginfo.lastWriteTime = item.LastWriteTime.Ticks; if (string.Compare(item.Name,_walloginfo.logfile,true) != 0) { _walloginfo.logfile = item.Name; _walloginfo.streamPoint = 0; } _walloginfo.lastWriteTime += 1; await ReadFile2Async(item.Open(FileMode.Open, FileAccess.Read, FileShare.ReadWrite), _walloginfo); File.WriteAllText(currentFile, JsonConvert.SerializeObject(_walloginfo)); } } reader?.CancelPendingRead(); } catch (Exception ex) { _logger?.LogError(ex.ToString()); } }
