100行代码实现了多线程,批量写入,文件分块的日志方法
一，您选择用什么样的日志组件
日志组件，不得不提大名鼎鼎的Log4Net。比较常用的还有 Enterprise Library Logging，ServiceStack Logging。当然您还可以补充，我就只用过这几款。
上边提到的3款日志组件，都要在.config里加代码，特别是Log4Net，还要把SQL写在配置里。我就是仅仅只写个日志，还要配置这么多信息，让人略有不爽。
所以在很长一段时间里，我用下边这个方法写日志：
private static void WriteText(string logPath, string logContent) { try { if (!File.Exists(logPath)) { File.CreateText(logPath).Close(); } StreamWriter sw = File.AppendText(logPath); sw.Write(logContent); sw.Close(); } catch (Exception ex) { } finally { } }
这个方法足够的简单，核心代码就只有那么5，6行，还包含容错机制。我就喜欢用这种简单的代码来处理简单的事。
二，多线程下引爆了问题
在多线程的情况下，比如100个线程同时需要写日志，上边提到的这个方法就力不从心了。
一个线程访问日志资源，另一个线程再去访问的时候，就会出现异常。
方法一：
public static Object _processLock = new Object(); private void Button_Click_1(object sender, RoutedEventArgs e) { lock (_processLock) { } }
方法二：
public static Object _processLock = new Object(); private void Button_Click_1(object sender, RoutedEventArgs e) { Monitor.Enter(_processLock); Monitor.Exit(_processLock); }
这样，你不得不承认，我已经解决了多线程的问题。
但是有瓶颈，这些需要写日志的线程，必须要等前一个释放了锁资源，后一个线程才能访问的情况。
三，重新设计日志组件
先看图，再说一下我的思路：
1，不管有多少线程同时需要写日志，我都用一个临时队列来存放这些日志信息。
2，再启用一个Task任务把队列的日志批量存放到.log文件里。
3，附加一个小功能，每个日志存储的大小限制，当日志太大了，查看打开的时候比较慢。
四，具体的代码实现
1，在多线程的情况下，我们首先把日志压到Queue队列里
static ConcurrentQueue<Tuple<string, string>> logQueue = new ConcurrentQueue<Tuple<string, string>>();
在这儿，我为什么选用 ConcurrentQueue 而不是 Queue 。因为ConcurrentQueue 表示线程安全的先进先出 (FIFO) 集合。
当然你一定要用Queue也是可以的，但是要自己去实现锁机制，何必自找麻烦呢？
2，把日志队列里的数据，批量持久化到.log文件里
这个问题，让我很头大。我最开始的方法是：
持久化日志方法一：
a，申明一个Task任务，当我Task任务没有实现化时，先实例化，然后再进行持久化日志写入。
b，当我的Task任务，已经实例化了，并且是处于 IsCompleted 状态，我重新实例化Task，再进行持久化日志的写入。
static Task writeTask = default(Task);
public static void WriteLog(String customDirectory, String preFile, String infoData)
{
string logPath = GetLogPath(customDirectory, preFile);
string logContent = String.Concat(DateTime.Now, " ", infoData);
logQueue.Enqueue(new Tuple<string, string>(logPath, logContent));
if (writeTask == null)
{
writeTask = new Task((object obj) =>
{
//pause.WaitOne(1000, true);
LogRepository();
}
, null
, TaskCreationOptions.LongRunning);
writeTask.Start();
}
if (writeTask.IsCompleted)
{
writeTask = new Task((object obj) =>
{
//pause.WaitOne(1000, true);
LogRepository();
}
, null
, TaskCreationOptions.LongRunning);
writeTask.Start();
}
}
异常信息：
理论是那么的美好，但是现实是那么残酷，当我跑单元测试的时候，一段时间后总是抛出如下错误。如果是有那位朋友知道其原因，把这个问题解决就完美了。
但是我不能因为这一个异常，导致我这个组件写不下去吧！活人不能被一泡尿给憋死。
追加：完整代码如下：
public class IOExtention { static ConcurrentQueue<Tuple<string, string>> logQueue = new ConcurrentQueue<Tuple<string, string>>(); static volatile Task writeTask = default(Task); static IOExtention() { } public static void WriteLog(String preFile, String infoData) { WriteLog(string.Empty, preFile, infoData); } static AutoResetEvent pause = new AutoResetEvent(false); public static void WriteLog(String customDirectory, String preFile, String infoData) { string logPath = GetLogPath(customDirectory, preFile); string logContent = String.Concat(DateTime.Now, " ", infoData); logQueue.Enqueue(new Tuple<string, string>(logPath, logContent)); if (writeTask == null) { writeTask = new Task((object obj) => { //pause.WaitOne(1000, true); LogRepository(); } , null , TaskCreationOptions.LongRunning); writeTask.Start(); } if (writeTask.IsCompleted) { writeTask = new Task((object obj) => { //pause.WaitOne(1000, true); LogRepository(); } , null , TaskCreationOptions.LongRunning); writeTask.Start(); } } public static void LogRepository() { List<string[]> temp = new List<string[]>(); foreach (var logItem in logQueue) { string logPath = logItem.Item1; string logMergeContent = String.Concat(logItem.Item2, Environment.NewLine, "-----------------------------------------------------------", Environment.NewLine); string[] logArr = temp.FirstOrDefault(d => d[0].Equals(logPath)); if (logArr != null) { logArr[1] = string.Concat(logArr[1], logMergeContent); } else { logArr = new string[] { logPath, logMergeContent }; temp.Add(logArr); } Tuple<string, string> val = default(Tuple<string, string>); logQueue.TryDequeue(out val); } foreach (string[] item in temp) { WriteText(item[0], item[1]); } } private static string GetLogPath(String customDirectory, String preFile) { string newFilePath = string.Empty; String logDir = string.IsNullOrEmpty(customDirectory) ? Path.Combine(Environment.CurrentDirectory, "logs") : customDirectory; if (!Directory.Exists(logDir)) { Directory.CreateDirectory(logDir); } string extension = ".log"; string fileNameNotExt = String.Concat(preFile, DateTime.Now.ToString("yyyyMMdd")); String fileName = String.Concat(fileNameNotExt, extension); string fileNamePattern = string.Concat(fileNameNotExt, "(*)", extension); List<string> filePaths = Directory.GetFiles(logDir, fileNamePattern, SearchOption.TopDirectoryOnly).ToList(); if (filePaths.Count > 0) { int fileMaxLen = filePaths.Max(d => d.Length); string lastFilePath = filePaths.Where(d => d.Length == fileMaxLen).OrderByDescending(d => d).FirstOrDefault(); if (new FileInfo(lastFilePath).Length > 1 * 1024 * 1024) { string no = new Regex(@"(?is)(?<=\()(.*)(?=\))").Match(Path.GetFileName(lastFilePath)).Value; int tempno = 0; bool parse = int.TryParse(no, out tempno); string formatno = String.Format("({0})", parse ? (tempno + 1) : tempno); string newFileName = String.Concat(fileNameNotExt, formatno, extension); newFilePath = Path.Combine(logDir, newFileName); } else { newFilePath = lastFilePath; } } else { string newFileName = String.Concat(fileNameNotExt, String.Format("({0})", 0), extension); newFilePath = Path.Combine(logDir, newFileName); } return newFilePath; } private static void WriteText(string logPath, string logContent) { try { if (!File.Exists(logPath)) { File.CreateText(logPath).Close(); } StreamWriter sw = File.AppendText(logPath); sw.Write(logContent); sw.Close(); } catch (Exception ex) { } finally { } } }
持久化日志方法二：
我采用了另外一种方法，在Task任务里我用信号量的方式来解决了些问题，完整代码如下：
static AutoResetEvent pause = new AutoResetEvent(false);
信号量法：
public class IOExtention { static ConcurrentQueue<Tuple<string, string>> logQueue = new ConcurrentQueue<Tuple<string, string>>(); static Task writeTask = default(Task); static IOExtention() { writeTask = new Task((object obj) => { while (true) { pause.WaitOne(1000, true); List<string[]> temp = new List<string[]>(); foreach (var logItem in logQueue) { string logPath = logItem.Item1; string logMergeContent = String.Concat(logItem.Item2, Environment.NewLine, "-----------------------------------------------------------", Environment.NewLine); string[] logArr = temp.FirstOrDefault(d => d[0].Equals(logPath)); if (logArr != null) { logArr[1] = string.Concat(logArr[1], logMergeContent); } else { logArr = new string[] { logPath, logMergeContent }; temp.Add(logArr); } Tuple<string, string> val = default(Tuple<string, string>); logQueue.TryDequeue(out val); } foreach (string[] item in temp) { WriteText(item[0], item[1]); } } } , null , TaskCreationOptions.LongRunning); writeTask.Start(); } public static void WriteLog(String preFile, String infoData) { WriteLog(string.Empty, preFile, infoData); } static AutoResetEvent pause = new AutoResetEvent(false); public static void WriteLog(String customDirectory, String preFile, String infoData) { string logPath = GetLogPath(customDirectory, preFile); string logContent = String.Concat(DateTime.Now, " ", infoData); logQueue.Enqueue(new Tuple<string, string>(logPath, logContent)); } private static string GetLogPath(String customDirectory, String preFile) { string newFilePath = string.Empty; String logDir = string.IsNullOrEmpty(customDirectory) ? Path.Combine(Environment.CurrentDirectory, "logs") : customDirectory; if (!Directory.Exists(logDir)) { Directory.CreateDirectory(logDir); } string extension = ".log"; string fileNameNotExt = String.Concat(preFile, DateTime.Now.ToString("yyyyMMdd")); String fileName = String.Concat(fileNameNotExt, extension); string fileNamePattern = string.Concat(fileNameNotExt, "(*)", extension); List<string> filePaths = Directory.GetFiles(logDir, fileNamePattern, SearchOption.TopDirectoryOnly).ToList(); if (filePaths.Count > 0) { int fileMaxLen = filePaths.Max(d => d.Length); string lastFilePath = filePaths.Where(d => d.Length == fileMaxLen).OrderByDescending(d => d).FirstOrDefault(); if (new FileInfo(lastFilePath).Length > 1 * 1024 * 1024) { string no = new Regex(@"(?is)(?<=\()(.*)(?=\))").Match(Path.GetFileName(lastFilePath)).Value; int tempno = 0; bool parse = int.TryParse(no, out tempno); string formatno = String.Format("({0})", parse ? (tempno + 1) : tempno); string newFileName = String.Concat(fileNameNotExt, formatno, extension); newFilePath = Path.Combine(logDir, newFileName); } else { newFilePath = lastFilePath; } } else { string newFileName = String.Concat(fileNameNotExt, String.Format("({0})", 0), extension); newFilePath = Path.Combine(logDir, newFileName); } return newFilePath; } private static void WriteText(string logPath, string logContent) { try { if (!File.Exists(logPath)) { File.CreateText(logPath).Close(); } StreamWriter sw = File.AppendText(logPath); sw.Write(logContent); sw.Close(); } catch (Exception ex) { } finally { } } }
持久化日志方法三：
如果你感觉写一个日志类还用什么信号量这些技术，太复杂了，那也可以用最简单的方式，定时器来解决。
有同学一听定时器，就默默的笑了，但是这儿的坑也很深，首先了解一下这几个定时器的使用场合，再用不迟！
System.Windows.Threading.DispatcherTimer
System.Windows.Forms.Timer
System.Timers.Timer
System.Threading.Timer
定时器法：
public class IOExtention { static ConcurrentQueue<Tuple<string, string>> logQueue = new ConcurrentQueue<Tuple<string, string>>(); static System.Timers.Timer logTimers = new System.Timers.Timer(); static IOExtention() { logTimers.Interval = 1000; logTimers.Elapsed += logTimers_Elapsed; logTimers.AutoReset = true; logTimers.Enabled = true; } public static void WriteLog(String preFile, String infoData) { WriteLog(string.Empty, preFile, infoData); } public static void WriteLog(String customDirectory, String preFile, String infoData) { string logPath = GetLogPath(customDirectory, preFile); string logContent = String.Concat(DateTime.Now, " ", infoData); logQueue.Enqueue(new Tuple<string, string>(logPath, logContent)); } private static string GetLogPath(String customDirectory, String preFile) { string newFilePath = string.Empty; String logDir = string.IsNullOrEmpty(customDirectory) ? Path.Combine(Environment.CurrentDirectory, "logs") : customDirectory; if (!Directory.Exists(logDir)) { Directory.CreateDirectory(logDir); } string extension = ".log"; string fileNameNotExt = String.Concat(preFile, DateTime.Now.ToString("yyyyMMdd")); String fileName = String.Concat(fileNameNotExt, extension); string fileNamePattern = string.Concat(fileNameNotExt, "(*)", extension); List<string> filePaths = Directory.GetFiles(logDir, fileNamePattern, SearchOption.TopDirectoryOnly).ToList(); if (filePaths.Count > 0) { int fileMaxLen = filePaths.Max(d => d.Length); string lastFilePath = filePaths.Where(d => d.Length == fileMaxLen).OrderByDescending(d => d).FirstOrDefault(); if (new FileInfo(lastFilePath).Length > 1 * 1024 * 1024) { string no = new Regex(@"(?is)(?<=\()(.*)(?=\))").Match(Path.GetFileName(lastFilePath)).Value; int tempno = 0; bool parse = int.TryParse(no, out tempno); string formatno = String.Format("({0})", parse ? (tempno + 1) : tempno); string newFileName = String.Concat(fileNameNotExt, formatno, extension); newFilePath = Path.Combine(logDir, newFileName); } else { newFilePath = lastFilePath; } } else { string newFileName = String.Concat(fileNameNotExt, String.Format("({0})", 0), extension); newFilePath = Path.Combine(logDir, newFileName); } return newFilePath; } static void logTimers_Elapsed(object sender, System.Timers.ElapsedEventArgs e) { System.Timers.Timer logTimers = (System.Timers.Timer)sender; logTimers.Enabled = false; List<string[]> temp = new List<string[]>(); foreach (var logItem in logQueue) { string logPath = logItem.Item1; string logMergeContent = String.Concat(logItem.Item2, Environment.NewLine, "-----------------------------------------------------------", Environment.NewLine); string[] logArr = temp.FirstOrDefault(d => d[0].Equals(logPath)); if (logArr != null) { logArr[1] = string.Concat(logArr[1], logMergeContent); } else { logArr = new string[] { logPath, logMergeContent }; temp.Add(logArr); } Tuple<string, string> val = default(Tuple<string, string>); logQueue.TryDequeue(out val); } foreach (string[] item in temp) { WriteText(item[0], item[1]); } logTimers.Enabled = true; } private static void WriteText(string logPath, string logContent) { try { if (!File.Exists(logPath)) { File.CreateText(logPath).Close(); } StreamWriter sw = File.AppendText(logPath); sw.Write(logContent); sw.Close(); } catch (Exception ex) { } finally { } } }
五，结语
重新设计的日志组件，思路还是非常清晰的。只是在持久化日志时遇上了问题了。
持久化日志方法一，其实是很完美的解决方法，但是在高并发的时候，总是抛出异常，找不出原因。
持久化日志方法二，是我目前采用的方法，能够有效的解决问题。
持久化日志方法三，采用定时器解决，也是可行的。只是代码看起来很别扭。
欢迎大家热烈讨论，看有没有更好的解决方案。
阿里云客户端的实现(支持文件分块,断点续传,进度,速度,倒计时显示)
信号量改进版：
using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; namespace LogTest { public class IOExtention { static ConcurrentQueue<Tuple<string, string>> logQueue = new ConcurrentQueue<Tuple<string, string>>(); static Task writeTask = default(Task); static ManualResetEvent pause = new ManualResetEvent(false); //Mutex mmm = new Mutex(); static IOExtention() { writeTask = new Task((object obj) => { while (true) { pause.WaitOne(); pause.Reset(); List<string[]> temp = new List<string[]>(); foreach (var logItem in logQueue) { string logPath = logItem.Item1; string logMergeContent = String.Concat(logItem.Item2, Environment.NewLine, "-----------------------------------------------------------", Environment.NewLine); string[] logArr = temp.FirstOrDefault(d => d[0].Equals(logPath)); if (logArr != null) { logArr[1] = string.Concat(logArr[1], logMergeContent); } else { logArr = new string[] { logPath, logMergeContent }; temp.Add(logArr); } Tuple<string, string> val = default(Tuple<string, string>); logQueue.TryDequeue(out val); } foreach (string[] item in temp) { WriteText(item[0], item[1]); } } } , null , TaskCreationOptions.LongRunning); writeTask.Start(); } public static void WriteLog(String preFile, String infoData) { WriteLog(string.Empty, preFile, infoData); } public static void WriteLog(String customDirectory, String preFile, String infoData) { string logPath = GetLogPath(customDirectory, preFile); string logContent = String.Concat(DateTime.Now, " ", infoData); logQueue.Enqueue(new Tuple<string, string>(logPath, logContent)); pause.Set(); } private static string GetLogPath(String customDirectory, String preFile) { string newFilePath = string.Empty; String logDir = string.IsNullOrEmpty(customDirectory) ? Path.Combine(Environment.CurrentDirectory, "logs") : customDirectory; if (!Directory.Exists(logDir)) { Directory.CreateDirectory(logDir); } string extension = ".log"; string fileNameNotExt = String.Concat(preFile, DateTime.Now.ToString("yyyyMMdd")); String fileName = String.Concat(fileNameNotExt, extension); string fileNamePattern = string.Concat(fileNameNotExt, "(*)", extension); List<string> filePaths = Directory.GetFiles(logDir, fileNamePattern, SearchOption.TopDirectoryOnly).ToList(); if (filePaths.Count > 0) { int fileMaxLen = filePaths.Max(d => d.Length); string lastFilePath = filePaths.Where(d => d.Length == fileMaxLen).OrderByDescending(d => d).FirstOrDefault(); if (new FileInfo(lastFilePath).Length > 1 * 1024 * 1024 * 1024) { string no = new Regex(@"(?is)(?<=\()(.*)(?=\))").Match(Path.GetFileName(lastFilePath)).Value; int tempno = 0; bool parse = int.TryParse(no, out tempno); string formatno = String.Format("({0})", parse ? (tempno + 1) : tempno); string newFileName = String.Concat(fileNameNotExt, formatno, extension); newFilePath = Path.Combine(logDir, newFileName); } else { newFilePath = lastFilePath; } } else { string newFileName = String.Concat(fileNameNotExt, String.Format("({0})", 0), extension); newFilePath = Path.Combine(logDir, newFileName); } return newFilePath; } private static void WriteText(string logPath, string logContent) { try { if (!File.Exists(logPath)) { File.CreateText(logPath).Close(); } StreamWriter sw = File.AppendText(logPath); sw.Write(logContent); sw.Close(); } catch (Exception ex) { } finally { } } } }
