自己最近写的一组日志处理类(支持高并发处理)
2012-01-21 21:55 java线程例子 阅读(365) 评论(0) 编辑 收藏 举报下面的日志处理我在3000个线程同时调用,写20个文件,相当于每个文件150个线程不间断大规模写.没问题,我提高到10000时,出了问题,但不是日志代码的问题,而是系统创建新的线程时报错,建到3500个线程的时候出现问题.后面我会贴出测试代码.
1)FileLogWorker
/// <summary> /// 文件日志处理类,利用队列机制,让写日志调用和日志写到文件分离,调用 /// 方将要写的日志和目标文件插入到日志队列中去就返回, /// 然后由内置线程去写到文件里去。这里用了单例模式。 /// </summary> public class FileLogWorker { private Queue<string> _logCaches = null; private Thread _writerThread = null; private StreamWriter _streamWriter; public string FilePath {get;private set;} public Thread LogWriteThread { get { return _writerThread; } } public FileLogWorker(string FilePath) { this.FilePath = FilePath; _logCaches = new Queue<string>(); _writerThread = new Thread(new ThreadStart(WriteLogToFile)); FStop = false; _streamWriter = new StreamWriter(FilePath,true); _writerThread.Start(); } private bool FStop = false; private void WriteLogToFile() { Int64 theCount = 0; while (FStop == false) { try { theCount++; string theLogContent = null; //队列操作时需要锁定,否则会报错.队列并不是线程安全的. //但多个队列可以同时写. lock (this) { theLogContent = _logCaches.Dequeue(); } if (theLogContent != null && theLogContent != "") { _streamWriter.WriteLine(theLogContent); _streamWriter.Flush(); LastExecTime = DateTime.Now; } if (theCount > 10000) { //GC.Collect(); theCount = 0; } } catch (Exception ex) { SysAppEventWriter.WriteEvent(-1, ex.Message, System.Diagnostics.EventLogEntryType.Error); } } } /// <summary> /// 向文件日志写日志内容 /// </summary> /// <param name="fileName">日志文件名</param> /// <param name="logContent">日志内容</param> public void WriteLogContent(string logContent) { lock (this) { _logCaches.Enqueue(logContent); } if (_logCaches.Count > 10000) { Thread.CurrentThread.Join(50); } } private DateTime _LastExecTime = DateTime.Now.AddDays(-1); public DateTime LastExecTime { get { return _LastExecTime; } set { _LastExecTime = value; } } /// <summary> /// 结束日志内置线程,并关闭所有文件流。程序正常退出时调用. /// </summary> public void CloseLogThread() { FStop = true; if (_writerThread != null) { _writerThread.Join(); } try { _streamWriter.Flush(); _streamWriter.Close(); _streamWriter.Dispose(); } catch (Exception ex) { SysAppEventWriter.WriteEvent(-1, ex.Message, System.Diagnostics.EventLogEntryType.Error); } } }
2)
/// <summary> /// 文件日志处理类,利用队列机制,让写日志调用和日志写到文件分离,调用 /// 方将要写的日志和目标文件插入到日志队列中去就返回, /// 然后由内置线程去写到文件里去。这里用了单例模式。 /// </summary> public class FileLogWriter : IDisposable { private class LocalLockObject { } private Dictionary<string, FileLogWorker> _Workers = null; private Thread _writerThread = null; static FileLogWriter() { Instance = new FileLogWriter(); } private FileLogWriter() { _Workers = new Dictionary<string, FileLogWorker>(); _writerThread = new Thread(new ThreadStart(Execute)); FStop = false; _writerThread.Start(); } private FileLogWorker GetFileLogWorker(string FullPath) { try { return _Workers[FullPath]; } catch { lock (typeof(LocalLockObject)) { if (_Workers.ContainsKey(FullPath) == false) { FileLogWorker theWorker = new FileLogWorker(FullPath); _Workers.Add(FullPath, theWorker); return theWorker; } else { return _Workers[FullPath]; } } } } private bool FStop = false; public static FileLogWriter Instance { get; private set; } private const int Timeout = 30; /// <summary> /// 向文件日志写日志内容 /// </summary> /// <param name="fileName">日志文件名</param> /// <param name="logContent">日志内容</param> public void WriteLogFile(string fileName, string logContent) { FileLogWorker theWorker = GetFileLogWorker(fileName); theWorker.WriteLogContent(logContent); } /// <summary> /// 清理20分钟没有发生读写的文件流。对于日志文件按天产生的非常有用。 /// </summary> public void Execute() { while (FStop == false) { try { lock (typeof(LocalLockObject)) { foreach (var theItem in _Workers) { if (DateTime.Now.Subtract(theItem.Value.LastExecTime).Minutes > Timeout) { theItem.Value.CloseLogThread(); _Workers.Remove(theItem.Key); } } } Thread.Sleep(100000); } catch (Exception ex) { SysAppEventWriter.WriteEvent(-1, ex.Message, System.Diagnostics.EventLogEntryType.Error); } } } /// <summary> /// 结束日志内置线程,并关闭所有文件流。程序正常退出时调用. /// </summary> public void CloseAll() { FStop = true; if (_writerThread != null) { _writerThread.Join(); } try { lock (typeof(LocalLockObject)) { if (_Workers != null) { foreach (var theItem in _Workers) { theItem.Value.CloseLogThread(); _Workers.Remove(theItem.Key); } } } } catch (Exception ex) { SysAppEventWriter.WriteEvent(-1, ex.Message, System.Diagnostics.EventLogEntryType.Error); } } public void Dispose() { try { CloseAll(); } catch { } } ~FileLogWriter() { try { CloseAll(); } catch { } } }
3)SysAppEventWriter
public static class SysAppEventWriter { public static void WriteEvent(long EventId,string EventContent,EventLogEntryType EntryType) { try { System.Diagnostics.EventInstance theEvtInst = new System.Diagnostics.EventInstance(EventId, 0, EntryType); System.Diagnostics.EventLog.WriteEvent(AppCfgs.CurrentAppCenterID + "KeDuoSysLogs",theEvtInst, EventContent,AppCfgs.ServiceBaseAddress); } catch { } } } }
4) 文件名从这个类取,当然也可以把方法封装
/// <summary> /// 系统常量性文件名规范 /// </summary> public static class SysConstFileName { /// <summary> /// 系统日志文件名 /// </summary> public static string SysLogFileName(string MerchantId) { return System.AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "logs\\sys" + MerchantId + DateTime.Now.ToString("yyyy-MM-dd") + ".log"; } /// <summary> /// 系统跟踪日志名 /// </summary> public static string TraceLogFileName(string MerchantId) { return System.AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "logs\\tra" + MerchantId + DateTime.Now.ToString("yyyy-MM-dd") + ".log"; } /// <summary> /// 系统事件日志文件名 /// </summary> public static string EventLogFileName(string MerchantId) { return System.AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "logs\\evt" + MerchantId + DateTime.Now.ToString("yyyy-MM-dd") + ".log"; } /// <summary> /// 系统事件日志文件名 /// </summary> public static string SysOptLogFileName(string MerchantId) { return System.AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "logs\\opt" + MerchantId + DateTime.Now.ToString("yyyy-MM-dd") + ".log"; } /// <summary> /// 系统登录日志文件名 /// </summary> public static string SysLoginFileName(string MerchantId) { return System.AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "logs\\login" + MerchantId + DateTime.Now.ToString("yyyy-MM-dd") + ".log"; } }
下面是测试出问题的时候的代码:
int theCount = 10000;
for (int i = 0; i < theCount; i++)
{
new Thread((o) =>
{
int theI = (int)o;
string file = "d:\\a"+( theI % 20) .ToString()+".txt";
for (int j = 0; j < 1000000; j++)
{
FileLogWriter.Instance.WriteLogFile(file, theI.ToString() + ":" + j.ToString());
}
FileLogWriter.Instance.WriteLogFile("d:\\a.txt", theI.ToString() + "结束!");
}).Start(i);
}
作为webApp的日志类,其实用不着这么高的并发,因为IIS的web应用,并不是多线程的,而是单线程的。
补充1:windows对单个进程所能创建的线程还是有一定限制的,否则windows的线程调度处理的压力会非常重。具体的等有机会再测试一下.