C#开源磁盘/内存缓存引擎
2013-07-04 14:33 灵感之源 阅读(6434) 评论(16) 编辑 收藏 举报前言
昨天写了个 《基于STSdb和fastJson的磁盘/内存缓存》,大家可以先看看。下午用到业务系统时候,觉得可以改进一下,昨晚想了一个晚上,刚才重新实现一下。
更新
1. 增加了对批量处理的支持,写操作速度提升5倍,读操作提升100倍
2. 增加了一个存储provider,可以选择不用STSdb做存储,而用物理文件/Dictionary。
3. 增加了空间回收
4. 增加了对并发的支持
需求
业务系统用的是数据库,数据量大,部分只读或相对稳定业务查询复杂,每次页面加载都要花耗不少时间(不讨论异步),觉得可以做一下高速缓存,譬如用nosql那种key/value快速存取结果
目的
提供一个简单易用的解决缓存方案,可以根据数据的大小缓存到内存或者磁盘。
实现
存取
方法1. 基于STSdb,提供高效的Key/Value存取,支持磁盘/内存,对Key无限制
方法2. 基于直接物理文件/Dictionary。Key必须是基本类型,譬如int/long/uint/ulong/DateTime/string等。
代码
代码比较简单,花了2个小时写的,很多情况没考虑,譬如磁盘空间/内存不足,自动回收过期缓存等,这些留给大家做家庭作业吧。另外,为了发布方便,STSdb和fastJson的代码都合并到一个项目里。
BaseCahce.cs
这是一个抽象基类,提供存取接口。
using System; using System.Collections.Generic; using System.Linq; using System.Text; namespace Com.SuperCache.Engine { public abstract class BaseCache { protected internal const string KeyExpiration = "Expiration"; public abstract void Add<K>(string Category, K Key, object Data); public abstract void Add<K, V>(string Category, IEnumerable<KeyValuePair<K, V>> Items, DateTime? ExpirationDate); public abstract void Add<K>(string Category, K Key, object Data, DateTime? ExpirationDate); public abstract List<KeyValuePair<K, V>> Get<K, V>(string Category, IEnumerable<K> Keys); public abstract V Get<K, V>(string Category, K Key); public abstract void Recycle<K>( string Category, long Count); } }
CahceEngine.cs
主要调用缓存引擎
using System; using System.Collections.Generic; using System.Linq; using System.Text; using STSdb4.Database; using fastJSON; using System.IO; namespace Com.SuperCache.Engine { public enum CacheProviders { Default = 1, Raw = 2 } public enum RecycleAlgorithms { None = 0, //LRU = 1, MRU = 2 } public enum RecycleModes { None = 0, Passive = 1, Active = 2 } public class CacheEngine { private BaseCache cacheProvider = null; public CacheEngine(string DataPath) : this(CacheProviders.Default, DataPath, RecycleAlgorithms.None, 0, 0, RecycleModes.None) { } public CacheEngine(CacheProviders Provider, string DataPath, RecycleAlgorithms RecycleAlgorithm, int MaxCount, int Threshold, RecycleModes RecycleMode) { switch (Provider) { case CacheProviders.Default: cacheProvider = new STSdbCache(DataPath, RecycleAlgorithm, MaxCount, Threshold, RecycleMode); break; case CacheProviders.Raw: cacheProvider = new RawCache(DataPath, RecycleAlgorithm, MaxCount, Threshold, RecycleMode); break; default: break; } } public void Add<K>(string Category, K Key, object Data) { cacheProvider.Add<K>(Category, Key, Data); } public void Add<K, V>(string Category, IEnumerable<KeyValuePair<K, V>> Items, DateTime? ExpirationDate) { cacheProvider.Add<K, V>(Category, Items, ExpirationDate); } public void Add<K>(string Category, K Key, object Data, DateTime? ExpirationDate) { cacheProvider.Add<K>(Category, Key, Data, ExpirationDate); } public List<KeyValuePair<K, V>> Get<K, V>(string Category, IEnumerable<K> Keys) { return cacheProvider.Get<K, V>(Category, Keys); } public V Get<K, V>(string Category, K Key) { return cacheProvider.Get<K, V>(Category, Key); } } }
STSdbCache.cs
STSdb存储引擎
using System; using System.Collections.Generic; using System.Linq; using System.Text; using STSdb4.Database; using fastJSON; using System.IO; namespace Com.SuperCache.Engine { public class STSdbCache : BaseCache { private const string UsageStat = "SuperCacheUsageStat"; private string dataPath; private static IStorageEngine memoryInstance = null; private static object syncRoot = new object(); private bool isMemory = false; private RecycleAlgorithms recycleAlgorithm; private int maxCount; private int threshold; private RecycleModes recycleMode; public STSdbCache(string DataPath, RecycleAlgorithms RecycleAlgorithm, int MaxCount, int Threshold, RecycleModes RecycleMode) { dataPath = DataPath; if (!dataPath.EndsWith(Path.DirectorySeparatorChar.ToString())) dataPath += Path.DirectorySeparatorChar; isMemory = string.IsNullOrEmpty(DataPath); recycleAlgorithm = RecycleAlgorithm; maxCount = MaxCount; threshold = Threshold; recycleMode = RecycleMode; } public override void Add<K>(string Category, K Key, object Data) { Add(Category, Key, Data, null); } private IStorageEngine Engine { get { if (isMemory) { lock (syncRoot) { if (memoryInstance == null) memoryInstance = STSdb.FromMemory(); } return memoryInstance; } else return STSdb.FromFile(GetFile(false), GetFile(true)); } } private string GetExpirationTable(string Category) { return KeyExpiration + "_" + Category; } public override void Add<K, V>(string Category, IEnumerable<KeyValuePair<K, V>> Items, DateTime? ExpirationDate) { long count = 0; lock (syncRoot) { var engine = Engine; var table = engine.OpenXIndex<K, string>(Category); var expiration = engine.OpenXIndex<K, DateTime>(GetExpirationTable(Category)); //track recycle IIndex<K, int> usage = null; if (recycleAlgorithm != RecycleAlgorithms.None) usage = engine.OpenXIndex<K, int>(UsageStat); Items.ForEach(i => { var key = i.Key; var data = i.Value; //will only serialize object other than string var result = typeof(V) == typeof(string) ? data as string : JSON.Instance.ToJSON(data); table[key] = result; table.Flush(); //specify expiration //default 30 mins to expire from now var expirationDate = ExpirationDate == null || ExpirationDate <= DateTime.Now ? DateTime.Now.AddMinutes(30) : (DateTime)ExpirationDate; expiration[key] = expirationDate; expiration.Flush(); //track recycle if (usage != null) { usage[key] = 0; if (recycleMode == RecycleModes.Active) Recycle<K>(Category, usage.Count()); } }); if (usage != null) count = usage.Count(); engine.Commit(); //only dispose disk-based engine if (!isMemory) engine.Dispose(); } if (recycleMode == RecycleModes.Passive) Recycle<K>(Category, count); } public override void Add<K>(string Category, K Key, object Data, DateTime? ExpirationDate) { Add<K, object>(Category, new List<KeyValuePair<K, object>> { new KeyValuePair<K, object>(Key, Data) }, ExpirationDate); } private string GetFile(bool IsData) { if (!Directory.Exists(dataPath)) Directory.CreateDirectory(dataPath); return dataPath + "SuperCache." + (IsData ? "dat" : "sys"); } public override List<KeyValuePair<K, V>> Get<K, V>(string Category, IEnumerable<K> Keys) { var result = new List<KeyValuePair<K, V>>(); lock (syncRoot) { var engine = Engine; var table = engine.OpenXIndex<K, string>(Category); var expiration = engine.OpenXIndex<K, DateTime>(GetExpirationTable(Category)); var isCommitRequired = false; //track recycle IIndex<K, int> usage = null; if (recycleAlgorithm != RecycleAlgorithms.None) usage = engine.OpenXIndex<K, int>(UsageStat); Keys.ForEach(key => { string buffer; V value; if (table.TryGet(key, out buffer)) { //will only deserialize object other than string value = typeof(V) == typeof(string) ? (V)(object)buffer : JSON.Instance.ToObject<V>(buffer); bool needUpdate = true; DateTime expirationDate; //get expiration date if (expiration.TryGet(key, out expirationDate)) { //expired if (expirationDate < DateTime.Now) { value = default(V); table.Delete(key); table.Flush(); expiration.Delete(key); expiration.Flush(); isCommitRequired = true; needUpdate = false; } } //track recycle if (usage != null && needUpdate) { usage[key]++; isCommitRequired = true; } } else value = default(V); result.Add(new KeyValuePair<K, V>(key, value)); }); //only need to commit write actions if (isCommitRequired) engine.Commit(); //only dispose disk-based engine if (!isMemory) engine.Dispose(); } return result; } public override V Get<K, V>(string Category, K Key) { var buffer = Get<K, V>(Category, new K[] { Key }); var result = buffer.FirstOrDefault(); return result.Value; } public override void Recycle<K>(string Category, long Count) { if (Count < maxCount) return; switch (recycleAlgorithm) { case RecycleAlgorithms.MRU: lock (syncRoot) { var engine = Engine; var table = engine.OpenXIndex<K, string>(Category); var expiration = engine.OpenXIndex<K, DateTime>(GetExpirationTable(Category)); var usage = engine.OpenXIndex<K, int>(UsageStat); //find out expired items var expired = expiration.Where(e => e.Value < DateTime.Now); expired.ForEach(e => { table.Delete(e.Key); expiration.Delete(e.Key); usage.Delete(e.Key); }); //find out least used items var leastUsed = usage.OrderByDescending(s => s.Value).Skip(maxCount - threshold); leastUsed.ForEach(u => { table.Delete(u.Key); expiration.Delete(u.Key); usage.Delete(u.Key); }); table.Flush(); expiration.Flush(); usage.Flush(); engine.Commit(); if (!isMemory) engine.Dispose(); } break; default: break; } } } }
RawCache.cs
物理文件/Dictionary引擎
using System; using System.Collections.Generic; using System.Linq; using System.Text; using STSdb4.Database; using fastJSON; using System.IO; namespace Com.SuperCache.Engine { public class RawCache : BaseCache { private const string ExpirationFileExtension = "exp"; private const string DataFileExtension = "dat"; private const string statFile = "SuperCache.sta"; private string dataPath; private static Dictionary<string, object> memoryData = new Dictionary<string, object>(); private static Dictionary<string, DateTime?> memoryExpiration = new Dictionary<string, DateTime?>(); private static object syncRoot = new object(); private bool isMemory = false; private RecycleAlgorithms recycleAlgorithm; private int maxCount; private int threshold; private static Dictionary<string, KeyValue> usageStat = new Dictionary<string, KeyValue>(); private Dictionary<string, KeyValuePair<DateTime, string>> expiredFiles = new Dictionary<string, KeyValuePair<DateTime, string>>(); private RecycleModes recycleMode; public RawCache(string DataPath, RecycleAlgorithms RecycleAlgorithm, int MaxCount, int Threshold, RecycleModes RecycleMode) { dataPath = DataPath; if (!dataPath.EndsWith(Path.DirectorySeparatorChar.ToString())) dataPath += Path.DirectorySeparatorChar; isMemory = string.IsNullOrEmpty(DataPath); recycleAlgorithm = RecycleAlgorithm; maxCount = MaxCount; threshold = Threshold; recycleMode = RecycleMode; } public override void Add<K>(string Category, K Key, object Data) { Add(Category, Key, Data, null); } private string GetExpirationTable(string Category) { return KeyExpiration + "_" + Category; } public override void Add<K, V>(string Category, IEnumerable<KeyValuePair<K, V>> Items, DateTime? ExpirationDate) { long count = 0; lock (syncRoot) { Items.ForEach(i => { var key = i.Key; var data = i.Value; var cacheKey = GetKey(Category, key.ToString()); if (isMemory) { memoryData[cacheKey] = data; memoryExpiration[cacheKey] = ExpirationDate; //recycle algo switch (recycleAlgorithm) { case RecycleAlgorithms.MRU: usageStat[cacheKey] = new KeyValue(string.Empty, 0); if (recycleMode == RecycleModes.Active) Recycle<K>(Category, memoryData.Count); break; default: break; } } else { //will only serialize object other than string var result = typeof(V) == typeof(string) ? data as string : JSON.Instance.ToJSON(data); var fileKey = key.ToString(); var dataFile = GetFile(Category, fileKey, true); bool exists = File.Exists(dataFile); File.WriteAllText(dataFile, result); //specify expiration //default 30 mins to expire from now var expirationDate = ExpirationDate == null || ExpirationDate <= DateTime.Now ? DateTime.Now.AddMinutes(30) : (DateTime)ExpirationDate; var expirationFile = GetFile(Category, fileKey, false); File.WriteAllText(expirationFile, expirationDate.ToString()); //recycle algo if (recycleAlgorithm != RecycleAlgorithms.None) { var statFilePath = dataPath + statFile; if (File.Exists(statFilePath)) { var buffer = File.ReadAllText(statFilePath); count = Convert.ToInt32(buffer); } if (!exists) { count++; File.WriteAllText(statFilePath, count.ToString()); } switch (recycleAlgorithm) { case RecycleAlgorithms.MRU: usageStat[cacheKey] = new KeyValue(expirationFile, 0); expiredFiles[cacheKey] = new KeyValuePair<DateTime, string>(expirationDate, expirationFile); if (recycleMode == RecycleModes.Active) Recycle<K>(Category, count); break; default: break; } } } }); if (recycleAlgorithm != RecycleAlgorithms.None && recycleMode == RecycleModes.Passive) { if (isMemory) count = memoryData.Count; Recycle<K>(Category, count); } } } public override void Add<K>(string Category, K Key, object Data, DateTime? ExpirationDate) { Add<K, object>(Category, new List<KeyValuePair<K, object>> { new KeyValuePair<K, object>(Key, Data) }, ExpirationDate); } private string GetFile(string Category, string FileName, bool IsData) { var path = dataPath + Category.NormalizeFileName() + @"\"; if (!Directory.Exists(path)) Directory.CreateDirectory(path); return path + FileName.NormalizeFileName() + "." + (IsData ? "dat" : ExpirationFileExtension); } private string GetKey(string Category, string Key) { return Category + "_" + Key; } public override List<KeyValuePair<K, V>> Get<K, V>(string Category, IEnumerable<K> Keys) { var result = new List<KeyValuePair<K, V>>(); lock (syncRoot) { Keys.ForEach(key => { string buffer; V value; var cacheKey = GetKey(Category, key.ToString()); if (isMemory) { object memBuffer; if (memoryData.TryGetValue(cacheKey, out memBuffer)) { //track recycle switch (recycleAlgorithm) { case RecycleAlgorithms.MRU: usageStat[cacheKey].Value++; break; default: break; } value = (V)memBuffer; DateTime? expirationDate; if (memoryExpiration.TryGetValue(cacheKey, out expirationDate)) { //expired if (expirationDate != null && (DateTime)expirationDate < DateTime.Now) { value = default(V); memoryData.Remove(cacheKey); memoryExpiration.Remove(cacheKey); } } } else value = default(V); } else { var dataFilePath = GetFile(Category, key.ToString(), true); if (File.Exists(dataFilePath)) { buffer = File.ReadAllText(dataFilePath); //track recycle switch (recycleAlgorithm) { case RecycleAlgorithms.MRU: usageStat[cacheKey].Value++; break; default: break; } //will only deserialize object other than string value = typeof(V) == typeof(string) ? (V)(object)buffer : JSON.Instance.ToObject<V>(buffer); DateTime expirationDate; var expirationFilePath = GetFile(Category, key.ToString(), false); if (File.Exists(expirationFilePath)) { buffer = File.ReadAllText(expirationFilePath); expirationDate = Convert.ToDateTime(buffer); //expired if (expirationDate < DateTime.Now) { value = default(V); File.Delete(dataFilePath); File.Delete(expirationFilePath); } } } else value = default(V); } result.Add(new KeyValuePair<K, V>(key, value)); }); } return result; } public override V Get<K, V>(string Category, K Key) { var buffer = Get<K, V>(Category, new K[] { Key }); var result = buffer.FirstOrDefault(); return result.Value; } public override void Recycle<K>(string Category, long Count) { if (Count < maxCount) return; switch (recycleAlgorithm) { case RecycleAlgorithms.MRU: lock (syncRoot) { var recycledFileCount = 0; if (isMemory) { //find out expired items var memExpired = memoryExpiration.Where(e => e.Value != null && (DateTime)e.Value < DateTime.Now); memExpired.ForEach(u => { memoryData.Remove(u.Key); memoryExpiration.Remove(u.Key); usageStat.Remove(u.Key); }); } else { if (expiredFiles.Count == 0) { Directory.GetFiles(dataPath, "*." + ExpirationFileExtension).ForEach(f => { var buffer = File.ReadAllText(f); var expirationDate = Convert.ToDateTime(buffer); expiredFiles[Path.GetFileNameWithoutExtension(f)] = new KeyValuePair<DateTime, string>(expirationDate, f); }); } //find out expired items var fileExpired = expiredFiles.Where(e => e.Value.Key < DateTime.Now); fileExpired.ForEach(u => { var dataFile = Path.ChangeExtension(u.Value.Value, DataFileExtension); File.Delete(dataFile); File.Delete(u.Value.Value); usageStat.Remove(u.Key); recycledFileCount++; }); } //find out least used items var leastUsed = usageStat.OrderByDescending(s => s.Value.Value).Skip(maxCount - threshold); leastUsed.ForEach(u => { if (isMemory) { memoryData.Remove(u.Key); memoryExpiration.Remove(u.Key); } else { var dataFile = Path.ChangeExtension(u.Value.Key, DataFileExtension); if (File.Exists(dataFile)) { recycledFileCount++; File.Delete(dataFile); } if (File.Exists(u.Value.Key)) File.Delete(u.Value.Key); } usageStat.Remove(u.Key); }); if (!isMemory) { var statFilePath = dataPath + statFile; var count = 0; if (File.Exists(statFilePath)) { var buffer = File.ReadAllText(statFilePath); count = Convert.ToInt32(buffer); } count = count - recycledFileCount; if (count < 0) count = 0; File.WriteAllText(statFilePath, count.ToString()); } } break; default: break; } } } }
Extensions.cs
扩展函数
using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.IO; namespace Com.SuperCache.Engine { public static class Extensions { public static void ForEach<T>(this IEnumerable<T> source, Action<T> action) { if (source != null) { foreach (var item in source) { action(item); } } } public static string NormalizeFileName(this string FileName) { var result = FileName; Path.GetInvalidFileNameChars().ForEach(c => { result = result.Replace(c.ToString(), string.Empty); }); return result; } } }
新建
构造CacheEngine需要传递缓存保存到哪个文件夹。
基于内存
如果你不喜欢基于磁盘的缓存,可以使用基于内存,构造函数传递空字符串便可。
增加/更新
同一个方法:Add。用户可以指定类型(Category),譬如User,Employee等。键(Key)支持泛型,值(Data)是object。有一个overload是过期日期(ExpirationDate),默认当前时间30分钟后
获取
Get方法需要指定类型(Category)和键(Key)。
例子
using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; using System.Diagnostics; using Com.SuperCache.Engine; namespace Com.SuperCache.Test { public class Foo { public string Name { get; set; } public int Age { get; set; } public double? Some { get; set; } public DateTime? Birthday { get; set; } } class Program { static void Main(string[] args) { //TestAddGet(); //Thread.Sleep(4000); //TestExpiration(); TestDefaultDiskPerformance(); TestDefaultMemoryPerformance(); TestRawDiskPerformance(); TestRawMemoryPerformance(); //TestConcurrent(); Console.Read(); } private static void TestConcurrent() { var w = new Stopwatch(); w.Start(); Parallel.For(1, 3, (a) => { var employees = Enumerable.Range((a - 1) * 1000, a * 1000).Select(i => new KeyValuePair<int, string>(i, "Wilson " + i + " Chen")); var engine = new CacheEngine(@"..\..\data"); engine.Add<int, string>("Employee", employees, DateTime.Now.AddMinutes(1)); }); w.Stop(); Console.WriteLine("add:" + w.Elapsed); var engine2 = new CacheEngine(@"..\..\data"); var o = engine2.Get<int, string>("Employee", 1005); Console.WriteLine(o); } private static void TestDefaultDiskPerformance() { TestPerformance(CacheProviders.Default, @"..\..\data"); } private static void TestDefaultMemoryPerformance() { TestPerformance(CacheProviders.Default, string.Empty); } private static void TestRawDiskPerformance() { TestPerformance(CacheProviders.Raw, @"..\..\data"); } private static void TestRawMemoryPerformance() { TestPerformance(CacheProviders.Raw, string.Empty); } private static void TestPerformance(CacheProviders Provider, string DataPath) { Console.WriteLine("Performance Test: " + Provider.ToString() + ", " + (string.IsNullOrEmpty(DataPath) ? "Memory" : DataPath)); var engine = new CacheEngine(Provider, DataPath, RecycleAlgorithms.MRU, 900, 100, RecycleModes.Passive); var w = new Stopwatch(); w.Start(); var employees = Enumerable.Range(0, 1000).Select(i => new KeyValuePair<int, string>(i, "Wilson " + i + " Chen")); engine.Add<int, string>("Employee", employees, DateTime.Now.AddMinutes(1)); w.Stop(); Console.WriteLine("add:" + w.Elapsed); /*w.Restart(); employees.ForEach(key => { var o1 = engine.Get<int, string>("Employee", key.Key); }); w.Stop(); Console.WriteLine("individual get:" + w.Elapsed);*/ w.Restart(); var keys = employees.Select(i => i.Key); var o = engine.Get<int, string>("Employee", keys); w.Stop(); Console.WriteLine("get:" + w.Elapsed); Console.WriteLine(); } private static void TestExpiration() { var engine = new CacheEngine(@"..\..\data"); var o = engine.Get<string, Foo>("User", "wchen"); Console.WriteLine(o != null ? o.Name : "wchen does not exist or expired"); } private static void TestAddGet() { var engine = new CacheEngine(@"..\..\data"); var f = new Foo { Name = "Wilson Chen", Age = 30, Birthday = DateTime.Now, Some = 123.456 }; engine.Add("User", "wchen", f, DateTime.Now.AddSeconds(5)); var o = engine.Get<string, Foo>("User", "wchen"); Console.WriteLine(o.Name); var o4 = engine.Get<string, Foo>("User", "foo"); Console.WriteLine(o4 != null ? o4.Name : "foo does not exist"); var o3 = engine.Get<string, string>("PlainText", "A"); Console.WriteLine(o3 ?? "A does not exist"); } } }
性能
通过上述性能测试例子,你会发现STSdb的磁盘存取速度要比一个记录对应一个物理文件快。想了解更多,请访问官方网站。
测试条件:1000条记录,7200RPM磁盘,i7。
引擎 | 介质 | 写入 | 读取 |
STSdb | 磁盘 | 0.4s | 0.07s |
内存 | 0.06s | 0.02s | |
Raw | 磁盘 | 1.0s | 0.56s |
内存 | 0.01s | 0.002s |
说明
项目中引用了System.Management是因为STSdb支持内存数据库,需要判断最大物理内存。如果不喜欢,大家可以移除引用,并且去掉STSdb4.Database.STSdb.FromMemory方法便可。
下载
点击这里下载