封装多线程处理大量数据操作（二）

接上文：封装多线程处理大量数据操作（一）

我们需要解决WaitAny和取得异步执行的返回值的问题。地球人都知道Thread和ThreadPool接受的委托都是没有返回值的。要想取的返回值，我们就得自己动手了，我们需要构造一个AsyncContext类，由这个类来保存异步执行的状态以并存储返回值。

代码如下：

?

1 2 3 4 5 6 7 8 9

using System; using System.Collections.Generic; using System.Text; using System.Collections; using System.Threading; using System.Diagnostics;   namespace AppUtility {

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356

public delegate object DoGetObjTask(object state);     public static class AsyncHelper     {         /// <summary>         /// 执行多线程操作任务         /// </summary>         /// <param name="dataCollection">多线程操作的数据集合</param>         /// <param name="threadCn">分多少个线程来做</param>         /// <param name="processItemMethod">处理数据集合中单个数据使用的处理方法</param>         public static void DoAsync(IList dataCollection, int threadCn, WaitCallback processItemMethod)         {             DoAsync(dataCollection, threadCn, processItemMethod, true);         }             /// <summary>         /// 执行多线程操作任务         /// </summary>         /// <param name="dataCollection">多线程操作的数据集合</param>         /// <param name="threadCn">分多少个线程来做</param>         /// <param name="processItemMethod">处理数据集合中单个数据使用的处理方法</param>         /// <param name="needWaitAll">是否需要等待所有线程执行完毕才返回，为true时会等待所有线程执行完毕，否则则是在有一个线程执行完毕就返回</param>         public static void DoAsync(IList dataCollection, int threadCn, DoGetObjTask processItemMethod, bool needWaitAll, out Hashtable processResult)         {             DoAsyncPrivate(dataCollection, threadCn, null, processItemMethod, needWaitAll, true, out processResult);         }           /// <summary>         /// 执行多线程操作任务         /// </summary>         /// <param name="dataCollection">多线程操作的数据集合</param>         /// <param name="threadCn">分多少个线程来做</param>         /// <param name="processItemMethod">处理数据集合中单个数据使用的处理方法</param>         /// <param name="needWaitAll">是否需要等待所有线程执行完毕才返回，为true时会等待所有线程执行完毕，否则则是在有一个线程执行完毕就返回</param>         public static void DoAsync(IList dataCollection, int threadCn, DoGetObjTask processItemMethod, out Hashtable processResult)         {             DoAsyncPrivate(dataCollection, threadCn, null, processItemMethod, true, true, out processResult);         }           /// <summary>         /// 执行多线程操作任务         /// </summary>         /// <param name="dataCollection">多线程操作的数据集合</param>         /// <param name="threadCn">分多少个线程来做</param>         /// <param name="processItemMethod">处理数据集合中单个数据使用的处理方法</param>         /// <param name="needWaitAll">是否需要等待所有线程执行完毕才返回，为true时会等待所有线程执行完毕，否则则是在有一个线程执行完毕就返回</param>         public static void DoAsync(IList dataCollection, int threadCn, WaitCallback processItemMethod, bool needWaitAll)         {             Hashtable hash;             DoAsyncPrivate(dataCollection, threadCn, processItemMethod, null, needWaitAll, false, out hash);         }           private static void DoAsyncPrivate(IList dataCollection, int threadCn, WaitCallback processItemMethod, DoGetObjTask getObjMethod, bool needWaitAll, bool hasReturnValue, out Hashtable processResult)         {             if (dataCollection == null) throw new ArgumentNullException("dataCollection");               if (threadCn >= 64 || threadCn < 2)             {                 throw new ArgumentOutOfRangeException("threadCn", "threadCn 参数必须在2和64之间");             }               if (threadCn > dataCollection.Count) threadCn = dataCollection.Count;               IList[] colls = new ArrayList[threadCn];               DataWithStateList dataWithStates = new DataWithStateList();             AutoResetEvent[] evts = new AutoResetEvent[threadCn];               for (int i = 0; i < threadCn; i++)             {                 colls[i] = new ArrayList();                 evts[i] = new AutoResetEvent(false);             }               for (int i = 0; i < dataCollection.Count; i++)             {                 object obj = dataCollection[i];                 int threadIndex = i % threadCn;                 colls[threadIndex].Add(obj);                 dataWithStates.Add(new DataWithState(obj, ProcessState.WaitForProcess));             }               AsyncContext context = AsyncContext.GetContext(threadCn, dataWithStates, needWaitAll, hasReturnValue, processItemMethod, getObjMethod);               for (int i = 0; i < threadCn; i++)             {                 ThreadPool.QueueUserWorkItem(DoPrivate, new object[] {                     colls[i],context,evts[i]                 });             }               if (needWaitAll)             {                 WaitHandle.WaitAll(evts);             }             else             {                 WaitHandle.WaitAny(evts);                 context.SetBreakSignal();             }             processResult = context.ProcessResult;         }           private class AsyncContext         {             static public AsyncContext GetContext(                 int threadCn,                 DataWithStateList dataWithStates,                 bool needWaitAll,                 bool hasReturnValue,                 WaitCallback processItemMethod,                 DoGetObjTask hasReturnValueMethod                 )             {                 AsyncContext context = new AsyncContext();                 context.ThreadCount = threadCn;                 context.DataWithStates = dataWithStates;                 context.NeedWaitAll = needWaitAll;                 if (hasReturnValue)                 {                     Hashtable processResult = Hashtable.Synchronized(new Hashtable());                     context.ProcessResult = processResult;                     context.HasReturnValueMethod = hasReturnValueMethod;                 }                 else                 {                     context.VoidMethod = processItemMethod;                 }                 context.HasReturnValue = hasReturnValue;                 return context;             }               internal int ThreadCount;               internal DataWithStateList DataWithStates;               internal bool NeedWaitAll;               internal bool HasReturnValue;               internal WaitCallback VoidMethod;               internal DoGetObjTask HasReturnValueMethod;               private bool _breakSignal;               private Hashtable _processResult;               internal Hashtable ProcessResult             {                 get { return _processResult; }                 set { _processResult = value; }             }               internal void SetReturnValue(object obj, object result)             {                 lock (_processResult.SyncRoot)                 {                     _processResult[obj] = result;                 }             }               internal void SetBreakSignal()             {                 if (NeedWaitAll) throw new NotSupportedException("设定为NeedWaitAll时不可设置BreakSignal");                   _breakSignal = true;             }               internal bool NeedBreak             {                 get                 {                     return !NeedWaitAll && _breakSignal;                 }             }               internal void Exec(object obj)             {                 if (HasReturnValue)                 {                     SetReturnValue(obj, HasReturnValueMethod(obj));                 }                 else                 {                     VoidMethod(obj);                 }                 DataWithStates.SetState(obj, ProcessState.Processed);             }         }           private enum ProcessState : byte         {             WaitForProcess = 0,             Processing = 1,             Processed = 2         }           private class DataWithStateList : List<DataWithState>         {             public void SetState(object obj, ProcessState state)             {                 lock (((ICollection)this).SyncRoot)                 {                     DataWithState dws = this.Find(delegate(DataWithState i) { return Object.Equals(i.Data, obj); });                       if (dws != null)                     {                         dws.State = state;                     }                 }             }               public ProcessState GetState(object obj)             {                 lock (((ICollection)this).SyncRoot)                 {                     DataWithState dws = this.Find(delegate(DataWithState i) { return Object.Equals(i.Data, obj); });                     return dws.State;                 }             }               private int GetCount(ProcessState state)             {                 List<DataWithState> datas = this.FindAll(delegate(DataWithState i) { return i.State == state; });                 if (datas == null) return 0;                 return datas.Count;             }               public int WaitForDataCount             {                 get                 {                     return GetCount(ProcessState.WaitForProcess);                 }             }               internal object GetWaitForObject()             {                 lock (((ICollection)this).SyncRoot)                 {                     DataWithState dws = this.Find(delegate(DataWithState i) { return i.State == ProcessState.WaitForProcess; });                     if (dws == null) return null;                     dws.State = ProcessState.Processing;                     return dws.Data;                 }             }               internal bool IsWaitForData(object obj, bool setState)             {                 lock (((ICollection)this).SyncRoot)                 {                     DataWithState dws = this.Find(delegate(DataWithState i) { return i.State == ProcessState.WaitForProcess; });                       if (setState && dws != null) dws.State = ProcessState.Processing;                       return dws != null;                 }             }         }           private class DataWithState         {             public readonly object Data;             public ProcessState State;               public DataWithState(object data, ProcessState state)             {                 Data = data;                 State = state;             }         }           private static int _threadNo = 0;           private static void DoPrivate(object state)         {             object[] objs = state as object[];               IList datas = objs[0] as IList;             AsyncContext context = objs[1] as AsyncContext;             AutoResetEvent evt = objs[2] as AutoResetEvent;               DataWithStateList objStates = context.DataWithStates;   #if DEBUG             Thread.CurrentThread.Name = "Thread " + _threadNo;               Interlocked.Increment(ref _threadNo);             string threadName = Thread.CurrentThread.Name + "[" + Thread.CurrentThread.ManagedThreadId + "]";             Trace.WriteLine("线程ID:" + threadName); #endif             if (datas != null)             {                 for (int i = 0; i < datas.Count; i++)                 {                     if (context.NeedBreak)                     { #if DEBUG                         Trace.WriteLine("线程" + threadName + "未执行完跳出"); #endif                         break;                     }                     object obj = datas[i];                     if (objStates.IsWaitForData(obj, true))                     {                         if (context.NeedBreak)                         { #if DEBUG                             Trace.WriteLine("线程" + threadName + "未执行完跳出"); #endif                             break;                         }                           context.Exec(obj);   #if DEBUG                         Trace.WriteLine(string.Format("线程{0}处理{1}", threadName, obj)); #endif                     }                 }             }               if (context.NeedWaitAll)             {                 //如果执行完当前进程的数据，还要查看剩下多少没有做，如果还剩下超过ThreadCount个没有做                 while (objStates.WaitForDataCount > context.ThreadCount)                 {                     if (context.NeedBreak) break;                       object obj = objStates.GetWaitForObject();                     if (obj != null && objStates.IsWaitForData(obj, false))                     {                         if (context.NeedBreak)                         { #if DEBUG                             Trace.WriteLine("线程" + threadName + "未执行完跳出"); #endif                             break;                         }                           context.Exec(obj);   #if DEBUG                         Trace.WriteLine(string.Format("线程{0}执行另一个进程的数据{1}", threadName, obj)); #endif                     }                 }             }               evt.Set();         }         } }

如何使用AsyncHelper类，请看下面的测试代码：

+ View Code?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101

using System; using System.Collections.Generic; using System.Text; using System.Diagnostics; using AppUtility; using System.IO; using System.Collections; using System.Threading;   namespace ConsoleApplication2 {     class Program     {         static void Main(string[] args)         {             Stopwatch sw = new Stopwatch();             sw.Start();             /*             List<string> testFiles = new List<string>();             for (int i = 0; i < 100; i++)             {                 testFiles.Add("D:\\test\\async\\file_" + i.ToString() + ".log");             }             AsyncHelper.DoAsync(testFiles, 10, WriteFile);               Console.WriteLine("异步写耗时"+sw.ElapsedMilliseconds + "ms");             */               List<string> testFiles = new List<string>();             for (int i = 0; i < 200; i++)             {                 testFiles.Add("D:\\test\\async\\file_" + i.ToString() + ".log");             }                           Hashtable result;               AsyncHelper.DoAsync(testFiles, 20, WriteFileAndReturnRowCount,false,out result);               Console.WriteLine("异步写耗时" + sw.ElapsedMilliseconds + "ms");               Thread.Sleep(10);               if (result != null)             {                 foreach (object key in result.Keys)                 {                     Console.WriteLine("{0}={1}",  key,result[key]);                 }               }               sw.Reset();             sw.Start();             for (int i = 0; i < 200; i++)             {                 WriteFile("D:\\test\\sync\\file_" + i.ToString() + ".log");             }               Console.WriteLine("同步写耗时" + sw.ElapsedMilliseconds + "ms");               Console.Read();         }           static void WriteFile(object objFilePath)         {             string filePath = (string)objFilePath;             string dir = Path.GetDirectoryName(filePath);             if (!Directory.Exists(dir))             {                 Directory.CreateDirectory(dir);             }               //Random r = new Random(DateTime.Now.Minute);             int rowCn = 10000;             using (StreamWriter writer = new StreamWriter(filePath, false, Encoding.Default))             {                 for (int i = 0; i < rowCn; i++) writer.WriteLine(Guid.NewGuid());             }         }           static object WriteFileAndReturnRowCount(object objFilePath)         {             string filePath = (string)objFilePath;             string dir = Path.GetDirectoryName(filePath);             if (!Directory.Exists(dir))             {                 Directory.CreateDirectory(dir);             }               //Random r = new Random(DateTime.Now.Minute);             int rowCn = 10000;             using (StreamWriter writer = new StreamWriter(filePath, false, Encoding.Default))             {                 for (int i = 0; i < rowCn ; i++) writer.WriteLine(Guid.NewGuid());             }             return DateTime.Now.ToLongTimeString();         }     } }   Sorry，代码太多，文字太少。发个牢骚，代码写完之后，再写思路是一件痛苦的事情！