C#中定义自己的消费队列（上）
一 背景
在我们的工作中我们经常有一种场景就是要使用到队列，特别是对于这样的一种情况：就是生产的速度明显快于消费的速度,而且在多线程的环境下消息的生产由多个线程产生消息的消费则没有这种限制，通过使用队列这种方式能够很大程度上将多线程的问题通过入队的方式变成单个线程内的消息的聚合，然后通过单独的线程进行消费，本篇文章我将介绍一种常见通过包装C#中Queue的方式实现一个能够通过外部增加队列元素，并且由外部进行队列的终止的一种常见的CustomQueue,后面会通过对源码的讲解来一步步加深对基础原理的理解，最后会通过几个单元测试来验证对应的使用。
二 原理讲解
2.1 源代码展示
using System;
using System.Collections.Generic;
using System.Threading;
namespace Pangea.Common.Utility.Buffer
{
public sealed class ConsumeQueue<T>
{
public static int _Counter_Instance;
public enum ConsumeState
{
Idle,
Consuming,
Terminated
}
private Queue<T> _queue;
private int _threadCounter = 0;
private object _lock = new object();
private Action<T> _consumeAction;
private Action _onTerminatedNotify;
private Func<bool> _shouldTerminateConsume;
public ConsumeState State { get; private set; }
public int PeedingItemsCount
{
get
{
lock (_lock)
{
if (_queue == null)
{
return 0;
}
else
{
return _queue.Count;
}
}
}
}
public ConsumeQueue(Action<T> consume, Func<bool> shouldTerminate, Action onTerminated)
{
Interlocked.Increment(ref _Counter_Instance);
_queue = new Queue<T>();
State = ConsumeState.Idle;
_consumeAction = consume;
_shouldTerminateConsume = shouldTerminate;
_onTerminatedNotify = onTerminated;
}
~ConsumeQueue()
{
Interlocked.Decrement(ref _Counter_Instance);
}
public void ProduceItem(T item)
{
lock (_lock)
{
if (State == ConsumeState.Terminated) return;
_queue.Enqueue(item);
if (State == ConsumeState.Idle)
{
State = ConsumeState.Consuming;
StartConsuming();
}
}
}
private void StartConsuming()
{
ThreadPool.QueueUserWorkItem(_ =>
{
bool terminatedFlag = false;
bool exitFlag = false;
++_threadCounter;
while (true)
{
T newData = default(T);
lock (_lock)
{
newData = _queue.Dequeue();
}
_consumeAction(newData);
lock (_lock)
{
if (_shouldTerminateConsume())
{
_queue.Clear();
_queue = null;
terminatedFlag =true;
State = ConsumeState.Terminated;
}
else if (_queue.Count == 0)
{
State = ConsumeState.Idle;
exitFlag =true;
}
}
if (terminatedFlag)
{
OnTerminated();
break;
}
if (exitFlag)
{
break;
}
}
--_threadCounter;
});
}
private void OnTerminated()
{
_consumeAction = null;
_shouldTerminateConsume = null;
_onTerminatedNotify?.Invoke();
}
}
}
2.2 代码解析
- 增加泛型定义和类类型
首先对于该类作为一个完整的工具类,所以该类设计为禁止被继承和重写，所以增加C#关键字sealed作为一个密封类，另外对于该类中定义的数据类型并没有明确的规定，所以该类设计成一个泛型类
- 定义ConsumeQueue中内部执行状态
在实际的代码中通过下面的一个枚举类型State来定义内部执行状态
public enum ConsumeState
{
Idle,
Consuming,
Terminated
}
- A 当CustomQueue初始化或者其内部的消息队列Queue被清除完毕的时候设置状态为
Idle状态并退出StartConsuming方法中的消费循环中 - B 当内部的消息队列中存在未被消费的项目时启动消费过程，并设置State为
Consuming - C 当外部传入的
ShouldTerminateConsume触发时则不论内部的待消费的队列是否为空都将退出当前消费过程，并调用内部的OnTerminated方法清除所有消费队列对象
2.3 对应的单元测试
单元测试部分主要是通过模拟随机产生1000条模拟数据，并在中途产生的数据大于900的时候去模拟终止CustomQueue的行为并断言最后的结果和行为。这里需要注意的是ConsumeQueue_TerminatedStatus除了模拟前面中断的行为以外还通过反射确认threadCounter==0确保当前的消费线程都能够得到正确的释放。
using NUnit.Framework;
using Pangea.Common.Utility.Buffer;
using System;
using System.Reflection;
using System.Threading;
namespace ACM.Framework.Test.Modules.Utils
{
[TestFixture]
internal class ConsumeQueueTests
{
[Test, Timeout(5000)]
public void ConsumeQueue_IdleProducing()
{
ManualResetEvent mre = new ManualResetEvent(false);
int prevData = -1;
Action<int> consume = data =>
{
Assert.IsTrue(data - prevData == 1, $"prev-{prevData}, current-{data}");
prevData = data;
};
Func<bool> func = () => prevData > 900;
Action terminated = () =>
{
mre.Set();
};
ConsumeQueue<int> queue = new ConsumeQueue<int>(consume, func, terminated);
GenerateIntData(data =>
{
queue.ProduceItem(data);
}, false);
mre.WaitOne();
int pendingCount = queue.PeedingItemsCount;
var currentState = queue.State;
Assert.IsTrue(currentState == ConsumeQueue<int>.ConsumeState.Terminated, $"current state : {currentState}");
Assert.IsTrue(pendingCount == 0, $"{pendingCount}");
}
[Test, Timeout(5000)]
public void ConsumeQueue_ContinueProducing()
{
ManualResetEvent mre = new ManualResetEvent(false);
int prevData = -1;
Action<int> consume = data =>
{
Assert.IsTrue(data - prevData == 1, $"prev-{prevData}, current-{data}");
prevData = data;
};
Func<bool> func = () => prevData > 900;
Action terminated = () =>
{
mre.Set();
};
ConsumeQueue<int> queue = new ConsumeQueue<int>(consume, func, terminated);
GenerateIntData(data =>
{
queue.ProduceItem(data);
}, true);
mre.WaitOne();
int pendingCount = queue.PeedingItemsCount;
var currentState = queue.State;
Assert.IsTrue(currentState == ConsumeQueue<int>.ConsumeState.Terminated, $"current state : {currentState}");
Assert.IsTrue(pendingCount == 0, $"{pendingCount}");
}
[Test, Timeout(5000)]
public void ConsumeQueue_TerminatedStatus()
{
ManualResetEvent mre = new ManualResetEvent(false);
int prevData = -1;
Action<int> consume = data =>
{
Assert.IsTrue(data - prevData == 1, $"prev-{prevData}, current-{data}");
prevData = data;
};
Func<bool> func = () => prevData > 500;
Action terminated = () => mre.Set();
ConsumeQueue<int> queue = new ConsumeQueue<int>(consume, func, terminated);
GenerateIntData(data =>
{
queue.ProduceItem(data);
if(queue.State == ConsumeQueue<int>.ConsumeState.Terminated)
{
Assert.IsTrue(queue.PeedingItemsCount == 0);
}
else
{
Assert.IsTrue(queue.PeedingItemsCount > 0);
}
}, true);
mre.WaitOne();
Thread.Sleep(1000); // wait one second for waiting consuming thread exit
int pendingCount = queue.PeedingItemsCount;
var currentState = queue.State;
int queueThreadNum = (int)queue.GetType().GetField("_threadCounter", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(queue);
Assert.IsTrue(currentState == ConsumeQueue<int>.ConsumeState.Terminated, $"current state : {currentState}");
Assert.IsTrue(pendingCount == 0, $"{pendingCount}");
Assert.IsTrue(queueThreadNum == 0, $"{queueThreadNum}");
}
private void GenerateIntData(Action<int> intData, bool withIdle)
{
ThreadPool.QueueUserWorkItem(state =>
{
int target = 1000;
int index = 0;
while (index < target)
{
intData(index++);
if (withIdle)
{
Thread.Sleep(new Random(Guid.NewGuid().GetHashCode()).Next(1, 5));
}
}
});
}
}
}
