高性能缓冲队列-BufferQueue

BufferQueue

[官网](https://github.com/eventhorizon-cli/BufferQueue/blob/cc53c3a49bd5511d56f4fce787c62dcada2884e5/README.zh-CN.md "官网")

高性能的缓冲队列实现，支持多线程并发操作
目前支持的缓冲区类型为**内存缓冲区***，后续会考虑支持更多类型的缓冲区
适用于生产者和消费者之间的速度不一致，需要并发批量处理数据的场景
详细介绍参见官网

代码演示

• 准备
  • 1.以WPF程序为例演示(Net6)
  • 2 Nuget
    • 2.1 BufferQueue
    • 2.2 Microsoft.Extensions.Hosting 依赖注入
    • 2.3 Mvvmlight wpf程序mvvm框架
  • 3 页面仅一个生产者按钮，点一下生产一个对象
  • 4 普通数据对象

 public class Foo
    {
        public Foo(int id, string name)
        {
            Id = id;
            Name = name;
        }
    
        public int Id { get; set; }
        public string Name { get; set; }
        public override string ToString()
        {
            return $"{Id}-{Name}";
        }
    }

- 5  app.xaml.cs
    - 5.1 app.xaml 设为page,删除startup
    - 5.2 服务注册

public partial class App : Application
{
    [STAThread]
    private static void Main(string[] args)
    {
        MainAsync(args).GetAwaiter().GetResult();
    }

    private static async Task MainAsync(string[] args)
    {
        using (IHost host = CreateHostBuilder(args).Build())
        {
            await host.StartAsync().ConfigureAwait(true);

            App app = new App();
            app.InitializeComponent();
            app.MainWindow = host.Services.GetRequiredService<MainWindow>();
            app.MainWindow.Visibility = Visibility.Visible;
            app.Run();

            await host.StopAsync().ConfigureAwait(true);
        }

    }

    public static IHostBuilder CreateHostBuilder(string[] args) =>
        Host.CreateDefaultBuilder(args)
            .ConfigureAppConfiguration((hostBuilderContext, configurationBuilder)
                =>
            {
            //加载配置文件
                configurationBuilder.AddJsonFile("appsetting.json", true, true);
                //加载数据文件
                configurationBuilder.AddJsonFile("data.json", true, true);
                configurationBuilder.AddUserSecrets(typeof(App).Assembly);
                configurationBuilder.SetBasePath(System.IO.Directory.GetCurrentDirectory());
            })
            .ConfigureLogging((ct, log) =>
            {
            //日志(可删除)
                log.ClearProviders();
                log.AddNLog(ct.Configuration);
                log.SetMinimumLevel(LogLevel.Information);
            })				
            .ConfigureServices((hostContext, services) =>
            {
                services.AddSingleton(sp => new MainWindow() { DataContext = sp.GetRequiredService<MainViewModel>() });
                services.AddSingleton<MainViewModel>();
                //bufferqueue
                services.AddBufferQueue(options =>
                {
                    options.UseMemory(bufferOptions =>
                        {
                            bufferOptions.AddTopic<Foo>("topic-foo1", partitionNumber: 10);
                          bufferOptions.AddTopic<Bar>("topic-bar1", partitionNumber: 2);
                        })
                        .AddPushCustomers(typeof(App).Assembly);
                });
                services.AddHostedService<Foo1PullConsumerHostService>();
                services.AddSingleton(_ => Current.Dispatcher);
            });
}

• 生产者

using GalaSoft.MvvmLight;
using GalaSoft.MvvmLight.CommandWpf;

namespace BufferQueue.Client;

public class MainViewModel:ViewModelBase
{
    private readonly IBufferQueue bufferQueue;
    public MainViewModel(IBufferQueue queue)
    {
        bufferQueue = queue;
    }

    private RelayCommand producerCommand;
    public RelayCommand ProducerCommand
    {
        get
        {
            producerCommand = producerCommand ?? new RelayCommand(() =>
            {
                var producer = bufferQueue.GetProducer<Foo>("topic-foo1");
                producer.ProduceAsync(new Foo(1, "foo1"));
                Parallel.For(0, 10, i =>
                {
                    producer.ProduceAsync(new Foo(i, "foo"+i));
                });

                // var producer = bufferQueue.GetProducer<Bar>("topic-bar1");
                // Parallel.For(1, 10, i =>
                // {
                //     producer.ProduceAsync(new Bar() { Name = DateTime.Now.ToString("h:mm:ss ") });
                // });

            });
            return producerCommand;
        }
    }
    
}

• 消费者
  • 自动提交

    using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;

namespace BufferQueue.Client.Service;

public class Foo1PullConsumerHostService(IBufferQueue bufferQueue, 
    ILogger<Foo1PullConsumerHostService> logger): IHostedService
{
    private readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();

    public async Task StartAsync(CancellationToken cancellationToken)
    {
        var token = CancellationTokenSource
            .CreateLinkedTokenSource(cancellationToken, cancellationTokenSource.Token)
            .Token;
        var consumers = bufferQueue.CreatePullConsumers<Foo>(
            new BufferPullConsumerOptions
            {
                TopicName = "topic-foo1", 
                GroupName = "group-foo1", AutoCommit = true, BatchSize = 100,
            }, consumerNumber: 10);

        foreach (IBufferPullConsumer<Foo> consumer in consumers)
        {
            _ = ConsumeAsync(consumer, token);
        }

 
    }
    
    public Task StopAsync(CancellationToken cancellationToken)
    {
        cancellationTokenSource.Cancel();
        return Task.CompletedTask;
    }

    private async Task ConsumeAsync(IBufferPullConsumer<Foo> consumer, CancellationToken cancellationToken)
    {
        await foreach (var buffer in consumer.ConsumeAsync(cancellationToken))
        {
            foreach (var foo in buffer)
            {
                Task.Delay(TimeSpan.FromSeconds(1));
                Console.WriteLine($"{DateTime.Now:HH:mm:ss ffff}-{foo.ToString()}");
            }
        }
    }
}

- 手动提交

using BufferQueue.PushConsumer;
using Microsoft.Extensions.DependencyInjection;

namespace BufferQueue.Client.Service;

[BufferPushCustomer(
    topicName: "topic-bar1",
    groupName: "group-2",
    batchSize: 100,
    serviceLifetime: ServiceLifetime.Scoped,
    concurrency: 2)]
public class BufferManualCommitPushConsumer:IBufferManualCommitPushConsumer<Bar>
{
    public async Task ConsumeAsync(IEnumerable<Bar> buffer, IBufferConsumerCommitter committer, CancellationToken cancellationToken)
    {
        foreach (var item in buffer)
        {
            Console.WriteLine(item.ToString());
            var ct = committer.CommitAsync();
            if (!ct.IsCompletedSuccessfully)
            {
                await ct.AsTask();
            }
        }
    }
}