.net core 下使用 Kafka 动态创建分区，创建主题(三)

Kafka 动态创建分区，创建主题，代码接上一篇 写在api里了

 

using Confluent.Kafka.Admin;
using Confluent.Kafka;
using KafkaCommom;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;

namespace KafkaWebAPI.Controllers
{
    /// <summary>
    /// kafka
    /// </summary>
    [Route("api/[controller]")]
    [ApiController]
    public class KafkaController : ControllerBase
    {
        private readonly KafKaSevice _kafKaSevice;
        private string kafkaAddr;
        public KafkaController(KafKaSevice kafKaSevice)
        {
            this._kafKaSevice = kafKaSevice;

            this.kafkaAddr = this._kafKaSevice._kafkaOptions.BootstrapServers;
        }




        /// <summary>
        /// 根据主题创建分区(更新分区)
        /// </summary>
        /// <param name="topic">主题</param>
        /// <param name="Partitions">分区</param>
        /// <returns></returns>
        [HttpGet("CreatePartitionsByTopicAsync")]
        public async Task<string> CreatePartitionsByTopicAsync(string topic, int PartitionCount)
        {
            AdminClientConfig adminClientConfig = new AdminClientConfig
            {
                BootstrapServers = this.kafkaAddr,
            };
            var builder = new AdminClientBuilder(adminClientConfig).Build();
            await builder.CreatePartitionsAsync(new PartitionsSpecification[] { new PartitionsSpecification { Topic = topic, IncreaseTo = PartitionCount } });
            return "创建主题成功";
        }

        /// <summary>
        /// 创建主题
        /// </summary>
        /// <param name="topic">主题</param>
        /// <returns></returns>
        [HttpGet("CreateTopic")]
        public async Task<string> CreateTopicAsync(string topic)
        {
            AdminClientConfig adminClientConfig = new AdminClientConfig
            {
                BootstrapServers = this.kafkaAddr,
            };
            var builder = new AdminClientBuilder(adminClientConfig).Build();
            await builder.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification { Name = topic } });
            return "创建主题成功";
        }


        /// <summary>
        /// 创建主题和分区
        /// </summary>
        /// <param name="topic">主题</param>
        /// <param name="PartitionCount">分区</param>
        /// <returns></returns>
        [HttpGet("CreateTopicAndPartitionAsync")]
        public async Task<string> CreateTopicAndPartitionAsync(string topic, int PartitionCount)
        {
            AdminClientConfig adminClientConfig = new AdminClientConfig
            {
                BootstrapServers = this.kafkaAddr,
            };
            var builder = new AdminClientBuilder(adminClientConfig).Build();
            await builder.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification { Name = topic, NumPartitions = PartitionCount } });
            return "创建成功";
        }


        /// <summary>
        /// 发送消息
        /// </summary>
        /// <param name="msg"></param>
        /// <returns></returns>
        [HttpGet]
        public string Get(string msg)
        {
            this._kafKaSevice.Send(msg);
            return "";
        }
    }
}

测试

 

 

 

测试创建主题

 

 

测试创建主题和分区

 

 

创建完分区 会有个问题分区解决什么问题？

 

知道分区处理的问题 ，那么 如何一一对应消费呢？ 就是下面代码指定 

consumer.Assign(new TopicPartition(KafkaTopic.Topic, new Partition(1)));
consumer.Assign(new TopicPartitionOffset(KafkaTopic.Topic, new Partition(0), offset + 1));

 接下来还有一个问题 消息是怎么均衡发送到各个分区的呢？

 kafka可以自定义负载均衡的方式 其实就是根据请求数做取模运算

 /// <summary>
        /// 分区轮询算法。两个分区得到消息是一致的
        /// </summary>
        /// <param name="topic"></param>
        /// <param name="partitionCount"></param>
        /// <param name="keyData"></param>
        /// <param name="keyIsNull"></param>
        /// <returns></returns>        
        static int requestCount = 0;//请求数
        private Partition RoundRobinPartitioner(string topic, int partitionCount, ReadOnlySpan<byte> keyData, bool keyIsNull)
        {
            int partition = requestCount % partitionCount;
            requestCount++;
            return new Partition(partition);
        }

生产者调用

            ProducerConfig producerConfig = this.CreateProducerConnection();
            producerConfig.MessageTimeoutMs = 5000;//失败重试时间
            producerConfig.EnableIdempotence = true;///幂等性:如果生产者发送失败不重复发消息失败重试
            var builder = new ProducerBuilder<string, string>(producerConfig);
            builder.SetDefaultPartitioner(RoundRobinPartitioner);//委托:调用负载均衡方法