.net core 下使用 Kafka 动态创建分区，创建主题(三)

Kafka 动态创建分区，创建主题，代码接上一篇 写在api里了

 

 1 using Confluent.Kafka.Admin;
 2 using Confluent.Kafka;
 3 using KafkaCommom;
 4 using Microsoft.AspNetCore.Http;
 5 using Microsoft.AspNetCore.Mvc;
 6 using Microsoft.Extensions.Options;
 7 
 8 namespace KafkaWebAPI.Controllers
 9 {
10     /// <summary>
11     /// kafka
12     /// </summary>
13     [Route("api/[controller]")]
14     [ApiController]
15     public class KafkaController : ControllerBase
16     {
17         private readonly KafKaSevice _kafKaSevice;
18         private string kafkaAddr;
19         public KafkaController(KafKaSevice kafKaSevice)
20         {
21             this._kafKaSevice = kafKaSevice;
22 
23             this.kafkaAddr = this._kafKaSevice._kafkaOptions.BootstrapServers;
24         }
25 
26 
27 
28 
29         /// <summary>
30         /// 根据主题创建分区(更新分区)
31         /// </summary>
32         /// <param name="topic">主题</param>
33         /// <param name="Partitions">分区</param>
34         /// <returns></returns>
35         [HttpGet("CreatePartitionsByTopicAsync")]
36         public async Task<string> CreatePartitionsByTopicAsync(string topic, int PartitionCount)
37         {
38             AdminClientConfig adminClientConfig = new AdminClientConfig
39             {
40                 BootstrapServers = this.kafkaAddr,
41             };
42             var builder = new AdminClientBuilder(adminClientConfig).Build();
43             await builder.CreatePartitionsAsync(new PartitionsSpecification[] { new PartitionsSpecification { Topic = topic, IncreaseTo = PartitionCount } });
44             return "创建主题成功";
45         }
46 
47         /// <summary>
48         /// 创建主题
49         /// </summary>
50         /// <param name="topic">主题</param>
51         /// <returns></returns>
52         [HttpGet("CreateTopic")]
53         public async Task<string> CreateTopicAsync(string topic)
54         {
55             AdminClientConfig adminClientConfig = new AdminClientConfig
56             {
57                 BootstrapServers = this.kafkaAddr,
58             };
59             var builder = new AdminClientBuilder(adminClientConfig).Build();
60             await builder.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification { Name = topic } });
61             return "创建主题成功";
62         }
63 
64 
65         /// <summary>
66         /// 创建主题和分区
67         /// </summary>
68         /// <param name="topic">主题</param>
69         /// <param name="PartitionCount">分区</param>
70         /// <returns></returns>
71         [HttpGet("CreateTopicAndPartitionAsync")]
72         public async Task<string> CreateTopicAndPartitionAsync(string topic, int PartitionCount)
73         {
74             AdminClientConfig adminClientConfig = new AdminClientConfig
75             {
76                 BootstrapServers = this.kafkaAddr,
77             };
78             var builder = new AdminClientBuilder(adminClientConfig).Build();
79             await builder.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification { Name = topic, NumPartitions = PartitionCount } });
80             return "创建成功";
81         }
82 
83 
84         /// <summary>
85         /// 发送消息
86         /// </summary>
87         /// <param name="msg"></param>
88         /// <returns></returns>
89         [HttpGet]
90         public string Get(string msg)
91         {
92             this._kafKaSevice.Send(msg);
93             return "";
94         }
95     }
96 }

测试

 

 

 

测试创建主题

 

 

测试创建主题和分区

 

 

创建完分区 会有个问题分区解决什么问题？

 

知道分区处理的问题 ，那么 如何一一对应消费呢？ 就是下面代码指定 

1 consumer.Assign(new TopicPartition(KafkaTopic.Topic, new Partition(1)));
2 consumer.Assign(new TopicPartitionOffset(KafkaTopic.Topic, new Partition(0), offset + 1));

 接下来还有一个问题 消息是怎么均衡发送到各个分区的呢？

 kafka可以自定义负载均衡的方式 其实就是根据请求数做取模运算

 1  /// <summary>
 2         /// 分区轮询算法。两个分区得到消息是一致的
 3         /// </summary>
 4         /// <param name="topic"></param>
 5         /// <param name="partitionCount"></param>
 6         /// <param name="keyData"></param>
 7         /// <param name="keyIsNull"></param>
 8         /// <returns></returns>        
 9         static int requestCount = 0;//请求数
10         private Partition RoundRobinPartitioner(string topic, int partitionCount, ReadOnlySpan<byte> keyData, bool keyIsNull)
11         {
12             int partition = requestCount % partitionCount;
13             requestCount++;
14             return new Partition(partition);
15         }

生产者调用

1             ProducerConfig producerConfig = this.CreateProducerConnection();
2             producerConfig.MessageTimeoutMs = 5000;//失败重试时间
3             producerConfig.EnableIdempotence = true;///幂等性:如果生产者发送失败不重复发消息失败重试
4             var builder = new ProducerBuilder<string, string>(producerConfig);
5             builder.SetDefaultPartitioner(RoundRobinPartitioner);//委托:调用负载均衡方法

 

posted on 2023-01-30 18:00  是水饺不是水饺  阅读(262)  评论(0)  编辑  收藏  举报

导航