go-kafka
go get github.com/Shopify/sarama
消息队列通讯模式
1.点对点
2.发布订阅
Broker
部署了kafka实例的服务器节点。
每个服务器上有一个或多个kafka的实例，broker对应一台服务器。每个kafka集群内的broker都有一个不重复的编号
Topic
消息的主题，可以理解为消息的分类，kafka的数据就保存在topic.在每个broker上都可以创建多个topic.实际应用中通常是一个业务线建一个topic.
Partition
Topic的分区，每个topic可以有多个分区，分区的作用是做负载，提高kafka的吞吐量。同一个topic在不同的分区的数据是不重复的，partition的表现形式就是一个一个的文件夹
Replication
每一个分区都有多个副本，副本的作用是做备胎。当主分区(leader)故障的时候会选择一个备胎(follower)上位，成为Leader.在kafka中默认副本的最大数量是10个，且副本的数量不能大于Broker的数量，follower和leader绝对是在不同的机器，同一机器对同一个分区也只可能存放一个副本(包括自己
Producer
SyncProducer
func SyncProducer(){
config := sarama.NewConfig() //实例化个sarama的Config
config.Producer.Return.Successes = true //是否开启消息发送成功后通知 successes channel
config.Producer.Partitioner = sarama.NewRandomPartitioner //随机分区器
client,err := sarama.NewClient([]string{"10.180.18.60:9092"}, config) //初始化客户端
defer client.Close()
if err != nil {panic(err)}
producer,err := sarama.NewSyncProducerFromClient(client)
if err!=nil {panic(err)}
partition, offset , err := producer.SendMessage(&sarama.ProducerMessage{Topic: "liangtian_topic", Key: nil, Value: sarama.StringEncoder("hahaha")})
if err != nil {
log.Fatalf("unable to produce message: %q", err)
}
fmt.Println("partition",partition)
fmt.Println("offset",offset)
}
AsyncProducer
func AsyncProducer(){
config := sarama.NewConfig()
// config.Producer.Return.Successes = true
//config.Producer.Return.Successes = false，那么在select等待的时候producer.Successes()不会返回，producer.Errors()也不会返回(假设没有错误发生)，就挂在这儿。当然可以加一个default分支绕过去，就不会挂住了
//config.Producer.Return.Successes = true,等同于同步方式
client, err := sarama.NewClient([]{"localhost:9092"}, config)
if err != nil {
log.Fatalf("unable to create kafka client: %q", err)
}
producer, err := sarama.NewAsyncProducerFromClient
if err != nil {
log.Fatalf("unable to create kafka producer: %q", err)
}
defer producer.Close()
text := fmt.Sprintf("message %08d", i)
producer.Input() <- &sarama.ProducerMessage{Topic: topic, Key: nil, Value: sarama.StringEncoder(text)}
// wait response
select {
//case msg := <-producer.Successes():
// log.Printf("Produced message successes: [%s]\n",msg.Value)
case err := <-producer.Errors():
log.Println("Produced message failure: ", err)
default:
log.Println("Produced message default",)
}
}
Consumer
group
type consumerGroupHandler struct{
name string
}
func main1() {
var wg sync.WaitGroup
config := sarama.NewConfig()
config.Consumer.Return.Errors = false
config.Version = sarama.V0_10_2_0
config.Consumer.Offsets.Initial = sarama.OffsetOldest
client,err := sarama.NewClient([]string{"10.180.18.60:9092"}, config)
defer client.Close()
if err != nil {
panic(err)
}
group1, err := sarama.NewConsumerGroupFromClient("c1", client)
if err != nil {
panic(err)
}
group2, err := sarama.NewConsumerGroupFromClient("c2", client)
if err != nil {
panic(err)
}
group3, err := sarama.NewConsumerGroupFromClient("c3", client)
if err != nil {
panic(err)
}
defer group1.Close()
defer group2.Close()
defer group3.Close()
wg.Add(3)
go consume(&group1,&wg,"c1")
go consume(&group2,&wg,"c2")
go consume(&group3,&wg,"c3")
wg.Wait()
signals := make(chan os.Signal, 1)
signal.Notify(signals, os.Interrupt)
select {
case <-signals:
}
}
func consume(group *sarama.ConsumerGroup,wg *sync.WaitGroup, name string) {
fmt.Println(name + "start")
wg.Done()
ctx := context.Background()
for {
//topic := []string{"tiantian_topic1","tiantian_topic2"} 可以消费多个topic
topics := []string{"liangtian_topic"}
handler := consumerGroupHandler{name: name}
err := (*group).Consume(ctx, topics, handler)
if err != nil {
panic(err)
}
}
}
func (consumerGroupHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil }
func (consumerGroupHandler) Cleanup(_ sarama.ConsumerGroupSession) error { return nil }
func (h consumerGroupHandler) ConsumeClaim(sess sarama.ConsumerGroupSession,
claim sarama.ConsumerGroupClaim) error {
for msg := range claim.Messages() {
fmt.Printf("%s Message topic:%q partition:%d offset:%d value:%s\n",h.name, msg.Topic, msg.Partition, msg.Offset, string(msg.Value))
// 手动确认消息
sess.MarkMessage(msg, "")
}
return nil
}
func handleErrors(group *sarama.ConsumerGroup,wg *sync.WaitGroup ){
wg.Done()
for err := range (*group).Errors() {
fmt.Println("ERROR", err)
}
}
consumer
func main() {
var wg sync.WaitGroup
//创建消费者
config := sarama.NewConfig()
config.Consumer.Return.Errors = true
client,err := sarama.NewClient([]string{"10.180.18.60:9092"}, config)
defer client.Close()
if err != nil {
panic(err)
}
consumer, err := sarama.NewConsumerFromClient(client)
defer consumer.Close()
if err != nil {panic(err)}
//设置分区
partitionList, err := consumer.Partitions("liangtian_topic")
if err != nil {
fmt.Println("faild to get the list of partitions",err)
}
//[0 1 2]
fmt.Println(partitionList)
//循环读取分区
for partition := range partitionList {
pc, err := consumer.ConsumePartition("liangtian_topic", int32(partition), sarama.OffsetNewest) //sarama.OffsetOldest
if err != nil {
fmt.Printf("Failed to start consumer for partition %d: %s\n", partition, err)
return
}
defer pc.AsyncClose()
wg.Add(1)
go func(pc sarama.PartitionConsumer) {
defer wg.Done()
for msg := range pc.Messages() {
fmt.Printf("Partition:%d, Offset:%d, Key:%s, Value:%s", msg.Partition, msg.Offset, string(msg.Key), string(msg.Value))
fmt.Println()
}
}(pc)
}
//time.Sleep(time.Hour)
wg.Wait()
consumer.Close()
}
重连
https://blog.csdn.net/stayfoolish_yj/article/details/104477637
