golang操作kafka

生产者

package main

import (
    "fmt"
    "time"

    "github.com/Shopify/sarama"
)

// 基于sarama第三方库开发的kafka client

func main() {
    config := sarama.NewConfig()
    config.Producer.RequiredAcks = sarama.WaitForAll          // 发送完数据需要leader和follow都确认
    config.Producer.Partitioner = sarama.NewRandomPartitioner // 新选出一个partition
    config.Producer.Return.Successes = true                   // 成功交付的消息将在success channel返回

    now := time.Now()                  //获取当前时间
    timestamp := now.Unix()            //时间戳
    timeObj := time.Unix(timestamp, 0) //将时间戳转为时间格式
    year := timeObj.Year()     //年
    month := timeObj.Month()   //月
    day := timeObj.Day()       //日
    hour := timeObj.Hour()     //小时
    minute := timeObj.Minute() //分钟
    second := timeObj.Second() //秒

    // 构造一个消息
    msg := &sarama.ProducerMessage{}
    msg.Topic = "test"
    msg.Value = sarama.StringEncoder("this is a test log " + fmt.Sprintf("%d-%02d-%02d %02d:%02d:%02d\n", year, month, day, hour, minute, second))
    // 连接kafka
    client, err := sarama.NewSyncProducer([]string{"10.128.172.9:11202"}, config)
    if err != nil {
        fmt.Println("producer closed, err:", err)
        return
    }
    defer client.Close()
    // 发送消息
    pid, offset, err := client.SendMessage(msg)
    if err != nil {
        fmt.Println("send msg failed, err:", err)
        return
    }
    fmt.Printf("pid:%v offset:%v\n", pid, offset)
}

 

消费者

package main

import (
    "context"
    "fmt"

    "github.com/Shopify/sarama"
)

var Consumer sarama.Consumer

func main() {
    var err error
    Consumer, err = sarama.NewConsumer([]string{"10.128.172.9:11202"}, nil)
    if err != nil {
        fmt.Printf("fail to start consumer,err:%v\n", err)
        return
    }
    topics := []string{"test"}
    //topics := []string{"nsq"}
    //topic := "nsq"
    for _, topic := range topics {
        ctx, _ := context.WithCancel(context.Background())
        go testTopic(topic, ctx)
    }
    select {}
}
func testTopic(topic string, ctx context.Context) {
    partitionList, err := Consumer.Partitions(topic)
    fmt.Println(partitionList)
    if err != nil {
        fmt.Printf("fail to start consumer partition,err:%v\n", err)
        return
    }
    for partition := range partitionList {
        //  遍历所有的分区，并且针对每一个分区建立对应的消费者
        pc, err := Consumer.ConsumePartition(topic, int32(partition), sarama.OffsetNewest)
        if err != nil {
            fmt.Printf("fail to start consumer for partition %d,err:%v\n", partition, err)
            return
        }
        defer pc.AsyncClose()
        go testGetMsg(pc, ctx)
    }
    for {
        select {
        case <-ctx.Done():
            return
        default:
            continue
        }
    }
}
func testGetMsg(partitionConsumer sarama.PartitionConsumer, ctx context.Context) {
    for msg := range partitionConsumer.Messages() {
        fmt.Printf("Partition:%d Offset:%v Key:%v Value:%v\n", msg.Partition, msg.Offset, msg.Key, string(msg.Value))
        select {
        case <-ctx.Done():
            return
        default:
            continue
        }
    }
}