go操作kafka

1、创建kafka

使用Docker Compose快速搭建，windows使用dockerdesktop部署，linux使用docker部署
以linux为例：创建docker-compose.yml文件，搭建单节点zookeeper和单节点kafka环境，并且在8080端口提供kafka-ui管理界面。

docker-compose.yml

version: '2.1'

services:
  zoo1:
    image: confluentinc/cp-zookeeper:7.3.2
    hostname: zoo1
    container_name: zoo1
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_SERVERS: zoo1:2888:3888

  kafka1:
    image: confluentinc/cp-kafka:7.3.2
    hostname: kafka1
    container_name: kafka1
    ports:
      - "9092:9092"
      - "29092:29092"
      - "9999:9999"
    environment:
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-192.168.xxx.xxx}:9092,DOCKER://host.docker.internal:29092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
      KAFKA_BROKER_ID: 1
      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_JMX_PORT: 9999
      KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-192.168.xxx.xxx}
      KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
    depends_on:
      - zoo1
  kafka-ui:
    container_name: kafka-ui
    image: provectuslabs/kafka-ui:latest
    ports:
      - 8080:8080
    depends_on:
      - kafka1
    environment:
      DYNAMIC_CONFIG_ENABLED: "TRUE"

注意：DOCKER_HOST_IP:-192.168.xxx.xxx表示如果DOCKER_HOST_IP没有配置，则使用宿主机ip:192.168.xxx.xxx 保存后执行

docker-compose up -d

容器启动后，使用浏览器打开192.168.xxx.xxx:8080 即可看到如下kafka-ui界面。

点击页面右侧的“Configure new cluster”按钮，配置kafka服务连接信息。

填写完信息后，点击页面下方的“Submit”按钮提交即可。

版本读取正常，"online"和"offline"用来描述Kafka集群中的分区（partition）状态。online为1表示分区状态正常，否则表示不正常。
创建topic（也可以在代码中创建）

2、go操作kafka

Go社区中目前有三个比较常用的kafka客户端库：confluent-kafka-go、sarama、kafka-go，本文使用kafka-go库。
下载依赖

go get github.com/segmentio/kafka-go

2.1连接kafka，发送消息

点击查看代码

// Kafka 配置
const (
	KafkaBroker = "192.168.xxx.xxx:9092" // 替换为你的 Kafka Broker 地址
	KafkaTopic  = "k8s-version"          // Kafka 主题
)

func main() {
	sendMesgKafka()
}

func sendMesgKafka() {
	w := kafka.NewWriter(kafka.WriterConfig{
		Brokers:  []string{KafkaBroker},
		Topic:    KafkaTopic,
		Balancer: &kafka.LeastBytes{},
	})

	err := w.WriteMessages(context.Background(),
		kafka.Message{
			Key:   []byte("Key-A"),
			Value: []byte("one!"),
		},
		kafka.Message{
			Key:   []byte("Key-B"),
			Value: []byte("two!"),
		},
		kafka.Message{
			Key:   []byte("Key-C"),
			Value: []byte("three!"),
		},
	)

	if err != nil {
		log.Fatal("failed to write messages:", err)
	}

	if err := w.Close(); err != nil {
		log.Fatal("failed to close writer:", err)
	}

	fmt.Println("Message sent successfully")

}

2.2消费消息

点击查看代码

func main() {
	readMesgKafka()
}
func readMesgKafka() {
	r := kafka.NewReader(kafka.ReaderConfig{
		Brokers:  []string{"192.168.xxx.xxx:9092"},
		GroupID:  "myGroup",
		Topic:    "k8s-version",
		MinBytes: 10e3, // 10KB
		MaxBytes: 10e6, // 10MB
	})

	for {
		msg, err := r.ReadMessage(context.Background())
		if err != nil {
			break
		}
		fmt.Println(string(msg.Key), string(msg.Value))
	}

	if err := r.Close(); err != nil {
		log.Fatal("failed to close reader:", err)
	}
}