docker 安装 kafka+zookeeper，golang操作kafka
环境:
- centos8的服务器
- 安装好docker和docker-compose
首先需要开放对外的ip端口,如阿里云的
docker-compose.yml
mkdir -p /usr/local/kafka
vim docker-compose.yml
写入下面的代码
version: '3.7'
services:
zookeeper:
image: wurstmeister/zookeeper
restart: always
volumes:
- ./data:/data
ports:
- 2181:2181
kafka:
image: wurstmeister/kafka
restart: always
ports:
- 9092:9092
environment:
KAFKA_BROKER_ID: 0
#kafka会在zookeeper中使用这个参数进行注册，如果不设置，zk就无法向外提供broker的信息,这里填写kafka的ip地址
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.16.0.110:9092
# KAFKA_ADVERTISED_HOST_NAME: 182.92.234.23 # 注册到zk里的kafka HOST_NAME dkafka 0.9.x以后弃用
KAFKA_CREATE_TOPICS: "test:2:0" #kafka启动后初始化一个有2个partition(分区)0个副本名叫test的topic
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
# kafka的tcp侦听ip地址 例如“127.0.0.1”，那么只有与该ip正确连接的客户端能成功连接到kafka；
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
KAFKA_HEAP_OPTS: "-Xmx256M -Xms128M"
volumes:
- ./kafka-logs:/kafka
depends_on:
- zookeeper
kafka listeners 和 advertised.listeners 的区别及应用
https://blog.csdn.net/weixin_38251332/article/details/105638535
启动服务
docker-compose up -d
A窗口 查看/发送消息到web_log里
# 进入kafka_kafka_1中
docker exec -it kafka_kafka_1 bash
#查看已经建好的topic （docker-compose.yml）
$KAFKA_HOME/bin/kafka-topics.sh --list --zookeeper kafka_zookeeper_1:2181
#发送消息
$KAFKA_HOME/bin/kafka-console-producer.sh --topic=web_log --broker-list kafka_kafka_1:9092
>1
>2
>3
>
B窗口接收消息web_log里信息
# 另开一个窗口,进入kafka_kafka_1中
docker exec -it kafka_kafka_1 bash
#接收消息
$KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server kafka_kafka_1:9092 --from-beginning --topic web_log
1
2
3
停止服务
docker-compose down
kafka其它命令
二进制安装方式 zookeeper启动
/usr/local/kafka/bin/zookeeper-server-start.sh -daemon /usr/local/kafka/config/zookeeper.properties
netstat -tplnu | grep 2181
二进制安装方式 kafka启动
nohup /usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties &
netstat -tplnu | grep 9092
查看kafka有哪些主题
/usr/local/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --list
./opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --list
crawler
wjes7
data_analysis
checkurl
grab
查看kafka 指定主题有没有消费
/usr/local/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --group wjes7 --describe
./opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --group grab --describe
TOPIC PARTITION CURRENT-OFFSET LOG-END-OFFSET LAG CONSUMER-ID HOST CLIENT-ID
es_insert 0 109 109 0 wjes7-e9e26bd0-4e4d-4d7a-9dd2-c2c45df38083 /172.34.1.162 wjes7
查看kafka 指定主题消费情况
/opt/bitnami/kafka/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic=es_insert_full --from-beginning
go操作kafka
producer 消费者
package main
import (
"fmt"
"github.com/Shopify/sarama"
)
var Topic = "web_log" //主题名称
// 基于sarama第三方库开发的kafka client
func main() {
config := sarama.NewConfig()
config.Producer.RequiredAcks = sarama.WaitForAll // 发送完数据需要leader和follow都确认
config.Producer.Partitioner = sarama.NewRandomPartitioner // 新选出一个partition
config.Producer.Return.Successes = true // 成功交付的消息将在success channel返回
// 连接kafka
client, err := sarama.NewSyncProducer([]string{"182.92.234.24:9092"}, config)
if err != nil {
fmt.Println("producer closed, err:", err)
return
}
defer client.Close()
//例子一发单个消息
// 构造一个消息
msg := &sarama.ProducerMessage{}
msg.Topic = Topic
content := "this is a test log"
send01(client,msg,content)
//例子二发多个消息
for _, word := range []string{"Welcome11", "to", "the", "Confluent", "Kafka", "Golang", "client"} {
send01(client,msg,word)
}
}
//发消息
func send01(client sarama.SyncProducer,msg *sarama.ProducerMessage,content string) {
msg.Value = sarama.StringEncoder(content)
// 发送消息
pid, offset, err := client.SendMessage(msg)
if err != nil {
fmt.Println("send msg failed, err:", err)
return
}
fmt.Printf("pid:%v offset:%v\n", pid, offset)
}
consumer 消费者
package main
import (
"fmt"
"github.com/Shopify/sarama"
)
// kafka consumer
var Topic = "web_log" //主题名称
func main() {
consumer, err := sarama.NewConsumer([]string{"182.92.234.24:9092"}, nil)
if err != nil {
fmt.Printf("fail to start consumer, err:%v\n", err)
return
}
partitionList, err := consumer.Partitions(Topic) // 根据topic取到所有的分区
if err != nil {
fmt.Printf("fail to get list of partition:err%v\n", err)
return
}
fmt.Println(partitionList)
for partition := range partitionList { // 遍历所有的分区
// 针对每个分区创建一个对应的分区消费者
pc, err := consumer.ConsumePartition(Topic, int32(partition), sarama.OffsetNewest)
if err != nil {
fmt.Printf("failed to start consumer for partition %d,err:%v\n", partition, err)
return
}
defer pc.AsyncClose()
// 异步从每个分区消费信息
go func(sarama.PartitionConsumer) {
for msg := range pc.Messages() {
fmt.Printf("Partition:%d Offset:%d Key:%v Value:%v\n", msg.Partition, msg.Offset, msg.Key, string(msg.Value))
}
}(pc)
}
select{} //阻塞进程
}
结合gin框架操作kafka
package main
import (
"fmt"
"github.com/Shopify/sarama"
"github.com/gin-gonic/gin"
"sync"
"time"
)
var Topic = "web_log" //主题名称
var kafkaIp = "182.92.234.24:9092"
func Test(ctx *gin.Context) {
//读取
ctx.JSON(200, gin.H{
"data": "product",
})
}
func main() {
//启动消息者
go InitConsumer()
r := gin.Default()
r.GET("/ping", func(c *gin.Context) {
c.JSON(200, gin.H{
"message": "pong",
})
})
r.GET("/send", SendMessage) //http://localhost:8082/send
r.Run("0.0.0.0:8082") // 监听并在 0.0.0.0:8080 上启动服务
}
//发消息到kakfa
func SendMessage(ctx *gin.Context) {
fmt.Println("SendMessage")
config := sarama.NewConfig()
config.Producer.RequiredAcks = sarama.WaitForAll // 发送完数据需要leader和follow都确认
config.Producer.Partitioner = sarama.NewRandomPartitioner // 新选出一个partition
config.Producer.Return.Successes = true // 成功交付的消息将在success channel返回
// 连接kafka
client, err := sarama.NewSyncProducer([]string{kafkaIp}, config)
if err != nil {
fmt.Println("producer closed, err:", err)
return
}
defer client.Close()
//例子一发单个消息
// 构造一个消息
msg := &sarama.ProducerMessage{}
msg.Topic = Topic
content := "this is a test log"
sendTokafka(client,msg,content)
//例子二发多个消息
for _, word := range []string{"Welcome11", "to", "the", "Confluent", "Kafka", "Golang", "client"} {
sendTokafka(client,msg,word)
}
}
//发消息函数
func sendTokafka(client sarama.SyncProducer,msg *sarama.ProducerMessage,content string) {
msg.Value = sarama.StringEncoder(content)
// 发送消息
pid, offset, err := client.SendMessage(msg)
if err != nil {
fmt.Println("send msg failed, err:", err)
return
}
fmt.Printf("pid:%v offset:%v\n", pid, offset)
}
func InitConsumer() {
time.Sleep(time.Second * 3)
fmt.Println("init Counsumer success")
var wg sync.WaitGroup
consumer, err := sarama.NewConsumer([]string{kafkaIp}, nil)
if err != nil {
fmt.Printf("fail to start consumer, err:%v\n", err)
return
}
partitionList, err := consumer.Partitions(Topic) // 根据topic取到所有的分区
if err != nil {
fmt.Printf("fail to get list of partition:err%v\n", err)
return
}
fmt.Println(partitionList)
for partition := range partitionList { // 遍历所有的分区
wg.Add(1)
// 针对每个分区创建一个对应的分区消费者
pc, err := consumer.ConsumePartition(Topic, int32(partition), sarama.OffsetNewest)
if err != nil {
fmt.Printf("failed to start consumer for partition %d,err:%v\n", partition, err)
return
}
// 异步从每个分区消费信息
go func(sarama.PartitionConsumer) {
for msg := range pc.Messages() {
fmt.Printf("Partition:%d Offset:%d Key:%v Value:%v\n", msg.Partition, msg.Offset, msg.Key, string(msg.Value))
}
defer pc.AsyncClose()
wg.Done()
}(pc)
}
//select{} //阻塞进程
wg.Wait()
consumer.Close()
}
访问:
http://localhost:8082/send
结果:
SendMessage
pid:0 offset:250
Partition:0 Offset:250 Key:[] Value:this is a test log
Partition:0 Offset:251 Key:[] Value:Welcome11
pid:0 offset:251
Partition:0 Offset:252 Key:[] Value:to
pid:0 offset:252
pid:0 offset:253
Partition:0 Offset:253 Key:[] Value:the
Partition:0 Offset:254 Key:[] Value:Confluent
pid:0 offset:254
Partition:0 Offset:255 Key:[] Value:Kafka
pid:0 offset:255
pid:0 offset:256
Partition:0 Offset:256 Key:[] Value:Golang
Partition:0 Offset:257 Key:[] Value:client
pid:0 offset:257
go-queue操作kafka
参考地址：
github.com/zeromicro/go-queue/kq
参考文档：
https://www.jianshu.com/p/97fa257622ee
https://segmentfault.com/a/1190000021746086
[Haima的博客]
http://www.cnblogs.com/haima/
