Python aiokafka生产者
0、简单介绍
AIOKafkaProducer是一个将记录发布到 Kafka 集群的客户端。
官方文档:https://aiokafka.readthedocs.io/en/stable/producer.html
1、测试的版本说明
测试的Python版本是Python3.6
测试的aiokafka版本是0.7.0
2、生产一条消息
import asyncio
import datetime
from aiokafka import AIOKafkaProducer
def now(fmt='%Y-%m-%d %H:%M:%S'):
string = datetime.datetime.now().strftime(fmt)
return string
# 发送一条消息
async def send_one(loop):
producer = AIOKafkaProducer(loop=loop, bootstrap_servers='192.168.1.3:9092')
await producer.start()
try:
await producer.send_and_wait(topic="my_topic", value=f"now:{now()}".encode())
print(f"发送信息到kafka成功")
except Exception as e:
print(f"发送信息到kafka失败, 异常:{e}")
finally:
await producer.stop()
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(send_one(loop))
运行
![image-20220304220955266](https://img2022.cnblogs.com/blog/1768648/202203/1768648-20220305113659406-2119516328.png)
3、生产多条消息(顺序发送)
正常的异步写法,嵌套在for循环中,await等待完成后,才会继续下一个,因此就是顺序的
import asyncio
from typing import List
from aiokafka import AIOKafkaProducer
"""
生产多条消息(顺序发送)
"""
SingleConn = dict()
async def send_one(producer, topic, msg: str):
await producer.send_and_wait(topic, str(msg).encode())
async def _send(loop, host, port, topic, msg_box: List[str]):
key = 'producer'
producer = SingleConn.get(key)
try:
if not producer:
bootstrap_servers = f"{host}:{port}"
print(f"连接kafka, 地址:{bootstrap_servers}")
producer = AIOKafkaProducer(loop=loop, bootstrap_servers=bootstrap_servers)
await producer.start()
SingleConn[key] = producer
# tasks = []
for msg in msg_box:
await send_one(producer, topic, msg)
# tasks.append(send_one(producer, topic, msg))
# await asyncio.wait(tasks)
except Exception as e:
print(f"发送消息失败, 消息:{msg_box}, 异常:{e}")
finally:
await producer.stop()
def debug():
local_ip = "192.168.1.3"
port = 9092
topic = "my_topic"
msg_box = [f"index:{i}" for i in range(20)]
loop = asyncio.get_event_loop()
loop.run_until_complete(_send(loop, local_ip, port, topic, msg_box))
if __name__ == '__main__':
debug()
运行
4、生产多条消息(无序发送)
收集所有的任务,然后使用asyncio.wait或者asyncio.gather来同时启动多个任务。因此是无序的。
import asyncio
import logging
from typing import List
from aiokafka import AIOKafkaProducer
SingleConn = dict()
async def send_one(producer, topic, msg: str):
await producer.send_and_wait(topic, str(msg).encode())
async def _send(loop, host, port, topic, msg_box: List[str]):
key = 'producer'
producer = SingleConn.get(key)
try:
if not producer:
bootstrap_servers = f"{host}:{port}"
print(f"连接kafka, 地址:{bootstrap_servers}")
producer = AIOKafkaProducer(loop=loop, bootstrap_servers=bootstrap_servers)
await producer.start()
SingleConn[key] = producer
tasks = []
for msg in msg_box:
# await send_one(producer, topic, msg)
tasks.append(send_one(producer, topic, msg))
await asyncio.wait(tasks)
except Exception as e:
print(f"发送消息失败, 消息:{msg_box}, 异常:{e}")
finally:
await producer.stop()
def debug():
local_ip = "192.168.1.3"
port = 9092
topic = "my_topic"
msg_box = [f"index:{i}" for i in range(20)]
logging.basicConfig(level=logging.DEBUG)
loop = asyncio.get_event_loop()
loop.run_until_complete(_send(loop, local_ip, port, topic, msg_box))
if __name__ == '__main__':
debug()
运行
5、用于自我调试使用(输入信息,然后回车发到kafka)
import asyncio
from aiokafka import AIOKafkaProducer
SingleConn = dict()
async def send_one(producer, topic, msg: str):
await producer.send_and_wait(topic, str(msg).encode())
async def send_input(loop, host, port, topic):
key = 'producer'
producer = SingleConn.get(key)
msg = ""
try:
if not producer:
bootstrap_servers = f"{host}:{port}"
print(f"连接kafka, 地址:{bootstrap_servers}")
producer = AIOKafkaProducer(loop=loop, bootstrap_servers=bootstrap_servers)
await producer.start()
SingleConn[key] = producer
while 1:
msg = input("请输入想要发送的字符串(输入quit退出):\n")
if msg.strip().lower() == "quit": # 输入quit退出
print("****告辞****")
break
await send_one(producer, topic, msg)
print("****发送成功****")
except Exception as e:
print(f"发送消息失败, 消息:{msg}, 异常:{e}")
finally:
await producer.stop()
def debug():
local_ip = "192.168.1.3"
port = 9092
topic = "my_topic"
loop = asyncio.get_event_loop()
loop.run_until_complete(send_input(loop, local_ip, port, topic))
if __name__ == '__main__':
debug()
运行
![image-20220304223158619](https://img2022.cnblogs.com/blog/1768648/202203/1768648-20220305113659673-1058372708.png)
学习链接 :https://yance.wiki/2020/06/08/pykafka/
docker kafka部署链接:https://www.cnblogs.com/rainbow-tan/p/15932030.html