Python aiokafka生产者

0、简单介绍

AIOKafkaProducer是一个将记录发布到 Kafka 集群的客户端。

官方文档：https://aiokafka.readthedocs.io/en/stable/producer.html

1、测试的版本说明

测试的Python版本是Python3.6

测试的aiokafka版本是0.7.0

2、生产一条消息

import asyncio
import datetime

from aiokafka import AIOKafkaProducer


def now(fmt='%Y-%m-%d %H:%M:%S'):
    string = datetime.datetime.now().strftime(fmt)
    return string


# 发送一条消息
async def send_one(loop):
    producer = AIOKafkaProducer(loop=loop, bootstrap_servers='192.168.1.3:9092')
    await producer.start()
    try:
        await producer.send_and_wait(topic="my_topic", value=f"now:{now()}".encode())
        print(f"发送信息到kafka成功")
    except Exception as e:
        print(f"发送信息到kafka失败, 异常:{e}")
    finally:
        await producer.stop()


if __name__ == '__main__':
    loop = asyncio.get_event_loop()
    loop.run_until_complete(send_one(loop))

运行

3、生产多条消息（顺序发送）

正常的异步写法，嵌套在for循环中，await等待完成后，才会继续下一个，因此就是顺序的

import asyncio
from typing import List

from aiokafka import AIOKafkaProducer

"""
生产多条消息（顺序发送）
"""
SingleConn = dict()


async def send_one(producer, topic, msg: str):
    await producer.send_and_wait(topic, str(msg).encode())


async def _send(loop, host, port, topic, msg_box: List[str]):
    key = 'producer'
    producer = SingleConn.get(key)
    try:
        if not producer:
            bootstrap_servers = f"{host}:{port}"
            print(f"连接kafka, 地址:{bootstrap_servers}")
            producer = AIOKafkaProducer(loop=loop, bootstrap_servers=bootstrap_servers)
            await producer.start()
            SingleConn[key] = producer
        # tasks = []
        for msg in msg_box:
            await send_one(producer, topic, msg)
            # tasks.append(send_one(producer, topic, msg))
        # await asyncio.wait(tasks)
    except Exception as e:
        print(f"发送消息失败, 消息:{msg_box}, 异常:{e}")
    finally:
        await producer.stop()


def debug():
    local_ip = "192.168.1.3"
    port = 9092
    topic = "my_topic"
    msg_box = [f"index:{i}" for i in range(20)]
    loop = asyncio.get_event_loop()
    loop.run_until_complete(_send(loop, local_ip, port, topic, msg_box))


if __name__ == '__main__':
    debug()

运行

4、生产多条消息（无序发送）

收集所有的任务，然后使用asyncio.wait或者asyncio.gather来同时启动多个任务。因此是无序的。

import asyncio
import logging
from typing import List

from aiokafka import AIOKafkaProducer

SingleConn = dict()


async def send_one(producer, topic, msg: str):
    await producer.send_and_wait(topic, str(msg).encode())


async def _send(loop, host, port, topic, msg_box: List[str]):
    key = 'producer'
    producer = SingleConn.get(key)
    try:
        if not producer:
            bootstrap_servers = f"{host}:{port}"
            print(f"连接kafka, 地址:{bootstrap_servers}")
            producer = AIOKafkaProducer(loop=loop, bootstrap_servers=bootstrap_servers)
            await producer.start()
            SingleConn[key] = producer
        tasks = []
        for msg in msg_box:
            # await send_one(producer, topic, msg)
            tasks.append(send_one(producer, topic, msg))
        await asyncio.wait(tasks)
    except Exception as e:
        print(f"发送消息失败, 消息:{msg_box}, 异常:{e}")
    finally:
        await producer.stop()


def debug():
    local_ip = "192.168.1.3"
    port = 9092
    topic = "my_topic"
    msg_box = [f"index:{i}" for i in range(20)]
    logging.basicConfig(level=logging.DEBUG)
    loop = asyncio.get_event_loop()
    loop.run_until_complete(_send(loop, local_ip, port, topic, msg_box))


if __name__ == '__main__':
    debug()

运行

5、用于自我调试使用（输入信息，然后回车发到kafka）

import asyncio

from aiokafka import AIOKafkaProducer

SingleConn = dict()


async def send_one(producer, topic, msg: str):
    await producer.send_and_wait(topic, str(msg).encode())


async def send_input(loop, host, port, topic):
    key = 'producer'
    producer = SingleConn.get(key)
    msg = ""
    try:
        if not producer:
            bootstrap_servers = f"{host}:{port}"
            print(f"连接kafka, 地址:{bootstrap_servers}")
            producer = AIOKafkaProducer(loop=loop, bootstrap_servers=bootstrap_servers)
            await producer.start()
            SingleConn[key] = producer
        while 1:
            msg = input("请输入想要发送的字符串(输入quit退出):\n")
            if msg.strip().lower() == "quit":  # 输入quit退出
                print("****告辞****")
                break
            await send_one(producer, topic, msg)
            print("****发送成功****")
    except Exception as e:
        print(f"发送消息失败, 消息:{msg}, 异常:{e}")
    finally:
        await producer.stop()


def debug():
    local_ip = "192.168.1.3"
    port = 9092
    topic = "my_topic"
    loop = asyncio.get_event_loop()
    loop.run_until_complete(send_input(loop, local_ip, port, topic))


if __name__ == '__main__':
    debug()

运行

学习链接 ：https://yance.wiki/2020/06/08/pykafka/

docker kafka部署链接：https://www.cnblogs.com/rainbow-tan/p/15932030.html