python操作kafka

1.生产者推送

# pip install kafka-python
import json
import time
import datetime
from kafka import KafkaProducer

topic='topic'
server="ip:port"
# 测试版：无用户名密码
# value_serializer 指定序列号方式，传入字典，转成json字符串



def producer():
    producer = KafkaProducer(bootstrap_servers=server,
                             value_serializer=lambda m: json.dumps(m).encode())
    for i in range(10):
        data = {'num': i, 'ts': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
        producer.send(topic, data)
        time.sleep(1)
        print(i)


if __name__ == '__main__':
    producer()

 

2.消费者拉取

from kafka import KafkaConsumer

def consumer():
    consumer = KafkaConsumer('top',
                             bootstrap_servers="IP:PORT",
                             group_id='test3',
                             auto_offset_reset='earliest')
    for msg in consumer:
        print(msg.value.decode())


if __name__ == '__main__':
    consumer()