python处理kafka数据

1、程序作用：从多个topic中读取数据--处理数据--写入新的kafkatopic中

pip3 install kafka-python

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37

import json from kafka import KafkaProducer from kafka import KafkaConsumer #消费者 def comsum():     print('start consumer')       consumer = KafkaConsumer(group_id='1',                     bootstrap_servers=["100.98.100.186:9092"],                     )     print(consumer)     consumer.subscribe(['test_topic', 'topic2'])     for message in consumer:         rs=json.loads(message.value)  # 这样就可以通过topic和value去做后边的操作了 #处理数据生成新的数据         dict1 = {}         dict1['name'] = rs['name']         dict1['msg'] = rs['msg']         print(dict1)         #生产新数据         kfk_produce(dict1)     print('end consumer') #生产者 def kfk_produce(data_dict):     """         发送 string 格式数据     :return:     """     producer = KafkaProducer(bootstrap_servers='100.98.100.186:9092')       msg: str = json.dumps(data_dict).encode('utf-8')       producer.send('new_topic', msg, partition=0)     producer.close()   if __name__ =='__main__':     comsum()

2、运行环境

?

1	docker run --name python-cron -d wukc/python3-cron:latest

3、执行方式

?

1	nohup python pyhon_kafka.py >kafka.log 2>&1 &