19 Kafka案例

1 基本操作

1.1查看topic列表:

kafka-topics.sh --zookeeper master:2181,slave1:2181,slave2:2181 --list

1.2创建topic:

kafka-topics.sh --zookeeper master:2181,slave1:2181,slave2:2181 --create --replication-factor 2 --partitions 3 --topic test

• （参数说明：
  • --replication-factor：指定每个分区的复制因子个数，默认1个
  • --partitions：指定当前创建的kafka分区数量，默认为1个
  • --topic：指定新建topic的名称）

1.3查看topic描述:

kafka-topics.sh --zookeeper master:2181,slave1:2181,slave2:2181 --describe --topic test

1.4删除topic:

kafka-topics.sh --delete --zookeeper master:2181,slave1:2181,slave2:2181 --topic test

1.5生产者消费者:

• 生产者（告知往哪个topic里写数据）
  kafka-console-producer.sh --broker-list master:9092,slave1:9092,slave2:9092 --topic test
• 消费者（告知往哪个topic里写数据）
  kafka-console-consumer.sh --zookeeper master:2181,slave1:2181,slave2:2181 --from-beginning --topic test
  

3 自己写producer、consumer(需要pip install kafka-python)

consumer_kafka.py

from kafka import KafkaConsumer


def main():
    consumer = KafkaConsumer(b"newyear_test", group_id=b"my_group_id",
                             bootstrap_servers=["master:9092", "slave1:9092", "slave2:9092"],
            auto_offset_reset='earliest')
            #auto_offset_reset='latest')
    for message in consumer:
        # This will wait and print messages as they become available
        print(message)


if __name__ == "__main__":
    main()

producer_kafka.py

import time

from kafka import SimpleProducer, KafkaClient
from kafka.common import LeaderNotAvailableError

def print_response(response=None):
    if response:
        print('Error: {0}'.format(response[0].error))
        print('Offset: {0}'.format(response[0].offset))


def main():
    kafka = KafkaClient("localhost:9092")
    producer = SimpleProducer(kafka)

    topic = b'newyear_test'
    msg = b'Hello World from 2019!'

    try:
        print_response(producer.send_messages(topic, msg))
    except LeaderNotAvailableError:
        time.sleep(1)
        print_response(producer.send_messages(topic, msg))

    kafka.close()

if __name__ == "__main__":
    main()

开启2个consumer执行producer看效果吧

5 自己写producer、consumer(指定partition)

consumer_kafka_2.py

#encoding=utf8
from kafka import KafkaConsumer
from kafka import TopicPartition
from kafka.structs import OffsetAndMetadata
from kafka.structs import TopicPartition

def main():
    consumer = KafkaConsumer('newyear_test', bootstrap_servers=['master:9092'])

    print consumer.partitions_for_topic("newyear_test")
    print consumer.topics()  #获取主题列表
    print consumer.subscription()  #获取当前消费者订阅的主题
    print consumer.assignment()  #获取当前消费者topic、分区信息
    print consumer.beginning_offsets(consumer.assignment()) #获取当前消费者可消费的偏移量

    consumer.seek(TopicPartition(topic=u'newyear_test', partition=2), 5)  #重置偏移量
    for message in consumer:
        print ("%s:%d:%d: key=%s value=%s" % (message.topic, message.partition,
            message.offset, message.key,
            message.value))


if __name__ == "__main__":
    main()

producer_kafka_2.py

from kafka import KafkaProducer

producer = KafkaProducer(bootstrap_servers='localhost:9092')

# # block until all pending messages are sent
# for _ in range(10):
#     producer.send('test_m_brokers', b'are you ok!!!')
# 
# producer.flush()


# key for hashed partitioning
producer.send('newyear_many_test', key=b'1', value=b'aaa')
producer.flush()

测试