kafka集群部署

2.1 部署前准备：

在部署 Kafka 之前，需要先部署 zookeeper 集群

主机名	IP	运行服务
zoo-server-node1	172.21.140.31	ZooKeeper、Kafka
zoo-server-node2	172.21.140.32	ZooKeeper、Kafka
zoo-server-node3	172.21.140.33	ZooKeeper、Kafka

2.2 获取安装包

curl -O https://mirror-hk.koddos.net/apache/kafka/2.4.1/kafka_2.11-2.4.1.tgz

2.3 解压并配置文件

node1配置

# tar xf kafka_2.11-2.4.1.tgz
# mv kafka_2.11-2.4.1 /opt/kafka

# vim /opt/kafka/config/server.properties
[root@localhost src]# grep "^[a-Z]" /opt/kafka/config/server.properties 
broker.id=1  #集群唯一标识，为正整数
listeners=PLAINTEXT://172.21.140.31:9092  #kafka监听地址和端口
num.network.threads=3  
num.io.threads=8 #IO线程数
socket.send.buffer.bytes=102400  #socket发送缓冲区大小
socket.receive.buffer.bytes=102400 #socket接收缓冲区大小
socket.request.max.bytes=104857600 #socket请求缓冲区大小
log.dirs=/data/kafka  #kafka用于保存所有数据的目录，所有的消息都会存储在该目录当中
num.partitions=1  #创建topic时默认存储其分区数
num.recovery.threads.per.data.dir=1  
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168  #设置kafka中保存消息的保存时间，默认为168小时即7天
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
#连接zk地址，zk存储broker的元数据
zookeeper.connect=172.21.140.31:2181,172.21.140.32:2181,172.21.140.33:2181
zookeeper.connection.timeout.ms=6000 #设置连接zk的超时时间，默认6秒钟
group.initial.rebalance.delay.ms=0

将kafka程序分发到其他主机：

# scp kafka_2.11-2.4.1.tgz 172.21.140.32:/usr/local/src
# scp kafka_2.11-2.4.1.tgz 172.21.140.33:/usr/local/src
# scp /opt/kafka/config/server.properties 172.21.140.32:/tmp
# scp /opt/kafka/config/server.properties 172.21.140.33:/tmp

node2节点配置

# tar xf kafka_2.11-2.4.1.tgz
# mv kafka_2.11-2.4.1 /opt/kafka
# cp /tmp/server.properties /opt/kafka/conf/ 
# sed -i "s/broker.id=1/broker.id=2/" /opt/kafka/config/server.properties
# sed -i "s/listeners=PLAINTEXT://172.21.140.31:9092/listeners=PLAINTEXT://172.21.140.32:9092/" /opt/kafka/config/server.properties

node3节点配置

# tar xf kafka_2.11-2.4.1.tgz
# mv kafka_2.11-2.4.1 /opt/kafka
# cp /tmp/server.properties /opt/kafka/conf/ 
# sed -i "s/broker.id=1/broker.id=3/" /opt/kafka/config/server.properties
# sed -i "s/listeners=PLAINTEXT://172.21.140.31:9092/listeners=PLAINTEXT://172.21.140.33:9092/" /opt/kafka/config/server.properties

2.4 各节点启动kafka

2.4.1 node1

# /opt/kafka/bin/kafka-server-start.sh -daemon /opt/kafka/config/server.properties

$查看日志
# tail -f logs/kafkaServer.out 
[2020-12-30 14:40:39,723] INFO [SocketServer brokerId=1] Started data-plane processors for 1 acceptors (kafka.network.SocketServer)
[2020-12-30 14:40:39,725] INFO Kafka version: 2.4.1 (org.apache.kafka.common.utils.AppInfoParser)
[2020-12-30 14:40:39,725] INFO Kafka commitId: c57222ae8cd7866b (org.apache.kafka.common.utils.AppInfoParser)
[2020-12-30 14:40:39,725] INFO Kafka startTimeMs: 1609310439724 (org.apache.kafka.common.utils.AppInfoParser)
[2020-12-30 14:40:39,727] INFO [KafkaServer id=1] started (kafka.server.KafkaServer)

2.4.1 node2

# /opt/kafka/bin/kafka-server-start.sh -daemon /opt/kafka/config/server.properties

$查看日志
# tail -f logs/kafkaServer.out 
[2020-12-30 14:40:39,723] INFO [SocketServer brokerId=1] Started data-plane processors for 1 acceptors (kafka.network.SocketServer)
[2020-12-30 14:40:39,725] INFO Kafka version: 2.4.1 (org.apache.kafka.common.utils.AppInfoParser)
[2020-12-30 14:40:39,725] INFO Kafka commitId: c57222ae8cd7866b (org.apache.kafka.common.utils.AppInfoParser)
[2020-12-30 14:40:39,725] INFO Kafka startTimeMs: 1609310439724 (org.apache.kafka.common.utils.AppInfoParser)
[2020-12-30 14:40:39,727] INFO [KafkaServer id=2] started (kafka.server.KafkaServer)

2.4.1 node1

# /opt/kafka/bin/kafka-server-start.sh -daemon /opt/kafka/config/server.properties

$查看日志
# tail -f logs/kafkaServer.out 
[2020-12-30 14:40:39,723] INFO [SocketServer brokerId=1] Started data-plane processors for 1 acceptors (kafka.network.SocketServer)
[2020-12-30 14:40:39,725] INFO Kafka version: 2.4.1 (org.apache.kafka.common.utils.AppInfoParser)
[2020-12-30 14:40:39,725] INFO Kafka commitId: c57222ae8cd7866b (org.apache.kafka.common.utils.AppInfoParser)
[2020-12-30 14:40:39,725] INFO Kafka startTimeMs: 1609310439724 (org.apache.kafka.common.utils.AppInfoParser)
[2020-12-30 14:40:39,727] INFO [KafkaServer id=3] started (kafka.server.KafkaServer)