kafka_2.12-2.5.0 集群安装
两台主机
| 主机 | hostname | zook | kafka |
| 192.168.0.21 | node-001 | zookeeper-001 | kafka-001 |
| 192.168.0.21 | node-002 | zookeeper-002 | kafka-002 |
| 192.168.0.22 | node-003 | zookeeper-003 | kafka-003 |
部署zookeeper
wget http://archive.apache.org/dist/zookeeper/zookeeper-3.4.5/zookeeper-3.4.5.tar.gz
配置hosts
vim /etc/hosts 192.168.0.21 es-test-001 node-001 node-002 192.168.0.22 es-test-002 es-test-003 node-003
开始部署
tar xf zookeeper-3.4.5.tar.gz -C /data/ mv /data/zookeeper /data/zookeeper-001 cp /data/zookeeper-001 /data/zookeeper-002
配置环境变量
vim /etc/profile export ZOOKEEPER_HOME=/data/zookeeper-001 export PATH=$PATH:$ZOOKEEPER_HOME/bin
添加配置文件
cp /data/zookeeper-001/conf/zoo_sample.cfg /data/zookeeper-001/conf/zoo.cfg cp /data/zookeeper-002/conf/zoo_sample.cfg /data/zookeeper-002/conf/zoo.cfg
修改配置文件
mkdir /data/zookeeper-001/{data,logs}
# 修改主机myid
echo 1 > /data/zookeeper-001/data/myid
vim /data/zookeeper-001/conf/zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
clientPort=2181
server.1=node-001:2888:3888
server.2=node-002:2889:3889
server.3=node-003:2888:3888
dataDir=/data/zookeeper-001/data
dataLogDir=/data/zookeeper-001/logs
mkdir /data/zookeeper-002/{data,logs}
# 修改主机myid
echo 2 > /data/zookeeper-002/data/myid
vim /data/zookeeper-002/conf/zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
clientPort=2182
server.1=node-001:2888:3888
server.2=node-002:2889:3889
server.3=node-003:2888:3888
dataDir=/data/zookeeper-002/data
dataLogDir=/data/zookeeper-002/logs
mkdir /data/zookeeper-003/{data,logs}
# 修改主机myid
echo 3 > /data/zookeeper-003/data/myid
vim /data/zookeeper-003/conf/zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
clientPort=2182
server.1=node-001:2888:3888
server.2=node-002:2889:3889
server.3=node-003:2888:3888
dataDir=/data/zookeeper-003/data
dataLogDir=/data/zookeeper-001/logs
启动zk
/data/zookeeper-001/bin/zkServer.sh start /data/zookeeper-002/bin/zkServer.sh start /data/zookeeper-003/bin/zkServer.sh start
查看三个节点状态
JMX enabled by default Using config: /data/zookeeper-001/bin/../conf/zoo.cfg Mode: leader JMX enabled by default Using config: /data/zookeeper-002/bin/../conf/zoo.cfg Mode: follower JMX enabled by default Using config: /data/zookeeper-003/bin/../conf/zoo.cfg Mode: follower
二、安装kafka
下载kafka
wget https://mirror.bit.edu.cn/apache/kafka/2.5.0/kafka_2.12-2.5.0.tgz
解压kafka
node-001 node-002 tar xf kafka_2.12-2.5.0.tgz -C /data/ mv /data/kafka_2.12-2.5.0 /data/kafka-001 cp -a /data/kafka-001 /data/kafka-002 node-003 tar xf kafka_2.12-2.5.0.tgz -C /data/ mv /data/kafka_2.12-2.5.0 /data/kafka-003
备份配置文件
cp /data/kafka-001/config/server.properties /data/kafka-001/config/server.properties.bak cp /data/kafka-002/config/server.properties /data/kafka-002/config/server.properties.bak cp /data/kafka-003/config/server.properties /data/kafka-003/config/server.properties.bak
修改配置文件
# node-001 # 唯一id broker.id=0 num.network.threads=3 num.io.threads=8 socket.send.buffer.bytes=102400 socket.receive.buffer.bytes=102400 socket.request.max.bytes=104857600 # 日志目录 log.dirs=/data/kafka-001/logs num.partitions=1 num.recovery.threads.per.data.dir=1 offsets.topic.replication.factor=1 transaction.state.log.replication.factor=1 transaction.state.log.min.isr=1 log.retention.hours=168 log.segment.bytes=1073741824 log.retention.check.interval.ms=300000 # zk集群 zookeeper.connect=192.168.0.21:2181,192.168.0.21:2181,192.168.0.22:2181 zookeeper.connection.timeout.ms=18000 group.initial.rebalance.delay.ms=0 # hostname 这个可随便填，配置过hosts就行 host.name=node-001 # 配置端口 单台多服务需要配置端口 listeners=PLAINTEXT://node-001:9092 # node-002 # 唯一id broker.id=1 num.network.threads=3 num.io.threads=8 socket.send.buffer.bytes=102400 socket.receive.buffer.bytes=102400 socket.request.max.bytes=104857600 # 日志目录 log.dirs=/data/kafka-002/logs num.partitions=1 num.recovery.threads.per.data.dir=1 offsets.topic.replication.factor=1 transaction.state.log.replication.factor=1 transaction.state.log.min.isr=1 log.retention.hours=168 log.segment.bytes=1073741824 log.retention.check.interval.ms=300000 # zk集群 zookeeper.connect=192.168.0.21:2181,192.168.0.21:2181,192.168.0.22:2181 zookeeper.connection.timeout.ms=18000 group.initial.rebalance.delay.ms=0 host.name=node-002 listeners=PLAINTEXT://node-002:9093 # node-003 broker.id=2 num.network.threads=3 num.io.threads=8 socket.send.buffer.bytes=102400 socket.receive.buffer.bytes=102400 socket.request.max.bytes=104857600 log.dirs=/data/kafka-003/logs num.partitions=1 num.recovery.threads.per.data.dir=1 offsets.topic.replication.factor=1 transaction.state.log.replication.factor=1 transaction.state.log.min.isr=1 log.retention.hours=168 log.segment.bytes=1073741824 log.retention.check.interval.ms=300000 zookeeper.connect=192.168.0.21:2181,192.168.0.21:2181,192.168.0.22:2181 zookeeper.connection.timeout.ms=18000 group.initial.rebalance.delay.ms=0 host.name=node-003 listeners=PLAINTEXT://node-003:9092
启动kafka
nohup /data/kafka-001/bin/kafka-server-start.sh /data/kafka-001/config/server.properties >/dev/null 2>&1 & nohup /data/kafka-002/bin/kafka-server-start.sh /data/kafka-002/config/server.properties nohup /data/kafka-003/bin/kafka-server-start.sh /data/kafka-003/config/server.properties
验证
在node-001中执行 发送消息 sh /data/kafka-001/bin/kafka-console-producer.sh --broker-list node-001:9092 --topic test-topic test-key test2 在node-002中执行 消费消息 sh /data/kafka-002/bin/kafka-console-consumer.sh --bootstrap-server node-001:9092 --topic test-topic test-key test2
