Kafka环境安装
wget https://downloads.apache.org/kafka/3.6.1/kafka_2.13-3.6.1.tgz
sudo mkdir /usr/local/kafka-server && cd $_
sudo tar -xvzf ~/kafka_2.13-3.6.1.tgz --strip 1
sudo useradd -r -d /usr/local/kafka-server -s /usr/sbin/nologin kafka
sudo -u kafka mkdir -p /usr/local/kafka-server/logs
sudo -u kafka mkdir -p /usr/local/kafka-server/data
sudo touch /usr/local/kafka-server/test.txt
sudo chmod a+w /usr/local/kafka-server/test.txt
sudo -u kafka echo -e "消息甲\n消息乙(文件流)" > /usr/local/kafka-server/test.txt
sudo vi /usr/local/kafka-server/config/server.properties
# log.dirs=/tmp/kafka-logs
log.dirs=/usr/local/kafka-server/logs
# Listener name, hostname and port the broker will advertise to clients.
# If not set, it uses the value for "listeners".
#advertised.listeners=PLAINTEXT://your.host.name:9092
# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details
#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL
sudo vi /usr/local/kafka-server/config/zookeeper.properties
# dataDir=/tmp/zookeeper
dataDir=/usr/local/kafka-server/data
sudo chown -R kafka:kafka /usr/local/kafka-server
sudo vi /etc/systemd/system/zookeeper.service
[Unit]
Requires=network.target remote-fs.target
After=network.target remote-fs.target
[Service]
Type=simple
User=kafka
ExecStart=/usr/local/kafka-server/bin/zookeeper-server-start.sh /usr/local/kafka-server/config/zookeeper.properties
ExecStop=/usr/local/kafka-server/bin/zookeeper-server-stop.sh
Restart=on-abnormal
[Install]
WantedBy=multi-user.target
sudo vi /etc/systemd/system/kafka.service
[Unit]
Requires=zookeeper.service
After=zookeeper.service
[Service]
Type=simple
User=kafka
ExecStart=/bin/sh -c '/usr/local/kafka-server/bin/kafka-server-start.sh /usr/local/kafka-server/config/server.properties > /usr/local/kafka-server/logs/start-kafka.log 2>&1'
ExecStop=/usr/local/kafka-server/bin/kafka-server-stop.sh
Restart=on-abnormal
[Install]
WantedBy=multi-user.target
sudo systemctl daemon-reload
sudo systemctl enable --now zookeeper
sudo systemctl enable --now kafka
sudo systemctl status zookeeper
sudo systemctl status kafka
sudo systemctl restart zookeeper
sudo systemctl restart kafka
# 查看主题
sudo -u kafka /usr/local/kafka-server/bin/kafka-topics.sh --list --bootstrap-server localhost:9092
sudo -u kafka /usr/local/kafka-server/bin/kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 --topic TestTopic
# 查看主题
sudo -u kafka /usr/local/kafka-server/bin/kafka-topics.sh --list --bootstrap-server localhost:9092
TestTopic
# 订阅消息
sudo -u kafka /usr/local/kafka-server/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic TestTopic --from-beginning
# 发送消息
sudo -u kafka /usr/local/kafka-server/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic TestTopic
# 删除主题
sudo -u kafka /usr/local/kafka-server/bin/kafka-topics.sh --bootstrap-server localhost:9092 --delete --topic TestTopic
# 文件流消息的使用
sudo -u kafka vi /usr/local/kafka-server/config/connect-standalone.properties
plugin.path=/usr/local/kafka-server/libs/connect-file-3.6.1.jar
sudo -u kafka vi /usr/local/kafka-server/config/connect-file-source.properties
name=local-file-source
connector.class=FileStreamSource
tasks.max=1
file=test.txt
topic=connect-test
cd /usr/local/kafka-server/
sudo -u kafka /usr/local/kafka-server/bin/connect-standalone.sh config/connect-standalone.properties config/connect-file-source.properties config/connect-file-sink.properties
sudo -u kafka /usr/local/kafka-server/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic connect-test --from-beginning
