docker下搭建kafka集群

参考博客：mac中使用docker搭建kafka集群

 

创建docker-compose.yml文件，分别是zookeeper、kafka、kafka-manager

version: '2'

services:
  kafka1:
    image: docker.io/wurstmeister/kafka
    #restart: always #自动重新启动
    hostname: kafka1
    container_name: kafka1
    ports:
      - 9093:9093
      - 9193:9193
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_LISTENERS: INSIDE://:9093,OUTSIDE://:9193
      #KAFKA_ADVERTISED_LISTENERS=INSIDE://<container>:9092,OUTSIDE://<host>:9094
      KAFKA_ADVERTISED_LISTENERS: INSIDE://192.168.0.112:9093,OUTSIDE://localhost:9193
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_ZOOKEEPER_CONNECT: zook1:2181
      ALLOW_PLAINTEXT_LISTENER : 'yes'
      JMX_PORT: 9999 #开放JMX监控端口，来监测集群数据
    volumes:
      - /Users/dengxing/Development/volume/kafka/kafka1/wurstmeister/kafka:/wurstmeister/kafka
      - /Users/dengxing/Development/volume/kafka/kafka1/kafka:/kafka
    external_links:
      - zook1
    networks:
      docker-net:
        ipv4_address: 192.168.0.112
  kafka2:
    image: docker.io/wurstmeister/kafka
    #restart: always #自动重新启动
    hostname: kafka2
    container_name: kafka2
    ports:
      - 9094:9094
      - 9194:9194
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_LISTENERS: INSIDE://:9094,OUTSIDE://:9194
      #KAFKA_ADVERTISED_LISTENERS=INSIDE://<container>:9092,OUTSIDE://<host>:9094
      KAFKA_ADVERTISED_LISTENERS: INSIDE://192.168.0.113:9094,OUTSIDE://localhost:9194
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_ZOOKEEPER_CONNECT: zook1:2181
      ALLOW_PLAINTEXT_LISTENER : 'yes'
      JMX_PORT: 9999 #开放JMX监控端口，来监测集群数据
    volumes:
      - /Users/dengxing/Development/volume/kafka/kafka2/wurstmeister/kafka:/wurstmeister/kafka
      - /Users/dengxing/Development/volume/kafka/kafka2/kafka:/kafka
    external_links:
      - zook1
    networks:
      docker-net:
        ipv4_address: 192.168.0.113
  kafka3:
    image: docker.io/wurstmeister/kafka
    #restart: always #自动重新启动
    hostname: kafka3
    container_name: kafka3
    ports:
      - 9095:9095
      - 9195:9195
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_LISTENERS: INSIDE://:9095,OUTSIDE://:9195
      #KAFKA_ADVERTISED_LISTENERS=INSIDE://<container>:9092,OUTSIDE://<host>:9094
      KAFKA_ADVERTISED_LISTENERS: INSIDE://192.168.0.114:9095,OUTSIDE://localhost:9195
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_ZOOKEEPER_CONNECT: zook1:2181
      ALLOW_PLAINTEXT_LISTENER : 'yes'
      JMX_PORT: 9999 #开放JMX监控端口，来监测集群数据
    volumes:
      - /Users/dengxing/Development/volume/kafka/kafka3/wurstmeister/kafka:/wurstmeister/kafka
      - /Users/dengxing/Development/volume/kafka/kafka3/kafka:/kafka
    external_links:
      - zook1
    networks:
      docker-net:
        ipv4_address: 192.168.0.114
networks:
  docker-net:
      name: docker-net

version: '3.4'

services:
  zook1:
    image: zookeeper:latest
    #restart: always #自动重新启动
    hostname: zook1
    container_name: zook1 #容器名称，方便在rancher中显示有意义的名称
    ports:
      - 2183:2181 #将本容器的zookeeper默认端口号映射出去
    volumes: # 挂载数据卷 前面是宿主机即本机的目录位置，后面是docker的目录
      - "/Users/dengxing/opt/kafka/zookeeper/volume/zkcluster/zook1/data:/data"
      - "/Users/dengxing/opt/kafka/zookeeper/volume/zkcluster/zook1/datalog:/datalog"
      - "/Users/dengxing/opt/kafka/zookeeper/volume/zkcluster/zook1/logs:/logs"
    networks:
        docker-net:
            ipv4_address: 192.168.0.115
networks:
  docker-net:
      name: docker-net

version: '2'

services:
  kafka-manager:
    image: scjtqs/kafka-manager:latest
    restart: always
    hostname: kafka-manager
    container_name: kafka-manager
    ports:
      - 9000:9000
    external_links:  # 连接本compose文件以外的container
      - zook1
      - kafka1
      - kafka2
      - kafka3
    environment:
      ZK_HOSTS: zook1:2181
      KAFKA_BROKERS: kafka1:9093,kafka2:9094,kafka3:9095
      APPLICATION_SECRET: letmein
      KM_ARGS: -Djava.net.preferIPv4Stack=true
    networks:
      docker-net:
        ipv4_address: 192.168.0.116
networks:
  docker-net:
    external:
      name: docker-net

依次执行yml文件，生成本地镜像与容器

docker compose -p zookeeper -f ./zk-docker-compose.yml up -d

docker compose -f ./kafka-docker-compose.yml up -d

docker compose -f ./kafka-manager-docker-compose.yml up -d

 

查看生成的结果

mac docker desktop中查看

 

 

执行kafka*.sh遇见以下错误

kafka Error: Exception thrown by the agent : java.rmi.server.ExportException: Port already in use: 9999; nested exception is: java.net.BindException: Address already in use (Bind failed)

 需要修改 kafka-run-class.sh

ISKAFKASERVER="false"
if [[ "$*" =~ "kafka.Kafka" ]]; then
    ISKAFKASERVER="true"
fi
if [  $JMX_PORT ] && [ -z "$ISKAFKASERVER" ]; then
  KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.port=$JMX_PORT "
fi

　

 

进入容器,创建topic

docker exec -it kafka1 /bin/bash

cd opt/kafka

./bin/kafka-topics.sh --create --zookeeper zook1:2181 --replication-factor 1 --partitions 1 --topic businessLogs

查询所有topic

 

创建生产者，写入消息

./kafka-console-producer.sh --broker-list localhost:9093 --topic businessLogs

创建消费者，消费消息（从头开始消费）

./bin/kafka-console-consumer.sh --bootstrap-server localhost:9095 --topic businessLogs --from-beginning

结果如下

 

kafka-manager

访问kafka-manager

添加集群

 

查看topics