Linux使用docker安装flink

 

 

配置文件不是原始的 修改过，可以根据自己的配置文件来 

 

flink-conf.yaml

#定义与作业管理器通信的地址 这里设置好像没用 要通过环境变量注入
jobmanager.rpc.address: 192.168.13.25
#定义与作业管理器通信的网络端口的配置参数
jobmanager.rpc.port: 6123
#TaskManager暴露的外部RPC端口
taskmanager.rpc.port: 6122
jobmanager.bind-host: 0.0.0.0
taskmanager.bind-host: 0.0.0.0
#TaskManager暴露的网络接口的外部地址
taskmanager.host: 192.168.13.35
#客户端用于连接到服务器的地址 和jobmanager地址一样
rest.address: 192.168.13.25
#服务器绑定的地址
rest.bind-address: 0.0.0.0
#TaskManager提供的插槽数（默认值：1） 但是这个这样设置没生效 通过环境变量可以
taskmanager.numberOfTaskSlots: 3
#JobManager的总进程内存大小
jobmanager.memory.process.size: 1500m
#TaskExecutor的总进程内存大小
taskmanager.memory.process.size: 20000m
#TaskExecutor框架的堆外内存大小。 默认值：128 mb
taskmanager.memory.framework.off-heap.size: 1024m
#TaskExecutor的任务堆外内存大小。 默认值：0 bytes
taskmanager.memory.task.off-heap.size: 384m
#TaskExecutor的最大网络内存大小
taskmanager.memory.network.max: 4096m
# TaskExecutor的最小网络内存大小
taskmanager.memory.network.min: 4096m
taskmanager.memory.network.fraction: 0.4
parallelism.default: 1

blob.server.port: 6124

#Flink内部指标查询服务使用的端口范围  默认情况下，Flink将选择一个随机端口
metrics.internal.query-service.port: 6125
#数据交换接口
taskmanager.data.port: 34037

#flink重启后，不会丢失jar
jobmanager.web.tmpdir: /tmp/flink-web
jobmanager.web.upload.dir: /tmp/flink-web

 

log4j-console.properties

# This affects logging for both user code and Flink
rootLogger.level = INFO
rootLogger.appenderRef.console.ref = ConsoleAppender
rootLogger.appenderRef.rolling.ref = RollingFileAppender

logger.akka.name = akka
logger.akka.level = INFO
logger.kafka.name= org.apache.kafka
logger.kafka.level = INFO
logger.hadoop.name = org.apache.hadoop
logger.hadoop.level = INFO
logger.zookeeper.name = org.apache.zookeeper
logger.zookeeper.level = INFO

# Log all infos to the console
appender.console.name = ConsoleAppender
appender.console.type = CONSOLE
appender.console.layout.type = PatternLayout
appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n

# Log all infos in the given rolling file
appender.rolling.name = RollingFileAppender
appender.rolling.type = RollingFile
appender.rolling.append = false
appender.rolling.fileName = ${sys:log.file}
appender.rolling.filePattern = ${sys:log.file}.%i
appender.rolling.layout.type = PatternLayout
appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
appender.rolling.policies.type = Policies
appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
appender.rolling.policies.size.size=100MB
appender.rolling.strategy.type = DefaultRolloverStrategy
appender.rolling.strategy.max = 10


# Suppress the irrelevant (wrong) warnings from the Netty channel handler
logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
logger.netty.level = OFF

 

 

 

docker-compose.yml

version: '3'
services:
  jobmanager:
    image: flink:1.14.2
    container_name: flink_jobmanager
    ports:
      - "6123:6123"
      - "8081:8081"
      - "6124:6124"
      - "6125:6125"
    volumes:
      - ./flink-conf.yaml:/opt/flink/conf/flink-conf.yaml
      - ./log4j-console.properties:/opt/flink/conf/log4j-console.properties
    environment:
      - JOB_MANAGER_RPC_ADDRESS=192.168.13.25
      - TZ=Asia/Shanghai
    command: jobmanager

  taskmanager:
    image: flink:1.14.2
    container_name: flink_taskmanager
    ports:
      - "6122:6122"
      - "6125:6125"
      - "34037:34037"
    volumes:
      - ./flink-conf.yaml:/opt/flink/conf/flink-conf.yaml
      - ./log4j-console.properties:/opt/flink/conf/log4j-console.properties
    environment:
      - JOB_MANAGER_RPC_ADDRESS=192.168.13.25
      - TASK_MANAGER_NUMBER_OF_TASK_SLOTS=10
      - TZ=Asia/Shanghai
    command: taskmanager

 

 

 

访问http://ip:8081/

 

 

这种方法部署有个问题没有解决就是

 

 这里可能看不到日志，但是 logs日志可以看到

 

dockerfile写法

jobmanager

FROM flink:1.14.2

# 设置工作目录
WORKDIR /opt/flink

# 复制配置文件到容器
COPY ./flink-conf.yaml /opt/flink/conf/flink-conf.yaml

# 复制log 文件
COPY ./log4j-console.properties /opt/flink/conf/log4j-console.properties


# 设置时区
ENV TZ=Asia/Shanghai

# 暴露端口
EXPOSE 6123 8081 6124 6125

# 启动命令
CMD ["jobmanager"]

 

taskmanager

FROM flink:1.14.2

# 设置工作目录
WORKDIR /opt/flink

# 复制配置文件到容器
COPY ./flink-conf.yaml /opt/flink/conf/flink-conf.yaml

# 复制log 文件
COPY ./log4j-console.properties /opt/flink/conf/log4j-console.properties



RUN ls /opt/flink/lib


# 设置时区
ENV TZ=Asia/Shanghai

# 暴露端口
EXPOSE 6122 6125 34037

# 启动命令
CMD ["taskmanager"]