大数据_调度平台_docker-compose部署DolphinScheduler

docker-compose 使用 dolphinscheduler

###修改配置文件：docker-compose.yml 
#~/apache-dolphinscheduler-2.0.2-src/docker/docker-swarm
## check
## config.env.sh
## docker-compose.back
## docker-compose.yml
## docker-stack.yml

修改配置文件：docker-compose.yml 解读

version: "3.1"

services:
  
   # PostgreSQL
  dolphinscheduler-postgresql:
    image: bitnami/postgresql:11.11.0
    environment:
	# 设置时区
      TZ: Asia/Shanghai
	   # PostgreSQL 相关的配置  用户名、密码 名为dolphinscheduler 的数据库
      POSTGRESQL_USERNAME: root
      POSTGRESQL_PASSWORD: root
      POSTGRESQL_DATABASE: dolphinscheduler
	 # 数据卷
    volumes:
    - dolphinscheduler-postgresql:/bitnami/postgresql
	# 重启策：在容器退出时总是重启容器
    restart: unless-stopped
	# 配置网络
    networks:
    - dolphinscheduler

  dolphinscheduler-zookeeper:
    image: bitnami/zookeeper:3.6.2
    environment:
      TZ: Asia/Shanghai
	  # Zookeeper 相关配置
      ALLOW_ANONYMOUS_LOGIN: "yes"
      ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
    volumes:
    - dolphinscheduler-zookeeper:/bitnami/zookeeper
    restart: unless-stopped
    networks:
    - dolphinscheduler

  # 自身的服务模块包括：api, alert, master, worker (有一个 logger 服务，运行在 worker 中)
  dolphinscheduler-api:
    image: apache/dolphinscheduler:2.0.2
    command: api-server
    ports:
    - 12345:12345
    environment:
      TZ: Asia/Shanghai
	 # 引入外部环境变量 config.env.sh 定义了用到的配置，通过 env_file 的方式传入容器，它会覆盖容器内的默认配置
    env_file: config.env.sh
     # 调用容器内的 checkpoint.sh，并传入服务名称，检查该 Java 进程是否存在。
     # 两次健康检查的间隔 30s，超时时间为 5s，如果超过这个时间，本次健康检查就被视为失败，
     # retries 重试次数设置为 3，当连续失败指定次数后，则将容器状态视为 unhealthy
    healthcheck:
      test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
      interval: 30s
      timeout: 5s
      retries: 3
	# 依赖 PostgreSQL 和 Zookeeper
    depends_on:
    - dolphinscheduler-postgresql
    - dolphinscheduler-zookeeper
    volumes:
    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
    - dolphinscheduler-shared-local:/opt/soft
    - dolphinscheduler-resource-local:/dolphinscheduler
    restart: unless-stopped
    networks:
    - dolphinscheduler

  dolphinscheduler-alert:
    image: apache/dolphinscheduler:2.0.2
    command: alert-server
    environment:
      TZ: Asia/Shanghai
    env_file: config.env.sh
    healthcheck:
      test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
      interval: 30s
      timeout: 5s
      retries: 3
    depends_on:
    - dolphinscheduler-postgresql
    volumes:
    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
    restart: unless-stopped
    networks:
    - dolphinscheduler

  dolphinscheduler-master:
    image: apache/dolphinscheduler:2.0.2
    command: master-server
    environment:
      TZ: Asia/Shanghai
    env_file: config.env.sh
    healthcheck:
      test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
      interval: 30s
      timeout: 5s
      retries: 3
    depends_on:
    - dolphinscheduler-postgresql
    - dolphinscheduler-zookeeper
    volumes:
    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
    - dolphinscheduler-shared-local:/opt/soft
    restart: unless-stopped
    networks:
    - dolphinscheduler

  dolphinscheduler-worker:
    image: apache/dolphinscheduler:2.0.2
    command: worker-server
    environment:
      TZ: Asia/Shanghai
    env_file: config.env.sh
    healthcheck:
      test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]
      interval: 30s
      timeout: 5s
      retries: 3
    depends_on:
    - dolphinscheduler-postgresql
    - dolphinscheduler-zookeeper
    volumes:
    - dolphinscheduler-worker-data:/tmp/dolphinscheduler
    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
    - dolphinscheduler-shared-local:/opt/soft
    - dolphinscheduler-resource-local:/dolphinscheduler
    restart: unless-stopped
    networks:
    - dolphinscheduler
		
  dolphinscheduler-worker2:
    image: apache/dolphinscheduler:2.0.2
    command: worker-server
    environment:
      TZ: Asia/Shanghai
    env_file: config.env.sh
    healthcheck:
      test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]
      interval: 30s
      timeout: 5s
      retries: 3
    depends_on:
    - dolphinscheduler-postgresql
    - dolphinscheduler-zookeeper
    volumes:
    - dolphinscheduler-worker-data:/tmp/dolphinscheduler
    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
    - dolphinscheduler-shared-local:/opt/soft
    - dolphinscheduler-resource-local:/dolphinscheduler
    restart: unless-stopped
    networks:
    - dolphinscheduler	
	
# 声明使用到的网络
networks:
  dolphinscheduler:
    driver: bridge
	
# 声明使用到的数据卷
volumes:
  dolphinscheduler-postgresql:
  dolphinscheduler-zookeeper:
  dolphinscheduler-worker-data:
  dolphinscheduler-logs:
  dolphinscheduler-shared-local:
  dolphinscheduler-resource-local:

构建生产环境中使用的

# 创建一个新的 Dockerfile，用于添加 mysql 驱动包:
#其中 mysql-connector-java-8.0.17.jar 是在soft 的目录下
cd ~/soft
vim Dockerfile

FROM apache/dolphinscheduler:2.0.3
COPY mysql-connector-java-8.0.17.jar /opt/dolphinscheduler/lib
COPY source.list /tmp/source.list
CMD  cat /tmp/source.list > /etc/apt/sources.list  
RUN apt-get update && \
    apt-get install -y --no-install-recommends ssh && \
    apt-get install -y --no-install-recommends python3 && \
    rm -rf /var/lib/apt/lists/*



# 构建一个包含 Oracle 驱动包的新镜像:
docker build -t apache/dolphinscheduler:sql-driver .


将 docker-compose.yml 文件中的所有 image 字段修改为 apache/dolphinscheduler:mysql-driver

暂时的方法

FROM apache/dolphinscheduler:2.0.3
COPY mysql-connector-java-8.0.17.jar /opt/dolphinscheduler/lib
COPY source.list /tmp/source.list
COPY hosts /tmp/hosts
COPY resolv.conf /tmp/resolv.conf
CMD cat /tmp/hosts >> /etc/hosts
RUN cat /tmp/source.list >/etc/apt/sources.list && \
    cat /tmp/resolv.conf >/etc/resolv.conf && \
    cat /tmp/hosts >> /etc/hosts && \
    apt-get update  --allow-insecure-repositories && \
    apt-get install -y --no-install-recommends python  --allow-unauthenticated && \
    apt-get install -y --no-install-recommends python-pip --allow-unauthenticated && \
    rm -rf /var/lib/apt/lists/*

Dockerfile

 /etc/hosts
 /etc/apt/sources.list
 /etc/resolv.conf
 
 /etc/default/docker
 /etc/docker/daemon.json

 Reading package lists... Done
 E: Could not get lock /var/lib/apt/lists/lock - open (11: Resource temporarily unavailable)
 E: Unable to lock directory /var/lib/apt/lists/
   rm -rf /var/lib/apt/lists/*
   
 “Repository does not have a release file”是因为添加的储存库不适用于当前Ubuntu版本
   

1.Docker容器修改  hosts 
     /etc/hosts, /etc/resolv.conf和/etc/hostname 容器中的这三个文件不存在于镜像,
           而是存在于于/var/lib/docker/containers/，在启动容器的时候，通过mount的形式将这些文件挂载到容器内部。
     因此，如果在容器中修改这些文件的话，修改部分不会存在于容器的top layer，而是直接写入这三个物理文件中。
2.为什么重启后修改内容不存在了？原因是：每次Docker在启动容器的时候，通过重新构建新的/etc/hosts文件，这又是为什么呢？
     原因是：容器重启，IP地址为改变，hosts文件中原来的IP地址无效，因此理应修改hosts文件，否则会产生脏数据。
3.有没有什么好的解决方法? 
 可以通过docker run命令的–add-host参数来为容器添加host与ip的映射关系。
 

Dockerfile文件
   COPY ADD
      Dockerfile中的COPY指令和ADD指令都可以将主机上的资源复制或加入到容器镜像中，都是在构建镜像的过程中完成的。
      COPY指令和ADD指令的唯一区别在于是否支持从远程URL获取资源。
      COPY指令只能从执行docker build所在的主机上读取资源并复制到镜像中。
      而ADD指令还支持通过URL从远程服务器读取资源并复制到镜像中
   RUN CMD  ENTRYPOINT
      CMD 在docker run 时运行
      CMD 指令指定的程序可被 docker run 命令行参数中指定要运行的程序所覆盖。
      如果 Dockerfile 中如果存在多个 CMD 指令，仅最后一个生效
   ENV  ARG
    FROM 
    VOLUME  EXPOSE  WORKDIR USER  LABEL

参考

 Apache Dolphin Scheduler - Docker Compose 详解  https://www.cnblogs.com/aaronlinv/p/15309275.html 
  Modify hosts file in dockerfile  https://serverfault.com/questions/777714/modify-hosts-file-in-dockerfile