dremio 集群简单部署

一个简单的测试，不算是ha （ha 后边会介绍使用）

环境准备

• docker-compose 文件

version: "3"

services:

  zookeeper:

    image: zookeeper

    ports:

    - "2181:2181"

    - "8080:8080"

  dremio1:

    image: dremio/dremio-oss:13.0

    environment:

     - DREMIO_JAVA_SERVER_EXTRA_OPTS=-Dsaffron.default.charset=UTF-16LE -Dsaffron.default.nationalcharset=UTF-16LE -Dsaffron.default.collation.name=UTF-16LE\$en_US

    volumes: 

    - "./dremio1.conf:/opt/dremio/conf/dremio.conf"

    - "./datas/data:/opt/dremio/data"

    ports:

      - "9047:9047"

      - "31010:31010"

  dremio2:

    image: dremio/dremio-oss:13.0

    environment:

     - DREMIO_JAVA_SERVER_EXTRA_OPTS=-Dsaffron.default.charset=UTF-16LE -Dsaffron.default.nationalcharset=UTF-16LE -Dsaffron.default.collation.name=UTF-16LE\$en_US

    volumes: 

    - "./dremio3.conf:/opt/dremio/conf/dremio.conf"

    ports:

      - "9048:9047"

      - "31011:31010"

  dremio3:

    image: dremio/dremio-oss:13.0

    environment:

     - DREMIO_JAVA_SERVER_EXTRA_OPTS=-Dsaffron.default.charset=UTF-16LE -Dsaffron.default.nationalcharset=UTF-16LE -Dsaffron.default.collation.name=UTF-16LE\$en_US

    volumes: 

    - "./dremio3.conf:/opt/dremio/conf/dremio.conf"

    ports:

      - "9049:9047"

      - "31012:31010"

  pg:

    image: postgres:12

    environment:

      - "POSTGRES_PASSWORD=dalong"

    ports:

      - "5432:5432"

  mongo:

    image: mongo

    ports:

    - "27017:27017"

 

• 简单说明

一个参考架构（使用了一个master，使用了一个独立的外部zk，使用了两个executor）

 

 

核心配置说明
主要是services 片段的配置
dremio 对于不同类型还是比较明确的，我们基于配置就可以了
参考配置
master dremio1.conf

 

paths: {

  # the local path for dremio to store data.

  local: ${DREMIO_HOME}"/data"

 

  # the distributed path Dremio data including job results, downloads, uploads, etc

  #dist: "pdfs://"${paths.local}"/pdfs"

}

services: {

  coordinator.enabled: true,

  coordinator.master.enabled: true,

  coordinator.master.embedded-zookeeper.enabled: false,

  executor.enabled: false

}

zookeeper: "zookeeper:2181"

 

 

executor 1 dremio2.conf

paths: {

  # the local path for dremio to store data.

  local: ${DREMIO_HOME}"/data"

 

  # the distributed path Dremio data including job results, downloads, uploads, etc

  #dist: "pdfs://"${paths.local}"/pdfs"

}

 

services: {

  coordinator.enabled: false,

  coordinator.master.enabled: false,

  coordinator.master.embedded-zookeeper.enabled: false,

  executor.enabled: true

}

zookeeper: "zookeeper:2181"

executor 2 dremio3.conf

paths: {

  # the local path for dremio to store data.

  local: ${DREMIO_HOME}"/data"

 

  # the distributed path Dremio data including job results, downloads, uploads, etc

  #dist: "pdfs://"${paths.local}"/pdfs"

}

 

services: {

  coordinator.enabled: false,

  coordinator.master.enabled: false,

  coordinator.master.embedded-zookeeper.enabled: false,

  executor.enabled: true

}

zookeeper: "zookeeper:2181"

运行效果

• 启动

docker-compose up -d

• 效果

 

 

• 一个参考执行效果

查看执行job

 

 

参考资料

https://docs.dremio.com/deployment/nodes-config.html
https://docs.dremio.com/deployment/high-availability-config.html