dremio 集群简单部署

一个简单的测试，不算是ha （ha 后边会介绍使用）

环境准备

  • docker-compose 文件
version: "3"
services:
  zookeeper:
    image: zookeeper
    ports:
    - "2181:2181"
    - "8080:8080"
  dremio1:
    image: dremio/dremio-oss:13.0
    environment:
     - DREMIO_JAVA_SERVER_EXTRA_OPTS=-Dsaffron.default.charset=UTF-16LE -Dsaffron.default.nationalcharset=UTF-16LE -Dsaffron.default.collation.name=UTF-16LE\$en_US
    volumes: 
    - "./dremio1.conf:/opt/dremio/conf/dremio.conf"
    - "./datas/data:/opt/dremio/data"
    ports:
      - "9047:9047"
      - "31010:31010"
  dremio2:
    image: dremio/dremio-oss:13.0
    environment:
     - DREMIO_JAVA_SERVER_EXTRA_OPTS=-Dsaffron.default.charset=UTF-16LE -Dsaffron.default.nationalcharset=UTF-16LE -Dsaffron.default.collation.name=UTF-16LE\$en_US
    volumes: 
    - "./dremio3.conf:/opt/dremio/conf/dremio.conf"
    ports:
      - "9048:9047"
      - "31011:31010"
  dremio3:
    image: dremio/dremio-oss:13.0
    environment:
     - DREMIO_JAVA_SERVER_EXTRA_OPTS=-Dsaffron.default.charset=UTF-16LE -Dsaffron.default.nationalcharset=UTF-16LE -Dsaffron.default.collation.name=UTF-16LE\$en_US
    volumes: 
    - "./dremio3.conf:/opt/dremio/conf/dremio.conf"
    ports:
      - "9049:9047"
      - "31012:31010"
  pg:
    image: postgres:12
    environment:
      - "POSTGRES_PASSWORD=dalong"
    ports:
      - "5432:5432"
  mongo:
    image: mongo
    ports:
    - "27017:27017"
 
  • 简单说明

一个参考架构（使用了一个master，使用了一个独立的外部zk，使用了两个executor）

 

 


核心配置说明
主要是services 片段的配置
dremio 对于不同类型还是比较明确的，我们基于配置就可以了
参考配置
master dremio1.conf

 
paths: {
  # the local path for dremio to store data.
  local: ${DREMIO_HOME}"/data"
 
  # the distributed path Dremio data including job results, downloads, uploads, etc
  #dist: "pdfs://"${paths.local}"/pdfs"
}
services: {
  coordinator.enabled: true,
  coordinator.master.enabled: true,
  coordinator.master.embedded-zookeeper.enabled: false,
  executor.enabled: false
}
zookeeper: "zookeeper:2181"
 
 

executor 1 dremio2.conf

paths: {
  # the local path for dremio to store data.
  local: ${DREMIO_HOME}"/data"
 
  # the distributed path Dremio data including job results, downloads, uploads, etc
  #dist: "pdfs://"${paths.local}"/pdfs"
}
 
services: {
  coordinator.enabled: false,
  coordinator.master.enabled: false,
  coordinator.master.embedded-zookeeper.enabled: false,
  executor.enabled: true
}
zookeeper: "zookeeper:2181"

executor 2 dremio3.conf

paths: {
  # the local path for dremio to store data.
  local: ${DREMIO_HOME}"/data"
 
  # the distributed path Dremio data including job results, downloads, uploads, etc
  #dist: "pdfs://"${paths.local}"/pdfs"
}
 
services: {
  coordinator.enabled: false,
  coordinator.master.enabled: false,
  coordinator.master.embedded-zookeeper.enabled: false,
  executor.enabled: true
}
zookeeper: "zookeeper:2181"

运行效果

  • 启动
docker-compose up -d
  • 效果

 

 

  • 一个参考执行效果

查看执行job

 

 

参考资料

https://docs.dremio.com/deployment/nodes-config.html
https://docs.dremio.com/deployment/high-availability-config.html

posted on 2021-02-07 23:25  荣锋亮  阅读(580)  评论(0)  编辑  收藏  举报

导航