dremio 21.1 分布式存储配置demo

主要演示下，具体参数以前我有说明过

环境准备

• dockerfile

FROM  dremio/dremio-oss:21.1

USER root

COPY dremio-env-profile /opt/dremio/conf/dremio-env

COPY --from=hengyunabc/arthas:latest /opt/arthas /opt/arthas

COPY core-site.xml /opt/dremio/conf/core-site.xml

COPY dremio.conf /opt/dremio/conf/dremio.conf

COPY jprofiler_linux_12_0_4.deb /opt/jprofiler_linux_12_0_4.deb

RUN dpkg -i /opt/jprofiler_linux_12_0_4.deb

COPY my-mysql-19.0.0-202110270341120595-3e3e05e7.jar /opt/dremio/jars/my-mysql-19.0.0-202110270341120595-3e3e05e7.jar

COPY mysql-connector-java-5.1.47.jar /opt/dremio/jars/3rdparty/mysql-connector-java-5.1.47.jar

COPY mysql-connector-java-8.0.20.jar /opt/dremio/jars/3rdparty/mysql-connector-java-8.0.20.jar

USER dremio

• core-site.xml

<?xml version="1.0"?>

<configuration>

    <property>

        <name>fs.dremioS3.impl</name>

        <description>The FileSystem implementation. Must be set to com.dremio.plugins.s3.store.S3FileSystem</description>

        <value>com.dremio.plugins.s3.store.S3FileSystem</value>

    </property>

    <property>

        <name>fs.s3a.access.key</name>

        <description>key</description>

        <value>minio</value>

    </property>

    <property>

        <name>fs.s3a.secret.key</name>

        <description>secret</description>

        <value>minio123</value>

    </property>

    <property>

        <name>fs.s3a.aws.credentials.provider</name>

        <description>The credential provider type.</description>

        <value>org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider</value>

    </property>

    <property>

        <name>fs.s3a.endpoint</name>

        <description>Endpoint can either be an IP or a hostname, where Minio server is running . However the endpoint value cannot contain the http(s) prefix. E.g. 175.1.2.3:9000 is a valid endpoint. </description>

        <value>minio:9000</value>

    </property>

    <property>

        <name>fs.s3a.path.style.access</name>

        <description>Value has to be set to true.</description>

        <value>true</value>

    </property>

    <property>

        <name>dremio.s3.compat</name>

        <description>Value has to be set to true.</description>

        <value>true</value>

    </property>

    <property>

        <name>fs.s3a.connection.ssl.enabled</name>

        <description>Value can either be true or false, set to true to use SSL with a secure Minio server.</description>

        <value>false</value>

    </property>

</configuration>

• dremio.conf

paths: {

  # the local path for dremio to store data.

  local: ${DREMIO_HOME}"/data"

  # the distributed path Dremio data including job results, downloads, uploads, etc

  #dist: "pdfs://"${paths.local}"/pdfs"

  dist: "dremioS3:///myappdemo/app1/"

  accelerator: ${paths.dist}/accelerator,

  downloads: ${paths.dist}/downloads,

  uploads: ${paths.dist}/uploads,

  results: ${paths.dist}/results

  scratch: ${paths.dist}/scratch

 

}

 

services: {

  coordinator.enabled: true,

  coordinator.master.enabled: true,

  executor.enabled: true,

  flight.use_session_service: true

}

• docker-compose 文件

version: '3'

services:

  pg:

    image: postgres:14.3-alpine3.15

    environment:

    - POSTGRES_PASSWORD=dalongdemo

    ports:

      - 5432:5432

  minio:

    image: minio/minio

    ports:

      - "9002:9000"

      - "19001:19001"

    environment:

      MINIO_ACCESS_KEY: minio

      MINIO_SECRET_KEY: minio123

    command: server --console-address :19001 --quiet /data

  dremio:

    build: ./

    image: dalongrong/dremio:21.1-jprofiler

    ports:

      - "9047:9047"

      - "8849:8849"

      - "31010:31010"

s3 效果

具体配置比较简单，我只截图说明下，因为21.1 之后反射必须使用分布式存储了

 

 

说明

以上只是演示下，因为21.1 必须使用分布式存储了，所以后续必须升级了，刚好进行下功能测试

参考资料

https://docs.dremio.com/software/advanced-administration/dremio-conf/
https://docs.dremio.com/software/deployment/dist-store-config/
https://github.com/rongfengliang/dremio-s3-dist-store