dremio 参考配置参考

实际上我以前简单说明过，下边包含一个官方文档完整的，方便参数学习

官方提供的参考配置

  • dremio-reference.conf
include classpath("dremio-reference-ext.conf")
 
paths: {
 
  # the local path for dremio to store data.
  local: "/tmp/dremio",
  # 此配置对于21 之后版本是比较重要的，需要使用分布式存储
  # the distributed path Dremio data including job results, downloads, uploads, etc
  dist: "pdfs://"${paths.local}"/pdfs"
 
  # location for catalog database (if master node)
  db: ${paths.local}/db,
 
  spilling: [${paths.local}/spill]
 
  # storage area for the accelerator cache.
  accelerator: ${paths.dist}/accelerator
 
  # staging area for json and csv ui downloads
  downloads: ${paths.dist}/downloads
 
  # stores uploaded data associated with user home directories
  uploads: ${paths.dist}/uploads
 
  # stores data associated with the job results cache.
  results: ${paths.dist}/results
 
  # shared scratch space for creation of tables.
  scratch: ${paths.dist}/scratch
 
  # storage area for internal Iceberg tables.
  metadata: ${paths.dist}/metadata
 
}
# dremio 服务定义的，包含了协调节点，执行节点，以及候选节点（会执行查询，但是不能进行元数据操作）
services: {
  node-tag: "",
  coordinator: {
    enabled: true,
 
    # Auto-upgrade Dremio at startup if needed
    auto-upgrade: false,
 
    master: {
      enabled: true,
      # configure an embedded ZooKeeper server on the same node as master
      embedded-zookeeper: {
        enabled: true,
        port: 2181,
        path: ${paths.local}/zk
      }
    },
   # web 配置
    web: {
      enabled: true,
      port: 9047,
      ssl: {
        # If SSL for communication path between browsers (or REST clients) and Dremio should be enabled.
        enabled: false,
 
        # Allow for auto-generated certificates if keyStore option is not set
        # Auto-generated self-signed certificates are considered insecure, and this
        # option should be set to false in production environment
        auto-certificate.enabled: true
 
        # KeyStore and TrustStore settings default to Java keystore and truststore JVM arguments.
        # If needed to be overridden, then change the below properties
 
        # KeyStore type
        keyStoreType: ${javax.net.ssl.keyStoreType},
 
        # Path to KeyStore file
        keyStore: ${javax.net.ssl.keyStore},
 
        # Password to access the keystore file
        keyStorePassword: ${javax.net.ssl.keyStorePassword},
 
        # Password to access the key
        keyPassword: ${javax.net.ssl.keyPassword},
 
        # TrustStore type
        trustStoreType: ${javax.net.ssl.trustStoreType},
 
        # Path to TrustStore file
        trustStore: ${javax.net.ssl.trustStore},
 
        # Password to access the truststore file
        trustStorePassword: ${javax.net.ssl.trustStorePassword}
      },
      auth: {
        type: "internal"
      }
      ui: {
        # Configuration for Intercom
        intercom: {
            enabled: true
            appid:  "@dremio.ui.intercom.appid@"
        }
      }
      tokens: {
        cache: {
          # number of tokens to store locally on this coordinator node
          # (set to 0, if all requests should be made directly to the database of tokens)
          size: 100
          # time (in minutes) after which the token needs to be obtained from the token store
          expiration_minutes: 5
        }
      }
    },
 
    client-endpoint: {
      port: 31010
    },
 
    scheduler: {
      threads: 24
    },
 
    command-pool: {
      releasable: true,
      enabled: true,
      size: 0 # 0 defaults to the machine's number of cores
    },
  },
 
  executor: {
    enabled: true
 
    # enable/disable local cache manager
    # storage space for cache manager
    # control max percentage of disk cache manager db instance and fs mount points can consume
    cache: {
      enabled: true,
      path: {
        db: ${paths.local},
        fs: [${services.executor.cache.path.db}]
      },
      pctquota: {
        db: 70,
        fs: [${services.executor.cache.pctquota.db}]
      },
      ensurefreespace: {
        fs: [10]
      }
    }
  },
 
  fabric: {
    port: 45678,
 
    memory: {
      reservation: 100M
    }
    ssl: {
      # If SSL for communication path between Dremio instances should be enabled.
      enabled: false,
 
      # Allow for auto-generated certificates if keyStore option is not set
      # Auto-generated self-signed certificates are considered insecure, and this
      # option should be set to false in production environment
      auto-certificate.enabled: false,
 
      # KeyStore and TrustStore settings default to Java keystore and truststore JVM arguments.
      # If needed to be overridden, then change the below properties
 
      # KeyStore type
      keyStoreType: ${javax.net.ssl.keyStoreType},
 
      # Path to KeyStore file
      keyStore: ${javax.net.ssl.keyStore},
 
      # Password to access the keystore file
      keyStorePassword: ${javax.net.ssl.keyStorePassword},
 
      # Password to access the key
      keyPassword: ${javax.net.ssl.keyPassword},
 
      # TrustStore type
      trustStoreType: ${javax.net.ssl.trustStoreType},
 
      # Path to TrustStore file
      trustStore: ${javax.net.ssl.trustStore},
 
      # Password to access the truststore file
      trustStorePassword: ${javax.net.ssl.trustStorePassword}
    }
  },
 
  nessie: {
    enabled: true,
 
    remote-uri: "",
 
    backend: {
      in-memory: false
    },
 
    kvstore: {
      commit-timeout-ms: 30000
    }
  },
 
  flight: {
    enabled: true
    port: 32010
    # Authentication mode to configure the FlightServer Endpoint to use.
    # There are two modes:
    #       - legacy.arrow.flight.auth
    #       - arrow.flight.auth2
    # legacy.arrow.flight.auth is backwards compatible and will be deprecated in the future.
    # arrow.flight.auth2 is the new and preferred Arrow Flight authentication method.
    auth.mode: "arrow.flight.auth2"
    use_session_service: true
 
    ssl: {
      # If SSL for communication path between Flight clients and Dremio should be enabled.
      enabled: false,
 
      # Allow for auto-generated certificates if keyStore option is not set
      # Auto-generated self-signed certificates are considered insecure, and this
      # option should be set to false in production environment
      auto-certificate.enabled: true
 
      # KeyStore and TrustStore settings default to Java keystore and truststore JVM arguments.
      # If needed to be overridden, then change the below properties
 
      # KeyStore type
      keyStoreType: ${javax.net.ssl.keyStoreType},
 
      # Path to KeyStore file
      keyStore: ${javax.net.ssl.keyStore},
 
      # Password to access the keystore file
      keyStorePassword: ${javax.net.ssl.keyStorePassword},
 
      # Password to access the key
      keyPassword: ${javax.net.ssl.keyPassword},
 
      # TrustStore type
      trustStoreType: ${javax.net.ssl.trustStoreType},
 
      # Path to TrustStore file
      trustStore: ${javax.net.ssl.trustStore},
 
      # Password to access the truststore file
      trustStorePassword: ${javax.net.ssl.trustStorePassword}
    }
  },
 
  usersessions.ttl.minutes: 120,
 
  conduit: {
    # If set to 0, a port is automatically allocated (typically in ephemeral range). Otherwise, the configured value
    # is used.
    port: 0
 
    ssl: {
      # If SSL for communication path between Dremio instances should be enabled.
      enabled: ${services.fabric.ssl.enabled},
 
      # Allow for auto-generated certificates if keyStore option is not set
      # Auto-generated self-signed certificates are considered insecure, and this
      # option should be set to false in production environment
      auto-certificate.enabled: ${services.fabric.ssl.auto-certificate.enabled},
 
      # KeyStore and TrustStore settings default to Java keystore and truststore JVM arguments.
      # If needed to be overridden, then change the below properties
 
      # KeyStore type
      keyStoreType: ${services.fabric.ssl.keyStoreType},
 
      # Path to KeyStore file
      keyStore: ${services.fabric.ssl.keyStore},
 
      # Password to access the keystore file
      keyStorePassword: ${services.fabric.ssl.keyStorePassword},
 
      # Password to access the key
      keyPassword: ${services.fabric.ssl.keyPassword},
 
      # TrustStore type
      trustStoreType: ${services.fabric.ssl.trustStoreType},
 
      # Path to TrustStore file
      trustStore: ${services.fabric.ssl.trustStore},
 
      # Password to access the truststore file
      trustStorePassword: ${services.fabric.ssl.trustStorePassword}
    }
  }
 
  # Set up kerberos credentials in server (applicable for both coordinator and executor)
  kerberos: {
    principal: "",
    keytab.file.path: ""
  }
 
  web-admin: {
    enabled: true,
    # IP of hostname to which liveness HTTP server will be bound to. Default is loopback interface.
    host: "127.0.0.1"
    # Port, on which the daemon responds to liveness HTTP requests (0 == auto-allocated)
    port: 0
  }
 
  jobs: {
    # if jobs server should be enabled in a coorinator
    enabled: true
  }
}
 
security: {
  # Password to access Dremio keystore
  keystore-password: ""
}
 
provisioning: {
  yarn: {
    jvmoptions: "",
    # list of jars to be added to the main container classpath
    classpath: []
    app {
      # list of jars to be added to the Dremio application classpath
      classpath: []
      # list of jars to be added to the Dremio application classpath (at the front)
      classpath-prefix: []
    }
    watchdog: {
      # how long to wait for server reply before considering it failed
      poll.timeout: 1000ms
      # interval between two server polls
      poll.interval: 10000ms
      # how many consecutive failed attempts before killing server
      missed.polls.before.kill: 6
      # maximum attempts at killing server
      max.kill.attempts: 10
      # how long before reattempting killing server
      kill.reattempt.interval: 1000ms
    }
    nodemanager: {
      # Enable certificate validation when connecting to Yarn Node-manager REST API
      certificate-validation.enabled: true
    }
    enabled:false
  }
}
 
# the zookeeper quorum for the cluster
zookeeper: "localhost:"${services.coordinator.master.embedded-zookeeper.port}
zk.client.session.timeout: 90000
zk.client.retry.unlimited: true
zk.client.retry.limit: -1
zk.client.retry.initial_timeout_ms: -1
zk.client.connection_handle.enabled: false
zk.client.supervisor.interval_ms: 30000
zk.client.supervisor.read_timeout_ms: 10000
zk.client.supervisor.max_failures: 5
# 此处一些配置还是比较有用的，比如开启debug 模式
debug: {
  enabled: false,
  autoPort: false,
  prepopulate: false,
  singleNode: false,
  verboseAccessLog: false,
  allowTestApis: false,
  forceRemote: false,
  useMemoryStorage: false,
  addDefaultUser: false,
  allowNewerKVStore: false,
  # to enable remote debugging of the DremioDaemon running in YARN container
  yarnremote.enabled: false
  # UI Red Screen Of Death
  rsod.enabled: false
  # UI File A Bug option
  bug.filing.enabled: false
  # DX-9126: enable on-idle load shedding
  task.on_idle_load_shed: true
  # DX-23620: enable eager load shedding
  task.eager_load_shed: true
  # DX-9911: enable rescheduling task on unblock
  task.reschedule_on_unblock: true
  # Use election service to elect between multiple master candidates
  # has to be set to false if multiple master candidates
  master.election.disabled: false,
  alloc.max_occupancy_percent: 75,
  alloc.est_heap_buf_size_bytes: 800,
  # Enable caching for distributed storage, it is turned off by default
  dist.caching.enabled: false,
  # Max percent of total available cache space to use when possible for distributed storage
  dist.max.cache.space.percent: 100,
  # Enable asynchronous access for distributed stores when possible
  dist.async.enabled: true,
  # Enable asynchronous access for uploaded data associated with user home directories
  uploads.async.enabled: true,
  # Enable asynchronous access for temporary support stores
  support.async.enabled: true,
  # Enable asynchronous access for the job results cache store
  results.async.enabled: true,
  # Enable asynchronous access for the create table scratch space
  scratch.async.enabled: true,
  # Enable asynchronous access for the json and csv download staging space
  download.async.enabled: true,
  # Enable asynchronous access for the logs store
  logs.async.enabled: true,
  # Enable asynchronous access for the metadata store
  debug.metadata.async.enabled: true,
  # NoOp cluster coordinator
  noop.cluster.coordinator.enabled: false,
  remote.datastore.rpc_timeout_seconds: 60,
  dist.s3_file_status_check.enabled: true,
}
 
# Plan Cache configuration
dremio.plan.cache.timeout_minutes: 14400
dremio.plan.cache.max_entries: 10000
 
# Statistics Cache configuration
dremio.statistics.cache.timeout_minutes: 1
dremio.statistics.cache.max_entries: 500
 
# These system properties are listed here to allow substitution of system property values for DAC Web SSL properties
# listed in services.web.ssl section. Currently we consider only the system properties listed in this file for
# substitution.
javax.net.ssl {
  keyStoreType: "",
  keyStore: "",
  keyStorePassword: "",
  keyPassword: "",
  trustStoreType: "",
  trustStore:"",
  trustStorePassword: ""
}
 
registration.publish-host: ""
 

说明

以上一些配置还是比较重要的，比如调度，rpc 配置，以及不同执行角色的

posted on 2022-12-29 19:02  荣锋亮  阅读(166)  评论(0)  编辑  收藏  举报

导航