dremio 配置文件
因为官方文档对于配置文件的介绍太简单了,比如默认zk 注册的时候会有主机名称的问题，不然我们就需要自己配置
hosts，以下是来自官方代码仓库的一个参考配置，方便使用(注意此配置主要是核心组件的，还有其他组件自己的配置)
配置文件
- 文件位置
源码的common resources dremio-reference.conf - 内容
## Copyright (C) 2017-2019 Dremio Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
include classpath("dremio-reference-ext.conf")
paths: {
# the local path for dremio to store data.
local: "/tmp/dremio",
# the distributed path Dremio data including job results, downloads, uploads, etc
dist: "pdfs://"${paths.local}"/pdfs"
# location for catalog database (if master node)
db: ${paths.local}/db,
spilling: [${paths.local}/spill]
# storage area for the accelerator cache.
accelerator: ${paths.dist}/accelerator
# staging area for json and csv ui downloads
downloads: ${paths.dist}/downloads
# stores uploaded data associated with user home directories
uploads: ${paths.dist}/uploads
# stores data associated with the job results cache.
results: ${paths.dist}/results
# shared scratch space for creation of tables.
scratch: ${paths.dist}/scratch
}
services: {
node-tag: "",
coordinator: {
enabled: true,
# Auto-upgrade Dremio at startup if needed
auto-upgrade: false,
master: {
enabled: true,
# configure an embedded ZooKeeper server on the same node as master
embedded-zookeeper: {
enabled: true,
port: 2181,
path: ${paths.local}/zk
}
},
web: {
enabled: true,
port: 9047,
ssl: {
# If SSL for communication path between browsers (or REST clients) and Dremio should be enabled.
enabled: false,
# Allow for auto-generated certificates if keyStore option is not set
# Auto-generated self-signed certificates are considered insecure, and this
# option should be set to false in production environment
auto-certificate.enabled: true
# KeyStore and TrustStore settings default to Java keystore and truststore JVM arguments.
# If needed to be overridden, then change the below properties
# KeyStore type
keyStoreType: ${javax.net.ssl.keyStoreType},
# Path to KeyStore file
keyStore: ${javax.net.ssl.keyStore},
# Password to access the keystore file
keyStorePassword: ${javax.net.ssl.keyStorePassword},
# Password to access the key
keyPassword: ${javax.net.ssl.keyPassword},
# TrustStore type
trustStoreType: ${javax.net.ssl.trustStoreType},
# Path to TrustStore file
trustStore: ${javax.net.ssl.trustStore},
# Password to access the truststore file
trustStorePassword: ${javax.net.ssl.trustStorePassword}
},
auth: {
type: "internal"
}
ui: {
# Configuration for Intercom
intercom: {
enabled: true
appid: "@dremio.ui.intercom.appid@"
}
}
tokens: {
cache: {
# number of tokens to store locally on this coordinator node
# (set to 0, if all requests should be made directly to the database of tokens)
size: 100
# time (in minutes) after which the token needs to be obtained from the token store
expiration_minutes: 5
}
}
},
client-endpoint: {
port: 31010
},
scheduler: {
threads: 24
},
command-pool: {
enabled: true
size: 0 # 0 defaults to the machine's number of cores
},
},
executor: {
enabled: true
# enable/disable local cache manager
# storage space for cache manager
# control max percentage of disk cache manager db instance and fs mount points can consume
cache: {
enabled: true,
path: {
db: ${paths.local},
fs: [${services.executor.cache.path.db}]
},
pctquota: {
db: 70,
fs: [${services.executor.cache.pctquota.db}]
},
ensurefreespace: {
fs: [10]
}
}
},
fabric: {
port: 45678,
memory: {
reservation: 100M
}
ssl: {
# If SSL for communication path between Dremio instances should be enabled.
enabled: false,
# Allow for auto-generated certificates if keyStore option is not set
# Auto-generated self-signed certificates are considered insecure, and this
# option should be set to false in production environment
auto-certificate.enabled: false,
# KeyStore and TrustStore settings default to Java keystore and truststore JVM arguments.
# If needed to be overridden, then change the below properties
# KeyStore type
keyStoreType: ${javax.net.ssl.keyStoreType},
# Path to KeyStore file
keyStore: ${javax.net.ssl.keyStore},
# Password to access the keystore file
keyStorePassword: ${javax.net.ssl.keyStorePassword},
# Password to access the key
keyPassword: ${javax.net.ssl.keyPassword},
# TrustStore type
trustStoreType: ${javax.net.ssl.trustStoreType},
# Path to TrustStore file
trustStore: ${javax.net.ssl.trustStore},
# Password to access the truststore file
trustStorePassword: ${javax.net.ssl.trustStorePassword}
}
},
flight: {
enabled: true
port: 32010
# Authentication mode to configure the FlightServer Endpoint to use.
# There are two modes:
# - legacy.arrow.flight.auth
# - arrow.flight.auth2
# legacy.arrow.flight.auth is backwards compatible and will be deprecated in the future.
# arrow.flight.auth2 is the new and preferred Arrow Flight authentication method.
auth.mode: "arrow.flight.auth2"
ssl: {
# If SSL for communication path between Flight clients and Dremio should be enabled.
enabled: false,
# Allow for auto-generated certificates if keyStore option is not set
# Auto-generated self-signed certificates are considered insecure, and this
# option should be set to false in production environment
auto-certificate.enabled: true
# KeyStore and TrustStore settings default to Java keystore and truststore JVM arguments.
# If needed to be overridden, then change the below properties
# KeyStore type
keyStoreType: ${javax.net.ssl.keyStoreType},
# Path to KeyStore file
keyStore: ${javax.net.ssl.keyStore},
# Password to access the keystore file
keyStorePassword: ${javax.net.ssl.keyStorePassword},
# Password to access the key
keyPassword: ${javax.net.ssl.keyPassword},
# TrustStore type
trustStoreType: ${javax.net.ssl.trustStoreType},
# Path to TrustStore file
trustStore: ${javax.net.ssl.trustStore},
# Password to access the truststore file
trustStorePassword: ${javax.net.ssl.trustStorePassword}
}
},
conduit: {
# If set to 0, a port is automatically allocated (typically in ephemeral range). Otherwise, the configured value
# is used.
port: 0
ssl: {
# If SSL for communication path between Dremio instances should be enabled.
enabled: ${services.fabric.ssl.enabled},
# Allow for auto-generated certificates if keyStore option is not set
# Auto-generated self-signed certificates are considered insecure, and this
# option should be set to false in production environment
auto-certificate.enabled: ${services.fabric.ssl.auto-certificate.enabled},
# KeyStore and TrustStore settings default to Java keystore and truststore JVM arguments.
# If needed to be overridden, then change the below properties
# KeyStore type
keyStoreType: ${services.fabric.ssl.keyStoreType},
# Path to KeyStore file
keyStore: ${services.fabric.ssl.keyStore},
# Password to access the keystore file
keyStorePassword: ${services.fabric.ssl.keyStorePassword},
# Password to access the key
keyPassword: ${services.fabric.ssl.keyPassword},
# TrustStore type
trustStoreType: ${services.fabric.ssl.trustStoreType},
# Path to TrustStore file
trustStore: ${services.fabric.ssl.trustStore},
# Password to access the truststore file
trustStorePassword: ${services.fabric.ssl.trustStorePassword}
}
}
# Set up kerberos credentials in server (applicable for both coordinator and executor)
kerberos: {
principal: "",
keytab.file.path: ""
}
web-admin: {
enabled: true,
# Port, bound to loopback interface, on which the daemon responds to liveness HTTP requests (0 == auto-allocated)
port: 0
}
jobs: {
# if jobs server should be enabled in a coorinator
enabled: true
}
}
provisioning: {
yarn: {
jvmoptions: "",
# list of jars to be added to the main container classpath
classpath: []
app {
# list of jars to be added to the Dremio application classpath
classpath: []
# list of jars to be added to the Dremio application classpath (at the front)
classpath-prefix: []
}
watchdog: {
# how long to wait for server reply before considering it failed
poll.timeout: 1000ms
# interval between two server polls
poll.interval: 10000ms
# how many consecutive failed attempts before killing server
missed.polls.before.kill: 6
# maximum attempts at killing server
max.kill.attempts: 10
# how long before reattempting killing server
kill.reattempt.interval: 1000ms
}
nodemanager: {
# Enable certificate validation when connecting to Yarn Node-manager REST API
certificate-validation.enabled: true
}
enabled:false
}
}
# the zookeeper quorum for the cluster
zookeeper: "localhost:"${services.coordinator.master.embedded-zookeeper.port}
zk.client.session.timeout: 90000
zk.client.retry.unlimited: true
zk.client.retry.limit: -1
zk.client.retry.initial_timeout_ms: -1
debug: {
enabled: false,
autoPort: false,
prepopulate: false,
singleNode: false,
verboseAccessLog: false,
allowTestApis: false,
forceRemote: false,
useMemoryStorage: false,
addDefaultUser: false,
allowNewerKVStore: true,
# to enable remote debugging of the DremioDaemon running in YARN container
yarnremote.enabled: false
# UI Red Screen Of Death
rsod.enabled: false
# UI File A Bug option
bug.filing.enabled: false
# DX-9126: enable on-idle load shedding
task.on_idle_load_shed: true
# DX-23620: enable eager load shedding
task.eager_load_shed: true
# DX-9911: enable rescheduling task on unblock
task.reschedule_on_unblock: true
# Use election service to elect between multiple master candidates
# has to be set to false if multiple master candidates
master.election.disabled: false,
alloc.max_occupancy_percent: 75,
alloc.est_heap_buf_size_bytes: 800,
# Enable caching for distributed storage, it is turned off by default
dist.caching.enabled: false,
# Max percent of total available cache space to use when possible for distributed storage
dist.max.cache.space.percent: 100,
# Enable asynchronous access for distributed stores when possible
dist.async.enabled: true,
# Enable asynchronous access for uploaded data associated with user home directories
uploads.async.enabled: true,
# Enable asynchronous access for temporary support stores
support.async.enabled: true,
# Enable asynchronous access for the job results cache store
results.async.enabled: true,
# Enable asynchronous access for the create table scratch space
scratch.async.enabled: true,
# Enable asynchronous access for the json and csv download staging space
download.async.enabled: true,
# Enable asynchronous access for the logs store
logs.async.enabled: true,
# NoOp cluster coordinator
noop.cluster.coordinator.enabled: false,
remote.datastore.rpc_timeout_seconds: 60,
dist.s3_file_status_check.enabled: true,
}
# These system properties are listed here to allow substitution of system property values for DAC Web SSL properties
# listed in services.web.ssl section. Currently we consider only the system properties listed in this file for
# substitution.
javax.net.ssl {
keyStoreType: "",
keyStore: "",
keyStorePassword: "",
keyPassword: "",
trustStoreType: "",
trustStore:"",
trustStorePassword: ""
}
registration.publish-host: ""
一个比较有用的配置
registration.publish-host 是一个比较方便的配置，在zk 注册的时候会使用主机名，我们通过此配置可以解决zk 注册主机名的问题
可以方便执行节点的运行（不需要配置hosts了）
