flink-conf.yaml

Flink 配置文件 对于管理员来说，差不多经常调整的就只有 conf 下的flink-conf.yaml ： 经过初步的调整，大约有以下模块的参数（未优化）

################################################################################

Licensed to the Apache Software Foundation (ASF) under one

or more contributor license agreements. See the NOTICE file

distributed with this work for additional information

regarding copyright ownership. The ASF licenses this file

to you under the Apache License, Version 2.0 (the

"License"); you may not use this file except in compliance

with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software

distributed under the License is distributed on an "AS IS" BASIS,

WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

See the License for the specific language governing permissions and

limitations under the License.

################################################################################

==============================================================================

Jobmanager && TaskManager 的基本参数

==============================================================================

jobmanager.rpc.address: flink-master

jobmanager.rpc.port: 6123
jobmanager.heap.size: 32768m
jobstore.cache-size: 52428800
taskmanager.heap.size: 92160m
taskmanager.numberOfTaskSlots: 30
parallelism.default: 3
io.tmp.dirs: /data1/data/flink/local
taskmanager.network.numberOfBuffers: 4096
taskmanager.memory.fraction: 0.7
taskmanager.memory.preallocate: false
taskmanager.debug.memory.startLogThread: 8
taskmanager.debug.memory.logIntervalMs: 3600000
blob.fetch.retries: 50
blob.fetch.num-concurrent: 200
blob.fetch.backlog: 500
task.cancellation-interval: 30000
taskmanager.exit-on-fatal-akka-error: false
taskkmanager.log.path:

==============================================================================

High Availability && and checkpointing

=============================================================================

high-availability: ZOOKEEPER
high-availability.cluster-id: /default_flink_dc
high-availability.zookeeper.quorum: VECS01455:2181,VECS01456:2181,VECS01457:2181,VECS01458:2181,VECS01459:2181
high-availability.storageDir: hdfs://flashHadoop/flink/recovery
high-availability.zookeeper.path.root: /flink
high-availability.zookeeper.path.checkpoint-counter: /checkpoint-counter
high-availability.zookeeper.path.checkpoints: /checkpoints
high-availability.zookeeper.path.jobgraphs: /jobgraphs
high-availability.zookeeper.path.latch: /leaderlatch
high-availability.zookeeper.path.leader: /leader
high-availability.zookeeper.path.mesos-workers: /mesos-workers
high-availability.zookeeper.path.running-registry: /running_job_registry/
high-availability.zookeeper.client.connection-timeout: 18000
high-availability.zookeeper.client.max-retry-attempts: 10
high-availability.zookeeper.client.retry-wait: 5000
high-availability.zookeeper.client.session-timeout: 60000

==============================================================================

HDFS and Fault tolerance and checkpointing

==============================================================================

fs.default-scheme: hdfs://flashHadoop/flink/data
fs.output.always-create-directory: false
fs.overwrite-files: false

state.backend: filesystem
state.backend.async: true
state.backend.fs.memory-threshold: 4096
state.backend.incremental: false
state.backend.local-recovery: false
state.checkpoints.dir: hdfs://flashHadoop/flink/checkpoints_data
state.checkpoints.num-retained: 1
state.savepoints.dir: hdfs://flashHadoop/flink/savepoints_data

==============================================================================

Web Frontend

==============================================================================

web.address: 0.0.0.0
web.access-control-allow-origin: "*"
web.backpressure.cleanup-interval: 600000
web.backpressure.delay-between-samples: 50
web.backpressure.num-samples: 100
web.backpressure.refresh-interval: 60000
web.checkpoints.history: 20
web.history: 20
web.log.path: /data1/data/flink/web/log
web.refresh-interval: 3000
web.submit.enable: true
web.timeout: 10000
web.tmpdir: /data1/data/flink/web/tmp

==============================================================================

Flink Cluster Security Configuration

==============================================================================

test is not set Security

==============================================================================

HistoryServer Config

==============================================================================

historyserver.web.address: 0.0.0.0
historyserver.web.port: 8082
historyserver.archive.fs.dir: hdfs://flashHadoop/flink/completed_jobs/
historyserver.archive.fs.refresh-interval: 2000
historyserver.web.tmpdir: /data1/data/flink/tmp

==============================================================================

==============================================================================

Environment 角色环境 申明

==============================================================================

env.java.opts: -XX:+UseG1GC -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+G1SummarizeConcMark -XX:MaxGCPauseMillis=100 -XX:-ResizePLAB -XX:+ParallelRefProcEnabled -XX:ParallelGCThreads=24 -XX:ConcGCThreads=16 -XX:G1HeapWastePercent=3 -XX:InitiatingHeapOccupancyPercent=35 -XX:G1MixedGCLiveThresholdPercent=85

env.java.opts.historyserver: -Xmx16g -Xms16g -Xmn4g -XX:+UseG1GC -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+G1SummarizeConcMark -XX:MaxGCPauseMillis=100 -XX:-ResizePLAB -XX:+ParallelRefProcEnabled -XX:+AlwaysPreTouch -XX:ParallelGCThreads=24 -XX:ConcGCThreads=16 -XX:G1HeapWastePercent=3 -XX:InitiatingHeapOccupancyPercent=35 -XX:G1MixedGCLiveThresholdPercent=85 -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:/data1/data/flink/gc/gc-historyserver-hostname.log

env.java.opts.jobmanager: -XX:+UseG1GC -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+G1SummarizeConcMark -XX:MaxGCPauseMillis=100 -XX:-ResizePLAB -XX:+ParallelRefProcEnabled -XX:ParallelGCThreads=24 -XX:ConcGCThreads=16 -XX:G1HeapWastePercent=3 -XX:InitiatingHeapOccupancyPercent=35 -XX:G1MixedGCLiveThresholdPercent=85 -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:/data1/data/flink/gc/gc-jobmanager-hostname.log

env.java.opts.taskmanager: -XX:+UseG1GC -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+G1SummarizeConcMark -XX:MaxGCPauseMillis=100 -XX:-ResizePLAB -XX:+ParallelRefProcEnabled -XX:ParallelGCThreads=24 -XX:ConcGCThreads=16 -XX:G1HeapWastePercent=3 -XX:InitiatingHeapOccupancyPercent=35 -XX:G1MixedGCLiveThresholdPercent=85 -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:/data1/data/flink/gc/gc-taskmanager-hostname.log

env.log.dir: /data1/data/flink/log

env.log.max: 1