flink启动脚本解析
本地环境编译flink源码 flink版本1.17
start-cluster.sh
1.加载config.sh函数以及配置属性
2.启动jobmanager
3.启动taskmanager
#判断执行的目录
bin=`dirname "$0"` bin=`cd "$bin"; pwd` # 加载配置属性 . "$bin"/config.sh # Start the JobManager instance(s) shopt -s nocasematch
#如果是HA高可用的话会读取zookeeper，高可用必须得配置zookeeper if [[ $HIGH_AVAILABILITY == "zookeeper" ]]; then #调用config.sh中的readMasters方法，读取maters文件
readMasters #输出有多少个masters，高可用 echo "Starting HA cluster with ${#MASTERS[@]} masters." for ((i=0;i<${#MASTERS[@]};++i)); do master=${MASTERS[i]} webuiport=${WEBUIPORTS[i]} if [ ${MASTERS_ALL_LOCALHOST} = true ] ; then "${FLINK_BIN_DIR}"/jobmanager.sh start "${master}" "${webuiport}" else ssh -n $FLINK_SSH_OPTS $master -- "nohup /bin/bash -l \"${FLINK_BIN_DIR}/jobmanager.sh\" start ${master} ${webuiport} &" fi done else echo "Starting cluster." # 启动standalone jobmanager "$FLINK_BIN_DIR"/jobmanager.sh start fi shopt -u nocasematch # 启动taskmanager TMWorkers start
jobmanager.sh脚本
USAGE="Usage: jobmanager.sh ((start|start-foreground) [args])|stop|stop-all" #判断是start stop 或者是stop-all参数 STARTSTOP=$1 if [ -z $2 ] || [[ $2 == "-D" ]]; then # start [-D ...] args=("${@:2}") elif [ -z $3 ] || [[ $3 == "-D" ]]; then # legacy path: start <host> [-D ...] HOST=$2 args=("${@:3}") else # legacy path: start <host> <port> [-D ...] HOST=$2 WEBUIPORT=$3 args=("${@:4}") fi if [[ $STARTSTOP != "start" ]] && [[ $STARTSTOP != "start-foreground" ]] && [[ $STARTSTOP != "stop" ]] && [[ $STARTSTOP != "stop-all" ]]; then echo $USAGE exit 1 fi bin=`dirname "$0"` bin=`cd "$bin"; pwd` . "$bin"/config.sh #默认是standalonesession模式 ENTRYPOINT=standalonesession if [[ $STARTSTOP == "start" ]] || [[ $STARTSTOP == "start-foreground" ]]; then # Add JobManager-specific JVM options export FLINK_ENV_JAVA_OPTS="${FLINK_ENV_JAVA_OPTS} ${FLINK_ENV_JAVA_OPTS_JM}" parseJmArgsAndExportLogs "${ARGS[@]}" args=("--configDir" "${FLINK_CONF_DIR}" "--executionMode" "cluster" "${args[@]}") if [ ! -z $HOST ]; then args+=("--host") args+=("${HOST}") fi if [ ! -z $WEBUIPORT ]; then args+=("--webui-port") args+=("${WEBUIPORT}") fi if [ ! -z "${DYNAMIC_PARAMETERS}" ]; then args=(${DYNAMIC_PARAMETERS[@]} "${args[@]}") fi fi if [[ $STARTSTOP == "start-foreground" ]]; then exec "${FLINK_BIN_DIR}"/flink-console.sh $ENTRYPOINT "${args[@]}" else
#启动flink 使用flink-daemon.sh 方式（后台启动） "${FLINK_BIN_DIR}"/flink-daemon.sh $STARTSTOP $ENTRYPOINT "${args[@]}" fi
taskmanager.sh
# Start/stop a Flink TaskManager. USAGE="Usage: taskmanager.sh (start|start-foreground|stop|stop-all)" STARTSTOP=$1 ARGS=("${@:2}") if [[ $STARTSTOP != "start" ]] && [[ $STARTSTOP != "start-foreground" ]] && [[ $STARTSTOP != "stop" ]] && [[ $STARTSTOP != "stop-all" ]]; then echo $USAGE exit 1 fi bin=`dirname "$0"` bin=`cd "$bin"; pwd` . "$bin"/config.sh ENTRYPOINT=taskexecutor if [[ $STARTSTOP == "start" ]] || [[ $STARTSTOP == "start-foreground" ]]; then # if no other JVM options are set, set the GC to G1 if [ -z "${FLINK_ENV_JAVA_OPTS}" ] && [ -z "${FLINK_ENV_JAVA_OPTS_TM}" ]; then export JVM_ARGS="$JVM_ARGS -XX:+UseG1GC" fi # Add TaskManager-specific JVM options export FLINK_ENV_JAVA_OPTS="${FLINK_ENV_JAVA_OPTS} ${FLINK_ENV_JAVA_OPTS_TM}" # Startup parameters parseTmArgsAndExportLogs "${ARGS[@]}" if [ ! -z "${DYNAMIC_PARAMETERS}" ]; then ARGS=(${DYNAMIC_PARAMETERS[@]} "${ARGS[@]}") fi ARGS=("--configDir" "${FLINK_CONF_DIR}" "${ARGS[@]}") fi if [[ $STARTSTOP == "start-foreground" ]]; then exec "${FLINK_BIN_DIR}"/flink-console.sh $ENTRYPOINT "${ARGS[@]}" else if [[ $FLINK_TM_COMPUTE_NUMA == "false" ]]; then # Start a single TaskManager "${FLINK_BIN_DIR}"/flink-daemon.sh $STARTSTOP $ENTRYPOINT "${ARGS[@]}" else # Example output from `numactl --show` on an AWS c4.8xlarge: # policy: default # preferred node: current # physcpubind: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 # cpubind: 0 1 # nodebind: 0 1 # membind: 0 1 read -ra NODE_LIST <<< $(numactl --show | grep "^nodebind: ") for NODE_ID in "${NODE_LIST[@]:1}"; do # Start a TaskManager for each NUMA node numactl --membind=$NODE_ID --cpunodebind=$NODE_ID -- "${FLINK_BIN_DIR}"/flink-daemon.sh $STARTSTOP $ENTRYPOINT "${ARGS[@]}" done fi fi
jobmanager.sh 和taskmanager.sh后台启动都是调用flink-damen.sh 来启动的
| 服务 | 入口类 | 描述 |
taskexecutor |
org.apache.flink.runtime.taskexecutor.TaskManagerRunner |
taskmanager服务 |
zookeeper |
org.apache.flink.runtime.zookeeper.FlinkZooKeeperQuorumPeer |
zookeeper高可用服务 |
historyserver |
org.apache.flink.runtime.webmonitor.history.HistoryServer |
历史服务器 |
standalonesession |
org.apache.flink.runtime.entrypoint.StandaloneSessionClusterEntrypoint |
Session模式启动 |
standalonejob |
org.apache.flink.container.entrypoint.StandaloneApplicationClusterEntryPoint |
application模式启动 |
sql-gateway |
org.apache.flink.table.gateway.SqlGateway |
于Netty实现的，允许其他应用程序通过REST API与Flink Cluster交互的服务 |
说明：Session模式：是多个应用共享一个集群，资源共享，所以可能会发生任务抢占的情况。
application模式：每一个提交的应用单独启动一个 JobManager，也就是创建一个集群。这 个 JobManager 只为执行这一个应用而存在，执行结束之后 JobManager 也就关闭了，这就是所谓的应 用模式。
per_job模式：和application类似，每个 提交的作业启动一个集群。但是flink本身不可以，需要使用大数据资源管理框架来控制。
flink-daemon.sh 脚本，所有的分析都要经过此脚本启动，部分代码截取
STARTSTOP=$1 DAEMON=$2 ARGS=("${@:3}") # get remaining arguments as array bin=`dirname "$0"` bin=`cd "$bin"; pwd` . "$bin"/config.sh case $DAEMON in (taskexecutor) CLASS_TO_RUN=org.apache.flink.runtime.taskexecutor.TaskManagerRunner ;; (zookeeper) CLASS_TO_RUN=org.apache.flink.runtime.zookeeper.FlinkZooKeeperQuorumPeer ;; (historyserver) CLASS_TO_RUN=org.apache.flink.runtime.webmonitor.history.HistoryServer ;; (standalonesession) CLASS_TO_RUN=org.apache.flink.runtime.entrypoint.StandaloneSessionClusterEntrypoint ;; (standalonejob) CLASS_TO_RUN=org.apache.flink.container.entrypoint.StandaloneApplicationClusterEntryPoint ;; (sql-gateway) CLASS_TO_RUN=org.apache.flink.table.gateway.SqlGateway SQL_GATEWAY_CLASSPATH=`findSqlGatewayJar` ;; (*) echo "Unknown daemon '${DAEMON}'. $USAGE." exit 1 ;; esac
