使用shell解析脚本依赖关系，并自动补数

将脚本依赖关系放到表中

使用shell解析脚本依赖关系，递归的计算各个脚本。

#!/bin/bash
# 2017.08.31 dm 补数

basepath=$(cd `dirname $0`; pwd)
cd $basepath

source /etc/profile
source ../etc/env.ini

 if [[ $# -ne 3 ]]; then
   echo "参数格式错误！"
   echo "$0 [运算日期，格式：'yyyy-mm-dd'] [运算类型：down/up] [表名列表] "
   echo "注：运算类型：down：表名是dm结果表，首先查找并调用其依赖的脚本，然后调用结果表。up：根据传入的前置表，查找其影响的结果表，然后执行。"
   echo "注：表名列表 (如果传入一个以上的表，需要用\" \"分割，\"\"包裹)：单表，例如“dm_event_meterial”；多表，例如“\dm_event_meterial,dm_voice_meterial\"”"
   exit 2
 fi

#运算类型 run_type down up
run_type=$2
#脚本名
shellNames=$3
#运算日期
sdate=$(date -d "-1 day" "+%Y-%m-%d")
if [[ $# -eq 3 ]]; then
  sdate=$(date -d "$1" "+%Y-%m-%d")
fi

#映射关系缓存
dmDic=""
#刷新映射关系缓存
function dic(){
  local dm=$1
  dmDic=$(spark-sql -e "
  use $HIVE_DB_ODS_S;
  select concat_ws(\",\",tmp.dm,tmp.shell) from
  (
    select dm,shell
    from yq_control_execute
    where dm=\"$dm\"
    union
    select c2.dm as dm,c2.shell as shell
    from yq_control_execute c1
    inner join
    yq_control_execute c2
    on c1.shell = c2.dm
    where c1.dm=\"$dm\"
    union
    select c3.dm as dm,c3.shell as shell
    from yq_control_execute c1
    inner join
    yq_control_execute c2
    on c1.shell = c2.dm
    inner join
    yq_control_execute c3
    on c2.shell = c3.dm
    where c1.dm=\"$dm\"
  ) tmp where tmp.shell!=tmp.dm
  ;
  ")
}

#将参数的shell列表转化为dm列表
function getDms(){
  local first=1
  shellNameSqlPart=""
  for _shellName in $shellNames
  do
    if [[ "$first" -ne "1" ]]; then
      shellNameSqlPart=$shellNameSqlPart",\"$_shellName\""
    else
      shellNameSqlPart=\"$_shellName\"
      ((first++))
    fi
  done
  echo $(spark-sql -S -e "
  use $HIVE_DB_ODS_S;
    select dm from
    (
      select dm
      from yq_control_execute
      where shell in ($shellNameSqlPart)
      union
      select c1.dm
      from yq_control_execute c1
      inner join
      yq_control_execute c2
      on c1.shell = c2.dm
      where c2.shell in ($shellNameSqlPart)
      union
      select c1.dm
      from yq_control_execute c1
      inner join
      yq_control_execute c2
      on c1.shell = c2.dm
      inner join
      yq_control_execute c3
      on c2.shell = c3.dm
      where c3.shell in ($shellNameSqlPart)
    ) tmp
  ;
  ")
}

#根据dm返回其依赖的shell列表
function getShellS() {
  if [[ $# -ne 1 ]]; then
    echo "$0 parameter length error."
    exit 2
  fi
  local target=$1
  local shells=""
  for line in $dmDic
  do
      local __dm=$(echo $line|cut -d ',' -f 1)
      local __shell=$(echo $line|cut -d ',' -f 2)
      if [[ "$target" == "$__dm" ]]; then
        local shells=$shells" "$__shell
      fi
  done
  echo $shells
}

runnedArray=""

runCount=0

#检查并执行脚本
#只执行dm开头并且没有执行过的脚本，执行过后存入已执行列表
function checkAndRun(){
  local flag=0
  local c_shell=$1
  for record in $runnedArray
  do
    if [[ "$record" ==  "$c_shell" ]]; then
      local flag=1
    fi
  done
#  echo "已执行的脚本"$runnedArray
  if [[ "$flag" == "0" && "${c_shell:0:2}" == "dm" ]]; then
     runnedArray="$runnedArray ${c_shell}"
     ((runCount++))
     echo "【$runCount】sh ./deal/${c_shell}.sh"
     sh ./deal/${c_shell}.sh $sdate
     sh ./export/main.sh $sdate ${c_shell}
  fi
}

#递归执行dm
function run(){
  local _dm=$1
  for shell in  $(getShellS $_dm)
  do
  # echo $shell"==>"$_dm
   if [[ "${shell:0:2}" == "dm" ]]; then
      run $shell
      checkAndRun $shell
   fi
  done
  if [[ "${_dm:0:2}" == "dm" ]]; then
     checkAndRun $_dm
  fi
}

if [[ "$run_type" == "up" ]]; then
  #找出其对应的结果表 遍历执行
  for dm in $(getDms $shellNames)
  do
    #执行前 更新依赖关系
    dic $dm
    run $dm
  done
else
  #遍历执行传入的dm列表
  for dm in $shellNames
  do
    #执行前 更新依赖关系
    dic $dm
    run $dm
  done
fi

脚本依赖表的格式：

这个依赖表除了有脚本的源表和目标表的关系，还有数据的来源类型（平台和渠道）；如果要实现基本功能的话，如下几列即可。

列	类型	注释
from	string	源表
to	string	目标表