dbt 项目依赖文件加载处理简单说明
核心是通过ManifestLoader 的load 方法中通过调用ReadFilesFromFileSystem 处理的，以前简单说明dbt 的一些任务执行是需要先生成manifest文件（比如run）
此任务数据的预处理是通过装饰器
ReadFilesFromFileSystem 类
@dataclass
class ReadFilesFromFileSystem:
all_projects: Mapping[str, Project]
files: MutableMapping[str, AnySourceFile] = field(default_factory=dict)
# saved_files is only used to compare schema files
saved_files: MutableMapping[str, AnySourceFile] = field(default_factory=dict)
# project_parser_files = {
# "my_project": {
# "ModelParser": ["my_project://models/my_model.sql"]
# }
# }
#
project_parser_files: Dict = field(default_factory=dict)
# 此处是load 中加载依赖以及项目信息的方法
def read_files(self):
for project in self.all_projects.values():
file_types = get_file_types_for_project(project)
self.read_files_for_project(project, file_types)
def read_files_for_project(self, project, file_types):
dbt_ignore_spec = generate_dbt_ignore_spec(project.project_root)
project_files = self.project_parser_files[project.project_name] = {}
for parse_ft, file_type_info in file_types.items():
project_files[file_type_info["parser"]] = read_files_for_parser(
project,
self.files,
parse_ft,
file_type_info,
self.saved_files,
dbt_ignore_spec,
)
流程参考处理
首选是基于manifest解析关联的，parse_manifest 通过 ManifestLoader.get_full_manifest 加载解析获取项目相关的依赖（比如deps）
之后基于依赖项目的路径遍历文件夹加载不同类型的资源（macro，model，source，seed，snapshot,analysis,schema,docs.... 基本就是dbt 的一些核心概念）
内部处理
- manifest 装饰器部分处理
# 通过parse_manifest 解析生成manifest
if ctx.obj.get("manifest") is None:
ctx.obj["manifest"] = parse_manifest(
runtime_config, write_perf_info, write, ctx.obj["flags"].write_json
)
- parse_manifest 处理
def parse_manifest(runtime_config, write_perf_info, write, write_json):
register_adapter(runtime_config, get_mp_context())
adapter = get_adapter(runtime_config)
# 处理macro 上下文，在macro 执行地方需要
adapter.set_macro_context_generator(generate_runtime_macro_context)
manifest = ManifestLoader.get_full_manifest(
runtime_config,
write_perf_info=write_perf_info,
)
if write and write_json:
write_manifest(manifest, runtime_config.project_target_path)
pm = plugins.get_plugin_manager(runtime_config.project_name)
plugin_artifacts = pm.get_manifest_artifacts(manifest)
for path, plugin_artifact in plugin_artifacts.items():
plugin_artifact.write(path)
return manifest
- generate_runtime_macro_context 处理
实际上就是dbt 对于macro 执行需要的context信息（方便jinja2 渲染处理）
def generate_runtime_macro_context(
macro: MacroProtocol,
config: RuntimeConfig,
manifest: Manifest,
package_name: Optional[str],
) -> Dict[str, Any]:
# context 都继承自ProviderContext ，后续会说明下
ctx = MacroContext(macro, config, manifest, OperationProvider(), package_name)
return ctx.to_dict()
- ManifestLoader.get_full_manifest
会通过runtime_config的配置信息进行进一步读取,runtime_config 也是通过装饰器预处理的(runtime_config 依赖的project 也是)
runtime_config 处理
def wrapper(*args, **kwargs):
ctx = args[0]
assert isinstance(ctx, Context)
req_strs = ["profile", "project"]
reqs = [ctx.obj.get(req_str) for req_str in req_strs]
if None in reqs:
raise DbtProjectError("profile and project required for runtime_config")
config = RuntimeConfig.from_parts(
ctx.obj["project"],
ctx.obj["profile"],
ctx.obj["flags"],
)
ctx.obj["runtime_config"] = config
project 处理
def load_project(
project_root: str,
version_check: bool,
profile: HasCredentials,
cli_vars: Optional[Dict[str, Any]] = None,
) -> Project:
# get the project with all of the provided information
project_renderer = DbtProjectYamlRenderer(profile, cli_vars)
## 注意此处还会处理项目依赖，比如packages，里边东西比较多，github 有一个简单的readme 说明，实际上就是基于dbt 的一些约定
（项目配置文件中的，加载相关的model，source，macro ）
project = Project.from_project_root(
project_root, project_renderer, verify_version=version_check
)
# Save env_vars encountered in rendering for partial parsing
project.project_env_vars = project_renderer.ctx_obj.env_vars
return project
get_full_manifest 核心处理
runtime 配置获取依赖信息，参考load_dependencies
projects = config.load_dependencies()
loader = cls(
config,
projects,
macro_hook=macro_hook,
file_diff=file_diff,
)
# 加载生成manifest
manifest = loader.load()
_check_manifest(manifest, config)
manifest.build_flat_graph()
# This needs to happen after loading from a partial parse,
# so that the adapter has the query headers from the macro_hook.
loader.save_macros_to_adapter(adapter)
依赖处理
def load_dependencies(self, base_only=False) -> Mapping[str, "RuntimeConfig"]:
if self.dependencies is None:
all_projects = {self.project_name: self}
# 会包含global，依赖的（packages的）以及dbt core 的
internal_packages = get_include_paths(self.credentials.type)
if base_only:
# Test setup -- we want to load macros without dependencies
project_paths = itertools.chain(internal_packages)
else:
# raise exception if fewer installed packages than in packages.yml
count_packages_specified = len(self.packages.packages) # type: ignore
count_packages_installed = len(tuple(self._get_project_directories()))
if count_packages_specified > count_packages_installed:
raise UninstalledPackagesFoundError(
count_packages_specified,
count_packages_installed,
self.packages_specified_path,
self.packages_install_path,
)
project_paths = itertools.chain(internal_packages, self._get_project_directories())
for project_name, project in self.load_projects(project_paths):
if project_name in all_projects:
raise NonUniquePackageNameError(project_name)
all_projects[project_name] = project
self.dependencies = all_projects
return self.dependencies
load 方法处理
里边代码比较多，核心是基于ReadFilesFromFileSystem 或者ReadFilesFromDiff 文件reader 读取项目以及依赖相关的文件（dbt 相关的，比如macro，modele，schema）
项目文件读取
def read_files(self):
for project in self.all_projects.values():
file_types = get_file_types_for_project(project)
self.read_files_for_project(project, file_types)
def read_files_for_project(self, project, file_types):
dbt_ignore_spec = generate_dbt_ignore_spec(project.project_root)
project_files = self.project_parser_files[project.project_name] = {}
for parse_ft, file_type_info in file_types.items():
# read_files_for_parser 基于项目，文件类型,进行分类查询
project_files[file_type_info["parser"]] = read_files_for_parser(
project,
self.files,
parse_ft,
file_type_info,
self.saved_files,
dbt_ignore_spec,
)
read_files_for_parser 处理
def read_files_for_parser(project, files, parse_ft, file_type_info, saved_files, ignore_spec):
dirs = file_type_info["paths"]
parser_files = []
for extension in file_type_info["extensions"]:
# get_source_files 是实际的文件夹遍历操作实现文件查找
source_files = get_source_files(
project, dirs, extension, parse_ft, saved_files, ignore_spec
)
for sf in source_files:
files[sf.file_id] = sf
parser_files.append(sf.file_id)
return parser_files
get_source_files 处理
def get_source_files(project, paths, extension, parse_file_type, saved_files, ignore_spec):
# file path list 基于os.walk 遍历文件夹
fp_list = filesystem_search(project, paths, extension, ignore_spec)
# file block list
fb_list = []
for fp in fp_list:
if parse_file_type == ParseFileType.Seed:
fb_list.append(load_seed_source_file(fp, project.project_name))
# singular tests live in /tests but only generic tests live
# in /tests/generic so we want to skip those
else:
if parse_file_type == ParseFileType.SingularTest:
path = pathlib.Path(fp.relative_path)
if path.parts[0] == "generic":
continue
file = load_source_file(fp, parse_file_type, project.project_name, saved_files)
# only append the list if it has contents. added to fix #3568
if file:
fb_list.append(file)
return fb_list
一个比较有意思的设计是dbt 没有直接使用绝对路径，而是自己弄了一套相对路径的格式,具体是BaseSourceFile 类
fid 格式处理如下
@property
def file_id(self):
if isinstance(self.path, RemoteFile):
return None
return f"{self.project_name}://{self.path.original_file_path}"
一个参考格式
{'MacroParser': ['dbt_utils://macros/web/get_url_host.sql', 'dbt_utils://macros/web/get_url_path.sql', 'dbt_utils://macros/web/get_url_parameter.sql', 'dbt_utils://macros/generic_tests/fewer_rows_than.sql', 'dbt_utils://macros/generic_tests/equal_rowcount.sql', 'dbt_utils://macros/generic_tests/relationships_where.sql', 'dbt_utils://macros/generic_tests/recency.sql', 'dbt_utils://macros/generic_tests/not_constant.sql', 'dbt_utils://macros/generic_tests/accepted_range.sql', 'dbt_utils://macros/generic_tests/not_accepted_values.sql', 'dbt_utils://macros/generic_tests/at_least_one.sql', 'dbt_utils://macros/generic_tests/unique_combination_of_columns.sql', 'dbt_utils://macros/generic_tests/
获取到文件列表之后还会对于文件进行进一步解析，以及macro进行处理
get_parsing_files (上边read_files 返回的文件，目前看是对于文件增删的处理)
说明
dbt 项目依赖文件 的处理主要是进行项目的依赖解析，然后对于项目依赖进行加载（macro，model，source），以上尽管时候关于依赖加载的
但是也有不少关于manifest的，manifest 处理属于dbt 比较核心的部分，后续会结合这个继续说明下
参考资料
core/dbt/parser/read_files.py
core/dbt/parser/manifest.py
core/dbt/config/project.py
core/dbt/parser/README.md
core/dbt/parser/search.py
core/dbt/parser/base.py
