全栈交叉编译X86完成过程经验分享
1 CMAKE的交叉编译配置
主要是C和C++编译器的配置和SYSROOT的配置。
set (CMAKE_SYSTEM_NAME "Linux") set (CMAKE_SYSTEM_PROCESSOR "x86_64") set (default_toolchain_folder /home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux) if (EXISTS "${default_toolchain_folder}/bin") set (toolchain_path "${default_toolchain_folder}/bin") else () message (FATAL_ERROR "No clang toolchain ${default_toolchain_folder} support on this platform") endif () set (CMAKE_C_COMPILER "${toolchain_path}/x86_64-linux-gnu-gcc") set (CMAKE_CXX_COMPILER "${toolchain_path}/x86_64-linux-gnu-g++") # Where is the target environment. set (CMAKE_FIND_ROOT_PATH "${default_toolchain_folder}") set (CMAKE_SYSROOT "${default_toolchain_folder}") #set (CMAKE_SYSROOT "${default_toolchain_folder}") #set (CMAKE_SYSROOT "${default_toolchain_folder}/x86_64-linux-gnu") set (ONLY_CMAKE_FIND_ROOT_PATH ON) # Search for programs in the build host directories. set (CMAKE_FIND_ROOT_PATH_MODE_PROGRAM BOTH) # Search for libraries and headers in both the target and host directories. set (CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) set (CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) # for opencv set (CV_DISABLE_OPTIMIZATION ON) set (CPU_BASELINE "") set (CPU_DISPATCH "") add_compile_options (--sysroot=/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux) add_link_options (--sysroot=/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux) set (CPP_STDLIB_FILE "${default_toolchain_folder}/x86_64-linux-gnu/lib64/libstdc++.so" "${default_toolchain_folder}/x86_64-linux-gnu/lib64/libstdc++.so.6" "${default_toolchain_folder}/x86_64-linux-gnu/lib64/libstdc++.so.6.0.21" ) # Use thin archives to save disk space if (NOT DISABLE_THIN) set (CMAKE_C_ARCHIVE_CREATE "<CMAKE_AR> Tcr <TARGET> <LINK_FLAGS> <OBJECTS>") set (CMAKE_C_ARCHIVE_APPEND "<CMAKE_AR> Tr <TARGET> <LINK_FLAGS> <OBJECTS>") set (CMAKE_CXX_ARCHIVE_CREATE "<CMAKE_AR> Tcr <TARGET> <LINK_FLAGS> <OBJECTS>") set (CMAKE_CXX_ARCHIVE_APPEND "<CMAKE_AR> Tr <TARGET> <LINK_FLAGS> <OBJECTS>") endif() set (MPI_mpi_LIBRARY "${default_toolchain_folder}/lib/libmpi.so") set (MPI_C_HEADER_DIR "${default_toolchain_folder}/x86_64-linux-gnu/include/") set (MPI_CXX_HEADER_DIR "${default_toolchain_folder}/x86_64-linux-gnu/include/") set (MPI_mpi_cxx_LIBRARY "${default_toolchain_folder}/lib/libmpi_cxx.so") set (MPI_LIBRARIES ${MPI_mpi_LIBRARY} ${MPI_mpi_cxx_LIBRARY}) set (CMAKE_TOOLCHAIN_FILE_FULL_PATH ${CMAKE_SOURCE_DIR}/infra/cmake/module/toolchain/linux_cross_gcc5_glibc2.17_x86_64.cmake) #message("CMAKE_TOOLCHAIN_FILE_FULL_PATH is ${CMAKE_TOOLCHAIN_FILE_FULL_PATH}") option (CROSS_COMPILE "if is cross compile" TRUE)
还有一些是第三方库的cmake宏定义，例如openmpi，zlib：
set (MPI_mpi_LIBRARY "${default_toolchain_folder}/lib/libmpi.so") set (MPI_C_HEADER_DIR "${default_toolchain_folder}/x86_64-linux-gnu/include/") set (MPI_CXX_HEADER_DIR "${default_toolchain_folder}/x86_64-linux-gnu/include/") set (MPI_mpi_cxx_LIBRARY "${default_toolchain_folder}/lib/libmpi_cxx.so") set (MPI_LIBRARIES ${MPI_mpi_LIBRARY} ${MPI_mpi_cxx_LIBRARY})
2 工具链本身的修改
2.1 sdk库和头文件的依赖
tops代码栈之前编译过程中依赖sdk的安装，所以建立了交叉编译工具链到sdk的lib库和dtu头文件路径的软链接，后面改成cmake_build下面的子目录传进来的做法之后，很多库在改变sysroot之后找不到了，所以在交叉编译工具链的根目录下面建立了到/home的软链接，确保改变sysroot之后也能找到相关的库和头文件。
2.2 x86_64-linux-gnu/lib和lib/x86_64-linux-gnu有什么差别
不同编译软件的搜索路径不一样，所以这些编译软件编译出来的库会存在搜索路径不一致的情况，这时为了统一，在某个目录放好源文件之后，另外一个目录需要建好软链接。
同样的差异还体现在include和x86_64-linux-gnu/include。
当前编译器的搜索路径可以用下面的命令查询。
库文件的搜索路径的查询命令“/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-gcc -print-search-dirs”，路径用“:”分隔：
/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-gcc -print-search-dirs install: /home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/ programs: =/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/libexec/gcc/x86_64-linux-gnu/5.5.0/:/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/libexec/gcc/x86_64-linux-gnu/5.5.0/:/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/libexec/gcc/x86_64-linux-gnu/:/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/:/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/:/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/../../../../x86_64-linux-gnu/bin/x86_64-linux-gnu/5.5.0/:/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/../../../../x86_64-linux-gnu/bin/ libraries: =/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/:/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/../../../../x86_64-linux-gnu/lib/x86_64-linux-gnu/5.5.0/:/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/../../../../x86_64-linux-gnu/lib/../lib64/:/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/../../../../x86_64-linux-gnu/lib/
头文件搜索路径的查询命令“/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-gcc -E -v -”：
/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-gcc -E -v - Using built-in specs. COLLECT_GCC=/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-gcc Target: x86_64-linux-gnu Configured with: ../configure --prefix=/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux --target=x86_64-linux-gnu --with-glibc-version=2.17 --enable-languages=c,c++ --disable-multilib Thread model: posix gcc version 5.5.0 (GCC) COLLECT_GCC_OPTIONS='-E' '-v' '-mtune=generic' '-march=x86-64' /home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/libexec/gcc/x86_64-linux-gnu/5.5.0/cc1 -E -quiet -v - -mtune=generic -march=x86-64 ignoring nonexistent directory "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/../../../../x86_64-linux-gnu/sys-include" #include "..." search starts here: #include <...> search starts here: /home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/include /home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/include-fixed /home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/../../../../x86_64-linux-gnu/include End of search list.
3 第三方组件找不到
一般需要添加到交叉编译工具链中。
3.1 大的第三方组件找不到
以python找不到为例。
需要在交叉编译工具链中增加python版本，当前增加了python3.5和python3.6，后续还需要增加python3.7和3.8，注意编译的时候需要增加sysroot的CFLAGS。
这类比较大的第三方组件需要用现有的交叉编译工具链进行编译。
3.2 小的第三方组件找不到
以sqlit找不到为例。
查了一下ubuntu1604上的sqlite包，发现里面不含glibc2.17以上的符号，直接下载增加到交叉编译工具链中。
查询：
apt search sqlite
下载：
apt download libsqlite0 //库文件 apt download libsqlite0-dev //头文件
解压：
deb包解压需要2步，先解压出data.tar.xz，再解压出数据文件：
ar -xf libsqlite0-dev_2.8.17-12fakesync1_amd64.deb tar -xf data.tar.xz ar -xf libsqlite0_2.8.17-12fakesync1_amd64.deb tar -xf data.tar.xz
解压完之后的目录结构如下，其中lib和include是我们需要的库和头文件，share部分文档不需要。将lib和include拷贝到交叉编译工具链的目录。
tree usr/ usr/ |-- include | `-- sqlite.h |-- lib | |-- libsqlite.a | |-- libsqlite.la | |-- libsqlite.so -> libsqlite.so.0.8.6 | |-- libsqlite.so.0 -> libsqlite.so.0.8.6 | |-- libsqlite.so.0.8.6 | `-- pkgconfig | `-- sqlite.pc `-- share `-- doc |-- libsqlite0 | |-- README.Debian | |-- changelog.Debian.gz | `-- copyright `-- libsqlite0-dev |-- changelog.Debian.gz -> ../libsqlite0/changelog.Debian.gz `-- copyright
3.3 判断库是否使用了高版本GLIBC接口的方法
很多对外发布的库里面是不含symbol的，所以用nm命令查不到任何内容，这时可以使用strings命令来查，下面的命令结果表面libsqlite.so.0.8.6中需要使用GLIBC_2.2.5 / GLIBC_2.3 / GLIBC_2.4 / GLIBC_2.15 这几个版本的接口，当前环境只要GLIBC高于2.14就可以正常运行，我们当前的交叉编译工具链的版本是2.17，这种情况下就不需要额外自己编译这个库了。
strings usr/lib/libsqlite.so.0.8.6 | grep "^GLIBC" GLIBC_2.3 GLIBC_2.14 GLIBC_2.4 GLIBC_2.2.5
4 非顶级tops的独立的cmake项目编译
主要是交叉编译工具链的传递，考虑到独立的cmake项目对应的infra的相对路径可能不一致，所以不能传相对路径的交叉编译工具链配置文件给独立的cmake项目，每个交叉编译工具链增加
CMAKE_TOOLCHAIN_FILE_FULL_PATH 变量类定义自己的全路径交叉工具链配置文件名。
set (CMAKE_TOOLCHAIN_FILE_FULL_PATH ${CMAKE_SOURCE_DIR}/xxx/linux_cross_gcc5_glibc2.17_x86_64.cmake)
5 bazel项目的适配
先在sdk/toolchain/下面配置好，其他bazel项目建立到这个目录的软链接，确保全局只有一套bazel的交叉编译工具链配置。
--crosstool_top=//toolchain/linux_cross_gcc5_glibc2.17_x86_64:cross_suite
对TensorFlow，如果配置了交叉编译工具链，还要配置host工具链，方便部分临时编译的host程序能通过执行来生成新的源代码，host工具链配置是bazel自带的。
--host_crosstool_top=@bazel_tools//tools/cpp:toolchain
bazel跨版本间的兼容性很差，一般网上别人给的toolchain配置，如果版本和当前不一样，一般都不能直接用，建议参考bazel编译过程中bazel自己生成的，一般在bazel编译路径的下面目录中可以找到：
external/bazel_tools/tools/cpp/BUILD external/bazel_tools/tools/cpp/cc_toolchain_config.bzl
bazel的配置项很多，大多数可以不改，但需要裁减掉不是相关host或者target的编译配置。
BUILD文件：
package(default_visibility = ["//visibility:public"]) licenses(["notice"]) # Apache 2.0 load(":cc_toolchain_config.bzl", "cc_toolchain_config") # The toolchain type used to distinguish cc toolchains. toolchain_type(name = "toolchain_type") cc_library( name = "malloc", ) filegroup( name = "grep-includes", srcs = ["grep-includes.sh"], ) filegroup( name = "empty", srcs = [] ) # This is the entry point for --crosstool_top. Toolchains are found # by lopping off the name of --crosstool_top and searching for # "cc-compiler-${CPU}" in this BUILD file, where CPU is the target CPU # specified in --cpu. # # This file group should include # * all cc_toolchain targets supported # * all file groups that said cc_toolchain might refer to # Hardcoded toolchain, legacy behaviour. cc_toolchain_suite( name = "crossx86-toolchain", toolchains = { "k8": ":cc-compiler-crossx86", }, ) cc_toolchain( name = "cc-compiler-crossx86", all_files = ":empty", ar_files = ":empty", as_files = ":empty", compiler_files = ":empty", dwp_files = ":empty", linker_files = ":empty", objcopy_files = ":empty", strip_files = ":empty", supports_param_files = 1, toolchain_config = ":local_linux", toolchain_identifier = "local_linux", ) cc_toolchain_config( name = "local_linux" ) toolchain( name = "cc-toolchain-local", exec_compatible_with = [ ], target_compatible_with = [ ], toolchain = ":cc-compiler-local", toolchain_type = ":toolchain_type", ) filegroup( name = "srcs", srcs = glob(["**"]) + [ "//tools/cpp/runfiles:srcs", ], ) filegroup( name = "embedded_tools", srcs = glob(["**"]) + [ "//tools/cpp/runfiles:embedded_tools", ], ) filegroup( name = "interface_library_builder", srcs = ["build_interface_so"], ) filegroup( name = "osx_wrapper", srcs = ["osx_cc_wrapper.sh"], ) filegroup( name = "link_dynamic_library", srcs = ["link_dynamic_library.sh"], ) filegroup( name = "lib_cc_configure", srcs = ["lib_cc_configure.bzl"], )
cc_toolchain_config.bzl：
# Copyright 2019 The Bazel Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """A Starlark cc_toolchain configuration rule""" load( "@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", "action_config", "artifact_name_pattern", "env_entry", "env_set", "feature", "feature_set", "flag_group", "flag_set", "make_variable", "tool", "tool_path", "variable_with_value", "with_feature_set", ) load( "@bazel_tools//tools/build_defs/cc:action_names.bzl", _ASSEMBLE_ACTION_NAME = "ASSEMBLE_ACTION_NAME", _CLIF_MATCH_ACTION_NAME = "CLIF_MATCH_ACTION_NAME", _CPP_COMPILE_ACTION_NAME = "CPP_COMPILE_ACTION_NAME", _CPP_HEADER_PARSING_ACTION_NAME = "CPP_HEADER_PARSING_ACTION_NAME", _CPP_LINK_DYNAMIC_LIBRARY_ACTION_NAME = "CPP_LINK_DYNAMIC_LIBRARY_ACTION_NAME", _CPP_LINK_EXECUTABLE_ACTION_NAME = "CPP_LINK_EXECUTABLE_ACTION_NAME", _CPP_LINK_NODEPS_DYNAMIC_LIBRARY_ACTION_NAME = "CPP_LINK_NODEPS_DYNAMIC_LIBRARY_ACTION_NAME", _CPP_LINK_STATIC_LIBRARY_ACTION_NAME = "CPP_LINK_STATIC_LIBRARY_ACTION_NAME", _CPP_MODULE_CODEGEN_ACTION_NAME = "CPP_MODULE_CODEGEN_ACTION_NAME", _CPP_MODULE_COMPILE_ACTION_NAME = "CPP_MODULE_COMPILE_ACTION_NAME", _C_COMPILE_ACTION_NAME = "C_COMPILE_ACTION_NAME", _LINKSTAMP_COMPILE_ACTION_NAME = "LINKSTAMP_COMPILE_ACTION_NAME", _LTO_BACKEND_ACTION_NAME = "LTO_BACKEND_ACTION_NAME", _LTO_INDEXING_ACTION_NAME = "LTO_INDEXING_ACTION_NAME", _PREPROCESS_ASSEMBLE_ACTION_NAME = "PREPROCESS_ASSEMBLE_ACTION_NAME", _STRIP_ACTION_NAME = "STRIP_ACTION_NAME", ) all_compile_actions = [ _C_COMPILE_ACTION_NAME, _CPP_COMPILE_ACTION_NAME, _LINKSTAMP_COMPILE_ACTION_NAME, _ASSEMBLE_ACTION_NAME, _PREPROCESS_ASSEMBLE_ACTION_NAME, _CPP_HEADER_PARSING_ACTION_NAME, _CPP_MODULE_COMPILE_ACTION_NAME, _CPP_MODULE_CODEGEN_ACTION_NAME, _CLIF_MATCH_ACTION_NAME, _LTO_BACKEND_ACTION_NAME, ] all_cpp_compile_actions = [ _CPP_COMPILE_ACTION_NAME, _LINKSTAMP_COMPILE_ACTION_NAME, _CPP_HEADER_PARSING_ACTION_NAME, _CPP_MODULE_COMPILE_ACTION_NAME, _CPP_MODULE_CODEGEN_ACTION_NAME, _CLIF_MATCH_ACTION_NAME, ] preprocessor_compile_actions = [ _C_COMPILE_ACTION_NAME, _CPP_COMPILE_ACTION_NAME, _LINKSTAMP_COMPILE_ACTION_NAME, _PREPROCESS_ASSEMBLE_ACTION_NAME, _CPP_HEADER_PARSING_ACTION_NAME, _CPP_MODULE_COMPILE_ACTION_NAME, _CLIF_MATCH_ACTION_NAME, ] codegen_compile_actions = [ _C_COMPILE_ACTION_NAME, _CPP_COMPILE_ACTION_NAME, _LINKSTAMP_COMPILE_ACTION_NAME, _ASSEMBLE_ACTION_NAME, _PREPROCESS_ASSEMBLE_ACTION_NAME, _CPP_MODULE_CODEGEN_ACTION_NAME, _LTO_BACKEND_ACTION_NAME, ] all_link_actions = [ _CPP_LINK_EXECUTABLE_ACTION_NAME, _CPP_LINK_DYNAMIC_LIBRARY_ACTION_NAME, _CPP_LINK_NODEPS_DYNAMIC_LIBRARY_ACTION_NAME, ] def _impl(ctx): toolchain_identifier = "local_linux" host_system_name = "local" target_system_name = "local" target_cpu = "local" target_libc = "GLIBC_2.17" abi_libc_version = "2.17" compiler = "compiler" abi_version = "gcc-5.5-cxx11" cc_target_os = None builtin_sysroot = "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux" objcopy_embed_data_action = action_config( action_name = "objcopy_embed_data", enabled = True, tools = [tool(path = "/usr/bin/objcopy")], ) action_configs = [objcopy_embed_data_action] random_seed_feature = feature(name = "random_seed", enabled = True) default_link_flags_feature = feature( name = "default_link_flags", enabled = True, flag_sets = [ flag_set( actions = all_link_actions, flag_groups = [ flag_group( flags = [ "-lstdc++", "-Wl,-z,relro,-z,now", "-no-canonical-prefixes", "-pass-exit-codes", ], ), ], ), flag_set( actions = all_link_actions, flag_groups = [flag_group(flags = ["-Wl,--gc-sections"])], with_features = [with_feature_set(features = ["opt"])], ), ], ) unfiltered_compile_flags_feature = feature( name = "unfiltered_compile_flags", enabled = True, flag_sets = [ flag_set( actions = [ _ASSEMBLE_ACTION_NAME, _PREPROCESS_ASSEMBLE_ACTION_NAME, _LINKSTAMP_COMPILE_ACTION_NAME, _C_COMPILE_ACTION_NAME, _CPP_COMPILE_ACTION_NAME, _CPP_HEADER_PARSING_ACTION_NAME, _CPP_MODULE_COMPILE_ACTION_NAME, _CPP_MODULE_CODEGEN_ACTION_NAME, _LTO_BACKEND_ACTION_NAME, _CLIF_MATCH_ACTION_NAME, ], flag_groups = [ flag_group( flags = [ "-no-canonical-prefixes", "-fno-canonical-system-headers", "-Wno-builtin-macro-redefined", "-D__DATE__=\"redacted\"", "-D__TIMESTAMP__=\"redacted\"", "-D__TIME__=\"redacted\"", ], ), ], ), ], ) supports_pic_feature = feature(name = "supports_pic", enabled = True) default_compile_flags_feature = feature( name = "default_compile_flags", enabled = True, flag_sets = [ flag_set( actions = [ _ASSEMBLE_ACTION_NAME, _PREPROCESS_ASSEMBLE_ACTION_NAME, _LINKSTAMP_COMPILE_ACTION_NAME, _C_COMPILE_ACTION_NAME, _CPP_COMPILE_ACTION_NAME, _CPP_HEADER_PARSING_ACTION_NAME, _CPP_MODULE_COMPILE_ACTION_NAME, _CPP_MODULE_CODEGEN_ACTION_NAME, _LTO_BACKEND_ACTION_NAME, _CLIF_MATCH_ACTION_NAME, ], flag_groups = [ flag_group( flags = [ "-U_FORTIFY_SOURCE", "-D_FORTIFY_SOURCE=1", "-fstack-protector", "-Wall", "-Wunused-but-set-parameter", "-Wno-free-nonheap-object", "-fno-omit-frame-pointer", ], ), ], ), flag_set( actions = [ _ASSEMBLE_ACTION_NAME, _PREPROCESS_ASSEMBLE_ACTION_NAME, _LINKSTAMP_COMPILE_ACTION_NAME, _C_COMPILE_ACTION_NAME, _CPP_COMPILE_ACTION_NAME, _CPP_HEADER_PARSING_ACTION_NAME, _CPP_MODULE_COMPILE_ACTION_NAME, _CPP_MODULE_CODEGEN_ACTION_NAME, _LTO_BACKEND_ACTION_NAME, _CLIF_MATCH_ACTION_NAME, ], flag_groups = [flag_group(flags = ["-g"])], with_features = [with_feature_set(features = ["dbg"])], ), flag_set( actions = [ _ASSEMBLE_ACTION_NAME, _PREPROCESS_ASSEMBLE_ACTION_NAME, _LINKSTAMP_COMPILE_ACTION_NAME, _C_COMPILE_ACTION_NAME, _CPP_COMPILE_ACTION_NAME, _CPP_HEADER_PARSING_ACTION_NAME, _CPP_MODULE_COMPILE_ACTION_NAME, _CPP_MODULE_CODEGEN_ACTION_NAME, _LTO_BACKEND_ACTION_NAME, _CLIF_MATCH_ACTION_NAME, ], flag_groups = [ flag_group( flags = [ "-g0", "-O2", "-DNDEBUG", "-ffunction-sections", "-fdata-sections", ], ), ], with_features = [with_feature_set(features = ["opt"])], ), flag_set( actions = [ _LINKSTAMP_COMPILE_ACTION_NAME, _CPP_COMPILE_ACTION_NAME, _CPP_HEADER_PARSING_ACTION_NAME, _CPP_MODULE_COMPILE_ACTION_NAME, _CPP_MODULE_CODEGEN_ACTION_NAME, _LTO_BACKEND_ACTION_NAME, _CLIF_MATCH_ACTION_NAME, ], flag_groups = [flag_group(flags = ["-std=c++0x"])], ), ], ) opt_feature = feature(name = "opt") supports_dynamic_linker_feature = feature(name = "supports_dynamic_linker", enabled = True) objcopy_embed_flags_feature = feature( name = "objcopy_embed_flags", enabled = True, flag_sets = [ flag_set( actions = ["objcopy_embed_data"], flag_groups = [flag_group(flags = ["-I", "binary"])], ), ], ) dbg_feature = feature(name = "dbg") user_compile_flags_feature = feature( name = "user_compile_flags", enabled = True, flag_sets = [ flag_set( actions = [ _ASSEMBLE_ACTION_NAME, _PREPROCESS_ASSEMBLE_ACTION_NAME, _LINKSTAMP_COMPILE_ACTION_NAME, _C_COMPILE_ACTION_NAME, _CPP_COMPILE_ACTION_NAME, _CPP_HEADER_PARSING_ACTION_NAME, _CPP_MODULE_COMPILE_ACTION_NAME, _CPP_MODULE_CODEGEN_ACTION_NAME, _LTO_BACKEND_ACTION_NAME, _CLIF_MATCH_ACTION_NAME, ], flag_groups = [ flag_group( flags = ["%{user_compile_flags}"], iterate_over = "user_compile_flags", expand_if_available = "user_compile_flags", ), ], ), ], ) sysroot_feature = feature( name = "sysroot", enabled = True, flag_sets = [ flag_set( actions = [ _PREPROCESS_ASSEMBLE_ACTION_NAME, _LINKSTAMP_COMPILE_ACTION_NAME, _C_COMPILE_ACTION_NAME, _CPP_COMPILE_ACTION_NAME, _CPP_HEADER_PARSING_ACTION_NAME, _CPP_MODULE_COMPILE_ACTION_NAME, _CPP_MODULE_CODEGEN_ACTION_NAME, _LTO_BACKEND_ACTION_NAME, _CLIF_MATCH_ACTION_NAME, _CPP_LINK_EXECUTABLE_ACTION_NAME, _CPP_LINK_DYNAMIC_LIBRARY_ACTION_NAME, _CPP_LINK_NODEPS_DYNAMIC_LIBRARY_ACTION_NAME, ], flag_groups = [ flag_group( flags = ["--sysroot=/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux"], ), ], ), ], ) fastbuild_feature = feature(name = "fastbuild") features = [ default_compile_flags_feature, default_link_flags_feature, supports_dynamic_linker_feature, supports_pic_feature, objcopy_embed_flags_feature, opt_feature, dbg_feature, user_compile_flags_feature, sysroot_feature, unfiltered_compile_flags_feature, ] cxx_builtin_include_directories = [ # NEW "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/include", "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/lib/gcc/x86_64-linux-gnu/5.5.0/include-fixed", "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/x86_64-linux-gnu/include" ] artifact_name_patterns = [] make_variables = [] tool_paths = [ # NEW tool_path( name = "gcc", path = "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-gcc", ), tool_path( name = "g++", path = "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-g++", ), tool_path( name = "ld", path = "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-ld", ), tool_path( name = "ar", path = "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/ar", ), tool_path( name = "cpp", path = "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-cpp", ), tool_path( name = "gcov", path = "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-gcov", ), tool_path( name = "nm", path = "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-gcc-nm", ), tool_path( name = "objdump", path = "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-objdump", ), tool_path( name = "strip", path = "/home/.devtools/efb/efb_x86_64_gcc-5.5.0_glibc-2.17_linux/bin/x86_64-linux-gnu-strip", ), ] out = ctx.actions.declare_file(ctx.label.name) ctx.actions.write(out, "Fake executable") return [ cc_common.create_cc_toolchain_config_info( ctx = ctx, features = features, action_configs = action_configs, artifact_name_patterns = artifact_name_patterns, cxx_builtin_include_directories = cxx_builtin_include_directories, toolchain_identifier = toolchain_identifier, host_system_name = host_system_name, target_system_name = target_system_name, target_cpu = target_cpu, target_libc = target_libc, compiler = compiler, abi_version = abi_version, abi_libc_version = abi_libc_version, tool_paths = tool_paths, make_variables = make_variables, builtin_sysroot = builtin_sysroot, cc_target_os = cc_target_os, ), DefaultInfo( executable = out, ), ] cc_toolchain_config = rule( implementation = _impl, attrs = {}, provides = [CcToolchainConfigInfo], executable = True, )
6 python项目的适配
部分项目的编译框架是python构建的，例如horovod，相关适配修改只能通过直接修改horovod的setup.py文件。
类似cmake的交叉编译工具链的配置，主要涉及CC/CXX/SYSROOT，要不然python项目中涉及的一些c或者c++代码的编译无法使用交叉编译工具链来编译。
如果python项目中有子项目是cmake或者bazel编译的话，需要参照其他cmake和bazel项目的编译配置方法进行配置。
7 其他注意事项
7.1 不要直接修改全局的CC或者CXX，除非知道修改的影响
特别的，TensorFlow目标编译过程中需要host编译器编译的临时文件，在修改CC和CXX为交叉工具链编译的可执行文件之后，有可能在host上是无法运行。
但python项目又必须要修改CC和CXX，这时必须通过临时修改环境变量的做法，确保python项目里面使用的是交叉的CC和CXX，但其他项目还是用默认的CC和CXX。
7.2 bazel target的符号缺失问题
使用交叉编译工具链编译出来的bazel tf_cc_test目标，部分target会从动态链接变成静态链接，如果依赖规则里面没有alwayslink编译选项的话，可能导致部分.o静态链接的时候符号缺失，这种情况下需要增加alwayslink=1的选项。
