(工具) 交叉编译 gperftools 及使用

交叉编译 gperftools 及使用

sudo apt-get install kcachegrind  # 导出为 callgrind 格式时需要
sudo apt install doxygen-latex
sudo apt install graphviz

# 编译 pprof
go install github.com/google/pprof@latest

1. 交叉编译 gperftools

# https://github.com/gperftools/gperftools/releases/tag/gperftools-2.9.1
CC=arm-linux-gnueabihf-gcc CXX=arm-linux-gnueabihf-g++ cmake .. -Dgperftools_enable_libunwind=OFF -Dgperftools_enable_frame_pointers=ON -DCMAKE_INSTALL_PREFIX=/home/hxf0223/tmp/gperftools/bbb
make -j && make install

2. ARM 测量待测试项目

2.1. 添加链接：libprofiler.so

file(GLOB SRCS *.cc)
ADD_EXECUTABLE(eigen_arm_test ${SRCS})

# SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/bin)
# target_link_libraries(eigen_arm_test gtest glog pthread)
target_link_libraries(eigen_arm_test dcl_utils pthread "-Wl,--no-as-needed" profiler "-Wl,--as-needed")

2.2. 在 ARM 板卡上运行 CPUPROFILE

# 生成 eigen_test.prof
CPUPROFILE=eigen_test.prof CPUPROFILE_FREQUENCY=500 ./eigen_arm_test

2.3. 转换 .prof 文件为 pdf 文件

拷贝ARM板卡上生成 eigen_test.prof 文件到PC主机上交叉编译生成的 eigen_arm_test 目录。
直接运行转换命令会导致找不到ARM测试程序依赖库文件：

/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/libc-2.28.so': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/libc-2.28.so': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/libc-2.28.so': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/libgcc_s.so.1': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/libgcc_s.so.1': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/libgcc_s.so.1': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/libm-2.28.so': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/libm-2.28.so': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/libm-2.28.so': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/usr/lib/arm-linux-gnueabihf/libstdc++.so.6.0.25': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/usr/lib/arm-linux-gnueabihf/libstdc++.so.6.0.25': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/usr/lib/arm-linux-gnueabihf/libstdc++.so.6.0.25': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: /usr/local/lib/libprofiler.so.0.5.4: file format not recognized
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: /usr/local/lib/libprofiler.so.0.5.4: file format not recognized
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: /usr/local/lib/libprofiler.so.0.5.4: file format not recognized
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/libpthread-2.28.so': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/libpthread-2.28.so': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/libpthread-2.28.so': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/ld-2.28.so': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/ld-2.28.so': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/ld-2.28.so': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-addr2line: '/usr/lib/arm-linux-gnueabihf/libstdc++.so.6.0.25': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-addr2line: '/lib/arm-linux-gnueabihf/libgcc_s.so.1': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-addr2line: '/lib/arm-linux-gnueabihf/libc-2.28.so': No such file

解决办法是，在主机上创建一个目录，在此目录下创建名称跟缺失的路径一样的软链接（该缺失的路径为 ARM 板卡上的路径）：

mkdir ~/tmp/fake_root && cd ~/tmp/fake_root
mkdir -p lib usr lib_gcc/lib
ln -s /usr/local/arm-linux-gnueabihf-8.3/arm-linux-gnueabihf/libc/lib/ lib/arm-linux-gnueabihf
ln -s /usr/local/arm-linux-gnueabihf-8.3/arm-linux-gnueabihf/libc/usr/lib usr/lib
ln -s /usr/local/arm-linux-gnueabihf-8.3/arm-linux-gnueabihf/lib lib_gcc/lib/arm-linux-gnueabihf

从 .prof 文件生成 pdf 文件：

pprof --tools=/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf- --lib_prefix=/home/hxf0223/tmp/fake_bbb_root,/home/hxf0223/tmp/fake_bbb_root/lib_gcc,/home/hxf0223/tmp/gperftools/bbb/lib ./eigen_arm_test eigen_test.prof --pdf > eigen_test.pdf

lib_prefix 添加了三个目录：ARM 运行时目录，arm-gcc 相关lib目录，及交叉编译 libprofiler.so 目录。
注意：lib_prefix 只能添加绝对路径。

2.4. PC 下转换命令格式

# 生成 pdf 格式的性能报告（层次调用节点有向图）
pprof --pdf ./eigen_arm_test eigen_test.prof > profile_eigen_x86.pdf
# pprof ----callgrind ./eigen_arm_test eigen_test.prof  > profile_eigen_x86.callgrind

3. 在项目中使用 ProfilerStart

链接使用 libprofiler.so, 在代码中添加 ProfilerStart, ProfilerStop

#include <gperftools/profiler.h>

ProfilerStart("test_capture.prof");
// 待分析代码
ProfilerStop();

运行编译的可执行程序，得到 test_capture.prof 文件。随后从 .prof 文件生成报告。

4. Ubuntu x86_64 编译 gperf-tools

使用如下链接，编译安装即可：

https://github.com/libunwind/libunwind.git
https://github.com/gperftools/gperftools.git

1. libunwind 按照官方说明编译即可；
2. gperftools 使用默认configure即可。如果 libunwind 安装到其他目录，gperftools 使用：

./configure CFLAGS=inc_dir LDFLAGS=lib_dir

5. 内存 Profile

export LD_PRELOAD=/usr/local/lib/libtcmalloc.so
export HEAPPROFILE=/home/q/perf-result/
export HEAP_PROFILE_ALLOCATION_INTERVAL=2000000000

• 使用google perf工具来排查堆外内存占用

7. 参考

• gperftools github io
• Analyze program performance with gperftools
• 用gperftools对C/C++程序进行profile

8. 其他

• Linux Trace Tool NG