【从零开始的eBPF】跑一个helloworld程序
最近在研究ebpf的应用，网上对较低版本的内核和centos操作系统的相关资料较少，这里记录一个自己环境配置&编译运行一个ebpf的helloworld程序的过程。
环境是centos7.9，虚拟机安装内存需要分配高一些，后续编译llvm很吃性能
基础依赖安装
升级内核版本
ebpf需要至少内核是4.9+以上的版本，这里选择了4.18版本的内核
下载4.18版本内核的安装包，下载链接
# 安装4.18版本的内核
yum install -y kernel-ml-4.18.0-1.el7.elrepo.x86_64.rpm
# 修改启动内核顺序
yum install -y grub2-pc
grub-set-default 'CentOS Linux (4.18.0-1.el7.elrepo.x86_64) 7 (Core)'
重启后可以确认下是否切换到4.18版本的内核
uname -sr
# Linux 4.18.0-1.el7.elrepo.x86_64
安装gcc
编译llvm 10+需要高版本gcc，这里使用scl软件集的方式升级gcc到7.3.1
yum install centos-release-scl
yum install devtoolset-7 -y
scl enable devtoolset-7 bash
echo "source /opt/rh/devtoolset-7/enable" >> ~/.bash_profile
source /opt/rh/devtoolset-7/enable
# gcc -v
Using built-in specs.
COLLECT_GCC=gcc
COLLECT_LTO_WRAPPER=/opt/rh/devtoolset-7/root/usr/libexec/gcc/x86_64-redhat-linux/7/lto-wrapper
Target: x86_64-redhat-linux
Configured with: ../configure --enable-bootstrap --enable-languages=c,c++,fortran,lto --prefix=/opt/rh/devtoolset-7/root/usr --mandir=/opt/rh/devtoolset-7/root/usr/share/man --infodir=/opt/rh/devtoolset-7/root/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-gcc-major-version-only --enable-plugin --with-linker-hash-style=gnu --enable-initfini-array --with-default-libstdcxx-abi=gcc4-compatible --with-isl=/builddir/build/BUILD/gcc-7.3.1-20180303/obj-x86_64-redhat-linux/isl-install --enable-libmpx --enable-gnu-indirect-function --with-tune=generic --with-arch_32=i686 --build=x86_64-redhat-linux
Thread model: posix
gcc version 7.3.1 20180303 (Red Hat 7.3.1-5) (GCC)
升级cmake
centos7默认安装的cmake版本较低，不满足llvm10+的需求，需要升级
这里选择了cmake 3.26.4版本
cd cmake-3.26.4
./configure
make
make install
ln -s /usr/local/bin/cmake /usr/bin/cmake
安装llvm10+
libbpf框架的最新版本需要llvm10+的编译支持，这里下载了11版本，解压后编译安装 链接
这里make的需要的时间很长，建议挂着编译去睡觉
unzip llvm-project-release-11.x.zip
cd llvm-project-release-11.x
mkdir build
cd build/
cmake -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_RTTI=ON -DLLVM_ENABLE_PROJECTS="clang;libcxx;libcxxabi" -G "Unix Makefiles" ../llvm
make
make install
借助libbpf-bootstrap脚手架跑一个helloworld程序
搭建脚手架
libbpf-bootstrap项目提供了一个快速构建ebpf程序的框架，包括libbpf和bpftool两大工具；项目包含一系列示例程序在examples/c文件夹中，并提供了一个相对通用的Makefile可以供我们了解一个ebpf程序是如何编译起来的
项目地址
克隆libbpf-bootstrap项目，并更新子项目
git clone https://github.com/libbpf/libbpf-bootstrap.git
git submodule update --init --recursive
编写一个helloworld程序
现在可以在examples/c文件夹下新建两个文件，分别命名为hello.bpf.c和hello.c
// hello.bpf.c
#define BPF_NO_GLOBAL_DATA
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
SEC("tracepoint/syscalls/sys_enter_execve")
int bpf_prog(void *ctx) {
char msg[] = "Hello, World!";
__bpf_printk("invoke bpf_prog: %s\n", msg);
return 0;
}
char LICENSE[] SEC("license") = "Dual BSD/GPL";
// hello.c
#include <stdio.h>
#include <unistd.h>
#include <sys/resource.h>
#include <bpf/libbpf.h>
#include "hello.skel.h"
static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
{
return vfprintf(stderr, format, args);
}
int main(int argc, char **argv)
{
struct hello_bpf *skel;
int err;
libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
/* Set up libbpf errors and debug info callback */
libbpf_set_print(libbpf_print_fn);
/* Open BPF application */
skel = hello_bpf__open();
if (!skel) {
fprintf(stderr, "Failed to open BPF skeleton\n");
return 1;
}
/* Load & verify BPF programs */
err = hello_bpf__load(skel);
if (err) {
fprintf(stderr, "Failed to load and verify BPF skeleton\n");
goto cleanup;
}
/* Attach tracepoint handler */
err = hello_bpf__attach(skel);
if (err) {
fprintf(stderr, "Failed to attach BPF skeleton\n");
goto cleanup;
}
printf("Successfully started! Please run `sudo cat /sys/kernel/debug/tracing/trace_pipe` "
"to see output of the BPF programs.\n");
for (;;) {
/* trigger our BPF program */
fprintf(stderr, ".");
sleep(1);
}
cleanup:
hello_bpf__destroy(skel);
return -err;
}
helloworld的程序hook了execve系统调用，在进程创建时会调用execve系统调用进入内核空间，此时被注入内核空间的ebpf字节码打印出一个字符串。
需要注意的是：
- 这个打印行为并不是在用户空间进行的，需要通过
/sys/kernel/debug/tracing/trace_pipe文件观察打印输出 - 在
hello.bpf.c文件中在#include <bpf/bpf_helpers.h>之前定义了一个#define BPF_NO_GLOBAL_DATA宏，这里是关闭ebpf的全局变量功能，因为低版本的内核不支持这个特性，项目的示例minimal和minimal_legacy很明确的标注了这个问题
This version of minimal is modified to allow running on even older kernels that do not allow global variables. bpf_printk uses global variables unless BPF_NO_GLOBAL_DATA is defined before including bpf_helpers.h. Additionally, the global variable my_pid has been replaced with an array of one element to hold the process pid.
编译运行helloworld程序
将自己的构建目标加入Makefile的APPS变量中，运行make $obj即可编译自己的ebpf程序
# Makefile
APPS = minimal minimal_legacy bootstrap uprobe kprobe fentry usdt sockfilter tc ksyscall hello
运行make hello后，打印出整个项目的编译流程，主要分为三个步骤如下
- 编译libbpf
- 编译bpftool
- 编译hello
当然编译一个ebpf程序的过程不是这么简单的过程，helloworld程序的具体编译的过程会在后文介绍
# make hello
which: no cargo in (/opt/rh/devtoolset-7/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin)
MKDIR .output
MKDIR .output/libbpf
LIB libbpf.a
MKDIR /root/libbpf-bootstrap/examples/c/.output//libbpf/staticobjs
CC /root/libbpf-bootstrap/examples/c/.output//libbpf/staticobjs/bpf.o
CC /root/libbpf-bootstrap/examples/c/.output//libbpf/staticobjs/btf.o
CC /root/libbpf-bootstrap/examples/c/.output//libbpf/staticobjs/libbpf.o
CC /root/libbpf-bootstrap/examples/c/.output//libbpf/staticobjs/libbpf_errno.o
CC /root/libbpf-bootstrap/examples/c/.output//libbpf/staticobjs/netlink.o
CC /root/libbpf-bootstrap/examples/c/.output//libbpf/staticobjs/nlattr.o
CC /root/libbpf-bootstrap/examples/c/.output//libbpf/staticobjs/str_error.o
CC /root/libbpf-bootstrap/examples/c/.output//libbpf/staticobjs/libbpf_probes.o
CC /root/libbpf-bootstrap/examples/c/.output//libbpf/staticobjs/bpf_prog_linfo.o
CC /root/libbpf-bootstrap/examples/c/.output//libbpf/staticobjs/btf_dump.o
CC /root/libbpf-bootstrap/examples/c/.output//libbpf/staticobjs/hashmap.o
CC /root/libbpf-bootstrap/examples/c/.output//libbpf/staticobjs/ringbuf.o
CC /root/libbpf-bootstrap/examples/c/.output//libbpf/staticobjs/strset.o
CC /root/libbpf-bootstrap/examples/c/.output//libbpf/staticobjs/linker.o
CC /root/libbpf-bootstrap/examples/c/.output//libbpf/staticobjs/gen_loader.o
CC /root/libbpf-bootstrap/examples/c/.output//libbpf/staticobjs/relo_core.o
CC /root/libbpf-bootstrap/examples/c/.output//libbpf/staticobjs/usdt.o
CC /root/libbpf-bootstrap/examples/c/.output//libbpf/staticobjs/zip.o
AR /root/libbpf-bootstrap/examples/c/.output//libbpf/libbpf.a
INSTALL bpf.h libbpf.h btf.h libbpf_common.h libbpf_legacy.h bpf_helpers.h bpf_helper_defs.h bpf_tracing.h bpf_endian.h bpf_core_read.h skel_internal.h libbpf_version.h usdt.bpf.h
INSTALL /root/libbpf-bootstrap/examples/c/.output//libbpf/libbpf.pc
INSTALL /root/libbpf-bootstrap/examples/c/.output//libbpf/libbpf.a
MKDIR bpftool
BPFTOOL bpftool/bootstrap/bpftool
... libbfd: [ OFF ]
... clang-bpf-co-re: [ on ]
... llvm: [ on ]
... libcap: [ OFF ]
MKDIR /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/libbpf/include/bpf
INSTALL /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/libbpf/include/bpf/hashmap.h
INSTALL /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/libbpf/include/bpf/relo_core.h
INSTALL /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/libbpf/include/bpf/libbpf_internal.h
MKDIR /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/
MKDIR /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/libbpf/
MKDIR /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/libbpf/staticobjs
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/libbpf/staticobjs/bpf.o
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/libbpf/staticobjs/btf.o
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/libbpf/staticobjs/libbpf.o
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/libbpf/staticobjs/libbpf_errno.o
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/libbpf/staticobjs/netlink.o
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/libbpf/staticobjs/nlattr.o
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/libbpf/staticobjs/str_error.o
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/libbpf/staticobjs/libbpf_probes.o
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/libbpf/staticobjs/bpf_prog_linfo.o
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/libbpf/staticobjs/btf_dump.o
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/libbpf/staticobjs/hashmap.o
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/libbpf/staticobjs/ringbuf.o
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/libbpf/staticobjs/strset.o
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/libbpf/staticobjs/linker.o
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/libbpf/staticobjs/gen_loader.o
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/libbpf/staticobjs/relo_core.o
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/libbpf/staticobjs/usdt.o
AR /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/libbpf/libbpf.a
INSTALL bpf.h libbpf.h btf.h libbpf_common.h libbpf_legacy.h bpf_helpers.h bpf_helper_defs.h bpf_tracing.h bpf_endian.h bpf_core_read.h skel_internal.h libbpf_version.h usdt.bpf.h
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/main.o
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/common.o
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/json_writer.o
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/gen.o
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/btf.o
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/xlated_dumper.o
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/btf_dumper.o
CC /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/disasm.o
LINK /root/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/bpftool
BPF .output/hello.bpf.o
GEN-SKEL .output/hello.skel.h
CC .output/hello.o
BINARY hello
这里我们的helloworld程序就编译好了，尝试运行一下，需要使用root用户：
# ./hello
libbpf: loading object 'hello_bpf' from buffer
libbpf: elf: section(2) .symtab, size 120, link 1, flags 0, type=2
libbpf: elf: section(3) tracepoint/syscalls/sys_enter_execve, size 200, link 0, flags 6, type=1
libbpf: sec 'tracepoint/syscalls/sys_enter_execve': found program 'bpf_prog' at insn offset 0 (0 bytes), code size 25 insns (200 bytes)
libbpf: elf: section(4) .rodata.str1.1, size 35, link 0, flags 32, type=1
libbpf: elf: section(5) license, size 13, link 0, flags 3, type=1
libbpf: license of hello_bpf is Dual BSD/GPL
libbpf: elf: section(6) .BTF, size 438, link 0, flags 0, type=1
libbpf: elf: section(7) .BTF.ext, size 160, link 0, flags 0, type=1
libbpf: looking for externs among 5 symbols...
libbpf: collected 0 externs total
libbpf: map '.rodata.str1.1' (global data): at sec_idx 4, offset 0, flags 80.
libbpf: map 0 is ".rodata.str1.1"
libbpf: map '.rodata.str1.1': skipped auto-creating...
Successfully started! Please run `sudo cat /sys/kernel/debug/tracing/trace_pipe` to see output of the BPF programs.
此时打开另一个终端查看/sys/kernel/debug/tracing/trace_pipe文件可以看到程序打印：
# cat /sys/kernel/debug/tracing/trace_pipe
<...>-79115 [000] .... 67971.749552: 0: invoke bpf_prog: Hello, World!
<...>-79117 [000] .... 67971.753866: 0: invoke bpf_prog: Hello, World!
<...>-79119 [000] .... 67971.757380: 0: invoke bpf_prog: Hello, World!
<...>-79120 [000] .... 67971.761711: 0: invoke bpf_prog: Hello, World!
<...>-79121 [000] .... 67971.764028: 0: invoke bpf_prog: Hello, World!
<...>-79123 [000] .... 67971.769068: 0: invoke bpf_prog: Hello, World!
<...>-79124 [000] .... 67971.771786: 0: invoke bpf_prog: Hello, World!
<...>-79126 [000] .... 67971.776243: 0: invoke bpf_prog: Hello, World!
<...>-79127 [000] .... 67971.779373: 0: invoke bpf_prog: Hello, World!
<...>-79129 [000] .... 67971.784824: 0: invoke bpf_prog: Hello, World!
<...>-79131 [000] .... 67971.791846: 0: invoke bpf_prog: Hello, World!
<...>-79132 [000] .... 67979.733858: 0: invoke bpf_prog: Hello, World!
这里我们的helloworld程序就运行成功了
to be continued
