eBPF基于LPM实现路由匹配

基于eBPF lpm map，icmp只有匹配上路由才能通。

最终目录结构

效果展示

启动应用前，可以ping通192.168.0.1和192.168.0.105。

启动应用后，无法ping通192.168.0.1，可以ping通192.168.0.105。

停止应用后，可以ping通192.168.0.1和192.168.0.105。

icmp/drop-icmp.c

#include "../headers/vmlinux.h"
#include "../headers/bpf_endian.h"
#include "../headers/bpf_helpers.h"

#define TC_ACT_SHOT 2
#define TC_ACT_OK 0

#define ETH_P_IP 0x0800

// 定义key
struct ipv4_lpm_key {
    __u32 prefixlen;
    __u32 data;
};

// 定义lpm map
struct {
    __uint(type, BPF_MAP_TYPE_LPM_TRIE);
    __type(key, struct ipv4_lpm_key);
    __type(value, __u32);
    __uint(map_flags, BPF_F_NO_PREALLOC);
    __uint(max_entries, 255);
    // 本地持久化map
    __uint(pinning, LIBBPF_PIN_BY_NAME);
} ipv4_lpm_map SEC(".maps");

struct ipv4_lpm_key *unused_ipv4_lpm_key __attribute__((unused));

char __license[] SEC("license") = "Dual MIT/GPL";

SEC("tc")
int tc_deny_icmp(struct __sk_buff *skb) {
    void *data_end = (void *)(long)skb->data_end;
    void *data = (void *)(long)skb->data;

    struct ethhdr *eth_hdr = data;
    if ((void *)eth_hdr + sizeof(*eth_hdr) > data_end || eth_hdr->h_proto != bpf_htons(ETH_P_IP)) {
        return TC_ACT_OK;
    }

    struct iphdr *ip_hdr = (void *)eth_hdr + sizeof(*eth_hdr);
    if ((void *)ip_hdr + sizeof(*ip_hdr) > data_end) {
        return TC_ACT_OK;
    }

    if (ip_hdr->protocol == IPPROTO_ICMP) {
        struct ipv4_lpm_key key = {
            .prefixlen = 32,
            .data = ip_hdr->daddr
        };

        if (!bpf_map_lookup_elem(&ipv4_lpm_map, &key)) {
            return TC_ACT_SHOT;
        }
    }

    return TC_ACT_OK;
}

cmd/main.go

package main

import (
	"fmt"
	"net"
	"os"
	"os/exec"
	"os/signal"
	"syscall"

	"github.com/cilium/ebpf"
	"github.com/cilium/ebpf/rlimit"
	"k8s.io/klog/v2"
)

//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -type ipv4_lpm_key bpf ../icmp/drop-icmp.c

func IPv4StringToUint32(ip string) (uint32, error) {
	p := net.ParseIP(ip).To4()
	if p == nil {
		return 0, fmt.Errorf("invalid ipv4 format")
	}

	return uint32(p[3])<<24 | uint32(p[2])<<16 | uint32(p[1])<<8 | uint32(p[0]), nil
}

func SetupSignalHandler() (stopCh <-chan struct{}) {
	stop := make(chan struct{})
	c := make(chan os.Signal, 2)
	signal.Notify(c, []os.Signal{os.Interrupt, syscall.SIGTERM}...)
	go func() {
		<-c
		close(c)
		close(stop)
	}()

	return stop
}

func DelTcEbpf() {
	cmds := []string{
		"tc qdisc del dev ens33 clsact",
	}
	for _, cmd := range cmds {
		exec.Command("bash", "-c", cmd).CombinedOutput()
	}
}

func main() {
	stopCh := SetupSignalHandler()

	DelTcEbpf()
	defer DelTcEbpf()

	// eBPF先删再加
	cmds := []string{
		"tc qdisc add dev ens33 clsact",
		"tc filter add dev ens33 egress bpf da obj icmp/drop-icmp.o sec tc",
		"tc filter add dev ens33 ingress bpf da obj icmp/drop-icmp.o sec tc",
	}
	for i, cmd := range cmds {
		if _, err := exec.Command("bash", "-c", cmd).CombinedOutput(); err != nil && i > 0 {
			klog.Errorf("exec %s failed, err is %v", cmd, err)
			return
		}
	}

	// Allow the current process to lock memory for eBPF resources.
	if err := rlimit.RemoveMemlock(); err != nil {
		klog.Errorf("rlimit remove memory lock failed, err is %v", err)
		return
	}

	ipv4LpmMap, err := ebpf.LoadPinnedMap("/sys/fs/bpf/tc/globals/ipv4_lpm_map", nil)
	if err != nil {
		klog.Errorf("load pinned map ipv4_lpm_map failed, err is %v", err)
		return
	}
	defer ipv4LpmMap.Close()

	ip, err := IPv4StringToUint32("192.168.0.105")
	if err != nil {
		klog.Errorf("ipv4 string to uint32 failed, err is %v", err)
		return
	}

	lpmKey := bpfIpv4LpmKey{
		Prefixlen: 32,
		Data:      ip,
	}
	if err := ipv4LpmMap.Put(lpmKey, uint32(0)); err != nil {
		klog.Errorf("put ipv4 lpm map failed, err is %v", err)
		return
	}

	<-stopCh
}

编译运行

rm -f icmp/drop-icmp.o
clang -c icmp/drop-icmp.c -o icmp/drop-icmp.o -target bpf -O2 -g
go generate ./cmd
go build -o ./cmd/ebpf-test ./cmd/.
./cmd/ebpf-test

内核LPM实现

访问eBPF LPM时，key是ip+mask。
不同的子网掩码对应1个hash表，最多32个。

for 子网掩码从指定mask到0
 ip & 子网掩码作为key，查询当前子网掩码对应的hash表。
 存在时返回。

参考资料

https://docs.kernel.org/next/bpf/map_lpm_trie.html