在鲲鹏916服务器上编译和安装dpdk踩坑
无法爬楼楼的可以从这里下:
[root@kunpeng82 data1]# cat /etc/default/grub |grep GRUB_CMDLINE_LINUX GRUB_CMDLINE_LINUX="crashkernel=auto" GRUB_CMDLINE_LINUX_DEFAULT="console=tty0 console=ttyAMA0,115200 no_timer_check nofb nomodeset vga=normal" [root@kunpeng82 data1]# mkdir /mnt/huge [root@kunpeng82 data1]# mount -t hugetlbfs nodev /mnt/huge [root@kunpeng82 data1]# grep Hugepagesize /proc/meminfo Hugepagesize: 524288 kB
[root@kunpeng82 devuser]# ls /sys/kernel/mm/hugepages/ hugepages-2048kB hugepages-524288kB [root@kunpeng82 devuser]#
[root@kunpeng82 devuser]# ls /sys/kernel/mm/hugepages/hugepages-524288kB/
free_hugepages nr_hugepages nr_hugepages_mempolicy nr_overcommit_hugepages resv_hugepages surplus_hugepages
[root@kunpeng82 devuser]# cat /sys/kernel/mm/hugepages/hugepages-524288kB/nr_hugepages
0
[root@kunpeng82 devuser]#
[root@kunpeng82 devuser]# echo 256 > /sys/kernel/mm/hugepages/hugepages-524288kB/nr_hugepages
[root@kunpeng82 devuser]# cat /sys/kernel/mm/hugepages/hugepages-524288kB/nr_hugepages
256
[root@kunpeng82 devuser]#
[root@kunpeng82 devuser]# cat /proc/meminfo |grep HugePages_
HugePages_Total: 256
HugePages_Free: 256
HugePages_Rsvd: 0
HugePages_Surp: 0
[root@kunpeng82 devuser]#
配置DPDK编译目标
DPDK支持多种平台(x86、x64、ARM),编译的目标由ARCH-MACHINE-EXECENV-TOOLCHAIN
给出
ARCH
can be:i686
,x86_64
,ppc_64
,arm64
MACHINE
can be:native
,power8
,armv8a
EXECENV
can be:linuxapp
,bsdapp
TOOLCHAIN
can be:gcc
,icc
[root@kunpeng82 data1]# lscpu Architecture: aarch64 Byte Order: Little Endian CPU(s): 64 On-line CPU(s) list: 0-63 Thread(s) per core: 1 Core(s) per socket: 32 Socket(s): 2 NUMA node(s): 4 Model: 2 BogoMIPS: 100.00 L1d cache: 32K L1i cache: 48K L2 cache: 1024K L3 cache: 16384K NUMA node0 CPU(s): 0-15 NUMA node1 CPU(s): 16-31 NUMA node2 CPU(s): 32-47 NUMA node3 CPU(s): 48-63 Flags: fp asimd evtstrm aes pmull sha1 sha2 crc32 cpuid
export RTE_TARGET=arm64-armv8a-linuxapp-gcc
export RTE_SDK=/data1/dpdk_sdk
make config T=arm64-armv8a-linuxapp-gcc
make -j 64
make install T=arm64-armv8a-linuxapp-gcc -j 64
[root@kunpeng82 dpdk-19.11]# make == Build lib == Build lib/librte_kvargs SYMLINK-FILE include/rte_kvargs.h CC rte_kvargs.o AR librte_kvargs.a INSTALL-LIB librte_kvargs.a == Build lib/librte_eal == Build lib/librte_eal/common SYMLINK-FILE include/rte_branch_prediction.h SYMLINK-FILE include/rte_common.h SYMLINK-FILE include/rte_compat.h SYMLINK-FILE include/rte_function_versioning.h SYMLINK-FILE include/rte_debug.h SYMLINK-FILE include/rte_eal.h SYMLINK-FILE include/rte_eal_interrupts.h SYMLINK-FILE include/rte_errno.h SYMLINK-FILE include/rte_launch.h SYMLINK-FILE include/rte_lcore.h SYMLINK-FILE include/rte_log.h SYMLINK-FILE include/rte_memory.h SYMLINK-FILE include/rte_memzone.h SYMLINK-FILE include/rte_per_lcore.h SYMLINK-FILE include/rte_random.h SYMLINK-FILE include/rte_tailq.h SYMLINK-FILE include/rte_interrupts.h SYMLINK-FILE include/rte_alarm.h SYMLINK-FILE include/rte_string_fns.h SYMLINK-FILE include/rte_version.h SYMLINK-FILE include/rte_eal_memconfig.h SYMLINK-FILE include/rte_hexdump.h SYMLINK-FILE include/rte_devargs.h SYMLINK-FILE include/rte_bus.h SYMLINK-FILE include/rte_dev.h SYMLINK-FILE include/rte_class.h SYMLINK-FILE include/rte_option.h SYMLINK-FILE include/rte_pci_dev_feature_defs.h SYMLINK-FILE include/rte_pci_dev_features.h SYMLINK-FILE include/rte_malloc.h SYMLINK-FILE include/rte_keepalive.h SYMLINK-FILE include/rte_time.h SYMLINK-FILE include/rte_service.h SYMLINK-FILE include/rte_service_component.h SYMLINK-FILE include/rte_bitmap.h SYMLINK-FILE include/rte_vfio.h SYMLINK-FILE include/rte_hypervisor.h SYMLINK-FILE include/rte_test.h SYMLINK-FILE include/rte_reciprocal.h SYMLINK-FILE include/rte_fbarray.h SYMLINK-FILE include/rte_uuid.h SYMLINK-FILE include/rte_atomic.h SYMLINK-FILE include/rte_atomic_32.h SYMLINK-FILE include/rte_atomic_64.h SYMLINK-FILE include/rte_byteorder.h SYMLINK-FILE include/rte_cpuflags.h SYMLINK-FILE include/rte_cpuflags_32.h SYMLINK-FILE include/rte_cpuflags_64.h SYMLINK-FILE include/rte_cycles.h SYMLINK-FILE include/rte_cycles_32.h SYMLINK-FILE include/rte_cycles_64.h SYMLINK-FILE include/rte_io.h SYMLINK-FILE include/rte_io_64.h SYMLINK-FILE include/rte_mcslock.h SYMLINK-FILE include/rte_memcpy.h SYMLINK-FILE include/rte_memcpy_32.h SYMLINK-FILE include/rte_memcpy_64.h SYMLINK-FILE include/rte_pause.h SYMLINK-FILE include/rte_pause_32.h SYMLINK-FILE include/rte_pause_64.h SYMLINK-FILE include/rte_prefetch.h SYMLINK-FILE include/rte_prefetch_32.h SYMLINK-FILE include/rte_prefetch_64.h SYMLINK-FILE include/rte_rwlock.h SYMLINK-FILE include/rte_spinlock.h SYMLINK-FILE include/rte_ticketlock.h SYMLINK-FILE include/rte_vect.h SYMLINK-FILE include/generic/rte_atomic.h SYMLINK-FILE include/generic/rte_byteorder.h SYMLINK-FILE include/generic/rte_cycles.h SYMLINK-FILE include/generic/rte_prefetch.h SYMLINK-FILE include/generic/rte_memcpy.h SYMLINK-FILE include/generic/rte_cpuflags.h SYMLINK-FILE include/generic/rte_mcslock.h SYMLINK-FILE include/generic/rte_spinlock.h SYMLINK-FILE include/generic/rte_rwlock.h SYMLINK-FILE include/generic/rte_ticketlock.h SYMLINK-FILE include/generic/rte_vect.h SYMLINK-FILE include/generic/rte_pause.h SYMLINK-FILE include/generic/rte_io.h == Build lib/librte_eal/linux == Build lib/librte_eal/linux/eal SYMLINK-FILE include/rte_kni_common.h SYMLINK-FILE include/rte_os.h CC eal.o CC eal_cpuflags.o CC eal_hugepage_info.o CC eal_memory.o /data1/dpdk-19.11/lib/librte_eal/linux/eal/eal_memory.c:32:18: fatal error: numa.h: No such file or directory #include <numa.h> ^ compilation terminated. make[5]: *** [eal_memory.o] Error 1 make[4]: *** [eal] Error 2 make[3]: *** [linux] Error 2 make[2]: *** [librte_eal] Error 2 make[1]: *** [lib] Error 2 make: *** [all] Error 2 [root@kunpeng82 dpdk-19.11]#
dpdk编译错误 numa.h
libnuma-dev的安装:yum install numactl libnuma-dev
安装
yum install numactl-devel*
make: *** /lib/modules/4.14.0-115.10.1.el7a.aarch64/build: No such file or directory. Stop
[root@kunpeng82 dpdk-19.11]# ls /lib/modules/4.14.0-115.10.1.el7a.aarch64/build /lib/modules/4.14.0-115.10.1.el7a.aarch64/build
查看链接状态,执行以上指令,可以看到build链接到了一个不存在的路径
[root@kunpeng82 dpdk-19.11]# rm -rfv /lib/modules/4.14.0-115.10.1.el7a.aarch64/build removed ‘/lib/modules/4.14.0-115.10.1.el7a.aarch64/build’
[root@kunpeng82 dpdk-19.11]# ls /usr/src/kernels/ 4.18.0-147.0.3.el7.aarch64 4.18.0-147.0.3.el7.aarch64+debug [root@kunpeng82 dpdk-19.11]# ls /usr/src/kernels/4.18.0-147.0.3.el7.aarch64 arch crypto fs ipc lib mm samples sound usr block drivers include Kconfig Makefile Module.symvers scripts System.map virt certs firmware init kernel Makefile.rhelver net security tools vmlinux.id [root@kunpeng82 dpdk-19.11]# ln -s /usr/src/kernels/4.18.0-147.0.3.el7.aarch64 /lib/modules/4.14.0-115.10.1.el7a.aarch64/build [root@kunpeng82 dpdk-19.11]#
编译DPDK的时候可以不用使用系统环境的编译 export RTE_KERNELDIR=$PWD
[root@kunpeng82 dpdk-19.11]# yum install kernel-devel*
Loaded plugins: fastestmirror
Repository base is listed more than once in the configuration
Repository updates is listed more than once in the configuration
Repository extras is listed more than once in the configuration
Repository centosplus is listed more than once in the configuration
Loading mirror speeds from cached hostfile
Package kernel-devel-4.18.0-147.0.3.el7.aarch64 already installed and latest version
Nothing to do
[root@kunpeng82 dpdk-19.11]# yum install kernel-headers*
Loaded plugins: fastestmirror
Repository base is listed more than once in the configuration
Repository updates is listed more than once in the configuration
Repository extras is listed more than once in the configuration
Repository centosplus is listed more than once in the configuration
Loading mirror speeds from cached hostfile
Package kernel-headers-4.18.0-147.0.3.el7.aarch64 already installed and latest version
Nothing to do
[root@kunpeng82 dpdk-19.11]#
[root@kunpeng82 dpdk-19.11]# modprobe vfio-pci
[root@kunpeng82 dpdk-19.11]# modprobe vfio
[root@kunpeng82 dpdk-19.11]# ls
一个设备都没有检测到
[root@kunpeng82 dpdk-19.11]# ./usertools/dpdk-devbind.py -s No 'Network' devices detected ============================= No 'Baseband' devices detected ============================== No 'Crypto' devices detected ============================ No 'Eventdev' devices detected ============================== No 'Mempool' devices detected ============================= No 'Compress' devices detected ============================== No 'Misc (rawdev)' devices detected =================================== [root@kunpeng82 dpdk-19.11]#
总结:
安装依赖包(如果已安装,可以忽略)
安装依赖包(如果已安装,可以忽略) yum install kernel-devel yum install numactl-devel
配置环境变量:
export RTE_SDK=/home/dpdk-19.11 (以当前环境DPDK源码实际路径为准)
export RTE_TARGET= arm64-armv8a-linuxapp-gcc (以当前环境CPU架构为准)
在当前DPDK源码目录下编译DPDK:
make install T=$RTE_TARGET -j 48
debug
export RTE_SDK=/data1/dpdk-19.11 export RTE_TARGET=arm64-armv8a-linuxapp-gcc //export CFLAGS+="-O0 -g" export EXTRA_CFLAGS="-O0 -g" make install T=$RTE_TARGET -j 128 # ingore Installation cannot run with T defined and DESTDIR undefined
=====================================================
[root@localhost dpdk-19.11]# mkdir /mnt/huge
[root@localhost dpdk-19.11]# mount -t hugetlbfs nodev /mnt/huge
[root@localhost dpdk-19.11]#
[root@localhost dpdk-19.11]# ls /sys/kernel/mm/hugepages/
hugepages-2048kB hugepages-524288kB
[root@localhost dpdk-19.11]# grep Hugepagesize /proc/meminfo
Hugepagesize: 524288 kB
[root@localhost dpdk-19.11]
cat /sys/kernel/mm/hugepages/hugepages-524288kB/nr_hugepages
echo 256 /sys/kernel/mm/hugepages/hugepages-524288kB/nr_hugepages
[root@localhost dpdk-19.11]# export RTE_TARGET= arm64-armv8a-linuxapp-gcc
-bash: export: `arm64-armv8a-linuxapp-gcc': not a valid identifier
[root@localhost dpdk-19.11]# export RTE_TARGET=arm64-armv8a-linuxapp-gcc
[root@localhost dpdk-19.11]#
yum install -y *gcc* *glibc* *stdc* *gfortran* m4 numactl* automake
[root@localhost dpdk-19.11]# ls /lib/modules/4.14.0-115.el7a.0.1.aarch64/build/
ls: cannot access /lib/modules/4.14.0-115.el7a.0.1.aarch64/build/: No such file or directory
[root@localhost data1]# ls CentOS-7-aarch64-Everything-1810.iso dpdk-19.11.kunpeng.tgz lost+found dpdk-19.11 hicn-master.zip Python-3.7.7.tgz [root@localhost data1]# mkdir -p /mnt/CentOS-7.6 [root@localhost data1]# mount -o loop CentOS-7-aarch64-Everything-1810.iso /mnt/CentOS-7.6 mount: /dev/loop0 is write-protected, mounting read-only [root@localhost data1]# vi /etc/yum. yum.conf yum.repos.d/ [root@localhost data1]# vi /etc/yum.repos.d/local.repo [local] name=local.repo baseurl=file:///mnt/CentOS-7.6 enabled=1 gpgcheck=0
备份yum.repos.d下的的其他repo
[root@localhost data1]# yum makecache Loaded plugins: fastestmirror Loading mirror speeds from cached hostfile local | 3.6 kB 00:00:00 Metadata Cache Created [root@localhost data1]# yum list | grep kernel-dev kernel-devel.aarch64 4.14.0-115.el7a.0.1 local [root@localhost data1]#
[root@localhost dpdk-19.11]# ls /lib/modules/4.14.0-115.el7a.0.1.aarch64/build/ arch drivers init lib net sound virt block firmware ipc Makefile samples System.map vmlinux.id certs fs Kconfig mm scripts tools crypto include kernel Module.symvers security usr [root@localhost dpdk-19.11]#