dpdk uio
root@gobgp:~# lspci | grep -i ether 01:00.0 Ethernet controller: Red Hat, Inc. Virtio network device (rev 01) 07:00.0 Ethernet controller: Red Hat, Inc. Virtio network device (rev 01) root@gobgp:~# lspci -s 07:00.0 07:00.0 Ethernet controller: Red Hat, Inc. Virtio network device (rev 01) root@gobgp:~# lspci -s 07:00.0 -v 07:00.0 Ethernet controller: Red Hat, Inc. Virtio network device (rev 01) Subsystem: Red Hat, Inc. Virtio network device Physical Slot: 0-6 Flags: bus master, fast devsel, latency 0, IRQ 38 Memory at 10c40000 (32-bit, non-prefetchable) [size=4K] Memory at 8000c00000 (64-bit, prefetchable) [size=16K] Expansion ROM at 10c00000 [disabled] [size=256K] Capabilities: [dc] MSI-X: Enable- Count=3 Masked- Capabilities: [c8] Vendor Specific Information: VirtIO: <unknown> Capabilities: [b4] Vendor Specific Information: VirtIO: Notify Capabilities: [a4] Vendor Specific Information: VirtIO: DeviceCfg Capabilities: [94] Vendor Specific Information: VirtIO: ISR Capabilities: [84] Vendor Specific Information: VirtIO: CommonCfg Capabilities: [7c] Power Management version 3 Capabilities: [40] Express Endpoint, MSI 00 Kernel driver in use: uio_pci_generic root@gobgp:~#
root@gobgp:~# cat /sys/bus/pci/drivers/uio_pci_generic/bind cat: /sys/bus/pci/drivers/uio_pci_generic/bind: Permission denied root@gobgp:~# ls -l /sys/bus/pci/devices/0000\:07\:00.0 lrwxrwxrwx 1 root root 0 Aug 24 14:20 /sys/bus/pci/devices/0000:07:00.0 -> ../../../devices/pci0000:00/0000:00:01.6/0000:07:00.0 root@gobgp:~# ls -l /sys/bus/pci/devices/0000\:07\:00.0/driver lrwxrwxrwx 1 root root 0 Aug 24 14:20 /sys/bus/pci/devices/0000:07:00.0/driver -> ../../../../bus/pci/drivers/uio_pci_generic root@gobgp:~#
root@gobgp:~# cat /dev/uio0 cat: /dev/uio0: Invalid argument root@gobgp:~#
Example code using uio_pci_generic Here is some sample userspace driver code using uio_pci_generic: #include <stdlib.h> #include <stdio.h> #include <unistd.h> #include <sys/types.h> #include <sys/stat.h> #include <fcntl.h> #include <errno.h> int main() { int uiofd; int configfd; int err; int i; unsigned icount; unsigned char command_high; uiofd = open("/dev/uio0", O_RDONLY); if (uiofd < 0) { perror("uio open:"); return errno; } configfd = open("/sys/class/uio/uio0/device/config", O_RDWR); if (configfd < 0) { perror("config open:"); return errno; } /* Read and cache command value */ err = pread(configfd, &command_high, 1, 5); if (err != 1) { perror("command config read:"); return errno; } command_high &= ~0x4; for(i = 0;; ++i) { /* Print out a message, for debugging. */ if (i == 0) fprintf(stderr, "Started uio test driver.\n"); else fprintf(stderr, "Interrupts: %d\n", icount); /****************************************/ /* Here we got an interrupt from the device. Do something to it. */ /****************************************/ /* Re-enable interrupts. */ err = pwrite(configfd, &command_high, 1, 5); if (err != 1) { perror("config write:"); break; } /* Wait for next interrupt. */ err = read(uiofd, &icount, 4); if (err != 4) { perror("uio read:"); break; } } return errno; }
#include <stdlib.h> #include <stdio.h> #include <unistd.h> #include <sys/types.h> #include <sys/stat.h> #include <fcntl.h> #include <errno.h> #include <stdint.h> #include <sys/mman.h> #define EDU_IO_SIZE 0x100 #define EDU_CARD_VERSION_ADDR 0x0 #define EDU_CARD_LIVENESS_ADDR 0x1 #define EDU_RAISE_INT_ADDR 0x18 #define EDU_CLEAR_INT_ADDR 0x19 int main() { int uiofd; int configfd; int bar0fd; int resetfd; int err; int i; unsigned icount; unsigned char command_high; volatile uint32_t *bar0; uiofd = open("/dev/uio0", O_RDWR); if (uiofd < 0) { perror("uio open:"); return errno; } configfd = open("/sys/class/uio/uio0/device/config", O_RDWR); if (configfd < 0) { perror("config open:"); return errno; } /* Read and cache command value */ err = pread(configfd, &command_high, 1, 5); if (err != 1) { perror("command config read:"); return errno; } command_high &= ~0x4; /* Map edu's MMIO */ bar0fd = open("/sys/class/uio/uio0/device/resource0", O_RDWR); if (bar0fd < 0) { perror("bar0fd open:"); return errno; } /* mmap the device's BAR */ bar0 = (volatile uint32_t *)mmap(NULL, EDU_IO_SIZE, PROT_READ|PROT_WRITE, MAP_SHARED, bar0fd, 0); if (bar0 == MAP_FAILED) { perror("Error mapping bar0!"); return errno; } fprintf(stdout, "Version = %08X\n", bar0[EDU_CARD_VERSION_ADDR]); /* Test the invertor function */ i = 0x12345678; bar0[EDU_CARD_LIVENESS_ADDR] = i; fprintf(stdout, "Inversion: %08X --> %08X\n", i, bar0[EDU_CARD_LIVENESS_ADDR]); /* Clear previous interrupt */ bar0[EDU_CLEAR_INT_ADDR] = 0xABCDABCD; /* Raise an interrupt */ bar0[EDU_RAISE_INT_ADDR] = 0xABCDABCD; for(i = 0;; ++i) { /* Print out a message, for debugging. */ if (i == 0) fprintf(stderr, "Started uio test driver.\n"); else fprintf(stderr, "Interrupts: %d\n", icount); /****************************************/ /* Here we got an interrupt from the device. Do something to it. */ /****************************************/ /* Re-enable interrupts. */ err = pwrite(configfd, &command_high, 1, 5); if (err != 1) { perror("config write:"); break; } /* Clear previous interrupt */ bar0[EDU_CLEAR_INT_ADDR] = 0xABCDABCD; /* Raise an interrupt */ bar0[EDU_RAISE_INT_ADDR] = 0xABCDABCD; /* Wait for next interrupt. */ err = read(uiofd, &icount, 4); if (err != 4) { perror("uio read:"); break; } } return errno; } The result looks something like this: Version = 010000ED Inversion: 12345678 --> EDCBA987 Started uio test driver. Interrupts: 3793548 Interrupts: 3793549 Interrupts: 3793550 Interrupts: 3793551 Interrupts: 3793552 Interrupts: 3793553 Interrupts: 3793554 Interrupts: 3793555 Interrupts: 3793556 ...
dpdk uio
./usertools/dpdk-devbind.py --bind=igb_uio 0000:06:00.0
[root@localhost dpdk-19.11]# ls /sys/bus/pci/devices/0000\:06\:00.0/u uevent uio/ [root@localhost dpdk-19.11]# ls /sys/bus/pci/devices/0000\:06\:00.0/uio/ uio0 [root@localhost dpdk-19.11]# ls /sys/bus/pci/devices/0000\:06\:00.0/uio/uio0/maps/ map0 map1 map2 [root@localhost dpdk-19.11]#
[root@localhost dpdk-19.11]# ls /sys/bus/pci/devices/0000\:06\:00.0/uio/uio0/maps/map0 addr name offset size [root@localhost dpdk-19.11]# cat /sys/bus/pci/devices/0000\:06\:00.0/uio/uio0/maps/map0/addr 0x0000080010400000 [root@localhost dpdk-19.11]# cat /sys/bus/pci/devices/0000\:06\:00.0/uio/uio0/maps/map0/name BAR0 [root@localhost dpdk-19.11]# cat /sys/bus/pci/devices/0000\:06\:00.0/uio/uio0/maps/map0/offset 0x0 [root@localhost dpdk-19.11]# cat /sys/bus/pci/devices/0000\:06\:00.0/uio/uio0/maps/map0/size 0x0000000000020000 [root@localhost dpdk-19.11]# cat /sys/bus/pci/devices/0000\:06\:00.0/uio/uio0/maps/map1/addr 0x0000080011320000 [root@localhost dpdk-19.11]# cat /sys/bus/pci/devices/0000\:06\:00.0/uio/uio0/maps/map2/addr 0x0000080008b00000 [root@localhost dpdk-19.11]# cat /sys/bus/pci/devices/0000\:06\:00.0/uio/uio0/maps/map1/name BAR2 [root@localhost dpdk-19.11]# cat /sys/bus/pci/devices/0000\:06\:00.0/uio/uio0/maps/map2/name BAR4 [root@localhost dpdk-19.11]#
[root@localhost dpdk-19.11]# lspci -s 0000\:06\:00.0 -v 06:00.0 Ethernet controller: Huawei Technologies Co., Ltd. Hi1822 Family (2*25GE) (rev 45) Subsystem: Huawei Technologies Co., Ltd. Device d139 Flags: bus master, fast devsel, latency 0, NUMA node 0 Memory at 80010400000 (64-bit, prefetchable) [size=128K] Memory at 80011320000 (64-bit, prefetchable) [size=32K] Memory at 80008b00000 (64-bit, prefetchable) [size=1M] Expansion ROM at e9300000 [disabled] [size=1M] Capabilities: [40] Express Endpoint, MSI 00 Capabilities: [80] MSI: Enable- Count=1/32 Maskable+ 64bit+ Capabilities: [a0] MSI-X: Enable- Count=32 Masked- Capabilities: [b0] Power Management version 3 Capabilities: [c0] Vital Product Data Capabilities: [100] Advanced Error Reporting Capabilities: [150] Alternative Routing-ID Interpretation (ARI) Capabilities: [200] Single Root I/O Virtualization (SR-IOV) Capabilities: [310] #19 Capabilities: [4e0] Device Serial Number 44-a1-91-ff-ff-a4-9b-ec Capabilities: [4f0] Transaction Processing Hints Capabilities: [600] Vendor Specific Information: ID=0000 Rev=0 Len=028 <?> Capabilities: [630] Access Control Services Kernel driver in use: igb_uio Kernel modules: hinic [root@localhost dpdk-19.11]#
[root@localhost dpdk-19.11]# cat /proc/iomem | grep 0000\:06\:00.0 e9300000-e93fffff : 0000:06:00.0 80008b00000-80008bfffff : 0000:06:00.0 80008c00000-800103fffff : 0000:06:00.0 80010400000-8001041ffff : 0000:06:00.0 80010420000-80010b9ffff : 0000:06:00.0 80010ba0000-8001131ffff : 0000:06:00.0 80011320000-80011327fff : 0000:06:00.0 [root@localhost dpdk-19.11]#
[root@localhost dpdk-19.11]# ls /sys/bus/pci/devices/0000\:06\:00.0/resource* /sys/bus/pci/devices/0000:06:00.0/resource /sys/bus/pci/devices/0000:06:00.0/resource2 /sys/bus/pci/devices/0000:06:00.0/resource0 /sys/bus/pci/devices/0000:06:00.0/resource4 [root@localhost dpdk-19.11]# cat /sys/bus/pci/devices/0000\:06\:00.0/resource* 0x0000080010400000 0x000008001041ffff 0x000000000014220c 0x0000000000000000 0x0000000000000000 0x0000000000000000 0x0000080011320000 0x0000080011327fff 0x000000000014220c 0x0000000000000000 0x0000000000000000 0x0000000000000000 0x0000080008b00000 0x0000080008bfffff 0x000000000014220c 0x0000000000000000 0x0000000000000000 0x0000000000000000 0x00000000e9300000 0x00000000e93fffff 0x0000000000046200 0x0000080010420000 0x0000080010b9ffff 0x000000000014220c 0x0000000000000000 0x0000000000000000 0x0000000000000000 0x0000080010ba0000 0x000008001131ffff 0x000000000014220c 0x0000000000000000 0x0000000000000000 0x0000000000000000 0x0000080008c00000 0x00000800103fffff 0x000000000014220c 0x0000000000000000 0x0000000000000000 0x0000000000000000 cat: /sys/bus/pci/devices/0000:06:00.0/resource0: Input/output error cat: /sys/bus/pci/devices/0000:06:00.0/resource2: Input/output error cat: /sys/bus/pci/devices/0000:06:00.0/resource4: Input/output error [root@localhost dpdk-19.11]#
[root@localhost data1]# cat uio.c #include <stdio.h> #include <sys/types.h> #include <sys/stat.h> #include <fcntl.h> #include <sys/mman.h> #include <stdint.h> #define PAGE_SIZE 0x1000 int main() { int fd; void *base; uint32_t *pmem; fd = open("/dev/uio0", O_RDWR); if (fd < 0) { fprintf(stderr, "Failed to open device\n"); exit(1); } base = mmap(NULL, PAGE_SIZE, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); if (base == MAP_FAILED) { fprintf(stderr, "Failed to mmap\n"); exit(1); } pmem = (uint32_t*)base; printf("pmem[0] = 0x%08x\n", pmem[0]); printf("pmem[1] = 0x%08x\n", pmem[1]);
[root@localhost data1]# ./uio pmem[0] = 0x01040080 pmem[1] = 0x002250c1
portio
Since these ioport regions can not be mapped, they will not appear under /sys/class/uio/uioX/maps/ like the normal memory described above. Without information about the port regions a hardware has to offer, it becomes difficult for the userspace part of the driver to find out which ports belong to which UIO device.
To address this situation, the new directory /sys/class/uio/uioX/portio/ was added. It only exists if the driver wants to pass information about one or more port regions to userspace. If that is the case, subdirectories named port0, port1, and so on, will appear underneath /sys/class/uio/uioX/portio/.
Each portX/ directory contains four read-only files that show name, start, size, and type of the port region:
-
name: A string identifier for this port region. The string is optional and can be empty. Drivers can set it to make it easier for userspace to find a certain port region. -
start: The first port of this region. -
size: The number of ports in this region. -
porttype: A string describing the type of port.
igb_uio & igb & ioremap & mmap
igb_uio 驱动会遍历该 PCI 设备的 BAR 空间，对于类型为存储器空间 IORESOURCE_MEM 的 BAR（Memory BAR），将其物理地址、大小等信息保存到 uio_info 结构的 mem 数组中；将类型为寄存器空间 IORESOURCE_IO 的 BAR（IO BAR），将其物理地址、大小等信息保存到 uio_info 结构的 port 数组中。
[root@localhost dpdk-19.11]# lspci -s 0000\:06\:00.0 -v 06:00.0 Ethernet controller: Huawei Technologies Co., Ltd. Hi1822 Family (2*25GE) (rev 45) Subsystem: Huawei Technologies Co., Ltd. Device d139 Flags: fast devsel, NUMA node 0 Memory at 80010400000 (64-bit, prefetchable) [size=128K] Memory at 80011320000 (64-bit, prefetchable) [size=32K] Memory at 80008b00000 (64-bit, prefetchable) [size=1M] Expansion ROM at e9300000 [disabled] [size=1M] Capabilities: [40] Express Endpoint, MSI 00 Capabilities: [80] MSI: Enable- Count=1/32 Maskable+ 64bit+ Capabilities: [a0] MSI-X: Enable- Count=32 Masked- Capabilities: [b0] Power Management version 3 Capabilities: [c0] Vital Product Data Capabilities: [100] Advanced Error Reporting Capabilities: [150] Alternative Routing-ID Interpretation (ARI) Capabilities: [200] Single Root I/O Virtualization (SR-IOV) Capabilities: [310] #19 Capabilities: [4e0] Device Serial Number 44-a1-91-ff-ff-a4-9b-ec Capabilities: [4f0] Transaction Processing Hints Capabilities: [600] Vendor Specific Information: ID=0000 Rev=0 Len=028 <?> Capabilities: [630] Access Control Services Kernel driver in use: igb_uio Kernel modules: hinic [root@localhost dpdk-19.11]#
而 igb 驱动同样也会遍历 BAR 空间，但是它不会记录空间的物理地址，而是调用 ioremap() 将物理地址映射为虚拟地址，然后驱动就可以在内核态中读写映射出来的虚拟地址，而不是像 igb_uio 驱动似的在用户态中进行读写。
