pci p2p
概述
在2018年，针对pci支持p2pdma的驱动合入主线，没记错的话应该是4.20.
补丁如下：
commit 52916982af48d9f9fc01ad825259de1eb3a9b25e
Author: Logan Gunthorpe <logang@deltatee.com>
Date: Thu Oct 4 15:27:35 2018 -0600
PCI/P2PDMA: Support peer-to-peer memory
一、pci p2p DMA的条件
1.1内核配置
需要开启 CONFIG_PCI_P2PDMA 才能使用p2p的DMA。
需要设备具备内存，并以 ZONE_DEVICE 方式注册到内核中，暴露给驱动，需要注意的是，arm 默认不开启 CONFIG_ZONE_DEVICE，需要手动打开。
1.2 host bridge要求
从代码看，要求两个device 同属于同一个host_bridge,除此之外，还有一个白名单。
static bool host_bridge_whitelist(struct pci_dev *a, struct pci_dev *b,
bool warn)
{
struct pci_host_bridge *host_a = pci_find_host_bridge(a->bus);
struct pci_host_bridge *host_b = pci_find_host_bridge(b->bus);
if (host_a == host_b)//caq:相同host brdige，没得说
return __host_bridge_whitelist(host_a, true, warn);//caq:此时白名单检查传参为true
if (__host_bridge_whitelist(host_a, false, warn) &&//caq:否则检查白名单
__host_bridge_whitelist(host_b, false, warn))//caq:此时传参为false
return true;
return false;
}
//caq:检查哪些host_bridge支持p2p
static bool __host_bridge_whitelist(struct pci_host_bridge *host,
bool same_host_bridge, bool warn)
{
struct pci_dev *root = pci_host_bridge_dev(host);
const struct pci_p2pdma_whitelist_entry *entry;
unsigned short vendor, device;
if (!root)
return false;
vendor = root->vendor;//caq:取vendorid
device = root->device;//caq:取deviceid
for (entry = pci_p2pdma_whitelist; entry->vendor; entry++) {
if (vendor != entry->vendor || device != entry->device)
continue;
if (entry->flags & REQ_SAME_HOST_BRIDGE && !same_host_bridge)//caq:入参same_host_bridge 就是 true的话，则跳过
return false;
return true;//caq:如果 same_host_bridge 传参为true，则只要是在白名单之内的pci_host_bridge设备，都支持p2pdma
}
if (warn)//caq:如果没找到，则打印warning
pci_warn(root, "Host bridge not in P2PDMA whitelist: %04x:%04x\n",
vendor, device);
return false;
}
那么，是哪些设备在白名单之内呢？
static const struct pci_p2pdma_whitelist_entry {
unsigned short vendor;
unsigned short device;
enum {
REQ_SAME_HOST_BRIDGE = 1 << 0,//caq:是否要求同一个host bridge
} flags;
} pci_p2pdma_whitelist[] = {//caq:白名单列表
/* Intel Xeon E5/Core i7 */
{PCI_VENDOR_ID_INTEL, 0x3c00, REQ_SAME_HOST_BRIDGE},
{PCI_VENDOR_ID_INTEL, 0x3c01, REQ_SAME_HOST_BRIDGE},
/* Intel Xeon E7 v3/Xeon E5 v3/Core i7 */
{PCI_VENDOR_ID_INTEL, 0x2f00, REQ_SAME_HOST_BRIDGE},
{PCI_VENDOR_ID_INTEL, 0x2f01, REQ_SAME_HOST_BRIDGE},
/* Intel SkyLake-E */
{PCI_VENDOR_ID_INTEL, 0x2030, 0},//caq:不要求同一个host_bridge,也支持p2p
{PCI_VENDOR_ID_INTEL, 0x2031, 0},
{PCI_VENDOR_ID_INTEL, 0x2032, 0},
{PCI_VENDOR_ID_INTEL, 0x2033, 0},
{PCI_VENDOR_ID_INTEL, 0x2020, 0},
{PCI_VENDOR_ID_INTEL, 0x09a2, 0},
{}
};
简单这么一看，只有intel的某些host_bridge支持p2pdma，那么，amd能忍么？
1.3 cpu要求
前面说到intel 通过白名单添加host_bridge的方式来支持跨host_bridge的p2p dma，amd一看，他也要有，然后提交了如下patch：
commit 7d5b10fcb81e511ddf79c1c6b7f6efb282f80680
Author: Alex Deucher <alexdeucher@gmail.com>
Date: Mon Apr 6 15:42:01 2020 -0400
PCI/P2PDMA: Add AMD Zen Raven and Renoir Root Ports to whitelist
According to the hardware architect, pre-Zen parts support p2p writes and
Zen parts support both p2p reads and writes.
Add entries for Zen parts Raven (0x15d0) and Renoir (0x1630).
Link: https://lore.kernel.org/r/20200406194201.846411-1-alexander.deucher@amd.com
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
Signed-off-by: Bjorn Helgaas <bhelgaas@google.com>
Acked-by: Christian König <christian.koenig@amd.com>
Acked-by: Huang Rui <ray.huang@amd.com>
diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c
index b73b10bce0df..e8e444eeb1cd 100644
--- a/drivers/pci/p2pdma.c
+++ b/drivers/pci/p2pdma.c
@@ -282,6 +282,8 @@ static const struct pci_p2pdma_whitelist_entry {
} pci_p2pdma_whitelist[] = {
/* AMD ZEN */
{PCI_VENDOR_ID_AMD, 0x1450, 0},
+ {PCI_VENDOR_ID_AMD, 0x15d0, 0},
+ {PCI_VENDOR_ID_AMD, 0x1630, 0},
/* Intel Xeon E5/Core i7 */
{PCI_VENDOR_ID_INTEL, 0x3c00, REQ_SAME_HOST_BRIDGE},
当前在此之前还添加了一个0x1450，但是加着加着amd缓过头来了，amd的zen架构之后，都支持跨host_bridge 来支持p2pdma，那岂不是白名单得一直加下去？
所以，amd干脆另起炉灶：
commit dea286bb71baded7d2fb4f090e3b9fd2c1ccac58
Author: Logan Gunthorpe <logang@deltatee.com>
Date: Wed Jul 29 17:18:44 2020 -0600
PCI/P2PDMA: Allow P2PDMA on AMD Zen and newer CPUs
Allow P2PDMA if the CPU vendor is AMD and family is 0x17 (Zen) or greater.
[bhelgaas: commit log, simplify #if/#else/#endif]
Link: https://lore.kernel.org/r/20200729231844.4653-1-logang@deltatee.com
Signed-off-by: Logan Gunthorpe <logang@deltatee.com>
Signed-off-by: Bjorn Helgaas <bhelgaas@google.com>
Reviewed-by: Alex Deucher <alexander.deucher@amd.com>
Cc: Christian König <christian.koenig@amd.com>
Cc: Huang Rui <ray.huang@amd.com>
diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c
index e8e444eeb1cd..1ec61fced4c3 100644
--- a/drivers/pci/p2pdma.c
+++ b/drivers/pci/p2pdma.c
@@ -273,6 +273,19 @@ static void seq_buf_print_bus_devfn(struct seq_buf *buf, struct pci_dev *pdev)
seq_buf_printf(buf, "%s;", pci_name(pdev));
}
+static bool cpu_supports_p2pdma(void)
+{
+#ifdef CONFIG_X86
+ struct cpuinfo_x86 *c = &cpu_data(0);
+
+ /* Any AMD CPU whose family ID is Zen or newer supports p2pdma */
+ if (c->x86_vendor == X86_VENDOR_AMD && c->x86 >= 0x17)//caq:amd另起炉灶，只要是zen或者更新的架构，都支持，但比如zen被授权出去，host_bridge被第三方修改，能保证么？
+ return true;
+#endif
+
+ return false;
+}
+
static const struct pci_p2pdma_whitelist_entry {
unsigned short vendor;
unsigned short device;
@@ -280,11 +293,6 @@ static const struct pci_p2pdma_whitelist_entry {
REQ_SAME_HOST_BRIDGE = 1 << 0,
} flags;
} pci_p2pdma_whitelist[] = {
- /* AMD ZEN */
- {PCI_VENDOR_ID_AMD, 0x1450, 0},
- {PCI_VENDOR_ID_AMD, 0x15d0, 0},
- {PCI_VENDOR_ID_AMD, 0x1630, 0},
-
/* Intel Xeon E5/Core i7 */
{PCI_VENDOR_ID_INTEL, 0x3c00, REQ_SAME_HOST_BRIDGE},
{PCI_VENDOR_ID_INTEL, 0x3c01, REQ_SAME_HOST_BRIDGE},
@@ -473,7 +481,8 @@ upstream_bridge_distance(struct pci_dev *provider, struct pci_dev *client,
acs_redirects, acs_list);
if (map_type == PCI_P2PDMA_MAP_THRU_HOST_BRIDGE) {
- if (!host_bridge_whitelist(provider, client))
+ if (!cpu_supports_p2pdma() &&
+ !host_bridge_whitelist(provider, client))
map_type = PCI_P2PDMA_MAP_NOT_SUPPORTED;
}
总而言之，就是zen系列之后的，全支持，不再跟intel那样加白名单了，有点霸气，但是不是还会改回来还未知，比如amd做了一款编号大于 0x17 的cpu，但是又不支持跨host_bridge的p2pdma咋办？
水平有限，如果有错误，请帮忙提醒我。如果您觉得本文对您有帮助，可以点击下面的 推荐 支持一下我。版权所有，需要转发请带上本文源地址，博客一直在更新，欢迎 关注 。
