[RFC 0/2] (Ongoing)Support multiple pci domains in pci_device
NOTE: This patch set is still ongoing and does not fully function as its goal. But it involves some API changes, therefore I post them for comments before moving on to make sure I'm on the right path. The corresponding qemu part can be found at https://gitlab.com/WhoisZihan/qemu-pci-domain/tree/master/qemu I will submit it to qemu list later. Currently seabios assumes there is only one pci domain(0), and almost everything operates on pci domain 0 by default. This patch aims to add multiple pci domain support for pci_device, while reserve the original API for compatibility. Some part of it is dirty and hardcoded, as I plan to make it support 2 pci domains first as a working POC. Many works remain to do when BIOS checks and initializes PCI devices, because almost everywhere assumes 1 pci domain by default. It compiles and runs, but the guest will issue a "Bus 0001:00 not in PCI Namespace" warning, and it does not correctly recognize the bus under host bridges other than q35. If you have any clue about this error, please point me out, I will appreciate it. Zihan Yang (2): fw/pciinit: Recognize pxb-pcie-dev device pci_device: Add pci domain support src/fw/coreboot.c | 2 +- src/fw/csm.c | 2 +- src/fw/paravirt.c | 3 +- src/fw/pciinit.c | 208 ++++++++++++++++++++++++++++++++++------------------- src/hw/pci.c | 69 +++++++++--------- src/hw/pci.h | 42 +++++++---- src/hw/pci_ids.h | 6 +- src/hw/pcidevice.c | 8 ++- src/hw/pcidevice.h | 10 ++- 9 files changed, 223 insertions(+), 127 deletions(-) -- 2.7.4
QEMU q35 uses pxb-pcie-dev to enable multiple host bridges, this patch recognizes such devices in seabios and add corresponding e820 entry. MCFG base and size are already setup in QEMU, we just need to read it Signed-off-by: Zihan Yang <whois.zihan.yang@gmail.com> --- src/fw/paravirt.c | 1 - src/fw/pciinit.c | 17 +++++++++++++++++ src/hw/pci_ids.h | 1 + 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/fw/paravirt.c b/src/fw/paravirt.c index 0770c47..6b14542 100644 --- a/src/fw/paravirt.c +++ b/src/fw/paravirt.c @@ -197,7 +197,6 @@ qemu_platform_setup(void) if (!loader_err) warn_internalerror(); } - acpi_setup(); } diff --git a/src/fw/pciinit.c b/src/fw/pciinit.c index 3a2f747..6e6a434 100644 --- a/src/fw/pciinit.c +++ b/src/fw/pciinit.c @@ -507,11 +507,28 @@ static void mch_mem_addr_setup(struct pci_device *dev, void *arg) pci_io_low_end = acpi_pm_base; } +static void pxb_mem_addr_setup(struct pci_device *dev, void *arg) +{ + union u64_u32_u mcfg_base; + mcfg_base.lo = pci_config_readl(dev->bdf, Q35_HOST_BRIDGE_PCIEXBAR); + mcfg_base.hi = pci_config_readl(dev->bdf, Q35_HOST_BRIDGE_PCIEXBAR + 4); + + // Fix me! Use another meaningful macro + u32 mcfg_size = pci_config_readl(dev->bdf, Q35_HOST_BRIDGE_PCIEXBAR + 8); + + /* Skip config write here as the qemu-level objects are already setup, we + * read mcfg_base and mcfg_size from it just now. Instead, we directly add + * this item to e820 */ + e820_add(mcfg_base.val, mcfg_size, E820_RESERVED); +} + static const struct pci_device_id pci_platform_tbl[] = { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82441, i440fx_mem_addr_setup), PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_Q35_MCH, mch_mem_addr_setup), + PCI_DEVICE(PCI_VENDOR_ID_REDHAT, PCI_DEVICE_ID_REDHAT_PXB_HOST, + pxb_mem_addr_setup), PCI_DEVICE_END }; diff --git a/src/hw/pci_ids.h b/src/hw/pci_ids.h index 38fa2ca..35096ea 100644 --- a/src/hw/pci_ids.h +++ b/src/hw/pci_ids.h @@ -2265,6 +2265,7 @@ #define PCI_VENDOR_ID_REDHAT 0x1b36 #define PCI_DEVICE_ID_REDHAT_ROOT_PORT 0x000C +#define PCI_DEVICE_ID_REDHAT_PXB_HOST 0x000B #define PCI_VENDOR_ID_TEKRAM 0x1de1 #define PCI_DEVICE_ID_TEKRAM_DC290 0xdc29 -- 2.7.4
Most part of seabios assume only PCI domain 0. This patch adds support for multiple domain in pci devices Signed-off-by: Zihan Yang <whois.zihan.yang@gmail.com> --- src/fw/coreboot.c | 2 +- src/fw/csm.c | 2 +- src/fw/paravirt.c | 2 +- src/fw/pciinit.c | 191 +++++++++++++++++++++++++++++++++-------------------- src/hw/pci.c | 69 ++++++++++--------- src/hw/pci.h | 42 ++++++++---- src/hw/pci_ids.h | 7 +- src/hw/pcidevice.c | 8 ++- src/hw/pcidevice.h | 10 ++- 9 files changed, 206 insertions(+), 127 deletions(-) diff --git a/src/fw/coreboot.c b/src/fw/coreboot.c index 7c0954b..c955dfd 100644 --- a/src/fw/coreboot.c +++ b/src/fw/coreboot.c @@ -254,7 +254,7 @@ coreboot_platform_setup(void) { if (!CONFIG_COREBOOT) return; - pci_probe_devices(); + pci_probe_devices(0); struct cb_memory *cbm = CBMemTable; if (!cbm) diff --git a/src/fw/csm.c b/src/fw/csm.c index 03b4bb8..e94f614 100644 --- a/src/fw/csm.c +++ b/src/fw/csm.c @@ -63,7 +63,7 @@ static void csm_maininit(struct bregs *regs) { interface_init(); - pci_probe_devices(); + pci_probe_devices(0); csm_compat_table.PnPInstallationCheckSegment = SEG_BIOS; csm_compat_table.PnPInstallationCheckOffset = get_pnp_offset(); diff --git a/src/fw/paravirt.c b/src/fw/paravirt.c index 6b14542..ef4d487 100644 --- a/src/fw/paravirt.c +++ b/src/fw/paravirt.c @@ -155,7 +155,7 @@ qemu_platform_setup(void) return; if (runningOnXen()) { - pci_probe_devices(); + pci_probe_devices(0); xen_hypercall_setup(); xen_biostable_setup(); return; diff --git a/src/fw/pciinit.c b/src/fw/pciinit.c index 6e6a434..71508d5 100644 --- a/src/fw/pciinit.c +++ b/src/fw/pciinit.c @@ -51,6 +51,7 @@ u64 pcimem_end = BUILD_PCIMEM_END; u64 pcimem64_start = BUILD_PCIMEM64_START; u64 pcimem64_end = BUILD_PCIMEM64_END; u64 pci_io_low_end = 0xa000; +u64 pxb_mcfg_size = 0; struct pci_region_entry { struct pci_device *dev; @@ -405,19 +406,21 @@ static void pci_bios_init_device(struct pci_device *pci) /* map the interrupt */ u16 bdf = pci->bdf; - int pin = pci_config_readb(bdf, PCI_INTERRUPT_PIN); + int pin = pci_config_readb_dom(bdf, PCI_INTERRUPT_PIN, pci->domain_nr); if (pin != 0) - pci_config_writeb(bdf, PCI_INTERRUPT_LINE, pci_slot_get_irq(pci, pin)); + pci_config_writeb_dom(bdf, PCI_INTERRUPT_LINE, pci_slot_get_irq(pci, pin), + pci->domain_nr); pci_init_device(pci_device_tbl, pci, NULL); /* enable memory mappings */ - pci_config_maskw(bdf, PCI_COMMAND, 0, - PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_SERR); + pci_config_maskw_dom(bdf, PCI_COMMAND, 0, + PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_SERR, + pci->domain_nr); /* enable SERR# for forwarding */ if (pci->header_type & PCI_HEADER_TYPE_BRIDGE) - pci_config_maskw(bdf, PCI_BRIDGE_CONTROL, 0, - PCI_BRIDGE_CTL_SERR); + pci_config_maskw_dom(bdf, PCI_BRIDGE_CONTROL, 0, + PCI_BRIDGE_CTL_SERR, pci->domain_nr); } static void pci_bios_init_devices(void) @@ -520,6 +523,10 @@ static void pxb_mem_addr_setup(struct pci_device *dev, void *arg) * read mcfg_base and mcfg_size from it just now. Instead, we directly add * this item to e820 */ e820_add(mcfg_base.val, mcfg_size, E820_RESERVED); + + /* Add PXBHosts so that we can can initialize them later */ + ++PXBHosts; + pxb_mcfg_size += mcfg_size; } static const struct pci_device_id pci_platform_tbl[] = { @@ -540,19 +547,21 @@ static void pci_bios_init_platform(void) } } -static u8 pci_find_resource_reserve_capability(u16 bdf) +static u8 pci_find_resource_reserve_capability(u16 bdf, int domain_nr) { - if (pci_config_readw(bdf, PCI_VENDOR_ID) == PCI_VENDOR_ID_REDHAT && - pci_config_readw(bdf, PCI_DEVICE_ID) == - PCI_DEVICE_ID_REDHAT_ROOT_PORT) { + if (pci_config_readw_dom(bdf, PCI_VENDOR_ID, domain_nr) == PCI_VENDOR_ID_REDHAT && + (pci_config_readw_dom(bdf, PCI_DEVICE_ID, domain_nr) == + PCI_DEVICE_ID_REDHAT_ROOT_PORT || + pci_config_readw_dom(bdf, PCI_DEVICE_ID, domain_nr) == + PCI_DEVICE_ID_REDHAT_PCIE_BRIDGE)) { u8 cap = 0; do { - cap = pci_find_capability(bdf, PCI_CAP_ID_VNDR, cap); + cap = pci_find_capability_dom(bdf, PCI_CAP_ID_VNDR, cap, domain_nr); } while (cap && - pci_config_readb(bdf, cap + PCI_CAP_REDHAT_TYPE_OFFSET) != + pci_config_readb_dom(bdf, cap + PCI_CAP_REDHAT_TYPE_OFFSET, domain_nr) != REDHAT_CAP_RESOURCE_RESERVE); if (cap) { - u8 cap_len = pci_config_readb(bdf, cap + PCI_CAP_FLAGS); + u8 cap_len = pci_config_readb_dom(bdf, cap + PCI_CAP_FLAGS, domain_nr); if (cap_len < RES_RESERVE_CAP_SIZE) { dprintf(1, "PCI: QEMU resource reserve cap length %d is invalid\n", cap_len); @@ -570,7 +579,7 @@ static u8 pci_find_resource_reserve_capability(u16 bdf) ****************************************************************/ static void -pci_bios_init_bus_rec(int bus, u8 *pci_bus) +pci_bios_init_bus_rec(int bus, u8 *pci_bus, int domain_nr) { int bdf; u16 class; @@ -578,54 +587,54 @@ pci_bios_init_bus_rec(int bus, u8 *pci_bus) dprintf(1, "PCI: %s bus = 0x%x\n", __func__, bus); /* prevent accidental access to unintended devices */ - foreachbdf(bdf, bus) { - class = pci_config_readw(bdf, PCI_CLASS_DEVICE); + foreachbdf_dom(bdf, bus, domain_nr) { + class = pci_config_readw_dom(bdf, PCI_CLASS_DEVICE, domain_nr); if (class == PCI_CLASS_BRIDGE_PCI) { - pci_config_writeb(bdf, PCI_SECONDARY_BUS, 255); - pci_config_writeb(bdf, PCI_SUBORDINATE_BUS, 0); + pci_config_writeb_dom(bdf, PCI_SECONDARY_BUS, 255, domain_nr); + pci_config_writeb_dom(bdf, PCI_SUBORDINATE_BUS, 0, domain_nr); } } - foreachbdf(bdf, bus) { - class = pci_config_readw(bdf, PCI_CLASS_DEVICE); + foreachbdf_dom(bdf, bus, domain_nr) { + class = pci_config_readw_dom(bdf, PCI_CLASS_DEVICE, domain_nr); if (class != PCI_CLASS_BRIDGE_PCI) { continue; } dprintf(1, "PCI: %s bdf = 0x%x\n", __func__, bdf); - u8 pribus = pci_config_readb(bdf, PCI_PRIMARY_BUS); + u8 pribus = pci_config_readb_dom(bdf, PCI_PRIMARY_BUS, domain_nr); if (pribus != bus) { dprintf(1, "PCI: primary bus = 0x%x -> 0x%x\n", pribus, bus); - pci_config_writeb(bdf, PCI_PRIMARY_BUS, bus); + pci_config_writeb_dom(bdf, PCI_PRIMARY_BUS, bus, domain_nr); } else { dprintf(1, "PCI: primary bus = 0x%x\n", pribus); } - u8 secbus = pci_config_readb(bdf, PCI_SECONDARY_BUS); + u8 secbus = pci_config_readb_dom(bdf, PCI_SECONDARY_BUS, domain_nr); (*pci_bus)++; if (*pci_bus != secbus) { dprintf(1, "PCI: secondary bus = 0x%x -> 0x%x\n", secbus, *pci_bus); secbus = *pci_bus; - pci_config_writeb(bdf, PCI_SECONDARY_BUS, secbus); + pci_config_writeb_dom(bdf, PCI_SECONDARY_BUS, secbus, domain_nr); } else { dprintf(1, "PCI: secondary bus = 0x%x\n", secbus); } /* set to max for access to all subordinate buses. later set it to accurate value */ - u8 subbus = pci_config_readb(bdf, PCI_SUBORDINATE_BUS); - pci_config_writeb(bdf, PCI_SUBORDINATE_BUS, 255); + u8 subbus = pci_config_readb_dom(bdf, PCI_SUBORDINATE_BUS, domain_nr); + pci_config_writeb_dom(bdf, PCI_SUBORDINATE_BUS, 255, domain_nr); - pci_bios_init_bus_rec(secbus, pci_bus); + pci_bios_init_bus_rec(secbus, pci_bus, domain_nr); if (subbus != *pci_bus) { u8 res_bus = *pci_bus; - u8 cap = pci_find_resource_reserve_capability(bdf); + u8 cap = pci_find_resource_reserve_capability(bdf, domain_nr); if (cap) { - u32 tmp_res_bus = pci_config_readl(bdf, - cap + RES_RESERVE_BUS_RES); + u32 tmp_res_bus = pci_config_readl_dom(bdf, + cap + RES_RESERVE_BUS_RES, domain_nr); if (tmp_res_bus != (u32)-1) { res_bus = tmp_res_bus & 0xFF; if ((u8)(res_bus + secbus) < secbus || @@ -648,7 +657,7 @@ pci_bios_init_bus_rec(int bus, u8 *pci_bus) } else { dprintf(1, "PCI: subordinate bus = 0x%x\n", subbus); } - pci_config_writeb(bdf, PCI_SUBORDINATE_BUS, subbus); + pci_config_writeb_dom(bdf, PCI_SUBORDINATE_BUS, subbus, domain_nr); } } @@ -658,16 +667,29 @@ pci_bios_init_bus(void) u8 extraroots = romfile_loadint("etc/extra-pci-roots", 0); u8 pci_bus = 0; - pci_bios_init_bus_rec(0 /* host bus */, &pci_bus); + pci_bios_init_bus_rec(0 /* host bus */, &pci_bus, 0); if (extraroots) { while (pci_bus < 0xff) { pci_bus++; - pci_bios_init_bus_rec(pci_bus, &pci_bus); + pci_bios_init_bus_rec(pci_bus, &pci_bus, 0); } } } +static void +pci_bios_init_pxb_bus(int domain_nr) +{ + u8 pci_bus = 0; + + // FIX ME! Hardcoded ports is not a good idea at all! + pci_bios_init_bus_rec(0, &pci_bus, domain_nr); + + while (pci_bus < 0xff) { + pci_bus++; + pci_bios_init_bus_rec(pci_bus, &pci_bus, domain_nr); + } +} /**************************************************************** * Bus sizing @@ -675,17 +697,18 @@ pci_bios_init_bus(void) static void pci_bios_get_bar(struct pci_device *pci, int bar, - int *ptype, u64 *psize, int *pis64) + int *ptype, u64 *psize, int *pis64, + int domain_nr) { u32 ofs = pci_bar(pci, bar); u16 bdf = pci->bdf; - u32 old = pci_config_readl(bdf, ofs); + u32 old = pci_config_readl_dom(bdf, ofs, domain_nr); int is64 = 0, type = PCI_REGION_TYPE_MEM; u64 mask; if (bar == PCI_ROM_SLOT) { mask = PCI_ROM_ADDRESS_MASK; - pci_config_writel(bdf, ofs, mask); + pci_config_writel_dom(bdf, ofs, mask, domain_nr); } else { if (old & PCI_BASE_ADDRESS_SPACE_IO) { mask = PCI_BASE_ADDRESS_IO_MASK; @@ -697,15 +720,15 @@ pci_bios_get_bar(struct pci_device *pci, int bar, is64 = ((old & PCI_BASE_ADDRESS_MEM_TYPE_MASK) == PCI_BASE_ADDRESS_MEM_TYPE_64); } - pci_config_writel(bdf, ofs, ~0); + pci_config_writel_dom(bdf, ofs, ~0, domain_nr); } - u64 val = pci_config_readl(bdf, ofs); - pci_config_writel(bdf, ofs, old); + u64 val = pci_config_readl_dom(bdf, ofs, domain_nr); + pci_config_writel_dom(bdf, ofs, old, domain_nr); if (is64) { - u32 hold = pci_config_readl(bdf, ofs + 4); - pci_config_writel(bdf, ofs + 4, ~0); - u32 high = pci_config_readl(bdf, ofs + 4); - pci_config_writel(bdf, ofs + 4, hold); + u32 hold = pci_config_readl_dom(bdf, ofs + 4, domain_nr); + pci_config_writel_dom(bdf, ofs + 4, ~0, domain_nr); + u32 high = pci_config_readl_dom(bdf, ofs + 4, domain_nr); + pci_config_writel_dom(bdf, ofs + 4, hold, domain_nr); val |= ((u64)high << 32); mask |= ((u64)0xffffffff << 32); *psize = (~(val & mask)) + 1; @@ -717,15 +740,20 @@ pci_bios_get_bar(struct pci_device *pci, int bar, } static int pci_bios_bridge_region_is64(struct pci_region *r, - struct pci_device *pci, int type) + struct pci_device *pci, int type, + int domain_nr) { if (type != PCI_REGION_TYPE_PREFMEM) return 0; - u32 pmem = pci_config_readl(pci->bdf, PCI_PREF_MEMORY_BASE); + u32 pmem = pci_config_readl_dom(pci->bdf, PCI_PREF_MEMORY_BASE, + domain_nr); if (!pmem) { - pci_config_writel(pci->bdf, PCI_PREF_MEMORY_BASE, 0xfff0fff0); - pmem = pci_config_readl(pci->bdf, PCI_PREF_MEMORY_BASE); - pci_config_writel(pci->bdf, PCI_PREF_MEMORY_BASE, 0x0); + pci_config_writel_dom(pci->bdf, PCI_PREF_MEMORY_BASE, 0xfff0fff0, + domain_nr); + pmem = pci_config_readl_dom(pci->bdf, PCI_PREF_MEMORY_BASE, + domain_nr); + pci_config_writel_dom(pci->bdf, PCI_PREF_MEMORY_BASE, 0x0, + domain_nr); } if ((pmem & PCI_PREF_RANGE_TYPE_MASK) != PCI_PREF_RANGE_TYPE_64) return 0; @@ -801,13 +829,15 @@ pci_region_create_entry(struct pci_bus *bus, struct pci_device *dev, return entry; } -static int pci_bus_hotplug_support(struct pci_bus *bus, u8 pcie_cap) +static int pci_bus_hotplug_support(struct pci_bus *bus, u8 pcie_cap, + int domain_nr) { u8 shpc_cap; if (pcie_cap) { - u16 pcie_flags = pci_config_readw(bus->bus_dev->bdf, - pcie_cap + PCI_EXP_FLAGS); + u16 pcie_flags = pci_config_readw_dom(bus->bus_dev->bdf, + pcie_cap + PCI_EXP_FLAGS, + domain_nr); u8 port_type = ((pcie_flags & PCI_EXP_FLAGS_TYPE) >> (__builtin_ffs(PCI_EXP_FLAGS_TYPE) - 1)); u8 downstream_port = (port_type == PCI_EXP_TYPE_DOWNSTREAM) || @@ -826,7 +856,8 @@ static int pci_bus_hotplug_support(struct pci_bus *bus, u8 pcie_cap) return downstream_port && slot_implemented; } - shpc_cap = pci_find_capability(bus->bus_dev->bdf, PCI_CAP_ID_SHPC, 0); + shpc_cap = pci_find_capability_dom(bus->bus_dev->bdf, PCI_CAP_ID_SHPC, 0, + domain_nr); return !!shpc_cap; } @@ -835,7 +866,8 @@ static int pci_bus_hotplug_support(struct pci_bus *bus, u8 pcie_cap) * Note: disables bridge's window registers as a side effect. */ static int pci_bridge_has_region(struct pci_device *pci, - enum pci_region_type region_type) + enum pci_region_type region_type, + int domain_nr) { u8 base; @@ -851,12 +883,12 @@ static int pci_bridge_has_region(struct pci_device *pci, return 1; } - pci_config_writeb(pci->bdf, base, 0xFF); + pci_config_writeb_dom(pci->bdf, base, 0xFF, domain_nr); - return pci_config_readb(pci->bdf, base) != 0; + return pci_config_readb_dom(pci->bdf, base, domain_nr) != 0; } -static int pci_bios_check_devices(struct pci_bus *busses) +static int pci_bios_check_devices(struct pci_bus *busses, int domain_nr) { dprintf(1, "PCI: check devices\n"); @@ -879,7 +911,7 @@ static int pci_bios_check_devices(struct pci_bus *busses) continue; int type, is64; u64 size; - pci_bios_get_bar(pci, i, &type, &size, &is64); + pci_bios_get_bar(pci, i, &type, &size, &is64, domain_nr); if (size == 0) continue; @@ -909,14 +941,14 @@ static int pci_bios_check_devices(struct pci_bus *busses) parent = &busses[0]; int type; u16 bdf = s->bus_dev->bdf; - u8 pcie_cap = pci_find_capability(bdf, PCI_CAP_ID_EXP, 0); - u8 qemu_cap = pci_find_resource_reserve_capability(bdf); + u8 pcie_cap = pci_find_capability_dom(bdf, PCI_CAP_ID_EXP, 0, domain_nr); + u8 qemu_cap = pci_find_resource_reserve_capability(bdf, domain_nr); - int hotplug_support = pci_bus_hotplug_support(s, pcie_cap); + int hotplug_support = pci_bus_hotplug_support(s, pcie_cap, domain_nr); for (type = 0; type < PCI_REGION_TYPE_COUNT; type++) { u64 align = (type == PCI_REGION_TYPE_IO) ? PCI_BRIDGE_IO_MIN : PCI_BRIDGE_MEM_MIN; - if (!pci_bridge_has_region(s->bus_dev, type)) + if (!pci_bridge_has_region(s->bus_dev, type, domain_nr)) continue; u64 size = 0; if (qemu_cap) { @@ -924,22 +956,25 @@ static int pci_bios_check_devices(struct pci_bus *busses) u64 tmp_size_64; switch(type) { case PCI_REGION_TYPE_IO: - tmp_size_64 = (pci_config_readl(bdf, qemu_cap + RES_RESERVE_IO) | - (u64)pci_config_readl(bdf, qemu_cap + RES_RESERVE_IO + 4) << 32); + tmp_size_64 = (pci_config_readl_dom(bdf, qemu_cap + RES_RESERVE_IO, domain_nr) | + (u64)pci_config_readl_dom(bdf, qemu_cap + RES_RESERVE_IO + 4, domain_nr) << 32); if (tmp_size_64 != (u64)-1) { size = tmp_size_64; } break; case PCI_REGION_TYPE_MEM: - tmp_size = pci_config_readl(bdf, qemu_cap + RES_RESERVE_MEM); + tmp_size = pci_config_readl_dom(bdf, qemu_cap + RES_RESERVE_MEM, domain_nr); if (tmp_size != (u32)-1) { size = tmp_size; } break; case PCI_REGION_TYPE_PREFMEM: - tmp_size = pci_config_readl(bdf, qemu_cap + RES_RESERVE_PREF_MEM_32); - tmp_size_64 = (pci_config_readl(bdf, qemu_cap + RES_RESERVE_PREF_MEM_64) | - (u64)pci_config_readl(bdf, qemu_cap + RES_RESERVE_PREF_MEM_64 + 4) << 32); + tmp_size = pci_config_readl_dom(bdf, qemu_cap + RES_RESERVE_PREF_MEM_32, + domain_nr); + tmp_size_64 = (pci_config_readl_dom(bdf, qemu_cap + RES_RESERVE_PREF_MEM_64, + domain_nr) | + (u64)pci_config_readl_dom(bdf, qemu_cap + RES_RESERVE_PREF_MEM_64 + 4, + domain_nr) << 32); if (tmp_size != (u32)-1 && tmp_size_64 == (u64)-1) { size = tmp_size; } else if (tmp_size == (u32)-1 && tmp_size_64 != (u64)-1) { @@ -970,7 +1005,7 @@ static int pci_bios_check_devices(struct pci_bus *busses) size = ALIGN(sum, align); } int is64 = pci_bios_bridge_region_is64(&s->r[type], - s->bus_dev, type); + s->bus_dev, type, domain_nr); // entry->bar is -1 if the entry represents a bridge region struct pci_region_entry *entry = pci_region_create_entry( parent, s->bus_dev, -1, size, align, type, is64); @@ -1036,6 +1071,7 @@ static int pci_bios_init_root_regions_mem(struct pci_bus *bus) align = pci_region_align(r_start); r_start->base = ALIGN_DOWN((r_end->base - sum), align); + dprintf(1, "==============\n\nr_start->base = 0x%llx, pcimem_start = 0x%llx, pcimem_end = 0x%llx\n============\n", r_start->base, pcimem_start, pcimem_end); if ((r_start->base < pcimem_start) || (r_start->base > pcimem_end)) // Memory range requested is larger than available. @@ -1127,8 +1163,8 @@ static void pci_bios_map_devices(struct pci_bus *busses) r64_pref.base = r64_mem.base + sum_mem; r64_pref.base = ALIGN(r64_pref.base, align_pref); r64_pref.base = ALIGN(r64_pref.base, (1LL<<30)); // 1G hugepage - pcimem64_start = r64_mem.base; - pcimem64_end = r64_pref.base + sum_pref; + pcimem64_start = r64_mem.base + pxb_mcfg_size; + pcimem64_end = r64_pref.base + sum_pref + pxb_mcfg_size; pcimem64_end = ALIGN(pcimem64_end, (1LL<<30)); // 1G hugepage dprintf(1, "PCI: 64: %016llx - %016llx\n", pcimem64_start, pcimem64_end); @@ -1165,9 +1201,12 @@ pci_setup(void) return; } pci_bios_init_bus(); + /* FIXME! Hardcode is just for debugging right now */ + pci_bios_init_pxb_bus(1); dprintf(1, "=== PCI device probing ===\n"); - pci_probe_devices(); + pci_probe_devices(0); + pci_probe_devices(1); pcimem_start = RamSize; pci_bios_init_platform(); @@ -1179,7 +1218,7 @@ pci_setup(void) return; } memset(busses, 0, sizeof(*busses) * (MaxPCIBus + 1)); - if (pci_bios_check_devices(busses)) + if (pci_bios_check_devices(busses, 0)) return; dprintf(1, "=== PCI new allocation pass #2 ===\n"); @@ -1187,6 +1226,14 @@ pci_setup(void) pci_bios_init_devices(); + /* then check pxb host busses */ + memset(busses, 0, sizeof(*busses) * (MaxPCIBus + 1)); + if (pci_bios_check_devices(busses, 1)) + return; + + //pci_bios_map_devices(busses); + //pci_bios_init_devices(); + free(busses); pci_enable_default_vga(); diff --git a/src/hw/pci.c b/src/hw/pci.c index 9855bad..cc1b6ec 100644 --- a/src/hw/pci.c +++ b/src/hw/pci.c @@ -11,72 +11,75 @@ #include "util.h" // udelay #include "x86.h" // outl -#define PORT_PCI_CMD 0x0cf8 -#define PORT_PCI_DATA 0x0cfc - -void pci_config_writel(u16 bdf, u32 addr, u32 val) +void pci_config_writel_dom(u16 bdf, u32 addr, u32 val, int domain_nr) { - outl(0x80000000 | (bdf << 8) | (addr & 0xfc), PORT_PCI_CMD); - outl(val, PORT_PCI_DATA); + outl(0x80000000 | (bdf << 8) | (addr & 0xfc), + domain_nr ? PORT_PXB_CMD_BASE + ((domain_nr - 1) << 3) : PORT_PCI_CMD); + outl(val, (domain_nr ? PORT_PXB_DATA_BASE + ((domain_nr - 1) << 3) : PORT_PCI_DATA)); } -void pci_config_writew(u16 bdf, u32 addr, u16 val) +void pci_config_writew_dom(u16 bdf, u32 addr, u16 val, int domain_nr) { - outl(0x80000000 | (bdf << 8) | (addr & 0xfc), PORT_PCI_CMD); - outw(val, PORT_PCI_DATA + (addr & 2)); + outl(0x80000000 | (bdf << 8) | (addr & 0xfc), + domain_nr == 0 ? PORT_PCI_CMD : PORT_PXB_CMD_BASE + (domain_nr << 3)); + outw(val, (domain_nr ? PORT_PXB_DATA_BASE + ((domain_nr - 1) << 3) : PORT_PCI_DATA) + (addr & 2)); } -void pci_config_writeb(u16 bdf, u32 addr, u8 val) +void pci_config_writeb_dom(u16 bdf, u32 addr, u8 val, int domain_nr) { - outl(0x80000000 | (bdf << 8) | (addr & 0xfc), PORT_PCI_CMD); - outb(val, PORT_PCI_DATA + (addr & 3)); + outl(0x80000000 | (bdf << 8) | (addr & 0xfc), + domain_nr ? PORT_PXB_CMD_BASE + ((domain_nr - 1) << 3) : PORT_PCI_CMD); + outb(val, (domain_nr ? PORT_PXB_DATA_BASE + ((domain_nr - 1) << 3) : PORT_PCI_DATA) + (addr & 3)); } -u32 pci_config_readl(u16 bdf, u32 addr) +u32 pci_config_readl_dom(u16 bdf, u32 addr, int domain_nr) { - outl(0x80000000 | (bdf << 8) | (addr & 0xfc), PORT_PCI_CMD); - return inl(PORT_PCI_DATA); + outl(0x80000000 | (bdf << 8) | (addr & 0xfc), + domain_nr ? PORT_PXB_CMD_BASE + ((domain_nr - 1) << 3) : PORT_PCI_CMD); + return inl((domain_nr ? PORT_PXB_DATA_BASE + ((domain_nr - 1) << 3) : PORT_PCI_DATA)); } -u16 pci_config_readw(u16 bdf, u32 addr) +u16 pci_config_readw_dom(u16 bdf, u32 addr, int domain_nr) { - outl(0x80000000 | (bdf << 8) | (addr & 0xfc), PORT_PCI_CMD); - return inw(PORT_PCI_DATA + (addr & 2)); + outl(0x80000000 | (bdf << 8) | (addr & 0xfc), + domain_nr ? PORT_PXB_CMD_BASE + ((domain_nr - 1) << 3) : PORT_PCI_CMD); + return inw((domain_nr ? PORT_PXB_DATA_BASE + ((domain_nr - 1) << 3) : PORT_PCI_DATA) + (addr & 2)); } -u8 pci_config_readb(u16 bdf, u32 addr) +u8 pci_config_readb_dom(u16 bdf, u32 addr, int domain_nr) { - outl(0x80000000 | (bdf << 8) | (addr & 0xfc), PORT_PCI_CMD); - return inb(PORT_PCI_DATA + (addr & 3)); + outl(0x80000000 | (bdf << 8) | (addr & 0xfc), + domain_nr ? PORT_PXB_CMD_BASE + ((domain_nr - 1) << 3) : PORT_PCI_CMD); + return inb((domain_nr ? PORT_PXB_DATA_BASE + ((domain_nr - 1) << 3) : PORT_PCI_DATA) + (addr & 3)); } void -pci_config_maskw(u16 bdf, u32 addr, u16 off, u16 on) +pci_config_maskw_dom(u16 bdf, u32 addr, u16 off, u16 on, int domain_nr) { - u16 val = pci_config_readw(bdf, addr); + u16 val = pci_config_readw_dom(bdf, addr, domain_nr); val = (val & ~off) | on; - pci_config_writew(bdf, addr, val); + pci_config_writew_dom(bdf, addr, val, domain_nr); } -u8 pci_find_capability(u16 bdf, u8 cap_id, u8 cap) +u8 pci_find_capability_dom(u16 bdf, u8 cap_id, u8 cap, int domain_nr) { int i; - u16 status = pci_config_readw(bdf, PCI_STATUS); + u16 status = pci_config_readw_dom(bdf, PCI_STATUS, domain_nr); if (!(status & PCI_STATUS_CAP_LIST)) return 0; if (cap == 0) { /* find first */ - cap = pci_config_readb(bdf, PCI_CAPABILITY_LIST); + cap = pci_config_readb_dom(bdf, PCI_CAPABILITY_LIST, domain_nr); } else { /* find next */ - cap = pci_config_readb(bdf, cap + PCI_CAP_LIST_NEXT); + cap = pci_config_readb_dom(bdf, cap + PCI_CAP_LIST_NEXT, domain_nr); } for (i = 0; cap && i <= 0xff; i++) { - if (pci_config_readb(bdf, cap + PCI_CAP_LIST_ID) == cap_id) + if (pci_config_readb_dom(bdf, cap + PCI_CAP_LIST_ID, domain_nr) == cap_id) return cap; - cap = pci_config_readb(bdf, cap + PCI_CAP_LIST_NEXT); + cap = pci_config_readb_dom(bdf, cap + PCI_CAP_LIST_NEXT, domain_nr); } return 0; @@ -84,10 +87,10 @@ u8 pci_find_capability(u16 bdf, u8 cap_id, u8 cap) // Helper function for foreachbdf() macro - return next device int -pci_next(int bdf, int bus) +pci_next_dom(int bdf, int bus, int domain_nr) { if (pci_bdf_to_fn(bdf) == 0 - && (pci_config_readb(bdf, PCI_HEADER_TYPE) & 0x80) == 0) + && (pci_config_readb_dom(bdf, PCI_HEADER_TYPE, domain_nr) & 0x80) == 0) // Last found device wasn't a multi-function device - skip to // the next device. bdf += 8; @@ -98,7 +101,7 @@ pci_next(int bdf, int bus) if (pci_bdf_to_bus(bdf) != bus) return -1; - u16 v = pci_config_readw(bdf, PCI_VENDOR_ID); + u16 v = pci_config_readw_dom(bdf, PCI_VENDOR_ID, domain_nr); if (v != 0x0000 && v != 0xffff) // Device is present. return bdf; diff --git a/src/hw/pci.h b/src/hw/pci.h index 2e30e28..4381563 100644 --- a/src/hw/pci.h +++ b/src/hw/pci.h @@ -3,7 +3,11 @@ #include "types.h" // u32 +#define PORT_PCI_CMD 0x0cf8 #define PORT_PCI_REBOOT 0x0cf9 +#define PORT_PCI_DATA 0x0cfc +#define PORT_PXB_CMD_BASE 0x1000 +#define PORT_PXB_DATA_BASE 0x1004 static inline u8 pci_bdf_to_bus(u16 bdf) { return bdf >> 8; @@ -27,20 +31,34 @@ static inline u16 pci_bus_devfn_to_bdf(int bus, u16 devfn) { return (bus << 8) | devfn; } -#define foreachbdf(BDF, BUS) \ - for (BDF=pci_next(pci_bus_devfn_to_bdf((BUS), 0)-1, (BUS)) \ +/* for compatibility */ +#define foreachbdf(BDF, BUS) foreachbdf_dom(BDF, BUS, 0) + +#define foreachbdf_dom(BDF, BUS, DOMAIN) \ + for (BDF=pci_next_dom(pci_bus_devfn_to_bdf((BUS), 0)-1, (BUS), (DOMAIN)) \ ; BDF >= 0 \ - ; BDF=pci_next(BDF, (BUS))) + ; BDF=pci_next_dom(BDF, (BUS), (DOMAIN))) -void pci_config_writel(u16 bdf, u32 addr, u32 val); -void pci_config_writew(u16 bdf, u32 addr, u16 val); -void pci_config_writeb(u16 bdf, u32 addr, u8 val); -u32 pci_config_readl(u16 bdf, u32 addr); -u16 pci_config_readw(u16 bdf, u32 addr); -u8 pci_config_readb(u16 bdf, u32 addr); -void pci_config_maskw(u16 bdf, u32 addr, u16 off, u16 on); -u8 pci_find_capability(u16 bdf, u8 cap_id, u8 cap); -int pci_next(int bdf, int bus); +#define pci_config_maskw(BDF, ADDR, OFF, ON) pci_config_maskw_dom((BDF), (ADDR), (OFF), (ON), 0) +#define pci_find_capability(BDF, CAP_ID, CAP) pci_find_capability_dom((BDF), (CAP_ID), (CAP), 0) +#define pci_next(BDF, BUS) pci_next_dom((BDF), (BUS), 0) + +#define pci_config_writel(BDF, ADDR, VAL) pci_config_writel_dom((BDF), (ADDR), (VAL), 0) +#define pci_config_writew(BDF, ADDR, VAL) pci_config_writew_dom((BDF), (ADDR), (VAL), 0) +#define pci_config_writeb(BDF, ADDR, VAL) pci_config_writeb_dom((BDF), (ADDR), (VAL), 0) +#define pci_config_readl(BDF, ADDR) pci_config_readl_dom((BDF), (ADDR), 0) +#define pci_config_readw(BDF, ADDR) pci_config_readw_dom((BDF), (ADDR), 0) +#define pci_config_readb(BDF, ADDR) pci_config_readb_dom((BDF), (ADDR), 0) + +void pci_config_writel_dom(u16 bdf, u32 addr, u32 val, int domain_nr); +void pci_config_writew_dom(u16 bdf, u32 addr, u16 val, int domain_nr); +void pci_config_writeb_dom(u16 bdf, u32 addr, u8 val, int domain_nr); +u32 pci_config_readl_dom(u16 bdf, u32 addr, int domain_nr); +u16 pci_config_readw_dom(u16 bdf, u32 addr, int domain_nr); +u8 pci_config_readb_dom(u16 bdf, u32 addr, int domain_nr); +void pci_config_maskw_dom(u16 bdf, u32 addr, u16 off, u16 on, int domain_nr); +u8 pci_find_capability_dom(u16 bdf, u8 cap_id, u8 cap, int domain_nr); +int pci_next_dom(int bdf, int bus, int domain_nr); int pci_probe_host(void); void pci_reboot(void); diff --git a/src/hw/pci_ids.h b/src/hw/pci_ids.h index 35096ea..1d4ddf6 100644 --- a/src/hw/pci_ids.h +++ b/src/hw/pci_ids.h @@ -2263,9 +2263,10 @@ #define PCI_DEVICE_ID_KORENIX_JETCARDF0 0x1600 #define PCI_DEVICE_ID_KORENIX_JETCARDF1 0x16ff -#define PCI_VENDOR_ID_REDHAT 0x1b36 -#define PCI_DEVICE_ID_REDHAT_ROOT_PORT 0x000C -#define PCI_DEVICE_ID_REDHAT_PXB_HOST 0x000B +#define PCI_VENDOR_ID_REDHAT 0x1b36 +#define PCI_DEVICE_ID_REDHAT_PXB_HOST 0x000B +#define PCI_DEVICE_ID_REDHAT_ROOT_PORT 0x000C +#define PCI_DEVICE_ID_REDHAT_PCIE_BRIDGE 0x000E #define PCI_VENDOR_ID_TEKRAM 0x1de1 #define PCI_DEVICE_ID_TEKRAM_DC290 0xdc29 diff --git a/src/hw/pcidevice.c b/src/hw/pcidevice.c index 8853cf7..ec21ec1 100644 --- a/src/hw/pcidevice.c +++ b/src/hw/pcidevice.c @@ -15,10 +15,11 @@ struct hlist_head PCIDevices VARVERIFY32INIT; int MaxPCIBus VARFSEG; +int PXBHosts VARFSEG; // Find all PCI devices and populate PCIDevices linked list. void -pci_probe_devices(void) +pci_probe_devices(int domain_nr) { dprintf(3, "PCI probe\n"); struct pci_device *busdevs[256]; @@ -29,7 +30,7 @@ pci_probe_devices(void) while (bus < 0xff && (bus < MaxPCIBus || rootbuses < extraroots)) { bus++; int bdf; - foreachbdf(bdf, bus) { + foreachbdf_dom(bdf, bus, domain_nr) { // Create new pci_device struct and add to list. struct pci_device *dev = malloc_tmp(sizeof(*dev)); if (!dev) { @@ -56,6 +57,7 @@ pci_probe_devices(void) } // Populate pci_device info. + dev->domain_nr = domain_nr; dev->bdf = bdf; dev->parent = parent; dev->rootbus = rootbus; @@ -69,7 +71,7 @@ pci_probe_devices(void) dev->header_type = pci_config_readb(bdf, PCI_HEADER_TYPE); u8 v = dev->header_type & 0x7f; if (v == PCI_HEADER_TYPE_BRIDGE || v == PCI_HEADER_TYPE_CARDBUS) { - u8 secbus = pci_config_readb(bdf, PCI_SECONDARY_BUS); + u8 secbus = pci_config_readb_dom(bdf, PCI_SECONDARY_BUS, domain_nr); dev->secondary_bus = secbus; if (secbus > bus && !busdevs[secbus]) busdevs[secbus] = dev; diff --git a/src/hw/pcidevice.h b/src/hw/pcidevice.h index 225d545..ddf60bb 100644 --- a/src/hw/pcidevice.h +++ b/src/hw/pcidevice.h @@ -5,6 +5,7 @@ #include "list.h" // hlist_node struct pci_device { + u32 domain_nr; u16 bdf; u8 rootbus; struct hlist_node node; @@ -22,6 +23,7 @@ struct pci_device { }; extern struct hlist_head PCIDevices; extern int MaxPCIBus; +extern int PXBHosts; static inline u32 pci_classprog(struct pci_device *pci) { return (pci->class << 8) | pci->prog_if; @@ -30,6 +32,12 @@ static inline u32 pci_classprog(struct pci_device *pci) { #define foreachpci(PCI) \ hlist_for_each_entry(PCI, &PCIDevices, node) +#define foreachpci_dom(PCI, DOMAIN) \ + hlist_for_each_entry(PCI, &PCIDevices, node) { \ + if (pci->domain_nr != (DOMAIN)) \ + continue; \ + } + #define PCI_ANY_ID (~0) struct pci_device_id { u32 vendid; @@ -62,7 +70,7 @@ struct pci_device_id { .vendid = 0, \ } -void pci_probe_devices(void); +void pci_probe_devices(int domain_nr); struct pci_device *pci_find_device(u16 vendid, u16 devid); struct pci_device *pci_find_class(u16 classid); int pci_init_device(const struct pci_device_id *ids -- 2.7.4
On Thu, Aug 02, 2018 at 10:39:13AM +0800, Zihan Yang wrote:
NOTE: This patch set is still ongoing and does not fully function as its goal. But it involves some API changes, therefore I post them for comments before moving on to make sure I'm on the right path.
The corresponding qemu part can be found at https://gitlab.com/WhoisZihan/qemu-pci-domain/tree/master/qemu I will submit it to qemu list later.
Currently seabios assumes there is only one pci domain(0), and almost everything operates on pci domain 0 by default. This patch aims to add multiple pci domain support for pci_device, while reserve the original API for compatibility.
Thanks. I understand the desire to support multiple PCI domains in QEMU and the guest OS. However, what is the high level reason for wanting the BIOS to be able to interact with the secondary PCI domains? -Kevin
participants (2)
-
Kevin O'Connor -
Zihan Yang