Jonathan Zhang has uploaded this change for review. ( https://review.coreboot.org/c/coreboot/+/44280 )
Change subject: soc/intel/xeon_sp/cpx: add VT-d support ......................................................................
soc/intel/xeon_sp/cpx: add VT-d support
Intel CPX-SP FSP added support for VT-d through adding UPD parameter X2apic. Based on devicetree.cb setting, enable VT-d programming through FSP-M.
When VT-d is enabled, add DMAR ACPI table.
Signed-off-by: Jonathan Zhang jonzhang@fb.com Change-Id: Ic66374af6e53fb847c1bdc324eb3f4e01c334a94 --- M src/soc/intel/xeon_sp/cpx/acpi.c M src/soc/intel/xeon_sp/cpx/chip.h M src/soc/intel/xeon_sp/cpx/romstage.c 3 files changed, 324 insertions(+), 2 deletions(-)
git pull ssh://review.coreboot.org:29418/coreboot refs/changes/80/44280/1
diff --git a/src/soc/intel/xeon_sp/cpx/acpi.c b/src/soc/intel/xeon_sp/cpx/acpi.c index 62569e6..0f310f6 100644 --- a/src/soc/intel/xeon_sp/cpx/acpi.c +++ b/src/soc/intel/xeon_sp/cpx/acpi.c @@ -448,12 +448,315 @@ return current; }
+/* + * Ports Stack Stack(HOB) IioConfigIou + * ========================================== + * 0 CSTACK stack 0 IOU0 + * 1A..1D PSTACK0 stack 1 IOU1 + * 2A..2D PSTACK1 stack 2 IOU2 + * 3A..3D PSTACK2 stack 4 IOU3 + */ +static int get_stack_for_port(int p) +{ + if (p == 0) + return CSTACK; + else if (p >= PORT_1A && p <= PORT_1D) + return PSTACK0; + else if (p >= PORT_2A && p <= PORT_2D) + return PSTACK1; + else //if (p >= PORT_3A && p <= PORT_3D) + return PSTACK2; +} + +static unsigned long acpi_create_drhd(unsigned long current, int socket, int stack) +{ + int IoApicID[] = { + // socket 0 + PC00_IOAPIC_ID, PC01_IOAPIC_ID, PC02_IOAPIC_ID, PC03_IOAPIC_ID, + PC04_IOAPIC_ID, PC05_IOAPIC_ID, + // socket 1 + PC06_IOAPIC_ID, PC07_IOAPIC_ID, PC08_IOAPIC_ID, PC09_IOAPIC_ID, + PC10_IOAPIC_ID, PC11_IOAPIC_ID, + }; + + uint32_t enum_id; + unsigned long tmp = current; + + size_t hob_size; + const uint8_t fsp_hob_iio_universal_data_guid[16] = FSP_HOB_IIO_UNIVERSAL_DATA_GUID; + const IIO_UDS *hob = fsp_find_extension_hob_by_guid( + fsp_hob_iio_universal_data_guid, &hob_size); + assert(hob != NULL && hob_size != 0); + + uint32_t bus = hob->PlatformData.IIO_resource[socket].StackRes[stack].BusBase; + uint32_t pcie_seg = hob->PlatformData.CpuQpiInfo[socket].PcieSegment; + uint32_t reg_base = + hob->PlatformData.IIO_resource[socket].StackRes[stack].VtdBarAddress; + printk(BIOS_SPEW, "%s socket: %d, stack: %d, bus: 0x%x, pcie_seg: 0x%x, reg_base: 0x%x\n", + __func__, socket, stack, bus, pcie_seg, reg_base); + + // Add DRHD Hardware Unit + if (socket == 0 && stack == CSTACK) { + printk(BIOS_DEBUG, "[Hardware Unit Definition] Flags: 0x%x, PCI Segment Number: 0x%x, " + "Register Base Address: 0x%x\n", + DRHD_INCLUDE_PCI_ALL, pcie_seg, reg_base); + current += acpi_create_dmar_drhd(current, DRHD_INCLUDE_PCI_ALL, + pcie_seg, reg_base); + } else { + printk(BIOS_DEBUG, "[Hardware Unit Definition] Flags: 0x%x, PCI Segment Number: 0x%x, " + "Register Base Address: 0x%x\n", 0, pcie_seg, reg_base); + current += acpi_create_dmar_drhd(current, 0, pcie_seg, reg_base); + } + + // Add PCH IOAPIC + if (socket == 0 && stack == CSTACK) { + printk(BIOS_DEBUG, " [IOAPIC Device] Enumeration ID: 0x%x, PCI Bus Number: 0x%x, " + "PCI Path: 0x%x, 0x%x\n", + PCH_IOAPIC_ID, PCH_IOAPIC_BUS_NUMBER, + PCH_IOAPIC_DEV_NUM, PCH_IOAPIC_FUNC_NUM); + current += acpi_create_dmar_ds_ioapic(current, PCH_IOAPIC_ID, + PCH_IOAPIC_BUS_NUMBER, PCH_IOAPIC_DEV_NUM, PCH_IOAPIC_FUNC_NUM); + } + + // Add IOAPIC entry + enum_id = IoApicID[(socket*MAX_IIO_STACK)+stack]; + printk(BIOS_DEBUG, " [IOAPIC Device] Enumeration ID: 0x%x, PCI Bus Number: 0x%x, " + "PCI Path: 0x%x, 0x%x\n", enum_id, bus, APIC_DEV_NUM, APIC_FUNC_NUM); + current += acpi_create_dmar_ds_ioapic(current, enum_id, bus, + APIC_DEV_NUM, APIC_FUNC_NUM); + + // Add CBDMA devices for CSTACK + if (socket != 0 && stack == CSTACK) { + for (int cbdma_func_id = 0; cbdma_func_id < 8; ++cbdma_func_id) { + printk(BIOS_DEBUG, " [PCI Endpoint Device] Enumeration ID: 0x%x, " + "PCI Bus Number: 0x%x, PCI Path: 0x%x, 0x%x\n", + 0, bus, CBDMA_DEV_NUM, cbdma_func_id); + current += acpi_create_dmar_ds_pci(current, + bus, CBDMA_DEV_NUM, cbdma_func_id); + } + } + + // Add PCIe Ports + if (socket != 0 || stack != CSTACK) { + IIO_RESOURCE_INSTANCE iio_resource = + hob->PlatformData.IIO_resource[socket]; + for (int p = 0; p < NUMBER_PORTS_PER_SOCKET; ++p) { + if (get_stack_for_port(p) != stack) + continue; + + uint32_t dev = iio_resource.PcieInfo.PortInfo[p].Device; + uint32_t func = iio_resource.PcieInfo.PortInfo[p].Function; + + uint32_t id = pci_mmio_read_config32(PCI_DEV(bus, dev, func), + PCI_VENDOR_ID); + if (id == 0xffffffff) + continue; + + printk(BIOS_DEBUG, " [PCI Bridge Device] Enumeration ID: 0x%x, " + "PCI Bus Number: 0x%x, PCI Path: 0x%x, 0x%x\n", + 0, bus, dev, func); + current += acpi_create_dmar_ds_pci_br(current, + bus, dev, func); + } + + // Add VMD + if (hob->PlatformData.VMDStackEnable[socket][stack] && + stack >= PSTACK0 && stack <= PSTACK2) { + printk(BIOS_DEBUG, " [PCI Endpoint Device] Enumeration ID: 0x%x, " + "PCI Bus Number: 0x%x, PCI Path: 0x%x, 0x%x\n", + 0, bus, VMD_DEV_NUM, VMD_FUNC_NUM); + current += acpi_create_dmar_ds_pci(current, + bus, VMD_DEV_NUM, VMD_FUNC_NUM); + } + } + + // Add HPET + if (socket == 0 && stack == CSTACK) { + uint16_t hpet_capid = read16((void *)HPET_BASE_ADDRESS); + uint16_t num_hpets = (hpet_capid >> 0x08) & 0x1F; // Bits [8:12] has hpet count + printk(BIOS_SPEW, "%s hpet_capid: 0x%x, num_hpets: 0x%x\n", + __func__, hpet_capid, num_hpets); + //BIT 15 + if (num_hpets && (num_hpets != 0x1f) && + (read32((void *)(HPET_BASE_ADDRESS + 0x100)) & (0x00008000))) { + printk(BIOS_DEBUG, " [Message-capable HPET Device] Enumeration ID: 0x%x, " + "PCI Bus Number: 0x%x, PCI Path: 0x%x, 0x%x\n", + 0, HPET_BUS_NUM, HPET_DEV_NUM, HPET0_FUNC_NUM); + current += acpi_create_dmar_ds_msi_hpet(current, 0, HPET_BUS_NUM, + HPET_DEV_NUM, HPET0_FUNC_NUM); + } + } + + acpi_dmar_drhd_fixup(tmp, current); + + return current; +} + +static unsigned long acpi_create_atsr(unsigned long current) +{ + size_t hob_size; + const uint8_t uds_guid[16] = FSP_HOB_IIO_UNIVERSAL_DATA_GUID; + const IIO_UDS *hob = fsp_find_extension_hob_by_guid(uds_guid, &hob_size); + assert(hob != NULL && hob_size != 0); + + for (int socket = 0; socket < hob->PlatformData.numofIIO; ++socket) { + uint32_t pcie_seg = hob->PlatformData.CpuQpiInfo[socket].PcieSegment; + unsigned long tmp = current; + bool first = true; + IIO_RESOURCE_INSTANCE iio_resource = + hob->PlatformData.IIO_resource[socket]; + + for (int stack = 0; stack <= PSTACK2; ++stack) { + uint32_t bus = hob->PlatformData.IIO_resource[socket].StackRes[stack].BusBase; + uint32_t vtd_base = iio_resource.StackRes[stack].VtdBarAddress; + if (!vtd_base) + continue; + uint64_t vtd_mmio_cap = read64((void *)(vtd_base + VTD_EXT_CAP_LOW)); + printk(BIOS_SPEW, "%s socket: %d, stack: %d, bus: 0x%x, vtd_base: 0x%x, " + "vtd_mmio_cap: 0x%llx\n", + __func__, socket, stack, bus, vtd_base, vtd_mmio_cap); + + // ATSR is applicable only for platform supporting device IOTLBs + // through the VT-d extended capability register + assert(vtd_mmio_cap != 0xffffffffffffffff); + if ((vtd_mmio_cap & 0x4) == 0) // BIT 2 + continue; + + for (int p = 0; p < NUMBER_PORTS_PER_SOCKET; ++p) { + if (socket == 0 && p == 0) + continue; + if (get_stack_for_port(p) != stack) + continue; + + uint32_t dev = iio_resource.PcieInfo.PortInfo[p].Device; + uint32_t func = iio_resource.PcieInfo.PortInfo[p].Function; + + u32 id = pci_mmio_read_config32(PCI_DEV(bus, dev, func), + PCI_VENDOR_ID); + if (id == 0xffffffff) + continue; + + if (first) { + printk(BIOS_DEBUG, "[Root Port ATS Capability] Flags: 0x%x, " + "PCI Segment Number: 0x%x\n", + 0, pcie_seg); + current += acpi_create_dmar_atsr(current, 0, pcie_seg); + first = 0; + } + + printk(BIOS_DEBUG, " [PCI Bridge Device] Enumeration ID: 0x%x, " + "PCI Bus Number: 0x%x, PCI Path: 0x%x, 0x%x\n", + 0, bus, dev, func); + current += acpi_create_dmar_ds_pci_br(current, bus, dev, func); + } + } + if (tmp != current) + acpi_dmar_atsr_fixup(tmp, current); + } + + return current; +} + +static unsigned long acpi_create_rmrr(unsigned long current) +{ + uint32_t size = ALIGN_UP(MEM_BLK_COUNT * sizeof(MEM_BLK), 0x1000); + + uint32_t *ptr; + + // reserve memory + ptr = cbmem_find(CBMEM_ID_STORAGE_DATA); + if (!ptr) { + ptr = cbmem_add(CBMEM_ID_STORAGE_DATA, size); + assert(ptr != NULL); + memset(ptr, 0, size); + } + + unsigned long tmp = current; + printk(BIOS_DEBUG, "[Reserved Memory Region] PCI Segment Number: 0x%x, Base Address: 0x%x, " + "End Address (limit): 0x%x\n", + 0, (uint32_t) ptr, (uint32_t) ((uint32_t) ptr + size - 1)); + current += acpi_create_dmar_rmrr(current, 0, (uint32_t) ptr, + (uint32_t) ((uint32_t) ptr + size - 1)); + + printk(BIOS_DEBUG, " [PCI Endpoint Device] Enumeration ID: 0x%x, PCI Bus Number: 0x%x, " + "PCI Path: 0x%x, 0x%x\n", + 0, XHCI_BUS_NUMBER, PCH_DEV_SLOT_XHCI, XHCI_FUNC_NUM); + current += acpi_create_dmar_ds_pci(current, XHCI_BUS_NUMBER, + PCH_DEV_SLOT_XHCI, XHCI_FUNC_NUM); + + acpi_dmar_rmrr_fixup(tmp, current); + + return current; +} + +static unsigned long acpi_create_rhsa(unsigned long current) +{ + size_t hob_size; + const uint8_t uds_guid[16] = FSP_HOB_IIO_UNIVERSAL_DATA_GUID; + const IIO_UDS *hob = fsp_find_extension_hob_by_guid(uds_guid, &hob_size); + assert(hob != NULL && hob_size != 0); + + for (int socket = 0; socket < hob->PlatformData.numofIIO; ++socket) { + IIO_RESOURCE_INSTANCE iio_resource = + hob->PlatformData.IIO_resource[socket]; + for (int stack = 0; stack <= PSTACK2; ++stack) { + uint32_t vtd_base = iio_resource.StackRes[stack].VtdBarAddress; + if (!vtd_base) + continue; + + printk(BIOS_DEBUG, "[Remapping Hardware Static Affinity] Base Address: 0x%x, " + "Proximity Domain: 0x%x\n", vtd_base, socket); + current += acpi_create_dmar_rhsa(current, vtd_base, socket); + } + } + + return current; +} + +static unsigned long acpi_fill_dmar(unsigned long current) +{ + size_t hob_size; + const uint8_t uds_guid[16] = FSP_HOB_IIO_UNIVERSAL_DATA_GUID; + const IIO_UDS *hob = fsp_find_extension_hob_by_guid(uds_guid, &hob_size); + assert(hob != NULL && hob_size != 0); + + // DRHD + for (int iio = 1; iio <= hob->PlatformData.numofIIO; ++iio) { + int socket = iio; + if (socket == hob->PlatformData.numofIIO) // socket 0 should be last DRHD entry + socket = 0; + + if (socket == 0) { + for (int stack = 1; stack <= PSTACK2; ++stack) + current = acpi_create_drhd(current, socket, stack); + current = acpi_create_drhd(current, socket, CSTACK); + } else { + for (int stack = 0; stack <= PSTACK2; ++stack) + current = acpi_create_drhd(current, socket, stack); + } + } + + // RMRR + current = acpi_create_rmrr(current); + + // ATSR - causes hang + current = acpi_create_atsr(current); + + // RHSA + current = acpi_create_rhsa(current); + + return current; +} + unsigned long northbridge_write_acpi_tables(const struct device *device, unsigned long current, struct acpi_rsdp *rsdp) { acpi_srat_t *srat; acpi_slit_t *slit; + acpi_dmar_t *dmar; + + const struct soc_intel_xeon_sp_cpx_config *const config = config_of(device);
/* SRAT */ current = ALIGN(current, 8); @@ -471,5 +774,18 @@ current += slit->header.length; acpi_add_table(rsdp, slit);
+ /* DMAR */ + if (config->vtd_support) { + current = ALIGN(current, 8); + dmar = (acpi_dmar_t *)current; + printk(BIOS_DEBUG, "ACPI: * DMAR\n"); + printk(BIOS_DEBUG, "[DMA Remapping table] Flags: 0x%x\n", + (DMAR_INTR_REMAP | DMAR_X2APIC_OPT_OUT)); + acpi_create_dmar(dmar, (DMAR_INTR_REMAP | DMAR_X2APIC_OPT_OUT), acpi_fill_dmar); + current += dmar->header.length; + current = acpi_align_current(current); + acpi_add_table(rsdp, dmar); + } + return current; } diff --git a/src/soc/intel/xeon_sp/cpx/chip.h b/src/soc/intel/xeon_sp/cpx/chip.h index e46f34f..5d12079 100644 --- a/src/soc/intel/xeon_sp/cpx/chip.h +++ b/src/soc/intel/xeon_sp/cpx/chip.h @@ -84,8 +84,9 @@
uint32_t pstate_req_ratio;
- uint32_t coherency_support; - uint32_t ats_support; + uint8_t vtd_support; + uint8_t vmx_enable; + uint8_t x2apic;
/* Generic IO decode ranges */ uint32_t gen1_dec; diff --git a/src/soc/intel/xeon_sp/cpx/romstage.c b/src/soc/intel/xeon_sp/cpx/romstage.c index 96949fd..e96b13e 100644 --- a/src/soc/intel/xeon_sp/cpx/romstage.c +++ b/src/soc/intel/xeon_sp/cpx/romstage.c @@ -13,6 +13,7 @@ void platform_fsp_memory_init_params_cb(FSPM_UPD *mupd, uint32_t version) { FSPM_CONFIG *m_cfg = &mupd->FspmConfig; + const config_t *config = config_of_soc();
/* ErrorLevel - 0 (disable) to 8 (verbose) */ m_cfg->DebugPrintLevel = 8; @@ -61,5 +62,9 @@ m_cfg->PEXPHIDE = 0x0; m_cfg->HidePEXPMenu = 0x0;
+ /* Enable VT-d according to DTB */ + m_cfg->VtdSupport = config->vtd_support; + m_cfg->X2apic = config->x2apic; + mainboard_memory_init_params(mupd); }
Angel Pons has posted comments on this change. ( https://review.coreboot.org/c/coreboot/+/44280 )
Change subject: soc/intel/xeon_sp/cpx: add VT-d support ......................................................................
Patch Set 1: Code-Review+1
(1 comment)
https://review.coreboot.org/c/coreboot/+/44280/1/src/soc/intel/xeon_sp/cpx/c... File src/soc/intel/xeon_sp/cpx/chip.h:
https://review.coreboot.org/c/coreboot/+/44280/1/src/soc/intel/xeon_sp/cpx/c... PS1, Line 88: vmx_enable This isn't used anywhere, is it intentional?
build bot (Jenkins) has posted comments on this change. ( https://review.coreboot.org/c/coreboot/+/44280 )
Change subject: soc/intel/xeon_sp/cpx: add VT-d support ......................................................................
Patch Set 1:
(1 comment)
https://review.coreboot.org/c/coreboot/+/44280/1/src/soc/intel/xeon_sp/cpx/a... File src/soc/intel/xeon_sp/cpx/acpi.c:
https://review.coreboot.org/c/coreboot/+/44280/1/src/soc/intel/xeon_sp/cpx/a... PS1, Line 610: uint32_t bus = hob->PlatformData.IIO_resource[socket].StackRes[stack].BusBase; line over 96 characters
Jonathan Zhang has posted comments on this change. ( https://review.coreboot.org/c/coreboot/+/44280 )
Change subject: soc/intel/xeon_sp/cpx: add VT-d support ......................................................................
Patch Set 1:
(1 comment)
https://review.coreboot.org/c/coreboot/+/44280/1/src/soc/intel/xeon_sp/cpx/c... File src/soc/intel/xeon_sp/cpx/chip.h:
https://review.coreboot.org/c/coreboot/+/44280/1/src/soc/intel/xeon_sp/cpx/c... PS1, Line 88: vmx_enable
This isn't used anywhere, is it intentional?
It is not used. I will remove it.
Hello build bot (Jenkins), Angel Pons, Patrick Rudolph,
I'd like you to reexamine a change. Please visit
https://review.coreboot.org/c/coreboot/+/44280
to look at the new patch set (#2).
Change subject: soc/intel/xeon_sp/cpx: add VT-d support ......................................................................
soc/intel/xeon_sp/cpx: add VT-d support
Intel CPX-SP FSP added support for VT-d through adding UPD parameter X2apic. Based on devicetree.cb setting, enable VT-d programming through FSP-M.
When VT-d is enabled, add DMAR ACPI table.
Signed-off-by: Jonathan Zhang jonzhang@fb.com Change-Id: Ic66374af6e53fb847c1bdc324eb3f4e01c334a94 --- M src/soc/intel/xeon_sp/cpx/acpi.c M src/soc/intel/xeon_sp/cpx/chip.h M src/soc/intel/xeon_sp/cpx/romstage.c 3 files changed, 323 insertions(+), 2 deletions(-)
git pull ssh://review.coreboot.org:29418/coreboot refs/changes/80/44280/2
Angel Pons has posted comments on this change. ( https://review.coreboot.org/c/coreboot/+/44280 )
Change subject: soc/intel/xeon_sp/cpx: add VT-d support ......................................................................
Patch Set 2: Code-Review+1
Needs a rebase, master broke *twice* recently.
Jonathan Zhang has posted comments on this change. ( https://review.coreboot.org/c/coreboot/+/44280 )
Change subject: soc/intel/xeon_sp/cpx: add VT-d support ......................................................................
Patch Set 3:
Patch Set 2: Code-Review+1
Needs a rebase, master broke *twice* recently.
Just did a rebase, tried on Deltalake server, no problem. But upstream gerrit build for Deltalake failed. So I simulated locally, no problem found though: 1024 2020-08-10 13:42:22 make clean 1025 2020-08-10 13:42:24 make defconfig KBUILD_DEFCONFIG=configs/builder/config.ocp.deltalake 1026 2020-08-10 13:42:26 make
Angel Pons has posted comments on this change. ( https://review.coreboot.org/c/coreboot/+/44280 )
Change subject: soc/intel/xeon_sp/cpx: add VT-d support ......................................................................
Patch Set 3:
Patch Set 3:
Patch Set 2: Code-Review+1
Needs a rebase, master broke *twice* recently.
Just did a rebase, tried on Deltalake server, no problem. But upstream gerrit build for Deltalake failed. So I simulated locally, no problem found though: 1024 2020-08-10 13:42:22 make clean 1025 2020-08-10 13:42:24 make defconfig KBUILD_DEFCONFIG=configs/builder/config.ocp.deltalake 1026 2020-08-10 13:42:26 make
Should be good to rebase on master now, I think.
Angel Pons has posted comments on this change. ( https://review.coreboot.org/c/coreboot/+/44280 )
Change subject: soc/intel/xeon_sp/cpx: add VT-d support ......................................................................
Patch Set 3:
(1 comment)
https://review.coreboot.org/c/coreboot/+/44280/3/src/soc/intel/xeon_sp/cpx/c... File src/soc/intel/xeon_sp/cpx/chip.h:
https://review.coreboot.org/c/coreboot/+/44280/3/src/soc/intel/xeon_sp/cpx/c... PS3, Line 87: uint32_t coherency_support; : uint32_t ats_support; Removing these seems to be undesired:
/cb-build/coreboot-gerrit.0/default/OCP_DELTALAKE/mainboard/ocp/deltalake/static.c:23:3: error: 'const struct soc_intel_xeon_sp_cpx_config' has no member named 'ats_support'; did you mean 'vtd_support'? .ats_support = 0, ^~~~~~~~~~~ vtd_support /cb-build/coreboot-gerrit.0/default/OCP_DELTALAKE/mainboard/ocp/deltalake/static.c:24:3: error: 'const struct soc_intel_xeon_sp_cpx_config' has no member named 'coherency_support' .coherency_support = 0,
Angel Pons has posted comments on this change. ( https://review.coreboot.org/c/coreboot/+/44280 )
Change subject: soc/intel/xeon_sp/cpx: add VT-d support ......................................................................
Patch Set 3:
(1 comment)
https://review.coreboot.org/c/coreboot/+/44280/3/src/soc/intel/xeon_sp/cpx/c... File src/soc/intel/xeon_sp/cpx/chip.h:
https://review.coreboot.org/c/coreboot/+/44280/3/src/soc/intel/xeon_sp/cpx/c... PS3, Line 87: uint32_t coherency_support; : uint32_t ats_support;
Removing these seems to be undesired: […]
I'd suggest removing these options from here and deltalake in its own commit, then add the VT-d options
Hello build bot (Jenkins), Angel Pons, Patrick Rudolph,
I'd like you to reexamine a change. Please visit
https://review.coreboot.org/c/coreboot/+/44280
to look at the new patch set (#4).
Change subject: soc/intel/xeon_sp/cpx: add VT-d support ......................................................................
soc/intel/xeon_sp/cpx: add VT-d support
Intel CPX-SP FSP added support for VT-d through adding UPD parameter X2apic. Based on devicetree.cb setting, enable VT-d programming through FSP-M.
When VT-d is enabled, add DMAR ACPI table.
Signed-off-by: Jonathan Zhang jonzhang@fb.com Change-Id: Ic66374af6e53fb847c1bdc324eb3f4e01c334a94 --- M src/soc/intel/xeon_sp/cpx/acpi.c M src/soc/intel/xeon_sp/cpx/chip.h M src/soc/intel/xeon_sp/cpx/romstage.c 3 files changed, 324 insertions(+), 0 deletions(-)
git pull ssh://review.coreboot.org:29418/coreboot refs/changes/80/44280/4
Jonathan Zhang has posted comments on this change. ( https://review.coreboot.org/c/coreboot/+/44280 )
Change subject: soc/intel/xeon_sp/cpx: add VT-d support ......................................................................
Patch Set 4:
(1 comment)
Thanks for the great point!
https://review.coreboot.org/c/coreboot/+/44280/3/src/soc/intel/xeon_sp/cpx/c... File src/soc/intel/xeon_sp/cpx/chip.h:
https://review.coreboot.org/c/coreboot/+/44280/3/src/soc/intel/xeon_sp/cpx/c... PS3, Line 87: uint32_t coherency_support; : uint32_t ats_support;
I'd suggest removing these options from here and deltalake in its own commit, then add the VT-d opti […]
Done
Angel Pons has posted comments on this change. ( https://review.coreboot.org/c/coreboot/+/44280 )
Change subject: soc/intel/xeon_sp/cpx: add VT-d support ......................................................................
Patch Set 4: Code-Review+2
Maxim Polyakov has posted comments on this change. ( https://review.coreboot.org/c/coreboot/+/44280 )
Change subject: soc/intel/xeon_sp/cpx: add VT-d support ......................................................................
Patch Set 4: Code-Review+2
It would be better if you didn't do such big commits. Please try to divide them into small patches, this makes the review easier. Thanks
Angel Pons has posted comments on this change. ( https://review.coreboot.org/c/coreboot/+/44280 )
Change subject: soc/intel/xeon_sp/cpx: add VT-d support ......................................................................
Patch Set 4:
Patch Set 4: Code-Review+2
It would be better if you didn't do such big commits. Please try to divide them into small patches, this makes the review easier. Thanks
The thing with this commit is that VT-d support requires generating DMAR tables, and this is added in a single commit.
Angel Pons has submitted this change. ( https://review.coreboot.org/c/coreboot/+/44280 )
Change subject: soc/intel/xeon_sp/cpx: add VT-d support ......................................................................
soc/intel/xeon_sp/cpx: add VT-d support
Intel CPX-SP FSP added support for VT-d through adding UPD parameter X2apic. Based on devicetree.cb setting, enable VT-d programming through FSP-M.
When VT-d is enabled, add DMAR ACPI table.
Signed-off-by: Jonathan Zhang jonzhang@fb.com Change-Id: Ic66374af6e53fb847c1bdc324eb3f4e01c334a94 Reviewed-on: https://review.coreboot.org/c/coreboot/+/44280 Tested-by: build bot (Jenkins) no-reply@coreboot.org Reviewed-by: Angel Pons th3fanbus@gmail.com Reviewed-by: Maxim Polyakov max.senia.poliak@gmail.com --- M src/soc/intel/xeon_sp/cpx/acpi.c M src/soc/intel/xeon_sp/cpx/chip.h M src/soc/intel/xeon_sp/cpx/romstage.c 3 files changed, 324 insertions(+), 0 deletions(-)
Approvals: build bot (Jenkins): Verified Angel Pons: Looks good to me, approved Maxim Polyakov: Looks good to me, approved
diff --git a/src/soc/intel/xeon_sp/cpx/acpi.c b/src/soc/intel/xeon_sp/cpx/acpi.c index 62569e6..4c73363 100644 --- a/src/soc/intel/xeon_sp/cpx/acpi.c +++ b/src/soc/intel/xeon_sp/cpx/acpi.c @@ -448,12 +448,315 @@ return current; }
+/* + * Ports Stack Stack(HOB) IioConfigIou + * ========================================== + * 0 CSTACK stack 0 IOU0 + * 1A..1D PSTACK0 stack 1 IOU1 + * 2A..2D PSTACK1 stack 2 IOU2 + * 3A..3D PSTACK2 stack 4 IOU3 + */ +static int get_stack_for_port(int p) +{ + if (p == 0) + return CSTACK; + else if (p >= PORT_1A && p <= PORT_1D) + return PSTACK0; + else if (p >= PORT_2A && p <= PORT_2D) + return PSTACK1; + else //if (p >= PORT_3A && p <= PORT_3D) + return PSTACK2; +} + +static unsigned long acpi_create_drhd(unsigned long current, int socket, int stack) +{ + int IoApicID[] = { + // socket 0 + PC00_IOAPIC_ID, PC01_IOAPIC_ID, PC02_IOAPIC_ID, PC03_IOAPIC_ID, + PC04_IOAPIC_ID, PC05_IOAPIC_ID, + // socket 1 + PC06_IOAPIC_ID, PC07_IOAPIC_ID, PC08_IOAPIC_ID, PC09_IOAPIC_ID, + PC10_IOAPIC_ID, PC11_IOAPIC_ID, + }; + + uint32_t enum_id; + unsigned long tmp = current; + + size_t hob_size; + const uint8_t fsp_hob_iio_universal_data_guid[16] = FSP_HOB_IIO_UNIVERSAL_DATA_GUID; + const IIO_UDS *hob = fsp_find_extension_hob_by_guid( + fsp_hob_iio_universal_data_guid, &hob_size); + assert(hob != NULL && hob_size != 0); + + uint32_t bus = hob->PlatformData.IIO_resource[socket].StackRes[stack].BusBase; + uint32_t pcie_seg = hob->PlatformData.CpuQpiInfo[socket].PcieSegment; + uint32_t reg_base = + hob->PlatformData.IIO_resource[socket].StackRes[stack].VtdBarAddress; + printk(BIOS_SPEW, "%s socket: %d, stack: %d, bus: 0x%x, pcie_seg: 0x%x, reg_base: 0x%x\n", + __func__, socket, stack, bus, pcie_seg, reg_base); + + // Add DRHD Hardware Unit + if (socket == 0 && stack == CSTACK) { + printk(BIOS_DEBUG, "[Hardware Unit Definition] Flags: 0x%x, PCI Segment Number: 0x%x, " + "Register Base Address: 0x%x\n", + DRHD_INCLUDE_PCI_ALL, pcie_seg, reg_base); + current += acpi_create_dmar_drhd(current, DRHD_INCLUDE_PCI_ALL, + pcie_seg, reg_base); + } else { + printk(BIOS_DEBUG, "[Hardware Unit Definition] Flags: 0x%x, PCI Segment Number: 0x%x, " + "Register Base Address: 0x%x\n", 0, pcie_seg, reg_base); + current += acpi_create_dmar_drhd(current, 0, pcie_seg, reg_base); + } + + // Add PCH IOAPIC + if (socket == 0 && stack == CSTACK) { + printk(BIOS_DEBUG, " [IOAPIC Device] Enumeration ID: 0x%x, PCI Bus Number: 0x%x, " + "PCI Path: 0x%x, 0x%x\n", + PCH_IOAPIC_ID, PCH_IOAPIC_BUS_NUMBER, + PCH_IOAPIC_DEV_NUM, PCH_IOAPIC_FUNC_NUM); + current += acpi_create_dmar_ds_ioapic(current, PCH_IOAPIC_ID, + PCH_IOAPIC_BUS_NUMBER, PCH_IOAPIC_DEV_NUM, PCH_IOAPIC_FUNC_NUM); + } + + // Add IOAPIC entry + enum_id = IoApicID[(socket*MAX_IIO_STACK)+stack]; + printk(BIOS_DEBUG, " [IOAPIC Device] Enumeration ID: 0x%x, PCI Bus Number: 0x%x, " + "PCI Path: 0x%x, 0x%x\n", enum_id, bus, APIC_DEV_NUM, APIC_FUNC_NUM); + current += acpi_create_dmar_ds_ioapic(current, enum_id, bus, + APIC_DEV_NUM, APIC_FUNC_NUM); + + // Add CBDMA devices for CSTACK + if (socket != 0 && stack == CSTACK) { + for (int cbdma_func_id = 0; cbdma_func_id < 8; ++cbdma_func_id) { + printk(BIOS_DEBUG, " [PCI Endpoint Device] Enumeration ID: 0x%x, " + "PCI Bus Number: 0x%x, PCI Path: 0x%x, 0x%x\n", + 0, bus, CBDMA_DEV_NUM, cbdma_func_id); + current += acpi_create_dmar_ds_pci(current, + bus, CBDMA_DEV_NUM, cbdma_func_id); + } + } + + // Add PCIe Ports + if (socket != 0 || stack != CSTACK) { + IIO_RESOURCE_INSTANCE iio_resource = + hob->PlatformData.IIO_resource[socket]; + for (int p = 0; p < NUMBER_PORTS_PER_SOCKET; ++p) { + if (get_stack_for_port(p) != stack) + continue; + + uint32_t dev = iio_resource.PcieInfo.PortInfo[p].Device; + uint32_t func = iio_resource.PcieInfo.PortInfo[p].Function; + + uint32_t id = pci_mmio_read_config32(PCI_DEV(bus, dev, func), + PCI_VENDOR_ID); + if (id == 0xffffffff) + continue; + + printk(BIOS_DEBUG, " [PCI Bridge Device] Enumeration ID: 0x%x, " + "PCI Bus Number: 0x%x, PCI Path: 0x%x, 0x%x\n", + 0, bus, dev, func); + current += acpi_create_dmar_ds_pci_br(current, + bus, dev, func); + } + + // Add VMD + if (hob->PlatformData.VMDStackEnable[socket][stack] && + stack >= PSTACK0 && stack <= PSTACK2) { + printk(BIOS_DEBUG, " [PCI Endpoint Device] Enumeration ID: 0x%x, " + "PCI Bus Number: 0x%x, PCI Path: 0x%x, 0x%x\n", + 0, bus, VMD_DEV_NUM, VMD_FUNC_NUM); + current += acpi_create_dmar_ds_pci(current, + bus, VMD_DEV_NUM, VMD_FUNC_NUM); + } + } + + // Add HPET + if (socket == 0 && stack == CSTACK) { + uint16_t hpet_capid = read16((void *)HPET_BASE_ADDRESS); + uint16_t num_hpets = (hpet_capid >> 0x08) & 0x1F; // Bits [8:12] has hpet count + printk(BIOS_SPEW, "%s hpet_capid: 0x%x, num_hpets: 0x%x\n", + __func__, hpet_capid, num_hpets); + //BIT 15 + if (num_hpets && (num_hpets != 0x1f) && + (read32((void *)(HPET_BASE_ADDRESS + 0x100)) & (0x00008000))) { + printk(BIOS_DEBUG, " [Message-capable HPET Device] Enumeration ID: 0x%x, " + "PCI Bus Number: 0x%x, PCI Path: 0x%x, 0x%x\n", + 0, HPET_BUS_NUM, HPET_DEV_NUM, HPET0_FUNC_NUM); + current += acpi_create_dmar_ds_msi_hpet(current, 0, HPET_BUS_NUM, + HPET_DEV_NUM, HPET0_FUNC_NUM); + } + } + + acpi_dmar_drhd_fixup(tmp, current); + + return current; +} + +static unsigned long acpi_create_atsr(unsigned long current) +{ + size_t hob_size; + const uint8_t uds_guid[16] = FSP_HOB_IIO_UNIVERSAL_DATA_GUID; + const IIO_UDS *hob = fsp_find_extension_hob_by_guid(uds_guid, &hob_size); + assert(hob != NULL && hob_size != 0); + + for (int socket = 0; socket < hob->PlatformData.numofIIO; ++socket) { + uint32_t pcie_seg = hob->PlatformData.CpuQpiInfo[socket].PcieSegment; + unsigned long tmp = current; + bool first = true; + IIO_RESOURCE_INSTANCE iio_resource = + hob->PlatformData.IIO_resource[socket]; + + for (int stack = 0; stack <= PSTACK2; ++stack) { + uint32_t bus = iio_resource.StackRes[stack].BusBase; + uint32_t vtd_base = iio_resource.StackRes[stack].VtdBarAddress; + if (!vtd_base) + continue; + uint64_t vtd_mmio_cap = read64((void *)(vtd_base + VTD_EXT_CAP_LOW)); + printk(BIOS_SPEW, "%s socket: %d, stack: %d, bus: 0x%x, vtd_base: 0x%x, " + "vtd_mmio_cap: 0x%llx\n", + __func__, socket, stack, bus, vtd_base, vtd_mmio_cap); + + // ATSR is applicable only for platform supporting device IOTLBs + // through the VT-d extended capability register + assert(vtd_mmio_cap != 0xffffffffffffffff); + if ((vtd_mmio_cap & 0x4) == 0) // BIT 2 + continue; + + for (int p = 0; p < NUMBER_PORTS_PER_SOCKET; ++p) { + if (socket == 0 && p == 0) + continue; + if (get_stack_for_port(p) != stack) + continue; + + uint32_t dev = iio_resource.PcieInfo.PortInfo[p].Device; + uint32_t func = iio_resource.PcieInfo.PortInfo[p].Function; + + u32 id = pci_mmio_read_config32(PCI_DEV(bus, dev, func), + PCI_VENDOR_ID); + if (id == 0xffffffff) + continue; + + if (first) { + printk(BIOS_DEBUG, "[Root Port ATS Capability] Flags: 0x%x, " + "PCI Segment Number: 0x%x\n", + 0, pcie_seg); + current += acpi_create_dmar_atsr(current, 0, pcie_seg); + first = 0; + } + + printk(BIOS_DEBUG, " [PCI Bridge Device] Enumeration ID: 0x%x, " + "PCI Bus Number: 0x%x, PCI Path: 0x%x, 0x%x\n", + 0, bus, dev, func); + current += acpi_create_dmar_ds_pci_br(current, bus, dev, func); + } + } + if (tmp != current) + acpi_dmar_atsr_fixup(tmp, current); + } + + return current; +} + +static unsigned long acpi_create_rmrr(unsigned long current) +{ + uint32_t size = ALIGN_UP(MEM_BLK_COUNT * sizeof(MEM_BLK), 0x1000); + + uint32_t *ptr; + + // reserve memory + ptr = cbmem_find(CBMEM_ID_STORAGE_DATA); + if (!ptr) { + ptr = cbmem_add(CBMEM_ID_STORAGE_DATA, size); + assert(ptr != NULL); + memset(ptr, 0, size); + } + + unsigned long tmp = current; + printk(BIOS_DEBUG, "[Reserved Memory Region] PCI Segment Number: 0x%x, Base Address: 0x%x, " + "End Address (limit): 0x%x\n", + 0, (uint32_t) ptr, (uint32_t) ((uint32_t) ptr + size - 1)); + current += acpi_create_dmar_rmrr(current, 0, (uint32_t) ptr, + (uint32_t) ((uint32_t) ptr + size - 1)); + + printk(BIOS_DEBUG, " [PCI Endpoint Device] Enumeration ID: 0x%x, PCI Bus Number: 0x%x, " + "PCI Path: 0x%x, 0x%x\n", + 0, XHCI_BUS_NUMBER, PCH_DEV_SLOT_XHCI, XHCI_FUNC_NUM); + current += acpi_create_dmar_ds_pci(current, XHCI_BUS_NUMBER, + PCH_DEV_SLOT_XHCI, XHCI_FUNC_NUM); + + acpi_dmar_rmrr_fixup(tmp, current); + + return current; +} + +static unsigned long acpi_create_rhsa(unsigned long current) +{ + size_t hob_size; + const uint8_t uds_guid[16] = FSP_HOB_IIO_UNIVERSAL_DATA_GUID; + const IIO_UDS *hob = fsp_find_extension_hob_by_guid(uds_guid, &hob_size); + assert(hob != NULL && hob_size != 0); + + for (int socket = 0; socket < hob->PlatformData.numofIIO; ++socket) { + IIO_RESOURCE_INSTANCE iio_resource = + hob->PlatformData.IIO_resource[socket]; + for (int stack = 0; stack <= PSTACK2; ++stack) { + uint32_t vtd_base = iio_resource.StackRes[stack].VtdBarAddress; + if (!vtd_base) + continue; + + printk(BIOS_DEBUG, "[Remapping Hardware Static Affinity] Base Address: 0x%x, " + "Proximity Domain: 0x%x\n", vtd_base, socket); + current += acpi_create_dmar_rhsa(current, vtd_base, socket); + } + } + + return current; +} + +static unsigned long acpi_fill_dmar(unsigned long current) +{ + size_t hob_size; + const uint8_t uds_guid[16] = FSP_HOB_IIO_UNIVERSAL_DATA_GUID; + const IIO_UDS *hob = fsp_find_extension_hob_by_guid(uds_guid, &hob_size); + assert(hob != NULL && hob_size != 0); + + // DRHD + for (int iio = 1; iio <= hob->PlatformData.numofIIO; ++iio) { + int socket = iio; + if (socket == hob->PlatformData.numofIIO) // socket 0 should be last DRHD entry + socket = 0; + + if (socket == 0) { + for (int stack = 1; stack <= PSTACK2; ++stack) + current = acpi_create_drhd(current, socket, stack); + current = acpi_create_drhd(current, socket, CSTACK); + } else { + for (int stack = 0; stack <= PSTACK2; ++stack) + current = acpi_create_drhd(current, socket, stack); + } + } + + // RMRR + current = acpi_create_rmrr(current); + + // ATSR - causes hang + current = acpi_create_atsr(current); + + // RHSA + current = acpi_create_rhsa(current); + + return current; +} + unsigned long northbridge_write_acpi_tables(const struct device *device, unsigned long current, struct acpi_rsdp *rsdp) { acpi_srat_t *srat; acpi_slit_t *slit; + acpi_dmar_t *dmar; + + const struct soc_intel_xeon_sp_cpx_config *const config = config_of(device);
/* SRAT */ current = ALIGN(current, 8); @@ -471,5 +774,18 @@ current += slit->header.length; acpi_add_table(rsdp, slit);
+ /* DMAR */ + if (config->vtd_support) { + current = ALIGN(current, 8); + dmar = (acpi_dmar_t *)current; + printk(BIOS_DEBUG, "ACPI: * DMAR\n"); + printk(BIOS_DEBUG, "[DMA Remapping table] Flags: 0x%x\n", + (DMAR_INTR_REMAP | DMAR_X2APIC_OPT_OUT)); + acpi_create_dmar(dmar, (DMAR_INTR_REMAP | DMAR_X2APIC_OPT_OUT), acpi_fill_dmar); + current += dmar->header.length; + current = acpi_align_current(current); + acpi_add_table(rsdp, dmar); + } + return current; } diff --git a/src/soc/intel/xeon_sp/cpx/chip.h b/src/soc/intel/xeon_sp/cpx/chip.h index aa605a4..434b343 100644 --- a/src/soc/intel/xeon_sp/cpx/chip.h +++ b/src/soc/intel/xeon_sp/cpx/chip.h @@ -84,6 +84,9 @@
uint32_t pstate_req_ratio;
+ uint8_t vtd_support; + uint8_t x2apic; + /* Generic IO decode ranges */ uint32_t gen1_dec; uint32_t gen2_dec; diff --git a/src/soc/intel/xeon_sp/cpx/romstage.c b/src/soc/intel/xeon_sp/cpx/romstage.c index 6c65994..a198c99 100644 --- a/src/soc/intel/xeon_sp/cpx/romstage.c +++ b/src/soc/intel/xeon_sp/cpx/romstage.c @@ -15,6 +15,7 @@ { FSPM_CONFIG *m_cfg = &mupd->FspmConfig; const struct device *dev; + const config_t *config = config_of_soc();
/* ErrorLevel - 0 (disable) to 8 (verbose) */ m_cfg->DebugPrintLevel = 8; @@ -68,5 +69,9 @@ dev = pcidev_path_on_root(PCH_DEVFN_THERMAL); m_cfg->ThermalDeviceEnable = dev && dev->enabled;
+ /* Enable VT-d according to DTB */ + m_cfg->VtdSupport = config->vtd_support; + m_cfg->X2apic = config->x2apic; + mainboard_memory_init_params(mupd); }
-
Angel Pons (Code Review) -
build bot (Jenkins) (Code Review)
-
Jonathan Zhang (Code Review)
-
Maxim Polyakov (Code Review)