Angel Pons has submitted this change. ( https://review.coreboot.org/c/coreboot/+/44280 )
Change subject: soc/intel/xeon_sp/cpx: add VT-d support ......................................................................
soc/intel/xeon_sp/cpx: add VT-d support
Intel CPX-SP FSP added support for VT-d through adding UPD parameter X2apic. Based on devicetree.cb setting, enable VT-d programming through FSP-M.
When VT-d is enabled, add DMAR ACPI table.
Signed-off-by: Jonathan Zhang jonzhang@fb.com Change-Id: Ic66374af6e53fb847c1bdc324eb3f4e01c334a94 Reviewed-on: https://review.coreboot.org/c/coreboot/+/44280 Tested-by: build bot (Jenkins) no-reply@coreboot.org Reviewed-by: Angel Pons th3fanbus@gmail.com Reviewed-by: Maxim Polyakov max.senia.poliak@gmail.com --- M src/soc/intel/xeon_sp/cpx/acpi.c M src/soc/intel/xeon_sp/cpx/chip.h M src/soc/intel/xeon_sp/cpx/romstage.c 3 files changed, 324 insertions(+), 0 deletions(-)
Approvals: build bot (Jenkins): Verified Angel Pons: Looks good to me, approved Maxim Polyakov: Looks good to me, approved
diff --git a/src/soc/intel/xeon_sp/cpx/acpi.c b/src/soc/intel/xeon_sp/cpx/acpi.c index 62569e6..4c73363 100644 --- a/src/soc/intel/xeon_sp/cpx/acpi.c +++ b/src/soc/intel/xeon_sp/cpx/acpi.c @@ -448,12 +448,315 @@ return current; }
+/* + * Ports Stack Stack(HOB) IioConfigIou + * ========================================== + * 0 CSTACK stack 0 IOU0 + * 1A..1D PSTACK0 stack 1 IOU1 + * 2A..2D PSTACK1 stack 2 IOU2 + * 3A..3D PSTACK2 stack 4 IOU3 + */ +static int get_stack_for_port(int p) +{ + if (p == 0) + return CSTACK; + else if (p >= PORT_1A && p <= PORT_1D) + return PSTACK0; + else if (p >= PORT_2A && p <= PORT_2D) + return PSTACK1; + else //if (p >= PORT_3A && p <= PORT_3D) + return PSTACK2; +} + +static unsigned long acpi_create_drhd(unsigned long current, int socket, int stack) +{ + int IoApicID[] = { + // socket 0 + PC00_IOAPIC_ID, PC01_IOAPIC_ID, PC02_IOAPIC_ID, PC03_IOAPIC_ID, + PC04_IOAPIC_ID, PC05_IOAPIC_ID, + // socket 1 + PC06_IOAPIC_ID, PC07_IOAPIC_ID, PC08_IOAPIC_ID, PC09_IOAPIC_ID, + PC10_IOAPIC_ID, PC11_IOAPIC_ID, + }; + + uint32_t enum_id; + unsigned long tmp = current; + + size_t hob_size; + const uint8_t fsp_hob_iio_universal_data_guid[16] = FSP_HOB_IIO_UNIVERSAL_DATA_GUID; + const IIO_UDS *hob = fsp_find_extension_hob_by_guid( + fsp_hob_iio_universal_data_guid, &hob_size); + assert(hob != NULL && hob_size != 0); + + uint32_t bus = hob->PlatformData.IIO_resource[socket].StackRes[stack].BusBase; + uint32_t pcie_seg = hob->PlatformData.CpuQpiInfo[socket].PcieSegment; + uint32_t reg_base = + hob->PlatformData.IIO_resource[socket].StackRes[stack].VtdBarAddress; + printk(BIOS_SPEW, "%s socket: %d, stack: %d, bus: 0x%x, pcie_seg: 0x%x, reg_base: 0x%x\n", + __func__, socket, stack, bus, pcie_seg, reg_base); + + // Add DRHD Hardware Unit + if (socket == 0 && stack == CSTACK) { + printk(BIOS_DEBUG, "[Hardware Unit Definition] Flags: 0x%x, PCI Segment Number: 0x%x, " + "Register Base Address: 0x%x\n", + DRHD_INCLUDE_PCI_ALL, pcie_seg, reg_base); + current += acpi_create_dmar_drhd(current, DRHD_INCLUDE_PCI_ALL, + pcie_seg, reg_base); + } else { + printk(BIOS_DEBUG, "[Hardware Unit Definition] Flags: 0x%x, PCI Segment Number: 0x%x, " + "Register Base Address: 0x%x\n", 0, pcie_seg, reg_base); + current += acpi_create_dmar_drhd(current, 0, pcie_seg, reg_base); + } + + // Add PCH IOAPIC + if (socket == 0 && stack == CSTACK) { + printk(BIOS_DEBUG, " [IOAPIC Device] Enumeration ID: 0x%x, PCI Bus Number: 0x%x, " + "PCI Path: 0x%x, 0x%x\n", + PCH_IOAPIC_ID, PCH_IOAPIC_BUS_NUMBER, + PCH_IOAPIC_DEV_NUM, PCH_IOAPIC_FUNC_NUM); + current += acpi_create_dmar_ds_ioapic(current, PCH_IOAPIC_ID, + PCH_IOAPIC_BUS_NUMBER, PCH_IOAPIC_DEV_NUM, PCH_IOAPIC_FUNC_NUM); + } + + // Add IOAPIC entry + enum_id = IoApicID[(socket*MAX_IIO_STACK)+stack]; + printk(BIOS_DEBUG, " [IOAPIC Device] Enumeration ID: 0x%x, PCI Bus Number: 0x%x, " + "PCI Path: 0x%x, 0x%x\n", enum_id, bus, APIC_DEV_NUM, APIC_FUNC_NUM); + current += acpi_create_dmar_ds_ioapic(current, enum_id, bus, + APIC_DEV_NUM, APIC_FUNC_NUM); + + // Add CBDMA devices for CSTACK + if (socket != 0 && stack == CSTACK) { + for (int cbdma_func_id = 0; cbdma_func_id < 8; ++cbdma_func_id) { + printk(BIOS_DEBUG, " [PCI Endpoint Device] Enumeration ID: 0x%x, " + "PCI Bus Number: 0x%x, PCI Path: 0x%x, 0x%x\n", + 0, bus, CBDMA_DEV_NUM, cbdma_func_id); + current += acpi_create_dmar_ds_pci(current, + bus, CBDMA_DEV_NUM, cbdma_func_id); + } + } + + // Add PCIe Ports + if (socket != 0 || stack != CSTACK) { + IIO_RESOURCE_INSTANCE iio_resource = + hob->PlatformData.IIO_resource[socket]; + for (int p = 0; p < NUMBER_PORTS_PER_SOCKET; ++p) { + if (get_stack_for_port(p) != stack) + continue; + + uint32_t dev = iio_resource.PcieInfo.PortInfo[p].Device; + uint32_t func = iio_resource.PcieInfo.PortInfo[p].Function; + + uint32_t id = pci_mmio_read_config32(PCI_DEV(bus, dev, func), + PCI_VENDOR_ID); + if (id == 0xffffffff) + continue; + + printk(BIOS_DEBUG, " [PCI Bridge Device] Enumeration ID: 0x%x, " + "PCI Bus Number: 0x%x, PCI Path: 0x%x, 0x%x\n", + 0, bus, dev, func); + current += acpi_create_dmar_ds_pci_br(current, + bus, dev, func); + } + + // Add VMD + if (hob->PlatformData.VMDStackEnable[socket][stack] && + stack >= PSTACK0 && stack <= PSTACK2) { + printk(BIOS_DEBUG, " [PCI Endpoint Device] Enumeration ID: 0x%x, " + "PCI Bus Number: 0x%x, PCI Path: 0x%x, 0x%x\n", + 0, bus, VMD_DEV_NUM, VMD_FUNC_NUM); + current += acpi_create_dmar_ds_pci(current, + bus, VMD_DEV_NUM, VMD_FUNC_NUM); + } + } + + // Add HPET + if (socket == 0 && stack == CSTACK) { + uint16_t hpet_capid = read16((void *)HPET_BASE_ADDRESS); + uint16_t num_hpets = (hpet_capid >> 0x08) & 0x1F; // Bits [8:12] has hpet count + printk(BIOS_SPEW, "%s hpet_capid: 0x%x, num_hpets: 0x%x\n", + __func__, hpet_capid, num_hpets); + //BIT 15 + if (num_hpets && (num_hpets != 0x1f) && + (read32((void *)(HPET_BASE_ADDRESS + 0x100)) & (0x00008000))) { + printk(BIOS_DEBUG, " [Message-capable HPET Device] Enumeration ID: 0x%x, " + "PCI Bus Number: 0x%x, PCI Path: 0x%x, 0x%x\n", + 0, HPET_BUS_NUM, HPET_DEV_NUM, HPET0_FUNC_NUM); + current += acpi_create_dmar_ds_msi_hpet(current, 0, HPET_BUS_NUM, + HPET_DEV_NUM, HPET0_FUNC_NUM); + } + } + + acpi_dmar_drhd_fixup(tmp, current); + + return current; +} + +static unsigned long acpi_create_atsr(unsigned long current) +{ + size_t hob_size; + const uint8_t uds_guid[16] = FSP_HOB_IIO_UNIVERSAL_DATA_GUID; + const IIO_UDS *hob = fsp_find_extension_hob_by_guid(uds_guid, &hob_size); + assert(hob != NULL && hob_size != 0); + + for (int socket = 0; socket < hob->PlatformData.numofIIO; ++socket) { + uint32_t pcie_seg = hob->PlatformData.CpuQpiInfo[socket].PcieSegment; + unsigned long tmp = current; + bool first = true; + IIO_RESOURCE_INSTANCE iio_resource = + hob->PlatformData.IIO_resource[socket]; + + for (int stack = 0; stack <= PSTACK2; ++stack) { + uint32_t bus = iio_resource.StackRes[stack].BusBase; + uint32_t vtd_base = iio_resource.StackRes[stack].VtdBarAddress; + if (!vtd_base) + continue; + uint64_t vtd_mmio_cap = read64((void *)(vtd_base + VTD_EXT_CAP_LOW)); + printk(BIOS_SPEW, "%s socket: %d, stack: %d, bus: 0x%x, vtd_base: 0x%x, " + "vtd_mmio_cap: 0x%llx\n", + __func__, socket, stack, bus, vtd_base, vtd_mmio_cap); + + // ATSR is applicable only for platform supporting device IOTLBs + // through the VT-d extended capability register + assert(vtd_mmio_cap != 0xffffffffffffffff); + if ((vtd_mmio_cap & 0x4) == 0) // BIT 2 + continue; + + for (int p = 0; p < NUMBER_PORTS_PER_SOCKET; ++p) { + if (socket == 0 && p == 0) + continue; + if (get_stack_for_port(p) != stack) + continue; + + uint32_t dev = iio_resource.PcieInfo.PortInfo[p].Device; + uint32_t func = iio_resource.PcieInfo.PortInfo[p].Function; + + u32 id = pci_mmio_read_config32(PCI_DEV(bus, dev, func), + PCI_VENDOR_ID); + if (id == 0xffffffff) + continue; + + if (first) { + printk(BIOS_DEBUG, "[Root Port ATS Capability] Flags: 0x%x, " + "PCI Segment Number: 0x%x\n", + 0, pcie_seg); + current += acpi_create_dmar_atsr(current, 0, pcie_seg); + first = 0; + } + + printk(BIOS_DEBUG, " [PCI Bridge Device] Enumeration ID: 0x%x, " + "PCI Bus Number: 0x%x, PCI Path: 0x%x, 0x%x\n", + 0, bus, dev, func); + current += acpi_create_dmar_ds_pci_br(current, bus, dev, func); + } + } + if (tmp != current) + acpi_dmar_atsr_fixup(tmp, current); + } + + return current; +} + +static unsigned long acpi_create_rmrr(unsigned long current) +{ + uint32_t size = ALIGN_UP(MEM_BLK_COUNT * sizeof(MEM_BLK), 0x1000); + + uint32_t *ptr; + + // reserve memory + ptr = cbmem_find(CBMEM_ID_STORAGE_DATA); + if (!ptr) { + ptr = cbmem_add(CBMEM_ID_STORAGE_DATA, size); + assert(ptr != NULL); + memset(ptr, 0, size); + } + + unsigned long tmp = current; + printk(BIOS_DEBUG, "[Reserved Memory Region] PCI Segment Number: 0x%x, Base Address: 0x%x, " + "End Address (limit): 0x%x\n", + 0, (uint32_t) ptr, (uint32_t) ((uint32_t) ptr + size - 1)); + current += acpi_create_dmar_rmrr(current, 0, (uint32_t) ptr, + (uint32_t) ((uint32_t) ptr + size - 1)); + + printk(BIOS_DEBUG, " [PCI Endpoint Device] Enumeration ID: 0x%x, PCI Bus Number: 0x%x, " + "PCI Path: 0x%x, 0x%x\n", + 0, XHCI_BUS_NUMBER, PCH_DEV_SLOT_XHCI, XHCI_FUNC_NUM); + current += acpi_create_dmar_ds_pci(current, XHCI_BUS_NUMBER, + PCH_DEV_SLOT_XHCI, XHCI_FUNC_NUM); + + acpi_dmar_rmrr_fixup(tmp, current); + + return current; +} + +static unsigned long acpi_create_rhsa(unsigned long current) +{ + size_t hob_size; + const uint8_t uds_guid[16] = FSP_HOB_IIO_UNIVERSAL_DATA_GUID; + const IIO_UDS *hob = fsp_find_extension_hob_by_guid(uds_guid, &hob_size); + assert(hob != NULL && hob_size != 0); + + for (int socket = 0; socket < hob->PlatformData.numofIIO; ++socket) { + IIO_RESOURCE_INSTANCE iio_resource = + hob->PlatformData.IIO_resource[socket]; + for (int stack = 0; stack <= PSTACK2; ++stack) { + uint32_t vtd_base = iio_resource.StackRes[stack].VtdBarAddress; + if (!vtd_base) + continue; + + printk(BIOS_DEBUG, "[Remapping Hardware Static Affinity] Base Address: 0x%x, " + "Proximity Domain: 0x%x\n", vtd_base, socket); + current += acpi_create_dmar_rhsa(current, vtd_base, socket); + } + } + + return current; +} + +static unsigned long acpi_fill_dmar(unsigned long current) +{ + size_t hob_size; + const uint8_t uds_guid[16] = FSP_HOB_IIO_UNIVERSAL_DATA_GUID; + const IIO_UDS *hob = fsp_find_extension_hob_by_guid(uds_guid, &hob_size); + assert(hob != NULL && hob_size != 0); + + // DRHD + for (int iio = 1; iio <= hob->PlatformData.numofIIO; ++iio) { + int socket = iio; + if (socket == hob->PlatformData.numofIIO) // socket 0 should be last DRHD entry + socket = 0; + + if (socket == 0) { + for (int stack = 1; stack <= PSTACK2; ++stack) + current = acpi_create_drhd(current, socket, stack); + current = acpi_create_drhd(current, socket, CSTACK); + } else { + for (int stack = 0; stack <= PSTACK2; ++stack) + current = acpi_create_drhd(current, socket, stack); + } + } + + // RMRR + current = acpi_create_rmrr(current); + + // ATSR - causes hang + current = acpi_create_atsr(current); + + // RHSA + current = acpi_create_rhsa(current); + + return current; +} + unsigned long northbridge_write_acpi_tables(const struct device *device, unsigned long current, struct acpi_rsdp *rsdp) { acpi_srat_t *srat; acpi_slit_t *slit; + acpi_dmar_t *dmar; + + const struct soc_intel_xeon_sp_cpx_config *const config = config_of(device);
/* SRAT */ current = ALIGN(current, 8); @@ -471,5 +774,18 @@ current += slit->header.length; acpi_add_table(rsdp, slit);
+ /* DMAR */ + if (config->vtd_support) { + current = ALIGN(current, 8); + dmar = (acpi_dmar_t *)current; + printk(BIOS_DEBUG, "ACPI: * DMAR\n"); + printk(BIOS_DEBUG, "[DMA Remapping table] Flags: 0x%x\n", + (DMAR_INTR_REMAP | DMAR_X2APIC_OPT_OUT)); + acpi_create_dmar(dmar, (DMAR_INTR_REMAP | DMAR_X2APIC_OPT_OUT), acpi_fill_dmar); + current += dmar->header.length; + current = acpi_align_current(current); + acpi_add_table(rsdp, dmar); + } + return current; } diff --git a/src/soc/intel/xeon_sp/cpx/chip.h b/src/soc/intel/xeon_sp/cpx/chip.h index aa605a4..434b343 100644 --- a/src/soc/intel/xeon_sp/cpx/chip.h +++ b/src/soc/intel/xeon_sp/cpx/chip.h @@ -84,6 +84,9 @@
uint32_t pstate_req_ratio;
+ uint8_t vtd_support; + uint8_t x2apic; + /* Generic IO decode ranges */ uint32_t gen1_dec; uint32_t gen2_dec; diff --git a/src/soc/intel/xeon_sp/cpx/romstage.c b/src/soc/intel/xeon_sp/cpx/romstage.c index 6c65994..a198c99 100644 --- a/src/soc/intel/xeon_sp/cpx/romstage.c +++ b/src/soc/intel/xeon_sp/cpx/romstage.c @@ -15,6 +15,7 @@ { FSPM_CONFIG *m_cfg = &mupd->FspmConfig; const struct device *dev; + const config_t *config = config_of_soc();
/* ErrorLevel - 0 (disable) to 8 (verbose) */ m_cfg->DebugPrintLevel = 8; @@ -68,5 +69,9 @@ dev = pcidev_path_on_root(PCH_DEVFN_THERMAL); m_cfg->ThermalDeviceEnable = dev && dev->enabled;
+ /* Enable VT-d according to DTB */ + m_cfg->VtdSupport = config->vtd_support; + m_cfg->X2apic = config->x2apic; + mainboard_memory_init_params(mupd); }