Lean Sheng Tan has submitted this change. ( https://review.coreboot.org/c/coreboot/+/83538?usp=email )
(
5 is the latest approved patch-set. No files were changed between the latest approved patch-set and the submitted one. )Change subject: soc/intel/xeon_sp: Reserve MMIO high range ......................................................................
soc/intel/xeon_sp: Reserve MMIO high range
Xeon-SP supports MMIO high range, a.k.a. MMIO range above 4G. FSP will assign domain MMIO high windows from this range.
However, there will be unassigned parts among these high windows for non-domain device usage (e.g. misc devices belonging to an IIO stack but not belonged to any PCIe domains under that stack). This will cause segmentation in MTRR UC coverage.
For example, in SPR-XCC where only CPM0/HQM0 are supported and instantiated to PCIe domains, MMIO ranges are still reserved for CPM1/HQM1. See more at src/soc/intel/xeon_sp/spr/ioat.c.
Reserve MMIO high range as a whole under domain0/00:0.0. During MTRR calculation, this reservation will connect the discontinued domain MMIO high windows together to form one continuous range, and save MTRR register usage from inadequacy.
This change is initially raised for SPR but could be effective for GNR as well.
TESTED = Build and boot in intel/archercity CRB, MTRR register usage decreases from 7 to 3 in 2S system.
TESTED = Only setting MTRR for below 4GB ranges test fails with LinuxBoot on SPR (through x86_setup_mtrrs_with_detect_no_above_4gb)
tsc: Detected 2000.000 MHz processor last_pfn = 0x2080000 max_arch_pfn = 0x10000000000 x86/PAT: Configuration [0-7]: WB WC UC- UC WB WP UC- WT WARNING: BIOS bug: CPU MTRRs don't cover all of memory, losing 129024MB of RAM. ------------[ cut here ]------------ WARNING: CPU: 0 PID: 0 at arch/x86/kernel/cpu/mtrr/cleanup.c:978 mtrr_trim_uncached_memory+0x2b9/0x2f9 ... Call Trace: ? 0xffffffff8f600000 ? setup_arch+0x4bb/0xaed ? printk+0x53/0x6a ? start_kernel+0x55/0x507 ? load_ucode_intel_bsp+0x1c/0x4d ? secondary_startup_64_no_verify+0xc2/0xcb random: get_random_bytes called from init_oops_id+0x1d/0x2c with crng_init=0 ---[ end trace 0e56686fd458f0c5 ]--- update e820 for mtrr modified physical RAM map: modified: [mem 0x0000000000000000-0x0000000000000fff] reserved ... modified: [mem 0x00000000ff000000-0x000000207fffffff] reserved last_pfn = 0x6354e max_arch_pfn = 0x10000000000 Memory KASLR using RDRAND RDTSC... x2apic: enabled by BIOS, switching to x2apic ops Using GB pages for direct mapping ... Initmem setup node 0 [mem 0x0000000000001000-0x000000006354dfff] DMA zone: 28769 pages in unavailable ranges DMA32 zone: 19122 pages in unavailable ranges BUG: unable to handle page fault for address: ff24b56eba60cff8 BAD Oops: 0000 [#1] SMP NOPTI CPU: 0 PID: 0 Comm: swapper Tainted: G W 5.10.50 #2 ... Call Trace: ? set_pte_vaddr_p4d+0x24/0x35 ? __native_set_fixmap+0x21/0x28 ? map_vsyscall+0x35/0x56 ? setup_arch+0xa00/0xaed ? printk+0x53/0x6a ? start_kernel+0x55/0x507 ? load_ucode_intel_bsp+0x1c/0x4d ? secondary_startup_64_no_verify+0xc2/0xcb CR2: ff24b56eba60cff8 ---[ end trace 0e56686fd458f0c6 ]--- RIP: 0010:fill_pud+0xa/0x62 ... Kernel panic - not syncing: Attempted to kill the idle task! ---[ end Kernel panic - not syncing: Attempted to kill the idle task! ]---
Change-Id: Ib2a0e1f1f13e797c1fab6aca589d060c4d3fa15b Signed-off-by: Shuo Liu shuo.liu@intel.com Reviewed-on: https://review.coreboot.org/c/coreboot/+/83538 Reviewed-by: Lean Sheng Tan sheng.tan@9elements.com Tested-by: build bot (Jenkins) no-reply@coreboot.org Reviewed-by: Nico Huber nico.h@gmx.de --- M src/soc/intel/xeon_sp/cpx/soc_util.c M src/soc/intel/xeon_sp/gnr/soc_util.c M src/soc/intel/xeon_sp/include/soc/util.h M src/soc/intel/xeon_sp/skx/soc_util.c M src/soc/intel/xeon_sp/spr/soc_util.c M src/soc/intel/xeon_sp/uncore.c 6 files changed, 51 insertions(+), 2 deletions(-)
Approvals: build bot (Jenkins): Verified Lean Sheng Tan: Looks good to me, approved Nico Huber: Looks good to me, but someone else must approve
diff --git a/src/soc/intel/xeon_sp/cpx/soc_util.c b/src/soc/intel/xeon_sp/cpx/soc_util.c index 3472d58..aac5589 100644 --- a/src/soc/intel/xeon_sp/cpx/soc_util.c +++ b/src/soc/intel/xeon_sp/cpx/soc_util.c @@ -141,3 +141,8 @@ { return 0; } + +bool get_mmio_high_base_size(resource_t *base, resource_t *size) +{ + return false; +} diff --git a/src/soc/intel/xeon_sp/gnr/soc_util.c b/src/soc/intel/xeon_sp/gnr/soc_util.c index a65e5fc..7ea65c2 100644 --- a/src/soc/intel/xeon_sp/gnr/soc_util.c +++ b/src/soc/intel/xeon_sp/gnr/soc_util.c @@ -148,3 +148,8 @@ { return true; } + +bool get_mmio_high_base_size(resource_t *base, resource_t *size) +{ + return false; +} diff --git a/src/soc/intel/xeon_sp/include/soc/util.h b/src/soc/intel/xeon_sp/include/soc/util.h index e694af3..a7b98f3 100644 --- a/src/soc/intel/xeon_sp/include/soc/util.h +++ b/src/soc/intel/xeon_sp/include/soc/util.h @@ -31,4 +31,6 @@ union p2sb_bdf soc_get_hpet_bdf(void); union p2sb_bdf soc_get_ioapic_bdf(void);
+bool get_mmio_high_base_size(resource_t *base, resource_t *size); + #endif diff --git a/src/soc/intel/xeon_sp/skx/soc_util.c b/src/soc/intel/xeon_sp/skx/soc_util.c index 0ccacfb..0938260 100644 --- a/src/soc/intel/xeon_sp/skx/soc_util.c +++ b/src/soc/intel/xeon_sp/skx/soc_util.c @@ -214,3 +214,8 @@ { return 0; } + +bool get_mmio_high_base_size(resource_t *base, resource_t *size) +{ + return false; +} diff --git a/src/soc/intel/xeon_sp/spr/soc_util.c b/src/soc/intel/xeon_sp/spr/soc_util.c index 5845327..6dd06ac 100644 --- a/src/soc/intel/xeon_sp/spr/soc_util.c +++ b/src/soc/intel/xeon_sp/spr/soc_util.c @@ -188,3 +188,12 @@ */ return (mem_type < MemTypeCxlAccVolatileMem); } + +bool get_mmio_high_base_size(resource_t *base, resource_t *size) +{ + const IIO_UDS *hob = get_iio_uds(); + *base = hob->PlatformData.PlatGlobalMmio64Base; + *size = hob->PlatformData.PlatGlobalMmio64Limit - (*base) + 1; + + return true; +} diff --git a/src/soc/intel/xeon_sp/uncore.c b/src/soc/intel/xeon_sp/uncore.c index 2e5023d..be5b38f 100644 --- a/src/soc/intel/xeon_sp/uncore.c +++ b/src/soc/intel/xeon_sp/uncore.c @@ -151,6 +151,9 @@ pci_write_config32(dev, VTD_LTDPR, dpr.raw); }
+#define MC_DRAM_RESOURCE_MMIO_HIGH 0x1000 +#define MC_DRAM_RESOURCE_ANON_START 0x1001 + /* * Host Memory Map: * @@ -350,7 +353,7 @@
static void mmapvtd_read_resources(struct device *dev) { - int index = 0; + int index = MC_DRAM_RESOURCE_ANON_START;
/* Read standard PCI resources. */ pci_dev_read_resources(dev); @@ -362,13 +365,33 @@ mc_add_dram_resources(dev, &index); }
+static void mmapvtd_set_resources(struct device *dev) +{ + /* + * The MMIO high window has to be added in set_resources() instead of + * read_resources(). Because adding in read_resources() would cause the + * whole window to be reserved, and it couldn't be used for resource + * allocation. + */ + if (is_domain0(dev->upstream->dev)) { + resource_t mmio64_base, mmio64_size; + if (get_mmio_high_base_size(&mmio64_base, &mmio64_size)) { + assert(!probe_resource(dev, MC_DRAM_RESOURCE_MMIO_HIGH)); + fixed_mem_range_flags(dev, MC_DRAM_RESOURCE_MMIO_HIGH, + mmio64_base, mmio64_size, IORESOURCE_STORED); + } + } + + pci_dev_set_resources(dev); +} + static void mmapvtd_init(struct device *dev) { }
static struct device_operations mmapvtd_ops = { .read_resources = mmapvtd_read_resources, - .set_resources = pci_dev_set_resources, + .set_resources = mmapvtd_set_resources, .enable_resources = pci_dev_enable_resources, .init = mmapvtd_init, .ops_pci = &soc_pci_ops,