Change in coreboot[master]: x86/mtrr: Enable Rd/WrDram mod in AMD fixed MTRRs

[Marshall Dawson (Code Review)]

Marshall Dawson has uploaded this change for review. ( https://review.coreboot.org/23722

Change subject: x86/mtrr: Enable Rd/WrDram mod in AMD fixed MTRRs
......................................................................

x86/mtrr: Enable Rd/WrDram mod in AMD fixed MTRRs

AMD fixed MTRRs have RdDram and WrDram bits that help route memory
accesses to DRAM or MMIO.  These are typically hidden for normal
operation by clearing SYS_CFG[19] (MtrrFixDramModEn).  According to
BKDGs and AMD64 Programmer's Manual vol 2, this bit is clear at
reset, should be set for configuration during POST, then cleared for
normal operation.

Attempting to modify the RdDram and WrDram settings without unhiding
them causes a General Protection Fault.  Add a function to enable or
disable MtrrFixDramModEn.

During commit_fixed_mtrrs() ensure the setting is enabled then
disabled after writing the registers.

In mp_init.c save_bsp_msrs() copies the fixed MTRR settings.  Enable
the bits so the complete MTRR contents may be read.  Restore the
setting when complete.

Finally, modify sipi_vector.S to enable the bits prior to writing
the fixed MTRRs and disable when complete.

BUG=b:68019051
TEST=Boot Kahlee, check steps with HDT

Change-Id: Ie195131ff752400eb886dfccc39b314b4fa6b3f3
Signed-off-by: Marshall Dawson 
---
M src/cpu/x86/mp_init.c
M src/cpu/x86/mtrr/mtrr.c
M src/cpu/x86/sipi_vector.S
M src/include/cpu/x86/mtrr.h
4 files changed, 50 insertions(+), 1 deletion(-)

git pull ssh://review.coreboot.org:29418/coreboot refs/changes/22/23722/1

diff --git a/src/cpu/x86/mp_init.c b/src/cpu/x86/mp_init.c
index baa6aec..40349aa 100644
--- a/src/cpu/x86/mp_init.c
+++ b/src/cpu/x86/mp_init.c
@@ -310,6 +310,8 @@

msr_entry = save_msr(MTRR_DEF_TYPE_MSR, msr_entry);

+	fixed_mtrrs_hide_rwdram(); /* hide Rd/WrDram bits if supported */
+
    return msr_count;
 }

diff --git a/src/cpu/x86/mtrr/mtrr.c b/src/cpu/x86/mtrr/mtrr.c
index c2c629c..a2a04fb 100644
--- a/src/cpu/x86/mtrr/mtrr.c
+++ b/src/cpu/x86/mtrr/mtrr.c
@@ -36,8 +36,8 @@
 #include <arch/cpu.h>
 #include <arch/acpi.h>
 #include <memrange.h>
-#if IS_ENABLED(CONFIG_X86_AMD_FIXED_MTRRS)
 #include <cpu/amd/mtrr.h>
+#if IS_ENABLED(CONFIG_X86_AMD_FIXED_MTRRS)
 #define MTRR_FIXED_WRBACK_BITS (MTRR_READ_MEM | MTRR_WRITE_MEM)
 #else
 #define MTRR_FIXED_WRBACK_BITS 0
@@ -83,6 +83,30 @@
    wrmsr(MTRR_DEF_TYPE_MSR, msr);
 }

+void fixed_mtrrs_expose_rwdram(void)
+{
+	msr_t syscfg;
+
+	if (!IS_ENABLED(CONFIG_X86_AMD_FIXED_MTRRS))
+		return;
+
+	syscfg = rdmsr(SYSCFG_MSR);
+	syscfg.lo |= SYSCFG_MSR_MtrrFixDramModEn;
+	wrmsr(SYSCFG_MSR, syscfg);
+}
+
+void fixed_mtrrs_hide_rwdram(void)
+{
+	msr_t syscfg;
+
+	if (!IS_ENABLED(CONFIG_X86_AMD_FIXED_MTRRS))
+		return;
+
+	syscfg = rdmsr(SYSCFG_MSR);
+	syscfg.lo &= ~SYSCFG_MSR_MtrrFixDramModEn;
+	wrmsr(SYSCFG_MSR, syscfg);
+}
+
 static void enable_var_mtrr(unsigned char deftype)
 {
    msr_t msr;
@@ -310,6 +334,8 @@
    msr_t fixed_msrs[NUM_FIXED_MTRRS];
    unsigned long msr_index[NUM_FIXED_MTRRS];

+	fixed_mtrrs_expose_rwdram(); /* unhide Rd/WrDram bits if supported */
+
    memset(&fixed_msrs, 0, sizeof(fixed_msrs));

msr_num = 0;
diff --git a/src/cpu/x86/sipi_vector.S b/src/cpu/x86/sipi_vector.S
index bd60c65..83606bd 100644
--- a/src/cpu/x86/sipi_vector.S
+++ b/src/cpu/x86/sipi_vector.S
@@ -15,6 +15,7 @@
  */

#include <cpu/x86/cr.h>
+#include <cpu/amd/mtrr.h>

/* The SIPI vector is responsible for initializing the APs in the sytem. It
  * loads microcode, sets up MSRs, and enables caching before calling into
@@ -172,6 +173,15 @@
    mov	msr_count, %ebx
    test	%ebx, %ebx
    jz	1f
+
+#if IS_ENABLED(CONFIG_X86_AMD_FIXED_MTRRS)
+	/* Allow modification of RdDram and WrDram bits */
+	mov	$SYSCFG_MSR, %ecx
+	rdmsr
+	or	$SYSCFG_MSR_MtrrFixDramModEn, %eax
+	wrmsr
+#endif
+
 load_msr:
    mov	(%edi), %ecx
    mov	4(%edi), %eax
@@ -181,6 +191,13 @@
    dec	%ebx
    jnz	load_msr

+#if IS_ENABLED(CONFIG_X86_AMD_FIXED_MTRRS)
+	mov	$SYSCFG_MSR, %ecx
+	rdmsr
+	and	$~SYSCFG_MSR_MtrrFixDramModEn, %eax
+	wrmsr
+#endif
+
 1:
    /* Enable caching. */
    mov	%cr0, %eax
diff --git a/src/include/cpu/x86/mtrr.h b/src/include/cpu/x86/mtrr.h
index a72d602..bc0f1fe 100644
--- a/src/include/cpu/x86/mtrr.h
+++ b/src/include/cpu/x86/mtrr.h
@@ -76,6 +76,10 @@
  */
 void x86_setup_var_mtrrs(unsigned int address_bits, unsigned int above4gb);
 void enable_fixed_mtrr(void);
+/* Unhide Rd/WrDram bits in AMD fixed MTRRs and allow modification. */
+void fixed_mtrrs_expose_rwdram(void);
+/* Hide Rd/WrDram bits in AMD fixed MTRRs and prevent modification */
+void fixed_mtrrs_hide_rwdram(void);
 void x86_setup_fixed_mtrrs(void);
 /* Set up fixed MTRRs but do not enable them. */
 void x86_setup_fixed_mtrrs_no_enable(void);