Patch set updated for coreboot: soc/apollolake: Tear down cache-as-ram very early in ramstage

[Andrey Petrov]

Andrey Petrov ([e-mail]) just uploaded a new patch set to gerrit, which you can find at https://review.coreboot.org/13364

-gerrit

commit efd91b5651a9af3fdd072d8f987f3fa2c87060a9
Author: Alexandru Gagniuc 
Date:   Thu Mar 10 15:02:02 2016 -0800

soc/apollolake: Tear down cache-as-ram very early in ramstage

Because of the reasons explained in the file, teardown of CAR needs to
    be done in ramstage. This patch addresses that issues in a way
    designed to minimize boot time. Teardown currently takes about 7 ms.

Change-Id: I6a0cb07f5b35ad42537da688e376ac96612990a6
    Signed-off-by: Alexandru Gagniuc 
    Signed-off-by: Andrey Petrov 
---
 src/soc/intel/apollolake/Makefile.inc             |   1 +
 src/soc/intel/apollolake/bootblock/cache_as_ram.S |   7 +-
 src/soc/intel/apollolake/car_teardown.c           | 135 ++++++++++++++++++++++
 src/soc/intel/apollolake/include/soc/cpu.h        |   7 +-
 4 files changed, 144 insertions(+), 6 deletions(-)

diff --git a/src/soc/intel/apollolake/Makefile.inc b/src/soc/intel/apollolake/Makefile.inc
index 4e05726..6bbfe57 100644
--- a/src/soc/intel/apollolake/Makefile.inc
+++ b/src/soc/intel/apollolake/Makefile.inc
@@ -25,6 +25,7 @@ romstage-y += memmap.c
 romstage-y += mmap_boot.c

smm-y += placeholders.c
+ramstage-y += car_teardown.c
 ramstage-y += cpu.c
 ramstage-y += chip.c
 ramstage-y += placeholders.c
diff --git a/src/soc/intel/apollolake/bootblock/cache_as_ram.S b/src/soc/intel/apollolake/bootblock/cache_as_ram.S
index c81fe0a..cc021dc 100644
--- a/src/soc/intel/apollolake/bootblock/cache_as_ram.S
+++ b/src/soc/intel/apollolake/bootblock/cache_as_ram.S
@@ -16,8 +16,7 @@
 #include <cpu/x86/cache.h>
 #include <cpu/x86/cr.h>
 #include <cpu/x86/post_code.h>
-
-#define EVICT_CTL_MSR		0x2e0
+#include <soc/cpu.h>

.global bootblock_pre_c_entry
 bootblock_pre_c_entry:
@@ -96,7 +95,7 @@ clear_var_mtrr:
    mov	%eax, %cr0

/* Disable cache eviction (setup stage) */
-	mov	$EVICT_CTL_MSR, %ecx
+	mov	$MSR_EVICT_CTL, %ecx
    rdmsr
    or	$0x1, %eax
    wrmsr
@@ -112,7 +111,7 @@ clear_var_mtrr:
    post_code(0x27)

/* Disable cache eviction (run stage) */
-	mov	$EVICT_CTL_MSR, %ecx
+	mov	$MSR_EVICT_CTL, %ecx
    rdmsr
    or	$0x2, %eax
    wrmsr
diff --git a/src/soc/intel/apollolake/car_teardown.c b/src/soc/intel/apollolake/car_teardown.c
new file mode 100644
index 0000000..d2df7f7
--- /dev/null
+++ b/src/soc/intel/apollolake/car_teardown.c
@@ -0,0 +1,135 @@
+/*
+ * Cache-as-ram teardown for apollolake SOC. See detailed explanation below.
+ *
+ * This file is part of the coreboot project.
+ *
+ * Copyright (C) 2015 Intel Corp.
+ * (Written by Alexandru Gagniuc  for Intel Corp.)
+ * (Written by Andrey Petrov  for Intel Corp.)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <bootstate.h>
+#include <console/console.h>
+#include <cpu/cpu.h>
+#include <cpu/x86/cache.h>
+#include <cpu/x86/mtrr.h>
+#include <cpu/x86/msr.h>
+#include <cpu/x86/tsc.h>
+#include <cbmem.h>
+#include <memrange.h>
+#include <symbols.h>
+
+#define EVICT_CTL_MSR		0x2e0
+#define CACHELINE_SIZE		64
+/*
+ * The CAR teardown on apollolake is more complicated than the standard x86
+ * teardown paths because romstage executes in CAR. That means romstage cannot
+ * do the teardown. For performance reasons, the cache is also used as a backing
+ * store for RAM after raminit.
+ *
+ * This results in a cache with both a CAR region, and memory-backed regions:
+ *   - Invalidating the entire cache will evict stack and code that has not yet
+ *     been commited, to RAM, resulting in a crash.
+ *   - A 'wbinvd' will result in a crash during the write-back phase because
+ *     the CAR region has no backing store.
+ *
+ * This teardown path resolves these problems by first flushing important data
+ * out of cache, to memory. After this operation, it is safe to invalidate the
+ * entire cache, which is then followed by standard MTRR housekeeping.
+ *
+ * For performance reasons, only the memory regions which may have been written
+ * to are flushed. This includes ramstage code and cbmem. A complete flush of
+ * all RAM ranges would take several seconds.
+ */
+
+static void clflush(uintptr_t addr)
+{
+	__asm__ volatile ("clflush (%0)"::"r" (addr));
+}
+
+static void clflush_mem_range(uintptr_t start, uintptr_t end)
+{
+	end = ALIGN_UP(end, CACHELINE_SIZE);
+
+	for (start = ALIGN_DOWN(start, CACHELINE_SIZE); start < end;
+						start += CACHELINE_SIZE)
+		clflush(start);
+}
+
+static void flush_ram_cache_and_invalidate(void)
+{
+	uintptr_t cbmem_base;
+	size_t cbmem_size;
+	struct memranges ctx;
+	struct range_entry entries[3];
+	const struct range_entry *r;
+	const struct cbmem_entry *fsp_rsvd_mem;
+
+	memranges_init_empty(&ctx, entries, ARRAY_SIZE(entries));
+
+	/* Figure out where ramstage resides; this is inclusive of the stack */
+	memranges_insert(&ctx, (uintptr_t)&_program, _program_size, 0);
+
+	/* It's okay if CBMEM and ramstage overlap. */
+	cbmem_region_used(&cbmem_base, &cbmem_size);
+	memranges_insert(&ctx, cbmem_base, cbmem_size, 0);
+
+	/*
+	 * FSP reserved memory was written before caching was enabled. Thus it
+	 * does not need to be flushed. This saves about 10ms of boot time.
+	 */
+	fsp_rsvd_mem = cbmem_entry_find(CBMEM_ID_FSP_RESERVED_MEMORY);
+	if (fsp_rsvd_mem) {
+		memranges_create_hole(&ctx,
+			(uintptr_t)cbmem_entry_start(fsp_rsvd_mem),
+			cbmem_entry_size(fsp_rsvd_mem)
+		);
+	}
+
+	/* Flush ramstage code and stack */
+	memranges_each_entry(r, &ctx)
+		clflush_mem_range(range_entry_base(r), range_entry_end(r));
+
+	/* Now that important data is flushed, invalidate all caches */
+	invd();
+}
+
+static void disable_car_region_and_no_evict_mode(void)
+{
+	msr_t msr;
+
+	/* Disable variable and fixed MTRRs */
+	msr.lo = msr.hi = 0;
+	wrmsr(MTRR_DEF_TYPE_MSR, msr);
+	/* Disable CAR MTRR */
+	wrmsr(MTRR_PHYS_BASE(0), msr);
+	wrmsr(MTRR_PHYS_MASK(0), msr);
+
+	/* Switch out of "no evict" mode */
+	msr = rdmsr(EVICT_CTL_MSR);
+	msr.lo &= ~2;
+	wrmsr(EVICT_CTL_MSR, msr);
+
+	msr.lo &= ~1;
+	wrmsr(EVICT_CTL_MSR, msr);
+
+	/* Re-enable remaining MTRRs */
+	msr.lo = MTRR_DEF_TYPE_EN;
+	msr.hi = 0;
+	wrmsr(MTRR_DEF_TYPE_MSR, msr);
+}
+
+static void tear_down_car(void *unused_arg)
+{
+	(void) unused_arg;
+
+	flush_ram_cache_and_invalidate();
+	disable_car_region_and_no_evict_mode();
+}
+
+BOOT_STATE_INIT_ENTRY(BS_PRE_DEVICE, BS_ON_ENTRY, tear_down_car, NULL);
diff --git a/src/soc/intel/apollolake/include/soc/cpu.h b/src/soc/intel/apollolake/include/soc/cpu.h
index 765be70..b7b2fcc 100644
--- a/src/soc/intel/apollolake/include/soc/cpu.h
+++ b/src/soc/intel/apollolake/include/soc/cpu.h
@@ -13,19 +13,22 @@
 #ifndef _SOC_APOLLOLAKE_CPU_H_
 #define _SOC_APOLLOLAKE_CPU_H_

+#ifndef __ASSEMBLER__
 #include <cpu/x86/msr.h>
 #include <device/device.h>

+void apollolake_init_cpus(struct device *dev);
+#endif
+
 #define CPUID_APOLLOLAKE_A0	0x506c8
 #define CPUID_APOLLOLAKE_B0	0x506c9

#define MSR_PLATFORM_INFO	0xce
 #define MSR_POWER_MISC		0x120
 #define MSR_CORE_THREAD_COUNT	0x35
+#define MSR_EVICT_CTL		0x2e0

#define BASE_CLOCK_MHZ		100

-void apollolake_init_cpus(struct device *dev);
-

#endif /* _SOC_APOLLOLAKE_CPU_H_ */