Patrick Georgi (pgeorgi@google.com) just uploaded a new patch set to gerrit, which you can find at http://review.coreboot.org/9341
-gerrit
commit b3e87f8168d24aeed3ea61cf01464b8e5946807d Author: Julius Werner jwerner@chromium.org Date: Thu Oct 9 17:31:45 2014 -0700
armv7: Add fine-grained page table support
This patch adds an mmu_config_range_kb() function, which can set memory types at the 4KB level by chaining a fine-grained page table to an existing superpage entry. It is only intended for special cases where this level of precision is really necessary and therefore comes with a few practical limitations (the area for each invocation must be confined within a single superpage, and you are not allowed to remap the same region with mmu_config_range() again later). Since the fine-grained page tables need some space, boards intending to use this feature must define a TTB_SUBTABLES() region in their memlayout.ld.
BUG=chrome-os-partner:32848 TEST=Booted both Veyron_Pinky (normal) and Nyan_Blaze (LPAE), ensured that they still work. Checksummed the page tables with and without this patch, confirmed that they end up equal. Hacked in some subtable test entries, hexdumped all tables and manually confirmed that they look as expected.
Change-Id: I8c3eb7c2eb9c82e2abc5f2c0dda91f5b2eee7023 Signed-off-by: Patrick Georgi pgeorgi@chromium.org Original-Commit-Id: 2f13e60cf5509b9a63fb7b8d84846daf889dc1b7 Original-Change-Id: Iedf7ca435ae337ead85115200d6987fb0d4828d7 Original-Signed-off-by: Julius Werner jwerner@chromium.org Original-Reviewed-on: https://chromium-review.googlesource.com/223781 --- src/arch/arm/armv7/mmu.c | 188 +++++++++++++++++++++----------- src/arch/arm/include/arch/memlayout.h | 5 + src/arch/arm/include/armv4/arch/cache.h | 19 ---- src/arch/arm/include/armv7/arch/cache.h | 11 +- src/include/symbols.h | 4 + 5 files changed, 140 insertions(+), 87 deletions(-)
diff --git a/src/arch/arm/armv7/mmu.c b/src/arch/arm/armv7/mmu.c index e9c6815..de7fa40 100644 --- a/src/arch/arm/armv7/mmu.c +++ b/src/arch/arm/armv7/mmu.c @@ -42,7 +42,7 @@ #if CONFIG_ARM_LPAE /* See B3.6.2 of ARMv7 Architecture Reference Manual */ /* TODO: Utilize the contiguous hint flag */ -#define ATTR_BASE (\ +#define ATTR_BLOCK (\ 0ULL << 54 | /* XN. 0:Not restricted */ \ 0ULL << 53 | /* PXN. 0:Not restricted */ \ 1 << 10 | /* AF. 1:Accessed. This is to prevent access \ @@ -52,16 +52,18 @@ 0 << 1 | /* block/table. 0:block entry */ \ 1 << 0 /* validity. 1:valid */ \ ) -#define ATTR_NC (ATTR_BASE | (MAIR_INDX_NC << 2) | \ - (1ULL << 53) | (1ULL << 54)) -#define ATTR_WT (ATTR_BASE | (MAIR_INDX_WT << 2)) -#define ATTR_WB (ATTR_BASE | (MAIR_INDX_WB << 2)) +#define ATTR_PAGE (ATTR_BLOCK | 1 << 1) +#define ATTR_NEXTLEVEL (0x3) +#define ATTR_NC ((MAIR_INDX_NC << 2) | (1ULL << 53) | (1ULL << 54)) +#define ATTR_WT (MAIR_INDX_WT << 2) +#define ATTR_WB (MAIR_INDX_WB << 2)
+#define PAGE_MASK 0x000ffffffffff000ULL +#define BLOCK_MASK 0x000fffffffe00000ULL +#define NEXTLEVEL_MASK PAGE_MASK #define BLOCK_SHIFT 21
-typedef uint64_t pgd_t; -typedef uint64_t pmd_t; -static const unsigned int denom = 2; +typedef uint64_t pte_t; #else /* CONFIG_ARM_LPAE */ /* * Section entry bits: @@ -79,27 +81,28 @@ static const unsigned int denom = 2; * 2 - B, 1 for bufferable * 1: 0 - 0b10 to indicate section entry */ -#define ATTR_BASE ((3 << 10) | 0x2) -#define ATTR_NC (ATTR_BASE | (1 << 4)) -#define ATTR_WT (ATTR_BASE | (1 << 3)) -#define ATTR_WB (ATTR_BASE | (1 << 3) | (1 << 2)) +#define ATTR_BLOCK ((3 << 10) | 0x2) +#define ATTR_PAGE ((3 << 4) | 0x2) +#define ATTR_NEXTLEVEL (0x1) +#define ATTR_NC (1 << 4) +#define ATTR_WT (1 << 3) +#define ATTR_WB ((1 << 3) | (1 << 2))
+#define PAGE_MASK 0xfffff000UL +#define BLOCK_MASK 0xfff00000UL +#define NEXTLEVEL_MASK 0xfffffc00UL #define BLOCK_SHIFT 20
-typedef uint32_t pgd_t; -typedef uint32_t pmd_t; -static const unsigned int denom = 1; +typedef uint32_t pte_t; #endif /* CONFIG_ARM_LPAE */
-static pmd_t *const ttb_buff = (pmd_t *)_ttb; - /* * mask/shift/size for pages and blocks */ #define PAGE_SHIFT 12 #define PAGE_SIZE (1UL << PAGE_SHIFT) -#define PAGE_MASK ~((1UL << PAGE_SHIFT) - 1) #define BLOCK_SIZE (1UL << BLOCK_SHIFT) +#define SUBTABLE_SIZE ((1 << (BLOCK_SHIFT - PAGE_SHIFT)) * sizeof(pte_t))
/* * MAIR Index @@ -108,69 +111,124 @@ static pmd_t *const ttb_buff = (pmd_t *)_ttb; #define MAIR_INDX_WT 1 #define MAIR_INDX_WB 2
-static void mmu_flush_page_table_entry_range( - unsigned long start_mb, unsigned long size_mb) +static pte_t *const ttb_buff = (void *)_ttb; +static int used_tables = 0; + +/* Not all boards want to use subtables and declare them in memlayout.ld. This + * outputs two 0x00000000 symbols if they don't, making _ttb_subtables_size 0. + * (I would like to explicitly assign them to 0 here, but that triggers + * https://sourceware.org/bugzilla/show_bug.cgi?id=1038 in GNU as.) */ +asm (".weak _ttb_subtables, _ettb_subtables"); + +static struct { + pte_t value; + const char *name; +} attrs[] = { + [DCACHE_OFF] = {.value = ATTR_NC, .name = "uncached"}, + [DCACHE_WRITEBACK] = {.value = ATTR_WB, .name = "writeback"}, + [DCACHE_WRITETHROUGH] = {.value = ATTR_WT, .name = "writethrough"}, +}; + +/* Fills page table entries in |table| from |start_idx| to |end_idx| with |attr| + * and performs necessary invalidations. |offset| is the start address of the + * area described by |table|, and |shift| is the size-shift of each frame. */ +static void mmu_fill_table(pte_t *table, u32 start_idx, u32 end_idx, + uintptr_t offset, u32 shift, pte_t attr) { int i;
+ /* Write out page table entries. */ + for (i = start_idx; i < end_idx; i++) + table[i] = (offset + (i << shift)) | attr; + /* Flush the page table entries from the dcache. */ - for (i = start_mb/denom; i*denom < start_mb + size_mb; i++) - dccmvac((uintptr_t)&ttb_buff[i]); + for (i = start_idx; i < end_idx; i++) + dccmvac((uintptr_t)&table[i]); dsb(); + /* Invalidate the TLB entries. */ - for (i = start_mb/denom; i*denom < start_mb + size_mb; i++) - tlbimvaa(i*denom*MiB); + for (i = start_idx; i < end_idx; i++) + tlbimvaa(offset + (i << shift)); dsb(); isb(); }
-void mmu_disable_range(unsigned long start_mb, unsigned long size_mb) +static pte_t *mmu_create_subtable(pte_t *pgd_entry) { - int i; + if (used_tables >= _ttb_subtables_size / SUBTABLE_SIZE) + die("Not enough room for another sub-pagetable!");
- printk(BIOS_DEBUG, "Disabling: [0x%08lx:0x%08lx)\n", - start_mb*MiB, start_mb*MiB + size_mb*MiB); + /* We assume that *pgd_entry must already be a valid block mapping. */ + uintptr_t start_addr = (uintptr_t)(*pgd_entry & BLOCK_MASK); + pte_t *table = (void *)(_ttb_subtables + used_tables++ * SUBTABLE_SIZE); + printk(BIOS_DEBUG, "Creating new subtable @%p for [%#.8x:%#.8lx)\n", + table, start_addr, start_addr + BLOCK_SIZE);
- for (i = start_mb/denom; i*denom < start_mb + size_mb; i++) - ttb_buff[i] = 0; + /* Initialize the new subtable with entries of the same attributes + * (XN bit moves from 4 to 0, set PAGE unless block was unmapped). */ + pte_t attr = *pgd_entry & ~(BLOCK_MASK); + if (!IS_ENABLED(CONFIG_ARM_LPAE) && (attr & (1 << 4))) + attr = ((attr & ~(1 << 4)) | (1 << 0)); + if (attr & ATTR_BLOCK) + attr = (attr & ~ATTR_BLOCK) | ATTR_PAGE; + mmu_fill_table(table, 0, SUBTABLE_SIZE / sizeof(pte_t), + start_addr, PAGE_SHIFT, attr);
- mmu_flush_page_table_entry_range(start_mb, size_mb); + /* Replace old entry in upper level table to point at subtable. */ + *pgd_entry = (pte_t)(uintptr_t)table | ATTR_NEXTLEVEL; + dccmvac((uintptr_t)pgd_entry); + dsb(); + tlbimvaa(start_addr); + dsb(); + isb(); + + return table; }
-void mmu_config_range(unsigned long start_mb, unsigned long size_mb, - enum dcache_policy policy) +void mmu_config_range_kb(u32 start_kb, u32 size_kb, enum dcache_policy policy) { - const char *str = NULL; - pmd_t attr; - int i; + pte_t *pgd_entry = &ttb_buff[start_kb / (BLOCK_SIZE/KiB)]; + pte_t *table = (void *)(uintptr_t)(*pgd_entry & NEXTLEVEL_MASK);
- switch(policy) { - case DCACHE_OFF: - /* XN set to avoid prefetches to uncached/unbuffered regions */ - attr = ATTR_NC; - str = "off"; - break; - case DCACHE_WRITEBACK: - attr = ATTR_WB; - str = "writeback"; - break; - case DCACHE_WRITETHROUGH: - attr = ATTR_WT; - str = "writethrough"; - break; - default: - printk(BIOS_ERR, "unknown dcache policy: %02x\n", policy); - return; - } - - printk(BIOS_DEBUG, "Setting dcache policy: [0x%08lx:0x%08lx) [%s]\n", - start_mb << 20, ((start_mb + size_mb) << 20), str); + /* Make sure the range is contained within a single superpage. */ + assert(((start_kb + size_kb - 1) & (BLOCK_MASK/KiB)) + == (start_kb & (BLOCK_MASK/KiB)) && start_kb < 4 * (GiB/KiB));
- /* Write out page table entries. */ - for (i = start_mb/denom; i*denom < start_mb + size_mb; i++) - ttb_buff[i] = ((pmd_t)i << BLOCK_SHIFT) | attr; + if ((*pgd_entry & ~NEXTLEVEL_MASK) != ATTR_NEXTLEVEL) + table = mmu_create_subtable(pgd_entry); + + /* Always _one_ _damn_ bit that won't fit... (XN moves from 4 to 0) */ + pte_t attr = attrs[policy].value; + if (!IS_ENABLED(CONFIG_ARM_LPAE) && (attr & (1 << 4))) + attr = ((attr & ~(1 << 4)) | (1 << 0)); + + /* Mask away high address bits that are handled by upper level table. */ + u32 mask = BLOCK_SIZE/KiB - 1; + printk(BIOS_DEBUG, "Mapping address range [%#.8x:%#.8x) as %s\n", + start_kb * KiB, (start_kb + size_kb) * KiB, attrs[policy].name); + mmu_fill_table(table, (start_kb & mask) / (PAGE_SIZE/KiB), + div_round_up((start_kb + size_kb) & mask, PAGE_SIZE/KiB), + (start_kb & ~mask) * KiB, PAGE_SHIFT, ATTR_PAGE | attr); +}
- mmu_flush_page_table_entry_range(start_mb, size_mb); +void mmu_disable_range(u32 start_mb, u32 size_mb) +{ + printk(BIOS_DEBUG, "Setting address range [%#.8x:%#.8x) as unmapped\n", + start_mb * MiB, (start_mb + size_mb) * MiB); + assert(start_mb + size_mb <= 4 * (GiB/MiB)); + mmu_fill_table(ttb_buff, start_mb / (BLOCK_SIZE/MiB), + div_round_up(start_mb + size_mb, BLOCK_SIZE/MiB), + 0, BLOCK_SHIFT, 0); +} + +void mmu_config_range(u32 start_mb, u32 size_mb, enum dcache_policy policy) +{ + printk(BIOS_DEBUG, "Mapping address range [%#.8x:%#.8x) as %s\n", + start_mb * MiB, (start_mb + size_mb) * MiB, attrs[policy].name); + assert(start_mb + size_mb <= 4 * (GiB/MiB)); + mmu_fill_table(ttb_buff, start_mb / (BLOCK_SIZE/MiB), + div_round_up(start_mb + size_mb, BLOCK_SIZE/MiB), + 0, BLOCK_SHIFT, ATTR_BLOCK | attrs[policy].value); }
/* @@ -187,8 +245,8 @@ void mmu_config_range(unsigned long start_mb, unsigned long size_mb, void mmu_init(void) { if (CONFIG_ARM_LPAE) { - pgd_t *const pgd_buff = (pgd_t*)(_ttb + 16*KiB); - pmd_t *pmd = ttb_buff; + pte_t *const pgd_buff = (pte_t*)(_ttb + 16*KiB); + pte_t *pmd = ttb_buff; int i;
printk(BIOS_DEBUG, "LPAE Translation tables are @ %p\n", @@ -214,8 +272,8 @@ void mmu_init(void) * See B3.6.1 of ARMv7 Architecture Reference Manual */ for (i = 0; i < 4; i++) { - pgd_buff[i] = ((uint32_t)pmd & PAGE_MASK) | - 3; /* 0b11: valid table entry */ + pgd_buff[i] = ((uint32_t)pmd & NEXTLEVEL_MASK) | + ATTR_NEXTLEVEL; pmd += BLOCK_SIZE / PAGE_SIZE; }
diff --git a/src/arch/arm/include/arch/memlayout.h b/src/arch/arm/include/arch/memlayout.h index 9f5b383..2c73394 100644 --- a/src/arch/arm/include/arch/memlayout.h +++ b/src/arch/arm/include/arch/memlayout.h @@ -27,6 +27,11 @@ _ = ASSERT(size >= 16K + IS_ENABLED(CONFIG_ARM_LPAE) * 32, \ "TTB must be 16K (+ 32 for LPAE)!");
+#define TTB_SUBTABLES(addr, size) \ + REGION(ttb_subtables, addr, size, IS_ENABLED(CONFIG_ARM_LPAE)*3K + 1K) \ + _ = ASSERT(size % (1K + 3K * IS_ENABLED(CONFIG_ARM_LPAE)) == 0, \ + "TTB subtable region must be evenly divisible by table size!"); + /* ARM stacks need 8-byte alignment and stay in one place through ramstage. */ #define STACK(addr, size) REGION(stack, addr, size, 8)
diff --git a/src/arch/arm/include/armv4/arch/cache.h b/src/arch/arm/include/armv4/arch/cache.h index e41ff9a..65f5966 100644 --- a/src/arch/arm/include/armv4/arch/cache.h +++ b/src/arch/arm/include/armv4/arch/cache.h @@ -72,23 +72,4 @@ void cache_sync_instructions(void); /* tlb invalidate all */ void tlb_invalidate_all(void);
-/* - * Generalized setup/init functions - */ - -/* mmu initialization (set page table address, set permissions, etc) */ -void mmu_init(void); - -enum dcache_policy { - DCACHE_OFF, - DCACHE_WRITEBACK, - DCACHE_WRITETHROUGH, -}; - -/* disable the mmu for a range. Primarily useful to lock out address 0. */ -void mmu_disable_range(unsigned long start_mb, unsigned long size_mb); -/* mmu range configuration (set dcache policy) */ -void mmu_config_range(unsigned long start_mb, unsigned long size_mb, - enum dcache_policy policy); - #endif /* ARM_CACHE_H */ diff --git a/src/arch/arm/include/armv7/arch/cache.h b/src/arch/arm/include/armv7/arch/cache.h index dde2c08..f23dec1 100644 --- a/src/arch/arm/include/armv7/arch/cache.h +++ b/src/arch/arm/include/armv7/arch/cache.h @@ -345,9 +345,14 @@ enum dcache_policy { };
/* disable the mmu for a range. Primarily useful to lock out address 0. */ -void mmu_disable_range(unsigned long start_mb, unsigned long size_mb); +void mmu_disable_range(u32 start_mb, u32 size_mb); /* mmu range configuration (set dcache policy) */ -void mmu_config_range(unsigned long start_mb, unsigned long size_mb, - enum dcache_policy policy); +void mmu_config_range(u32 start_mb, u32 size_mb, enum dcache_policy policy); + +/* Reconfigure memory mappings at the fine-grained (4K) page level. Must be + * called on a range contained within a single, already mapped block/superpage. + * Careful: Do NOT map over this address range with mmu_config_range() again + * later, or you will leak resources and may desync your TLB! */ +void mmu_config_range_kb(u32 start_kb, u32 size_kb, enum dcache_policy policy);
#endif /* ARM_CACHE_H */ diff --git a/src/include/symbols.h b/src/include/symbols.h index c343fd9..2259dcb 100644 --- a/src/include/symbols.h +++ b/src/include/symbols.h @@ -69,6 +69,10 @@ extern u8 _ttb[]; extern u8 _ettb[]; #define _ttb_size (_ettb - _ttb)
+extern u8 _ttb_subtables[]; +extern u8 _ettb_subtables[]; +#define _ttb_subtables_size (_ettb_subtables - _ttb_subtables) + extern u8 _dma_coherent[]; extern u8 _edma_coherent[]; #define _dma_coherent_size (_edma_coherent - _dma_coherent)