Signed-off-by: Kevin O'Connor kevin@koconnor.net --- src/Kconfig | 7 +++ src/malloc.c | 3 ++ src/memmap.c | 129 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- src/memmap.h | 17 ++++++-- src/romlayout.S | 35 ++++++++++++--- src/stacks.c | 15 ++++++- src/x86.h | 23 ++++++++++ 7 files changed, 217 insertions(+), 12 deletions(-)
diff --git a/src/Kconfig b/src/Kconfig index b873cd3..c28f671 100644 --- a/src/Kconfig +++ b/src/Kconfig @@ -122,6 +122,13 @@ endchoice selected, the memory is instead allocated from the "9-segment" (0x90000-0xa0000).
+ config PAGING + bool "PAE paging support" + default y + help + Enable paging in 32bit mode on processors that support PAE + paging. + config ROM_SIZE int "ROM size (in KB)" default 0 diff --git a/src/malloc.c b/src/malloc.c index ec7a21f..c6b460e 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -418,6 +418,9 @@ malloc_preinit(void) // Mark known areas as reserved. e820_add(BUILD_BIOS_ADDR, BUILD_BIOS_SIZE, E820_RESERVED);
+ char memmap_early_data[MEMMAP_EARLY_DATA_SIZE]; + memmap_early_preinit(memmap_early_data); + // Populate temp high ram u32 highram = 0; int i; diff --git a/src/memmap.c b/src/memmap.c index b08313a..4e83112 100644 --- a/src/memmap.c +++ b/src/memmap.c @@ -64,8 +64,8 @@ handle_pf(struct x86_exception_stack *data) }
// Setup a 32bit General Protection (GP) and Page Fault (PF) handler -void -memmap_preinit(void) +static void +idt_preinit(void) { u32 ilen = sizeof(u64)*(IDT_VECTOR_PF+1); u64 *pidt = malloc_high(ilen); @@ -85,3 +85,128 @@ memmap_preinit(void) dprintf(1, "Installing pmode exception handler\n"); lidt(&pmode_IDT_info); } + + +/**************************************************************** + * PAE based paging + ****************************************************************/ + +u32 PageDirPtr VARFSEG; +u64 *PageDir; + +#define PAE_MAP_SIZE (2*1024*1024) +#define PAE_MAP_COUNT 2048 + +#define PG_P (1<<0) +#define PG_RW (1<<1) +#define PG_A (1<<5) +#define PG_D (1<<6) +#define PG_PS (1<<7) +#define PG_FLAGS (PG_P | PG_RW | PG_A | PG_D | PG_PS) + +u64 +memmap_virt_to_phys(void *v) +{ + if (!CONFIG_PAGING || !PageDirPtr) + return (u32)v; + int idx = (u32)v / PAE_MAP_SIZE; + int rem = (u32)v % PAE_MAP_SIZE; + return (PageDir[idx] & ~(PAE_MAP_SIZE-1)) + rem; +} + +void * +memmap_vmap(u64 addr, u32 len) +{ + if (!CONFIG_PAGING || !PageDirPtr) + return (void*)(u32)addr; + u64 newmap = (addr & ~(PAE_MAP_SIZE-1)) | PG_FLAGS; + u32 rem = addr % PAE_MAP_SIZE; + u64 key = addr / PAE_MAP_SIZE; + int pages = DIV_ROUND_UP(rem+len, PAE_MAP_SIZE); + int max; + for (max=64; max; max--, key*=33) { + u32 pos = key % PAE_MAP_COUNT; + if (pos + pages > PAE_MAP_COUNT) + continue; + int j; + for (j=0; j<pages; j++) + if (PageDir[pos+j] && PageDir[pos+j] != newmap + j*PAE_MAP_SIZE) + break; + if (j != pages) + continue; + for (j=0; j<pages; j++) + if (!PageDir[pos+j]) + PageDir[pos+j] = newmap + j*PAE_MAP_SIZE; + return (void*)(pos*PAE_MAP_SIZE + rem); + } + // It's not worth updating all callers to handle a failed mapping + panic("Unable to map address %llx with size %d\n", addr, len); +} + +// Detect PAE support and enable it if present +void +memmap_early_preinit(char *data) +{ + if (!CONFIG_PAGING) + return; + // Check if CPU supports PAE + u32 eax, ebx, ecx, edx, cpuid_features = 0; + cpuid(0, &eax, &ebx, &ecx, &edx); + if (!eax) + return; + cpuid(1, &eax, &ebx, &ecx, &cpuid_features); + if (!(cpuid_features & (1<<6))) + return; + + // Create page table in temporary ram + dprintf(1, "PAE support found\n"); + char *aligned_data = (void*)ALIGN((u32)data, PAGE_SIZE); + u64 *dir = (void*)aligned_data; + u64 *pdp = (void*)aligned_data + sizeof(u64) * PAE_MAP_COUNT; + if (!pdp || !dir) { + warn_noalloc(); + free(pdp); + free(dir); + return; + } + memset(dir, 0, sizeof(u64) * PAE_MAP_COUNT); + dir[0] = 0x0 | PG_FLAGS; + int i; + for (i=0; i<4; i++) + pdp[i] = ((u32)dir + i*4096) | PG_P; + + // Enable PAE + PageDirPtr = (u32)pdp; + PageDir = dir; + cr3_write(PageDirPtr); + cr4_mask(0, CR4_PAE); + cr0_mask(0, CR0_PG); +} + +void +memmap_preinit(void) +{ + if (!CONFIG_PAGING || !PageDirPtr) + return; + // Move temporary page tables to permanent storage + u64 *dir = memalign_high(PAGE_SIZE, sizeof(u64) * PAE_MAP_COUNT); + u64 *pdp = malloc_high(sizeof(u64)*4); + if (!pdp || !dir) { + warn_noalloc(); + free(pdp); + free(dir); + // XXX - may not be able to turn off paging at this point + PageDirPtr = 0; + PageDir = NULL; + return; + } + memcpy(dir, PageDir, sizeof(u64) * PAE_MAP_COUNT); + int i; + for (i=0; i<4; i++) + pdp[i] = ((u32)dir + i*4096) | PG_P; + PageDirPtr = (u32)pdp; + PageDir = dir; + cr3_write(PageDirPtr); + + idt_preinit(); +} diff --git a/src/memmap.h b/src/memmap.h index d4f150a..850d726 100644 --- a/src/memmap.h +++ b/src/memmap.h @@ -7,13 +7,22 @@ #define PAGE_SIZE 4096 #define PAGE_SHIFT 12
+extern u32 PageDirPtr; +u64 memmap_virt_to_phys(void *v); +void *memmap_vmap(u64 addr, u32 len); +#define MEMMAP_EARLY_DATA_SIZE (5*PAGE_SIZE + 4*sizeof(u64) - 1) +void memmap_early_preinit(char *data); void memmap_preinit(void);
-static inline u32 virt_to_phys(void *v) { - return (u32)v; +static inline u64 virt_to_phys(void *v) { + if (MODESEGMENT) + return (u32)v; + return memmap_virt_to_phys(v); } -static inline void *memremap(u32 addr, u32 len) { - return (void*)addr; +static inline void *memremap(u64 addr, u32 len) { + if (MODESEGMENT) + return (void*)(u32)addr; + return memmap_vmap(addr, len); } static inline void *ioremap(u64 addr, u32 len) { return memremap(addr, len); diff --git a/src/romlayout.S b/src/romlayout.S index acf0f32..d931516 100644 --- a/src/romlayout.S +++ b/src/romlayout.S @@ -51,12 +51,26 @@ transition32_nmi_off: orl $CR0_PE, %ecx movl %ecx, %cr0
+#if CONFIG_PAGING + movl %cs:PageDirPtr, %ecx + cmpl $0, %ecx + jz 1f + // Enable PAE paging + movl %ecx, %cr3 + movl %cr4, %ecx + orl $CR4_PAE, %ecx + movl %ecx, %cr4 + movl %cr0, %ecx + orl $CR0_PG, %ecx + movl %ecx, %cr0 +#endif + // start 32bit protected mode code - ljmpl $SEG32_MODE32_CS, $(BUILD_BIOS_ADDR + 1f) +1: ljmpl $SEG32_MODE32_CS, $(BUILD_BIOS_ADDR + 2f)
.code32 // init data segments -1: movl $SEG32_MODE32_DS, %ecx +2: movl $SEG32_MODE32_DS, %ecx movw %cx, %ds movw %cx, %es movw %cx, %ss @@ -97,14 +111,25 @@ transition16big: .code16 // Disable protected mode 1: movl %cr0, %ecx - andl $~CR0_PE, %ecx + andl $~(CR0_PE|CR0_PG), %ecx movl %ecx, %cr0
+#if CONFIG_PAGING + cmpl $0, %cs:PageDirPtr + jz 2f + // Disable PAE paging + movl %cr4, %ecx + andl $~CR4_PAE, %ecx + movl %ecx, %cr4 + xorl %ecx, %ecx + movl %ecx, %cr3 +#endif + // far jump to flush CPU queue after transition to real mode - ljmpw $SEG_BIOS, $2f +2: ljmpw $SEG_BIOS, $3f
// restore IDT to normal real-mode defaults -2: lidtl %cs:rmode_IDT_info +3: lidtl %cs:rmode_IDT_info
// Clear segment registers xorw %cx, %cx diff --git a/src/stacks.c b/src/stacks.c index fa9c7db..a3de397 100644 --- a/src/stacks.c +++ b/src/stacks.c @@ -10,6 +10,7 @@ #include "hw/rtc.h" // rtc_use #include "list.h" // hlist_node #include "malloc.h" // free +#include "memmap.h" // PageDirPtr #include "output.h" // dprintf #include "romfile.h" // romfile_loadint #include "stacks.h" // struct mutex_s @@ -28,7 +29,7 @@ struct { u8 cmosindex; u8 a20; u16 ss, fs, gs; - u32 cr0; + u32 cr0, cr3, cr4; struct descloc_s gdt; } Call16Data VARLOW;
@@ -48,6 +49,10 @@ call32_prep(u8 method) // Called in 16bit protected mode?! return -1; SET_LOW(Call16Data.cr0, cr0); + if (CONFIG_PAGING && GET_GLOBAL(PageDirPtr)) { + SET_LOW(Call16Data.cr3, cr3_read()); + SET_LOW(Call16Data.cr4, cr4_read()); + }
// Backup fs/gs and gdt SET_LOW(Call16Data.fs, GET_SEG(FS)); @@ -98,6 +103,14 @@ call32_post(void) u32 cr0_caching = GET_LOW(Call16Data.cr0) & (CR0_CD|CR0_NW); if (cr0_caching) cr0_mask(CR0_CD|CR0_NW, cr0_caching); + if (CONFIG_PAGING && GET_GLOBAL(PageDirPtr)) { + u32 cr4_pae = GET_LOW(Call16Data.cr4) & CR4_PAE; + if (cr4_pae) + cr4_mask(CR4_PAE, cr4_pae); + u32 cr3 = GET_LOW(Call16Data.cr3); + if (cr3) + cr3_write(cr3); + } }
// Restore cmos index register diff --git a/src/x86.h b/src/x86.h index 5de149a..58cc20c 100644 --- a/src/x86.h +++ b/src/x86.h @@ -14,6 +14,8 @@ #define CR0_NW (1<<29) // Not Write-through #define CR0_PE (1<<0) // Protection enable
+#define CR4_PAE (1<<5) + // PORT_A20 bitdefs #define PORT_A20 0x0092 #define A20_ENABLE_BIT 0x02 @@ -98,6 +100,27 @@ static inline u32 cr2_read(void) { return cr2; }
+static inline u32 cr3_read(void) { + u32 cr3; + asm("movl %%cr3, %0" : "=r"(cr3)); + return cr3; +} +static inline void cr3_write(u32 cr3) { + asm("movl %0, %%cr3" : : "r"(cr3)); +} + +static inline u32 cr4_read(void) { + u32 cr4; + asm("movl %%cr4, %0" : "=r"(cr4)); + return cr4; +} +static inline void cr4_write(u32 cr4) { + asm("movl %0, %%cr4" : : "r"(cr4)); +} +static inline void cr4_mask(u32 off, u32 on) { + cr4_write((cr4_read() & ~off) | on); +} + static inline u64 rdmsr(u32 index) { u64 ret;