Introduce helper for finding temp space to hold an "allocation detail struct" and use it in both alloc_add() and _malloc().
Signed-off-by: Kevin O'Connor kevin@koconnor.net --- src/list.h | 10 +++++++++ src/malloc.c | 72 +++++++++++++++++++++++++++++++----------------------------- 2 files changed, 47 insertions(+), 35 deletions(-)
diff --git a/src/list.h b/src/list.h index de656b9..94512e3 100644 --- a/src/list.h +++ b/src/list.h @@ -61,6 +61,16 @@ hlist_add_after(struct hlist_node *n, struct hlist_node *prev) hlist_add(n, &prev->next); }
+static inline void +hlist_replace(struct hlist_node *old, struct hlist_node *new) +{ + new->next = old->next; + if (new->next) + new->next->pprev = &new->next; + new->pprev = old->pprev; + *new->pprev = new; +} + #define hlist_for_each_entry(pos, head, member) \ for (pos = container_of((head)->first, typeof(*pos), member) \ ; pos != container_of(NULL, typeof(*pos), member) \ diff --git a/src/malloc.c b/src/malloc.c index 3aed014..6f8332e 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -57,8 +57,6 @@ alloc_new(struct zone_s *zone, u32 size, u32 align, struct allocinfo_s *fill) void *newallocend = (void*)ALIGN_DOWN((u32)allocend - size, align); if (newallocend >= dataend && newallocend <= allocend) { // Found space - now reserve it. - if (!fill) - fill = newallocend; fill->data = newallocend; fill->dataend = newallocend + size; fill->allocend = allocend; @@ -71,6 +69,28 @@ alloc_new(struct zone_s *zone, u32 size, u32 align, struct allocinfo_s *fill) return NULL; }
+// Reserve space for a 'struct allocdetail_s' and fill +static struct allocdetail_s * +alloc_new_detail(struct allocdetail_s *temp) +{ + struct allocdetail_s *detail = alloc_new( + &ZoneTmpHigh, sizeof(*detail), MALLOC_MIN_ALIGN, &temp->detailinfo); + if (!detail) { + detail = alloc_new(&ZoneTmpLow, sizeof(*detail) + , MALLOC_MIN_ALIGN, &temp->detailinfo); + if (!detail) { + warn_noalloc(); + return NULL; + } + } + + // Fill final 'detail' allocation from data in 'temp' + memcpy(detail, temp, sizeof(*detail)); + hlist_replace(&temp->detailinfo.node, &detail->detailinfo.node); + hlist_replace(&temp->datainfo.node, &detail->datainfo.node); + return detail; +} + // Add new memory to a zone static void alloc_add(struct zone_s *zone, void *start, void *end) @@ -85,29 +105,15 @@ alloc_add(struct zone_s *zone, void *start, void *end)
// Add space using temporary allocation info. struct allocdetail_s tempdetail; + tempdetail.handle = MALLOC_DEFAULT_HANDLE; tempdetail.datainfo.data = tempdetail.datainfo.dataend = start; tempdetail.datainfo.allocend = end; hlist_add(&tempdetail.datainfo.node, pprev);
// Allocate final allocation info. - struct allocdetail_s *detail = alloc_new( - &ZoneTmpHigh, sizeof(*detail), MALLOC_MIN_ALIGN, NULL); - if (!detail) { - detail = alloc_new(&ZoneTmpLow, sizeof(*detail) - , MALLOC_MIN_ALIGN, NULL); - if (!detail) { - hlist_del(&tempdetail.datainfo.node); - warn_noalloc(); - return; - } - } - - // Replace temp alloc space with final alloc space - pprev = tempdetail.datainfo.node.pprev; - hlist_del(&tempdetail.datainfo.node); - memcpy(&detail->datainfo, &tempdetail.datainfo, sizeof(detail->datainfo)); - detail->handle = MALLOC_DEFAULT_HANDLE; - hlist_add(&detail->datainfo.node, pprev); + struct allocdetail_s *detail = alloc_new_detail(&tempdetail); + if (!detail) + hlist_del(&tempdetail.datainfo.node); }
// Release space allocated with alloc_new() @@ -232,23 +238,19 @@ _malloc(struct zone_s *zone, u32 size, u32 align) if (!size) return NULL;
- // Find and reserve space for bookkeeping. - struct allocdetail_s *detail = alloc_new( - &ZoneTmpHigh, sizeof(*detail), MALLOC_MIN_ALIGN, NULL); - if (!detail) { - detail = alloc_new(&ZoneTmpLow, sizeof(*detail) - , MALLOC_MIN_ALIGN, NULL); - if (!detail) - return NULL; - } - detail->handle = MALLOC_DEFAULT_HANDLE; - // Find and reserve space for main allocation - void *data = alloc_new(zone, size, align, &detail->datainfo); + struct allocdetail_s tempdetail; + tempdetail.handle = MALLOC_DEFAULT_HANDLE; + void *data = alloc_new(zone, size, align, &tempdetail.datainfo); if (!CONFIG_MALLOC_UPPERMEMORY && !data && zone == &ZoneLow) - data = zonelow_expand(size, align, &detail->datainfo); - if (!data) { - alloc_free(&detail->detailinfo); + data = zonelow_expand(size, align, &tempdetail.datainfo); + if (!data) + return NULL; + + // Find and reserve space for bookkeeping. + struct allocdetail_s *detail = alloc_new_detail(&tempdetail); + if (!detail) { + alloc_free(&tempdetail.datainfo); return NULL; }
Consistently use 'u32' for physical addresses and pointers for virtual addresses in the malloc code. Introduce and use memremap() where a physical address needs to be converted to a virtual address. Use virt_to_phys() for the inverse.
Signed-off-by: Kevin O'Connor kevin@koconnor.net --- src/malloc.c | 168 ++++++++++++++++++++++++++++++++--------------------------- src/malloc.h | 11 ++-- src/memmap.h | 3 ++ src/pmm.c | 16 +++--- 4 files changed, 107 insertions(+), 91 deletions(-)
diff --git a/src/malloc.c b/src/malloc.c index 6f8332e..093b165 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -18,7 +18,7 @@ // Information on a reserved area. struct allocinfo_s { struct hlist_node node; - void *data, *dataend, *allocend; + u32 range_start, range_end, alloc_size; };
// Information on a tracked memory allocation. @@ -47,42 +47,43 @@ static struct zone_s *Zones[] VARVERIFY32INIT = { ****************************************************************/
// Find and reserve space from a given zone -static void * +static u32 alloc_new(struct zone_s *zone, u32 size, u32 align, struct allocinfo_s *fill) { struct allocinfo_s *info; hlist_for_each_entry(info, &zone->head, node) { - void *dataend = info->dataend; - void *allocend = info->allocend; - void *newallocend = (void*)ALIGN_DOWN((u32)allocend - size, align); - if (newallocend >= dataend && newallocend <= allocend) { + u32 alloc_end = info->range_start + info->alloc_size; + u32 range_end = info->range_end; + u32 new_range_end = ALIGN_DOWN(range_end - size, align); + if (new_range_end >= alloc_end && new_range_end <= range_end) { // Found space - now reserve it. - fill->data = newallocend; - fill->dataend = newallocend + size; - fill->allocend = allocend; + fill->range_start = new_range_end; + fill->range_end = range_end; + fill->alloc_size = size;
- info->allocend = newallocend; + info->range_end = new_range_end; hlist_add_before(&fill->node, &info->node); - return newallocend; + return new_range_end; } } - return NULL; + return 0; }
// Reserve space for a 'struct allocdetail_s' and fill static struct allocdetail_s * alloc_new_detail(struct allocdetail_s *temp) { - struct allocdetail_s *detail = alloc_new( - &ZoneTmpHigh, sizeof(*detail), MALLOC_MIN_ALIGN, &temp->detailinfo); - if (!detail) { - detail = alloc_new(&ZoneTmpLow, sizeof(*detail) - , MALLOC_MIN_ALIGN, &temp->detailinfo); - if (!detail) { + u32 detail_addr = alloc_new(&ZoneTmpHigh, sizeof(struct allocdetail_s) + , MALLOC_MIN_ALIGN, &temp->detailinfo); + if (!detail_addr) { + detail_addr = alloc_new(&ZoneTmpLow, sizeof(struct allocdetail_s) + , MALLOC_MIN_ALIGN, &temp->detailinfo); + if (!detail_addr) { warn_noalloc(); return NULL; } } + struct allocdetail_s *detail = memremap(detail_addr, sizeof(*detail));
// Fill final 'detail' allocation from data in 'temp' memcpy(detail, temp, sizeof(*detail)); @@ -93,21 +94,22 @@ alloc_new_detail(struct allocdetail_s *temp)
// Add new memory to a zone static void -alloc_add(struct zone_s *zone, void *start, void *end) +alloc_add(struct zone_s *zone, u32 start, u32 end) { // Find position to add space struct allocinfo_s *info; struct hlist_node **pprev; hlist_for_each_entry_pprev(info, pprev, &zone->head, node) { - if (info->data < start) + if (info->range_start < start) break; }
// Add space using temporary allocation info. struct allocdetail_s tempdetail; tempdetail.handle = MALLOC_DEFAULT_HANDLE; - tempdetail.datainfo.data = tempdetail.datainfo.dataend = start; - tempdetail.datainfo.allocend = end; + tempdetail.datainfo.range_start = start; + tempdetail.datainfo.range_end = end; + tempdetail.datainfo.alloc_size = 0; hlist_add(&tempdetail.datainfo.node, pprev);
// Allocate final allocation info. @@ -122,20 +124,20 @@ alloc_free(struct allocinfo_s *info) { struct allocinfo_s *next = container_of_or_null( info->node.next, struct allocinfo_s, node); - if (next && next->allocend == info->data) - next->allocend = info->allocend; + if (next && next->range_end == info->range_start) + next->range_end = info->range_end; hlist_del(&info->node); }
// Search all zones for an allocation obtained from alloc_new() static struct allocinfo_s * -alloc_find(void *data) +alloc_find(u32 data) { int i; for (i=0; i<ARRAY_SIZE(Zones); i++) { struct allocinfo_s *info; hlist_for_each_entry(info, &Zones[i]->head, node) { - if (info->data == data) + if (info->range_start == data) return info; } } @@ -178,22 +180,22 @@ relocate_ebda(u32 newebda, u32 oldebda, u8 ebda_size) }
// Support expanding the ZoneLow dynamically. -static void * +static u32 zonelow_expand(u32 size, u32 align, struct allocinfo_s *fill) { // Make sure to not move ebda while an optionrom is running. if (unlikely(wait_preempt())) { - void *data = alloc_new(&ZoneLow, size, align, fill); + u32 data = alloc_new(&ZoneLow, size, align, fill); if (data) return data; }
struct allocinfo_s *info = alloc_get_sentinel(&ZoneLow); if (!info) - return NULL; - u32 oldpos = (u32)info->allocend; + return 0; + u32 oldpos = info->range_end; u32 newpos = ALIGN_DOWN(oldpos - size, align); - u32 bottom = (u32)info->dataend; + u32 bottom = info->range_start + info->alloc_size; if (newpos >= bottom && newpos <= oldpos) // Space already present. return alloc_new(&ZoneLow, size, align, fill); @@ -208,19 +210,18 @@ zonelow_expand(u32 size, u32 align, struct allocinfo_s *fill) u32 newebda = ALIGN_DOWN(newbottom - ebda_size * 1024, 1024); if (newebda < BUILD_EBDA_MINIMUM) // Not enough space. - return NULL; + return 0;
// Move ebda int ret = relocate_ebda(newebda, ebda_pos, ebda_size); if (ret) - return NULL; + return 0;
// Update zone - if (ebda_end == bottom) { - info->data = (void*)newbottom; - info->dataend = (void*)newbottom; - } else - alloc_add(&ZoneLow, (void*)newbottom, (void*)ebda_end); + if (ebda_end == bottom) + info->range_start = newbottom; + else + alloc_add(&ZoneLow, newbottom, ebda_end);
return alloc_new(&ZoneLow, size, align, fill); } @@ -230,52 +231,65 @@ zonelow_expand(u32 size, u32 align, struct allocinfo_s *fill) * tracked memory allocations ****************************************************************/
-// Allocate memory from the given zone and track it as a PMM allocation -void * __malloc -_malloc(struct zone_s *zone, u32 size, u32 align) +// Allocate physical memory from the given zone and track it as a PMM allocation +u32 +phys_alloc(struct zone_s *zone, u32 size, u32 align) { ASSERT32FLAT(); if (!size) - return NULL; + return 0;
// Find and reserve space for main allocation struct allocdetail_s tempdetail; tempdetail.handle = MALLOC_DEFAULT_HANDLE; - void *data = alloc_new(zone, size, align, &tempdetail.datainfo); + u32 data = alloc_new(zone, size, align, &tempdetail.datainfo); if (!CONFIG_MALLOC_UPPERMEMORY && !data && zone == &ZoneLow) data = zonelow_expand(size, align, &tempdetail.datainfo); if (!data) - return NULL; + return 0;
// Find and reserve space for bookkeeping. struct allocdetail_s *detail = alloc_new_detail(&tempdetail); if (!detail) { alloc_free(&tempdetail.datainfo); - return NULL; + return 0; }
- dprintf(8, "_malloc zone=%p size=%d align=%x ret=%p (detail=%p)\n" + dprintf(8, "phys_alloc zone=%p size=%d align=%x ret=%x (detail=%p)\n" , zone, size, align, data, detail);
return data; }
-// Free a data block allocated with _malloc +// Allocate virtual memory from the given zone +void * __malloc +_malloc(struct zone_s *zone, u32 size, u32 align) +{ + return memremap(phys_alloc(zone, size, align), size); +} + +// Free a data block allocated with phys_alloc int -_free(void *data) +phys_free(u32 data) { ASSERT32FLAT(); struct allocinfo_s *info = alloc_find(data); - if (!info || data == (void*)info || data == info->dataend) + if (!info || data == virt_to_phys(info) || !info->alloc_size) return -1; struct allocdetail_s *detail = container_of( info, struct allocdetail_s, datainfo); - dprintf(8, "_free %p (detail=%p)\n", data, detail); + dprintf(8, "phys_free %x (detail=%p)\n", data, detail); alloc_free(info); alloc_free(&detail->detailinfo); return 0; }
+// Free memory allocated with _malloc +void free(void *data) +{ + phys_free(virt_to_phys(data)); +} + // Find the amount of free space in a given zone. u32 malloc_getspace(struct zone_s *zone) @@ -285,7 +299,7 @@ malloc_getspace(struct zone_s *zone) u32 maxspace = 0; struct allocinfo_s *info; hlist_for_each_entry(info, &zone->head, node) { - u32 space = info->allocend - info->dataend; + u32 space = info->range_end - info->range_start - info->alloc_size; if (space > maxspace) maxspace = space; } @@ -301,34 +315,34 @@ malloc_getspace(struct zone_s *zone)
// Set a handle associated with an allocation. void -malloc_sethandle(void *data, u32 handle) +malloc_sethandle(u32 data, u32 handle) { ASSERT32FLAT(); struct allocinfo_s *info = alloc_find(data); - if (!info || data == (void*)info || data == info->dataend) + if (!info || data == virt_to_phys(info) || !info->alloc_size) return; struct allocdetail_s *detail = container_of( info, struct allocdetail_s, datainfo); detail->handle = handle; }
-// Find the data block allocated with _malloc with a given handle. -void * +// Find the data block allocated with phys_alloc with a given handle. +u32 malloc_findhandle(u32 handle) { int i; for (i=0; i<ARRAY_SIZE(Zones); i++) { struct allocinfo_s *info; hlist_for_each_entry(info, &Zones[i]->head, node) { - if (info->data != (void*)info) + if (info->range_start != virt_to_phys(info)) continue; struct allocdetail_s *detail = container_of( info, struct allocdetail_s, detailinfo); if (detail->handle == handle) - return detail->datainfo.data; + return detail->datainfo.range_start; } } - return NULL; + return 0; }
@@ -346,7 +360,7 @@ u32 rom_get_max(void) { if (CONFIG_MALLOC_UPPERMEMORY) - return ALIGN_DOWN((u32)RomBase->allocend - OPROM_HEADER_RESERVE + return ALIGN_DOWN(RomBase->range_end - OPROM_HEADER_RESERVE , OPTION_ROM_ALIGN); extern u8 final_readonly_start[]; return (u32)final_readonly_start; @@ -369,7 +383,7 @@ rom_reserve(u32 size) if (CONFIG_MALLOC_UPPERMEMORY) { if (newend < (u32)zonelow_base) newend = (u32)zonelow_base; - RomBase->data = RomBase->dataend = (void*)newend + OPROM_HEADER_RESERVE; + RomBase->range_start = newend + OPROM_HEADER_RESERVE; } return (void*)RomEnd; } @@ -422,14 +436,13 @@ malloc_preinit(void) e = newe; } } - alloc_add(&ZoneTmpHigh, (void*)s, (void*)e); + alloc_add(&ZoneTmpHigh, s, e); }
// Populate regions - alloc_add(&ZoneTmpLow, (void*)BUILD_STACK_ADDR, (void*)BUILD_EBDA_MINIMUM); + alloc_add(&ZoneTmpLow, BUILD_STACK_ADDR, BUILD_EBDA_MINIMUM); if (highram) { - alloc_add(&ZoneHigh, (void*)highram - , (void*)highram + BUILD_MAX_HIGHTABLE); + alloc_add(&ZoneHigh, highram, highram + BUILD_MAX_HIGHTABLE); e820_add(highram, BUILD_MAX_HIGHTABLE, E820_RESERVED); } } @@ -440,13 +453,13 @@ csm_malloc_preinit(u32 low_pmm, u32 low_pmm_size, u32 hi_pmm, u32 hi_pmm_size) ASSERT32FLAT();
if (hi_pmm_size > BUILD_MAX_HIGHTABLE) { - void *hi_pmm_end = (void*)hi_pmm + hi_pmm_size; - alloc_add(&ZoneTmpHigh, (void*)hi_pmm, hi_pmm_end - BUILD_MAX_HIGHTABLE); + u32 hi_pmm_end = hi_pmm + hi_pmm_size; + alloc_add(&ZoneTmpHigh, hi_pmm, hi_pmm_end - BUILD_MAX_HIGHTABLE); alloc_add(&ZoneHigh, hi_pmm_end - BUILD_MAX_HIGHTABLE, hi_pmm_end); } else { - alloc_add(&ZoneTmpHigh, (void*)hi_pmm, (void*)hi_pmm + hi_pmm_size); + alloc_add(&ZoneTmpHigh, hi_pmm, hi_pmm + hi_pmm_size); } - alloc_add(&ZoneTmpLow, (void*)low_pmm, (void*)low_pmm + low_pmm_size); + alloc_add(&ZoneTmpLow, low_pmm, low_pmm + low_pmm_size); }
u32 LegacyRamSize VARFSEG; @@ -490,18 +503,18 @@ malloc_init(void) extern u8 varlow_start[], varlow_end[], final_varlow_start[]; memmove(final_varlow_start, varlow_start, varlow_end - varlow_start); if (CONFIG_MALLOC_UPPERMEMORY) { - alloc_add(&ZoneLow, zonelow_base + OPROM_HEADER_RESERVE - , final_varlow_start); + alloc_add(&ZoneLow, (u32)zonelow_base + OPROM_HEADER_RESERVE + , (u32)final_varlow_start); RomBase = alloc_get_sentinel(&ZoneLow); } else { - alloc_add(&ZoneLow, (void*)ALIGN_DOWN((u32)final_varlow_start, 1024) - , final_varlow_start); + alloc_add(&ZoneLow, ALIGN_DOWN((u32)final_varlow_start, 1024) + , (u32)final_varlow_start); }
// Add space available in f-segment to ZoneFSeg extern u8 zonefseg_start[], zonefseg_end[]; memset(zonefseg_start, 0, zonefseg_end - zonefseg_start); - alloc_add(&ZoneFSeg, zonefseg_start, zonefseg_end); + alloc_add(&ZoneFSeg, (u32)zonefseg_start, (u32)zonefseg_end);
calcRamSize(); } @@ -528,15 +541,16 @@ malloc_prepboot(void)
// Clear unused f-seg ram. struct allocinfo_s *info = alloc_get_sentinel(&ZoneFSeg); - memset(info->dataend, 0, info->allocend - info->dataend); + u32 size = info->range_end - info->range_start; + memset(memremap(info->range_start, size), 0, size); dprintf(1, "Space available for UMB: %x-%x, %x-%x\n" - , RomEnd, base, (u32)info->dataend, (u32)info->allocend); + , RomEnd, base, info->range_start, info->range_end);
// Give back unused high ram. info = alloc_get_sentinel(&ZoneHigh); if (info) { - u32 giveback = ALIGN_DOWN(info->allocend - info->dataend, PAGE_SIZE); - e820_add((u32)info->dataend, giveback, E820_RAM); + u32 giveback = ALIGN_DOWN(info->range_end-info->range_start, PAGE_SIZE); + e820_add(info->range_start, giveback, E820_RAM); dprintf(1, "Returned %d bytes of ZoneHigh\n", giveback); }
diff --git a/src/malloc.h b/src/malloc.h index 2bcb5bf..84c285e 100644 --- a/src/malloc.h +++ b/src/malloc.h @@ -15,11 +15,13 @@ void malloc_preinit(void); extern u32 LegacyRamSize; void malloc_init(void); void malloc_prepboot(void); +u32 phys_alloc(struct zone_s *zone, u32 size, u32 align); void *_malloc(struct zone_s *zone, u32 size, u32 align); -int _free(void *data); +int phys_free(u32 data); +void free(void *data); u32 malloc_getspace(struct zone_s *zone); -void malloc_sethandle(void *data, u32 handle); -void *malloc_findhandle(u32 handle); +void malloc_sethandle(u32 data, u32 handle); +u32 malloc_findhandle(u32 handle);
#define MALLOC_DEFAULT_HANDLE 0xFFFFFFFF // Minimum alignment of malloc'd memory @@ -64,8 +66,5 @@ static inline void *memalign_tmp(u32 align, u32 size) { return ret; return memalign_tmplow(align, size); } -static inline void free(void *data) { - _free(data); -}
#endif // malloc.h diff --git a/src/memmap.h b/src/memmap.h index 9a59024..092e2ad 100644 --- a/src/memmap.h +++ b/src/memmap.h @@ -10,5 +10,8 @@ static inline u32 virt_to_phys(void *v) { return (u32)v; } +static inline void *memremap(u32 addr, u32 len) { + return (void*)addr; +}
#endif // memmap.h diff --git a/src/pmm.c b/src/pmm.c index 304faab..85f35b5 100644 --- a/src/pmm.c +++ b/src/pmm.c @@ -65,26 +65,26 @@ handle_pmm00(u16 *args) if (align < MALLOC_MIN_ALIGN) align = MALLOC_MIN_ALIGN; } - void *data; + u32 data; switch (flags & 3) { default: case 0: return 0; case 1: - data = _malloc(lowzone, size, align); + data = phys_alloc(lowzone, size, align); break; case 2: - data = _malloc(highzone, size, align); + data = phys_alloc(highzone, size, align); break; case 3: { - data = _malloc(lowzone, size, align); + data = phys_alloc(lowzone, size, align); if (!data) - data = _malloc(highzone, size, align); + data = phys_alloc(highzone, size, align); } } if (data && handle != MALLOC_DEFAULT_HANDLE) malloc_sethandle(data, handle); - return (u32)data; + return data; }
// PMM - find @@ -95,7 +95,7 @@ handle_pmm01(u16 *args) dprintf(3, "pmm01: handle=%x\n", handle); if (handle == MALLOC_DEFAULT_HANDLE) return 0; - return (u32)malloc_findhandle(handle); + return malloc_findhandle(handle); }
// PMM - deallocate @@ -104,7 +104,7 @@ handle_pmm02(u16 *args) { u32 buffer = *(u32*)&args[1]; dprintf(3, "pmm02: buffer=%x\n", buffer); - int ret = _free((void*)buffer); + int ret = phys_free(buffer); if (ret) // Error return 1;
Signed-off-by: Kevin O'Connor kevin@koconnor.net --- Makefile | 2 +- src/memmap.c | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ src/misc.c | 45 --------------------------------------------- 3 files changed, 54 insertions(+), 46 deletions(-) create mode 100644 src/memmap.c
diff --git a/Makefile b/Makefile index e5f28d4..3448598 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ SRCBOTH=misc.c stacks.c output.c string.c block.c cdrom.c disk.c mouse.c kbd.c \ hw/lsi-scsi.c hw/esp-scsi.c hw/megasas.c SRC16=$(SRCBOTH) SRC32FLAT=$(SRCBOTH) post.c e820map.c malloc.c romfile.c x86.c optionroms.c \ - pmm.c font.c boot.c bootsplash.c jpeg.c bmp.c tcgbios.c sha1.c \ + pmm.c font.c boot.c bootsplash.c jpeg.c bmp.c tcgbios.c sha1.c memmap.c \ hw/ahci.c hw/pvscsi.c hw/usb-xhci.c hw/usb-hub.c hw/sdcard.c \ fw/coreboot.c fw/lzmadecode.c fw/multiboot.c fw/csm.c fw/biostables.c \ fw/paravirt.c fw/shadow.c fw/pciinit.c fw/smm.c fw/smp.c fw/mtrr.c fw/xen.c \ diff --git a/src/memmap.c b/src/memmap.c new file mode 100644 index 0000000..51a0123 --- /dev/null +++ b/src/memmap.c @@ -0,0 +1,53 @@ +// CPU protected mode memory map handling +// +// Copyright (C) 2008-2015 Kevin O'Connor kevin@koconnor.net +// +// This file may be distributed under the terms of the GNU LGPLv3 license. + +#include "biosvar.h" // struct rmode_IVT +#include "x86.h" // struct descloc_s + + +/**************************************************************** + * GDT and IDT tables + ****************************************************************/ + +// Real mode IDT descriptor +struct descloc_s rmode_IDT_info VARFSEG = { + .length = sizeof(struct rmode_IVT) - 1, + .addr = (u32)MAKE_FLATPTR(SEG_IVT, 0), +}; + +// Dummy IDT that forces a machine shutdown if an irq happens in +// protected mode. +u8 dummy_IDT VARFSEG; + +// Protected mode IDT descriptor +struct descloc_s pmode_IDT_info VARFSEG = { + .length = sizeof(dummy_IDT) - 1, + .addr = (u32)&dummy_IDT, +}; + +// GDT +u64 rombios32_gdt[] VARFSEG __aligned(8) = { + // First entry can't be used. + 0x0000000000000000LL, + // 32 bit flat code segment (SEG32_MODE32_CS) + GDT_GRANLIMIT(0xffffffff) | GDT_CODE | GDT_B, + // 32 bit flat data segment (SEG32_MODE32_DS) + GDT_GRANLIMIT(0xffffffff) | GDT_DATA | GDT_B, + // 16 bit code segment base=0xf0000 limit=0xffff (SEG32_MODE16_CS) + GDT_LIMIT(BUILD_BIOS_SIZE-1) | GDT_CODE | GDT_BASE(BUILD_BIOS_ADDR), + // 16 bit data segment base=0x0 limit=0xffff (SEG32_MODE16_DS) + GDT_LIMIT(0x0ffff) | GDT_DATA, + // 16 bit code segment base=0xf0000 limit=0xffffffff (SEG32_MODE16BIG_CS) + GDT_GRANLIMIT(0xffffffff) | GDT_CODE | GDT_BASE(BUILD_BIOS_ADDR), + // 16 bit data segment base=0 limit=0xffffffff (SEG32_MODE16BIG_DS) + GDT_GRANLIMIT(0xffffffff) | GDT_DATA, +}; + +// GDT descriptor +struct descloc_s rombios32_gdt_48 VARFSEG = { + .length = sizeof(rombios32_gdt) - 1, + .addr = (u32)rombios32_gdt, +}; diff --git a/src/misc.c b/src/misc.c index 8caaf31..99b2970 100644 --- a/src/misc.c +++ b/src/misc.c @@ -133,51 +133,6 @@ struct bios_config_table_s BIOS_CONFIG_TABLE VARFSEGFIXED(0xe6f5) = {
/**************************************************************** - * GDT and IDT tables - ****************************************************************/ - -// Real mode IDT descriptor -struct descloc_s rmode_IDT_info VARFSEG = { - .length = sizeof(struct rmode_IVT) - 1, - .addr = (u32)MAKE_FLATPTR(SEG_IVT, 0), -}; - -// Dummy IDT that forces a machine shutdown if an irq happens in -// protected mode. -u8 dummy_IDT VARFSEG; - -// Protected mode IDT descriptor -struct descloc_s pmode_IDT_info VARFSEG = { - .length = sizeof(dummy_IDT) - 1, - .addr = (u32)&dummy_IDT, -}; - -// GDT -u64 rombios32_gdt[] VARFSEG __aligned(8) = { - // First entry can't be used. - 0x0000000000000000LL, - // 32 bit flat code segment (SEG32_MODE32_CS) - GDT_GRANLIMIT(0xffffffff) | GDT_CODE | GDT_B, - // 32 bit flat data segment (SEG32_MODE32_DS) - GDT_GRANLIMIT(0xffffffff) | GDT_DATA | GDT_B, - // 16 bit code segment base=0xf0000 limit=0xffff (SEG32_MODE16_CS) - GDT_LIMIT(BUILD_BIOS_SIZE-1) | GDT_CODE | GDT_BASE(BUILD_BIOS_ADDR), - // 16 bit data segment base=0x0 limit=0xffff (SEG32_MODE16_DS) - GDT_LIMIT(0x0ffff) | GDT_DATA, - // 16 bit code segment base=0xf0000 limit=0xffffffff (SEG32_MODE16BIG_CS) - GDT_GRANLIMIT(0xffffffff) | GDT_CODE | GDT_BASE(BUILD_BIOS_ADDR), - // 16 bit data segment base=0 limit=0xffffffff (SEG32_MODE16BIG_DS) - GDT_GRANLIMIT(0xffffffff) | GDT_DATA, -}; - -// GDT descriptor -struct descloc_s rombios32_gdt_48 VARFSEG = { - .length = sizeof(rombios32_gdt) - 1, - .addr = (u32)rombios32_gdt, -}; - - -/**************************************************************** * Misc fixed vars ****************************************************************/
Signed-off-by: Kevin O'Connor kevin@koconnor.net --- src/memmap.h | 3 +++ src/optionroms.c | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/src/memmap.h b/src/memmap.h index 092e2ad..c927ab9 100644 --- a/src/memmap.h +++ b/src/memmap.h @@ -13,5 +13,8 @@ static inline u32 virt_to_phys(void *v) { static inline void *memremap(u32 addr, u32 len) { return (void*)addr; } +static inline void *ioremap(u64 addr, u32 len) { + return memremap(addr, len); +}
#endif // memmap.h diff --git a/src/optionroms.c b/src/optionroms.c index c81eff2..0c55765 100644 --- a/src/optionroms.c +++ b/src/optionroms.c @@ -12,14 +12,15 @@ #include "hw/pci_ids.h" // PCI_CLASS_DISPLAY_VGA #include "hw/pci_regs.h" // PCI_ROM_ADDRESS #include "malloc.h" // rom_confirm +#include "memmap.h" // ioremap #include "output.h" // dprintf #include "romfile.h" // romfile_loadint #include "stacks.h" // farcall16big #include "std/optionrom.h" // struct rom_header #include "std/pnpbios.h" // PNP_SIGNATURE #include "string.h" // memset -#include "util.h" // get_pnp_offset #include "tcgbios.h" // tpm_* +#include "util.h" // get_pnp_offset
static int EnforceChecksum, S3ResumeVga, RunPCIroms;
@@ -272,7 +273,7 @@ map_pcirom(struct pci_device *pci) // Looks like a rom - enable it. pci_config_writel(bdf, PCI_ROM_ADDRESS, orig | PCI_ROM_ADDRESS_ENABLE);
- struct rom_header *rom = (void*)orig; + struct rom_header *rom = ioremap(orig, sz); for (;;) { dprintf(5, "Inspecting possible rom at %p (vd=%04x:%04x" " bdf=%02x:%02x.%x)\n"
Signed-off-by: Kevin O'Connor kevin@koconnor.net --- src/fw/smp.c | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-)
diff --git a/src/fw/smp.c b/src/fw/smp.c index 579acdb..711eb27 100644 --- a/src/fw/smp.c +++ b/src/fw/smp.c @@ -7,16 +7,18 @@
#include "config.h" // CONFIG_* #include "hw/rtc.h" // CMOS_BIOS_SMP_COUNT +#include "memmap.h" // ioremap #include "output.h" // dprintf #include "romfile.h" // romfile_loadint #include "stacks.h" // yield #include "util.h" // smp_setup #include "x86.h" // wrmsr
-#define APIC_ICR_LOW ((u8*)BUILD_APIC_ADDR + 0x300) -#define APIC_SVR ((u8*)BUILD_APIC_ADDR + 0x0F0) -#define APIC_LINT0 ((u8*)BUILD_APIC_ADDR + 0x350) -#define APIC_LINT1 ((u8*)BUILD_APIC_ADDR + 0x360) +#define APIC_ICR_LOW 0x300 +#define APIC_SVR 0x0F0 +#define APIC_LINT0 0x350 +#define APIC_LINT1 0x360 +#define APIC_SIZE 0x400
#define APIC_ENABLED 0x0100
@@ -105,23 +107,24 @@ smp_setup(void) *(u64*)BUILD_AP_BOOT_ADDR = new;
// enable local APIC - u32 val = readl(APIC_SVR); - writel(APIC_SVR, val | APIC_ENABLED); + void *apic = ioremap(BUILD_APIC_ADDR, APIC_SIZE); + u32 val = readl(apic + APIC_SVR); + writel(apic + APIC_SVR, val | APIC_ENABLED);
/* Set LINT0 as Ext_INT, level triggered */ - writel(APIC_LINT0, 0x8700); + writel(apic + APIC_LINT0, 0x8700);
/* Set LINT1 as NMI, level triggered */ - writel(APIC_LINT1, 0x8400); + writel(apic + APIC_LINT1, 0x8400);
// Init the lock. writel(&SMPLock, 1);
// broadcast SIPI barrier(); - writel(APIC_ICR_LOW, 0x000C4500); + writel(apic + APIC_ICR_LOW, 0x000C4500); u32 sipi_vector = BUILD_AP_BOOT_ADDR >> 12; - writel(APIC_ICR_LOW, 0x000C4600 | sipi_vector); + writel(apic + APIC_ICR_LOW, 0x000C4600 | sipi_vector);
// Wait for other CPUs to process the SIPI. u8 cmos_smp_count = rtc_read(CMOS_BIOS_SMP_COUNT) + 1;
Signed-off-by: Kevin O'Connor kevin@koconnor.net --- src/hw/tpm_drivers.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/src/hw/tpm_drivers.c b/src/hw/tpm_drivers.c index 444eac3..8c92401 100644 --- a/src/hw/tpm_drivers.c +++ b/src/hw/tpm_drivers.c @@ -8,11 +8,12 @@ // This file may be distributed under the terms of the GNU LGPLv3 license.
#include "config.h" // CONFIG_TPM_TIS_SHA1THRESHOLD +#include "hw/tpm_drivers.h" // struct tpm_driver +#include "memmap.h" // ioremap #include "string.h" // memcpy +#include "tcgbios.h" // TCG_* #include "util.h" // msleep #include "x86.h" // readl -#include "hw/tpm_drivers.h" // struct tpm_driver -#include "tcgbios.h" // TCG_*
static const u32 tis_default_timeouts[4] = { TIS_DEFAULT_TIMEOUT_A, @@ -39,6 +40,7 @@ static u32 tis_probe(void) return 0;
u32 rc = 0; + ioremap(TPM_TIS_BASE_ADDRESS, 1); // XXX u32 didvid = readl(TIS_REG(0, TIS_REG_DID_VID));
if ((didvid != 0) && (didvid != 0xffffffff))
Support simple exception handlers for General Protection (GP) and Page Fault (PF) faults. The handlers will report the issue and halt the machine.
Signed-off-by: Kevin O'Connor kevin@koconnor.net --- src/malloc.c | 2 ++ src/memmap.c | 66 +++++++++++++++++++++++++++++++++++++++++++-------------- src/memmap.h | 2 ++ src/romlayout.S | 21 +++++++++++++++--- src/x86.h | 22 ++++++++++++++++++- 5 files changed, 93 insertions(+), 20 deletions(-)
diff --git a/src/malloc.c b/src/malloc.c index 093b165..ec7a21f 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -445,6 +445,8 @@ malloc_preinit(void) alloc_add(&ZoneHigh, highram, highram + BUILD_MAX_HIGHTABLE); e820_add(highram, BUILD_MAX_HIGHTABLE, E820_RESERVED); } + + memmap_preinit(); }
void diff --git a/src/memmap.c b/src/memmap.c index 51a0123..b08313a 100644 --- a/src/memmap.c +++ b/src/memmap.c @@ -5,6 +5,10 @@ // This file may be distributed under the terms of the GNU LGPLv3 license.
#include "biosvar.h" // struct rmode_IVT +#include "malloc.h" // malloc_high +#include "memmap.h" // memmap_preinit +#include "string.h" // memset +#include "output.h" // panic #include "x86.h" // struct descloc_s
@@ -12,22 +16,6 @@ * GDT and IDT tables ****************************************************************/
-// Real mode IDT descriptor -struct descloc_s rmode_IDT_info VARFSEG = { - .length = sizeof(struct rmode_IVT) - 1, - .addr = (u32)MAKE_FLATPTR(SEG_IVT, 0), -}; - -// Dummy IDT that forces a machine shutdown if an irq happens in -// protected mode. -u8 dummy_IDT VARFSEG; - -// Protected mode IDT descriptor -struct descloc_s pmode_IDT_info VARFSEG = { - .length = sizeof(dummy_IDT) - 1, - .addr = (u32)&dummy_IDT, -}; - // GDT u64 rombios32_gdt[] VARFSEG __aligned(8) = { // First entry can't be used. @@ -51,3 +39,49 @@ struct descloc_s rombios32_gdt_48 VARFSEG = { .length = sizeof(rombios32_gdt) - 1, .addr = (u32)rombios32_gdt, }; + +// Real mode IDT descriptor +struct descloc_s rmode_IDT_info VARFSEG = { + .length = sizeof(struct rmode_IVT) - 1, + .addr = (u32)MAKE_FLATPTR(SEG_IVT, 0), +}; + +// Protected mode IDT descriptor +struct descloc_s pmode_IDT_info VARFSEG; + +void VISIBLE32FLAT +handle_gp(struct x86_exception_stack *data) +{ + panic("#GP exception: %x %x %x %x\n", data->error_code, data->eip + , data->cs, data->eflags); +} + +void VISIBLE32FLAT +handle_pf(struct x86_exception_stack *data) +{ + panic("#PF exception: %x %x %x %x %x\n", data->error_code, data->eip + , data->cs, data->eflags, cr2_read()); +} + +// Setup a 32bit General Protection (GP) and Page Fault (PF) handler +void +memmap_preinit(void) +{ + u32 ilen = sizeof(u64)*(IDT_VECTOR_PF+1); + u64 *pidt = malloc_high(ilen); + if (!pidt) { + warn_noalloc(); + return; + } + memset(pidt, 0, ilen); + extern void entry_gp(void); + extern void entry_pf(void); + pidt[IDT_VECTOR_GP] = IDT_IRQ | IDT_SEGMENT(SEG32_MODE32_CS) + | IDT_OFFSET((u32)&entry_gp); + pidt[IDT_VECTOR_PF] = IDT_IRQ | IDT_SEGMENT(SEG32_MODE32_CS) + | IDT_OFFSET((u32)&entry_pf); + pmode_IDT_info.addr = virt_to_phys(pidt); + pmode_IDT_info.length = ilen-1; + dprintf(1, "Installing pmode exception handler\n"); + lidt(&pmode_IDT_info); +} diff --git a/src/memmap.h b/src/memmap.h index c927ab9..d4f150a 100644 --- a/src/memmap.h +++ b/src/memmap.h @@ -7,6 +7,8 @@ #define PAGE_SIZE 4096 #define PAGE_SHIFT 12
+void memmap_preinit(void); + static inline u32 virt_to_phys(void *v) { return (u32)v; } diff --git a/src/romlayout.S b/src/romlayout.S index 823188b..acf0f32 100644 --- a/src/romlayout.S +++ b/src/romlayout.S @@ -42,8 +42,8 @@ transition32:
transition32_nmi_off: // Set segment descriptors - lidtw %cs:pmode_IDT_info - lgdtw %cs:rombios32_gdt_48 + lidtl %cs:pmode_IDT_info + lgdtl %cs:rombios32_gdt_48
// Enable protected mode movl %cr0, %ecx @@ -104,7 +104,7 @@ transition16big: ljmpw $SEG_BIOS, $2f
// restore IDT to normal real-mode defaults -2: lidtw %cs:rmode_IDT_info +2: lidtl %cs:rmode_IDT_info
// Clear segment registers xorw %cx, %cx @@ -412,6 +412,21 @@ __csm_return: popfw lretw
+// Entry point for protected mode exceptions + DECLFUNC entry_gp + .code32 +entry_gp: + movl %esp, %eax + calll _cfunc32flat_handle_gp - BUILD_BIOS_ADDR + .code16 + + DECLFUNC entry_pf + .code32 +entry_pf: + movl %esp, %eax + calll _cfunc32flat_handle_pf - BUILD_BIOS_ADDR + .code16 +
/**************************************************************** * Interrupt entry points diff --git a/src/x86.h b/src/x86.h index 53378e9..5de149a 100644 --- a/src/x86.h +++ b/src/x86.h @@ -92,6 +92,12 @@ static inline u16 cr0_vm86_read(void) { return cr0; }
+static inline u32 cr2_read(void) { + u32 cr2; + asm("movl %%cr2, %0" : "=r"(cr2)); + return cr2; +} + static inline u64 rdmsr(u32 index) { u64 ret; @@ -228,7 +234,7 @@ static inline u8 readb(const void *addr) {
// GDT bits #define GDT_CODE (0x9bULL << 40) // Code segment - P,R,A bits also set -#define GDT_DATA (0x93ULL << 40) // Data segment - W,A bits also set +#define GDT_DATA (0x93ULL << 40) // Data segment - P,W,A bits also set #define GDT_B (0x1ULL << 54) // Big flag #define GDT_G (0x1ULL << 55) // Granularity flag // GDT bits for segment base @@ -239,6 +245,17 @@ static inline u8 readb(const void *addr) { | (((u64)(v) & 0x0000ffff) << 0)) // GDT bits for segment limit (0-4Gig in 4K chunks) #define GDT_GRANLIMIT(v) (GDT_G | GDT_LIMIT((v) >> 12)) +// IDT bits +#define IDT_IRQ (0x8eULL << 40) // IRQ gate - P,D bits also set +#define IDT_OFFSET(v) ((((u64)(v) & 0xffff0000) << 32) \ + | ((u64)(v) & 0x0000ffff)) +#define IDT_SEGMENT(v) ((u64)(v) << 16) +#define IDT_VECTOR_GP 13 +#define IDT_VECTOR_PF 14 + +struct x86_exception_stack { + u32 error_code, eip, cs, eflags; +};
struct descloc_s { u16 length; @@ -251,6 +268,9 @@ static inline void sgdt(struct descloc_s *desc) { static inline void lgdt(struct descloc_s *desc) { asm("lgdtl %0" : : "m"(*desc) : "memory"); } +static inline void lidt(struct descloc_s *desc) { + asm("lidtl %0" : : "m"(*desc) : "memory"); +}
static inline u8 get_a20(void) { return (inb(PORT_A20) & A20_ENABLE_BIT) != 0;
Signed-off-by: Kevin O'Connor kevin@koconnor.net --- src/Kconfig | 7 +++ src/malloc.c | 3 ++ src/memmap.c | 129 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- src/memmap.h | 17 ++++++-- src/romlayout.S | 35 ++++++++++++--- src/stacks.c | 15 ++++++- src/x86.h | 23 ++++++++++ 7 files changed, 217 insertions(+), 12 deletions(-)
diff --git a/src/Kconfig b/src/Kconfig index b873cd3..c28f671 100644 --- a/src/Kconfig +++ b/src/Kconfig @@ -122,6 +122,13 @@ endchoice selected, the memory is instead allocated from the "9-segment" (0x90000-0xa0000).
+ config PAGING + bool "PAE paging support" + default y + help + Enable paging in 32bit mode on processors that support PAE + paging. + config ROM_SIZE int "ROM size (in KB)" default 0 diff --git a/src/malloc.c b/src/malloc.c index ec7a21f..c6b460e 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -418,6 +418,9 @@ malloc_preinit(void) // Mark known areas as reserved. e820_add(BUILD_BIOS_ADDR, BUILD_BIOS_SIZE, E820_RESERVED);
+ char memmap_early_data[MEMMAP_EARLY_DATA_SIZE]; + memmap_early_preinit(memmap_early_data); + // Populate temp high ram u32 highram = 0; int i; diff --git a/src/memmap.c b/src/memmap.c index b08313a..4e83112 100644 --- a/src/memmap.c +++ b/src/memmap.c @@ -64,8 +64,8 @@ handle_pf(struct x86_exception_stack *data) }
// Setup a 32bit General Protection (GP) and Page Fault (PF) handler -void -memmap_preinit(void) +static void +idt_preinit(void) { u32 ilen = sizeof(u64)*(IDT_VECTOR_PF+1); u64 *pidt = malloc_high(ilen); @@ -85,3 +85,128 @@ memmap_preinit(void) dprintf(1, "Installing pmode exception handler\n"); lidt(&pmode_IDT_info); } + + +/**************************************************************** + * PAE based paging + ****************************************************************/ + +u32 PageDirPtr VARFSEG; +u64 *PageDir; + +#define PAE_MAP_SIZE (2*1024*1024) +#define PAE_MAP_COUNT 2048 + +#define PG_P (1<<0) +#define PG_RW (1<<1) +#define PG_A (1<<5) +#define PG_D (1<<6) +#define PG_PS (1<<7) +#define PG_FLAGS (PG_P | PG_RW | PG_A | PG_D | PG_PS) + +u64 +memmap_virt_to_phys(void *v) +{ + if (!CONFIG_PAGING || !PageDirPtr) + return (u32)v; + int idx = (u32)v / PAE_MAP_SIZE; + int rem = (u32)v % PAE_MAP_SIZE; + return (PageDir[idx] & ~(PAE_MAP_SIZE-1)) + rem; +} + +void * +memmap_vmap(u64 addr, u32 len) +{ + if (!CONFIG_PAGING || !PageDirPtr) + return (void*)(u32)addr; + u64 newmap = (addr & ~(PAE_MAP_SIZE-1)) | PG_FLAGS; + u32 rem = addr % PAE_MAP_SIZE; + u64 key = addr / PAE_MAP_SIZE; + int pages = DIV_ROUND_UP(rem+len, PAE_MAP_SIZE); + int max; + for (max=64; max; max--, key*=33) { + u32 pos = key % PAE_MAP_COUNT; + if (pos + pages > PAE_MAP_COUNT) + continue; + int j; + for (j=0; j<pages; j++) + if (PageDir[pos+j] && PageDir[pos+j] != newmap + j*PAE_MAP_SIZE) + break; + if (j != pages) + continue; + for (j=0; j<pages; j++) + if (!PageDir[pos+j]) + PageDir[pos+j] = newmap + j*PAE_MAP_SIZE; + return (void*)(pos*PAE_MAP_SIZE + rem); + } + // It's not worth updating all callers to handle a failed mapping + panic("Unable to map address %llx with size %d\n", addr, len); +} + +// Detect PAE support and enable it if present +void +memmap_early_preinit(char *data) +{ + if (!CONFIG_PAGING) + return; + // Check if CPU supports PAE + u32 eax, ebx, ecx, edx, cpuid_features = 0; + cpuid(0, &eax, &ebx, &ecx, &edx); + if (!eax) + return; + cpuid(1, &eax, &ebx, &ecx, &cpuid_features); + if (!(cpuid_features & (1<<6))) + return; + + // Create page table in temporary ram + dprintf(1, "PAE support found\n"); + char *aligned_data = (void*)ALIGN((u32)data, PAGE_SIZE); + u64 *dir = (void*)aligned_data; + u64 *pdp = (void*)aligned_data + sizeof(u64) * PAE_MAP_COUNT; + if (!pdp || !dir) { + warn_noalloc(); + free(pdp); + free(dir); + return; + } + memset(dir, 0, sizeof(u64) * PAE_MAP_COUNT); + dir[0] = 0x0 | PG_FLAGS; + int i; + for (i=0; i<4; i++) + pdp[i] = ((u32)dir + i*4096) | PG_P; + + // Enable PAE + PageDirPtr = (u32)pdp; + PageDir = dir; + cr3_write(PageDirPtr); + cr4_mask(0, CR4_PAE); + cr0_mask(0, CR0_PG); +} + +void +memmap_preinit(void) +{ + if (!CONFIG_PAGING || !PageDirPtr) + return; + // Move temporary page tables to permanent storage + u64 *dir = memalign_high(PAGE_SIZE, sizeof(u64) * PAE_MAP_COUNT); + u64 *pdp = malloc_high(sizeof(u64)*4); + if (!pdp || !dir) { + warn_noalloc(); + free(pdp); + free(dir); + // XXX - may not be able to turn off paging at this point + PageDirPtr = 0; + PageDir = NULL; + return; + } + memcpy(dir, PageDir, sizeof(u64) * PAE_MAP_COUNT); + int i; + for (i=0; i<4; i++) + pdp[i] = ((u32)dir + i*4096) | PG_P; + PageDirPtr = (u32)pdp; + PageDir = dir; + cr3_write(PageDirPtr); + + idt_preinit(); +} diff --git a/src/memmap.h b/src/memmap.h index d4f150a..850d726 100644 --- a/src/memmap.h +++ b/src/memmap.h @@ -7,13 +7,22 @@ #define PAGE_SIZE 4096 #define PAGE_SHIFT 12
+extern u32 PageDirPtr; +u64 memmap_virt_to_phys(void *v); +void *memmap_vmap(u64 addr, u32 len); +#define MEMMAP_EARLY_DATA_SIZE (5*PAGE_SIZE + 4*sizeof(u64) - 1) +void memmap_early_preinit(char *data); void memmap_preinit(void);
-static inline u32 virt_to_phys(void *v) { - return (u32)v; +static inline u64 virt_to_phys(void *v) { + if (MODESEGMENT) + return (u32)v; + return memmap_virt_to_phys(v); } -static inline void *memremap(u32 addr, u32 len) { - return (void*)addr; +static inline void *memremap(u64 addr, u32 len) { + if (MODESEGMENT) + return (void*)(u32)addr; + return memmap_vmap(addr, len); } static inline void *ioremap(u64 addr, u32 len) { return memremap(addr, len); diff --git a/src/romlayout.S b/src/romlayout.S index acf0f32..d931516 100644 --- a/src/romlayout.S +++ b/src/romlayout.S @@ -51,12 +51,26 @@ transition32_nmi_off: orl $CR0_PE, %ecx movl %ecx, %cr0
+#if CONFIG_PAGING + movl %cs:PageDirPtr, %ecx + cmpl $0, %ecx + jz 1f + // Enable PAE paging + movl %ecx, %cr3 + movl %cr4, %ecx + orl $CR4_PAE, %ecx + movl %ecx, %cr4 + movl %cr0, %ecx + orl $CR0_PG, %ecx + movl %ecx, %cr0 +#endif + // start 32bit protected mode code - ljmpl $SEG32_MODE32_CS, $(BUILD_BIOS_ADDR + 1f) +1: ljmpl $SEG32_MODE32_CS, $(BUILD_BIOS_ADDR + 2f)
.code32 // init data segments -1: movl $SEG32_MODE32_DS, %ecx +2: movl $SEG32_MODE32_DS, %ecx movw %cx, %ds movw %cx, %es movw %cx, %ss @@ -97,14 +111,25 @@ transition16big: .code16 // Disable protected mode 1: movl %cr0, %ecx - andl $~CR0_PE, %ecx + andl $~(CR0_PE|CR0_PG), %ecx movl %ecx, %cr0
+#if CONFIG_PAGING + cmpl $0, %cs:PageDirPtr + jz 2f + // Disable PAE paging + movl %cr4, %ecx + andl $~CR4_PAE, %ecx + movl %ecx, %cr4 + xorl %ecx, %ecx + movl %ecx, %cr3 +#endif + // far jump to flush CPU queue after transition to real mode - ljmpw $SEG_BIOS, $2f +2: ljmpw $SEG_BIOS, $3f
// restore IDT to normal real-mode defaults -2: lidtl %cs:rmode_IDT_info +3: lidtl %cs:rmode_IDT_info
// Clear segment registers xorw %cx, %cx diff --git a/src/stacks.c b/src/stacks.c index fa9c7db..a3de397 100644 --- a/src/stacks.c +++ b/src/stacks.c @@ -10,6 +10,7 @@ #include "hw/rtc.h" // rtc_use #include "list.h" // hlist_node #include "malloc.h" // free +#include "memmap.h" // PageDirPtr #include "output.h" // dprintf #include "romfile.h" // romfile_loadint #include "stacks.h" // struct mutex_s @@ -28,7 +29,7 @@ struct { u8 cmosindex; u8 a20; u16 ss, fs, gs; - u32 cr0; + u32 cr0, cr3, cr4; struct descloc_s gdt; } Call16Data VARLOW;
@@ -48,6 +49,10 @@ call32_prep(u8 method) // Called in 16bit protected mode?! return -1; SET_LOW(Call16Data.cr0, cr0); + if (CONFIG_PAGING && GET_GLOBAL(PageDirPtr)) { + SET_LOW(Call16Data.cr3, cr3_read()); + SET_LOW(Call16Data.cr4, cr4_read()); + }
// Backup fs/gs and gdt SET_LOW(Call16Data.fs, GET_SEG(FS)); @@ -98,6 +103,14 @@ call32_post(void) u32 cr0_caching = GET_LOW(Call16Data.cr0) & (CR0_CD|CR0_NW); if (cr0_caching) cr0_mask(CR0_CD|CR0_NW, cr0_caching); + if (CONFIG_PAGING && GET_GLOBAL(PageDirPtr)) { + u32 cr4_pae = GET_LOW(Call16Data.cr4) & CR4_PAE; + if (cr4_pae) + cr4_mask(CR4_PAE, cr4_pae); + u32 cr3 = GET_LOW(Call16Data.cr3); + if (cr3) + cr3_write(cr3); + } }
// Restore cmos index register diff --git a/src/x86.h b/src/x86.h index 5de149a..58cc20c 100644 --- a/src/x86.h +++ b/src/x86.h @@ -14,6 +14,8 @@ #define CR0_NW (1<<29) // Not Write-through #define CR0_PE (1<<0) // Protection enable
+#define CR4_PAE (1<<5) + // PORT_A20 bitdefs #define PORT_A20 0x0092 #define A20_ENABLE_BIT 0x02 @@ -98,6 +100,27 @@ static inline u32 cr2_read(void) { return cr2; }
+static inline u32 cr3_read(void) { + u32 cr3; + asm("movl %%cr3, %0" : "=r"(cr3)); + return cr3; +} +static inline void cr3_write(u32 cr3) { + asm("movl %0, %%cr3" : : "r"(cr3)); +} + +static inline u32 cr4_read(void) { + u32 cr4; + asm("movl %%cr4, %0" : "=r"(cr4)); + return cr4; +} +static inline void cr4_write(u32 cr4) { + asm("movl %0, %%cr4" : : "r"(cr4)); +} +static inline void cr4_mask(u32 off, u32 on) { + cr4_write((cr4_read() & ~off) | on); +} + static inline u64 rdmsr(u32 index) { u64 ret;
This is a test. Changes to paravirt.c and stacks.c are not correct. --- src/fw/paravirt.c | 8 ++++++++ src/malloc.c | 48 +++++++++++++++++++++++++++++------------------- src/malloc.h | 15 +++++++++++---- src/stacks.c | 2 +- 4 files changed, 49 insertions(+), 24 deletions(-)
diff --git a/src/fw/paravirt.c b/src/fw/paravirt.c index acb44b9..6c71366 100644 --- a/src/fw/paravirt.c +++ b/src/fw/paravirt.c @@ -120,6 +120,14 @@ qemu_preinit(void) e820_add(0xfffc0000, 256*1024, E820_RESERVED);
dprintf(1, "RamSize: 0x%08x [cmos]\n", RamSize); + + // XXX + u64 high = ((rtc_read(CMOS_MEM_HIGHMEM_LOW) << 16) + | ((u32)rtc_read(CMOS_MEM_HIGHMEM_MID) << 24) + | ((u64)rtc_read(CMOS_MEM_HIGHMEM_HIGH) << 32)); + RamSizeOver4G = high; + e820_add(0x100000000ull, high, E820_RAM); + dprintf(1, "RamSizeOver4G: 0x%016llx [cmos]\n", RamSizeOver4G); }
void diff --git a/src/malloc.c b/src/malloc.c index c6b460e..9a684d6 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -18,7 +18,8 @@ // Information on a reserved area. struct allocinfo_s { struct hlist_node node; - u32 range_start, range_end, alloc_size; + u64 range_start, range_end; + u32 alloc_size; };
// Information on a tracked memory allocation. @@ -36,9 +37,10 @@ struct zone_s { struct zone_s ZoneLow VARVERIFY32INIT, ZoneHigh VARVERIFY32INIT; struct zone_s ZoneFSeg VARVERIFY32INIT; struct zone_s ZoneTmpLow VARVERIFY32INIT, ZoneTmpHigh VARVERIFY32INIT; +struct zone_s ZoneTmpHigh64 VARVERIFY32INIT;
static struct zone_s *Zones[] VARVERIFY32INIT = { - &ZoneTmpLow, &ZoneLow, &ZoneFSeg, &ZoneTmpHigh, &ZoneHigh + &ZoneTmpLow, &ZoneLow, &ZoneFSeg, &ZoneTmpHigh, &ZoneHigh, &ZoneTmpHigh64 };
@@ -47,14 +49,14 @@ static struct zone_s *Zones[] VARVERIFY32INIT = { ****************************************************************/
// Find and reserve space from a given zone -static u32 +static u64 alloc_new(struct zone_s *zone, u32 size, u32 align, struct allocinfo_s *fill) { struct allocinfo_s *info; hlist_for_each_entry(info, &zone->head, node) { - u32 alloc_end = info->range_start + info->alloc_size; - u32 range_end = info->range_end; - u32 new_range_end = ALIGN_DOWN(range_end - size, align); + u64 alloc_end = info->range_start + info->alloc_size; + u64 range_end = info->range_end; + u64 new_range_end = ALIGN_DOWN(range_end - size, align); if (new_range_end >= alloc_end && new_range_end <= range_end) { // Found space - now reserve it. fill->range_start = new_range_end; @@ -73,14 +75,18 @@ alloc_new(struct zone_s *zone, u32 size, u32 align, struct allocinfo_s *fill) static struct allocdetail_s * alloc_new_detail(struct allocdetail_s *temp) { - u32 detail_addr = alloc_new(&ZoneTmpHigh, sizeof(struct allocdetail_s) + u64 detail_addr = alloc_new(&ZoneTmpHigh64, sizeof(struct allocdetail_s) , MALLOC_MIN_ALIGN, &temp->detailinfo); if (!detail_addr) { - detail_addr = alloc_new(&ZoneTmpLow, sizeof(struct allocdetail_s) + detail_addr = alloc_new(&ZoneTmpHigh, sizeof(struct allocdetail_s) , MALLOC_MIN_ALIGN, &temp->detailinfo); if (!detail_addr) { - warn_noalloc(); - return NULL; + detail_addr = alloc_new(&ZoneTmpLow, sizeof(struct allocdetail_s) + , MALLOC_MIN_ALIGN, &temp->detailinfo); + if (!detail_addr) { + warn_noalloc(); + return NULL; + } } } struct allocdetail_s *detail = memremap(detail_addr, sizeof(*detail)); @@ -94,7 +100,7 @@ alloc_new_detail(struct allocdetail_s *temp)
// Add new memory to a zone static void -alloc_add(struct zone_s *zone, u32 start, u32 end) +alloc_add(struct zone_s *zone, u64 start, u64 end) { // Find position to add space struct allocinfo_s *info; @@ -131,7 +137,7 @@ alloc_free(struct allocinfo_s *info)
// Search all zones for an allocation obtained from alloc_new() static struct allocinfo_s * -alloc_find(u32 data) +alloc_find(u64 data) { int i; for (i=0; i<ARRAY_SIZE(Zones); i++) { @@ -232,7 +238,7 @@ zonelow_expand(u32 size, u32 align, struct allocinfo_s *fill) ****************************************************************/
// Allocate physical memory from the given zone and track it as a PMM allocation -u32 +u64 phys_alloc(struct zone_s *zone, u32 size, u32 align) { ASSERT32FLAT(); @@ -242,7 +248,7 @@ phys_alloc(struct zone_s *zone, u32 size, u32 align) // Find and reserve space for main allocation struct allocdetail_s tempdetail; tempdetail.handle = MALLOC_DEFAULT_HANDLE; - u32 data = alloc_new(zone, size, align, &tempdetail.datainfo); + u64 data = alloc_new(zone, size, align, &tempdetail.datainfo); if (!CONFIG_MALLOC_UPPERMEMORY && !data && zone == &ZoneLow) data = zonelow_expand(size, align, &tempdetail.datainfo); if (!data) @@ -255,7 +261,7 @@ phys_alloc(struct zone_s *zone, u32 size, u32 align) return 0; }
- dprintf(8, "phys_alloc zone=%p size=%d align=%x ret=%x (detail=%p)\n" + dprintf(8, "phys_alloc zone=%p size=%d align=%x ret=%llx (detail=%p)\n" , zone, size, align, data, detail);
return data; @@ -270,7 +276,7 @@ _malloc(struct zone_s *zone, u32 size, u32 align)
// Free a data block allocated with phys_alloc int -phys_free(u32 data) +phys_free(u64 data) { ASSERT32FLAT(); struct allocinfo_s *info = alloc_find(data); @@ -278,7 +284,7 @@ phys_free(u32 data) return -1; struct allocdetail_s *detail = container_of( info, struct allocdetail_s, datainfo); - dprintf(8, "phys_free %x (detail=%p)\n", data, detail); + dprintf(8, "phys_free %llx (detail=%p)\n", data, detail); alloc_free(info); alloc_free(&detail->detailinfo); return 0; @@ -429,8 +435,12 @@ malloc_preinit(void) u64 end = en->start + en->size; if (end < 1024*1024) break; - if (en->type != E820_RAM || end > 0xffffffff) + if (en->type != E820_RAM) continue; + if (end > 0xffffffff) { + alloc_add(&ZoneTmpHigh64, en->start, end); + continue; + } u32 s = en->start, e = end; if (!highram) { u32 newe = ALIGN_DOWN(e - BUILD_MAX_HIGHTABLE, MALLOC_MIN_ALIGN); @@ -548,7 +558,7 @@ malloc_prepboot(void) struct allocinfo_s *info = alloc_get_sentinel(&ZoneFSeg); u32 size = info->range_end - info->range_start; memset(memremap(info->range_start, size), 0, size); - dprintf(1, "Space available for UMB: %x-%x, %x-%x\n" + dprintf(1, "Space available for UMB: %x-%x, %llx-%llx\n" , RomEnd, base, info->range_start, info->range_end);
// Give back unused high ram. diff --git a/src/malloc.h b/src/malloc.h index 84c285e..d964361 100644 --- a/src/malloc.h +++ b/src/malloc.h @@ -5,6 +5,7 @@
// malloc.c extern struct zone_s ZoneLow, ZoneHigh, ZoneFSeg, ZoneTmpLow, ZoneTmpHigh; +extern struct zone_s ZoneTmpHigh64; u32 rom_get_max(void); u32 rom_get_last(void); struct rom_header *rom_reserve(u32 size); @@ -15,9 +16,9 @@ void malloc_preinit(void); extern u32 LegacyRamSize; void malloc_init(void); void malloc_prepboot(void); -u32 phys_alloc(struct zone_s *zone, u32 size, u32 align); +u64 phys_alloc(struct zone_s *zone, u32 size, u32 align); void *_malloc(struct zone_s *zone, u32 size, u32 align); -int phys_free(u32 data); +int phys_free(u64 data); void free(void *data); u32 malloc_getspace(struct zone_s *zone); void malloc_sethandle(u32 data, u32 handle); @@ -43,7 +44,10 @@ static inline void *malloc_tmphigh(u32 size) { return _malloc(&ZoneTmpHigh, size, MALLOC_MIN_ALIGN); } static inline void *malloc_tmp(u32 size) { - void *ret = malloc_tmphigh(size); + void *ret = _malloc(&ZoneTmpHigh64, size, MALLOC_MIN_ALIGN); + if (ret) + return ret; + ret = malloc_tmphigh(size); if (ret) return ret; return malloc_tmplow(size); @@ -61,7 +65,10 @@ static inline void *memalign_tmphigh(u32 align, u32 size) { return _malloc(&ZoneTmpHigh, size, align); } static inline void *memalign_tmp(u32 align, u32 size) { - void *ret = memalign_tmphigh(align, size); + void *ret = _malloc(&ZoneTmpHigh64, size, align); + if (ret) + return ret; + ret = memalign_tmphigh(align, size); if (ret) return ret; return memalign_tmplow(align, size); diff --git a/src/stacks.c b/src/stacks.c index a3de397..583b456 100644 --- a/src/stacks.c +++ b/src/stacks.c @@ -563,7 +563,7 @@ run_thread(void (*func)(void*), void *data) if (! CONFIG_THREADS || ! ThreadControl) goto fail; struct thread_info *thread; - thread = memalign_tmphigh(THREADSTACKSIZE, THREADSTACKSIZE); + thread = memalign_tmp(THREADSTACKSIZE, THREADSTACKSIZE); // XXX if (!thread) goto fail;
-
Kevin O'Connor