[SeaBIOS] [PATCHv3] vgabios: Don't use extra stack if it appears a modern OS is in use

[Kevin O'Connor]

If the last mode set (while not in vm86 mode) was done from a VBE mode
set call then disable the extra stack.  This works under the premise
that only a modern OS would invoke the VBE mode changing facilities
and a modern OS would always call the vgabios with sufficient stack
space.

This is an ugly hack to work around a problem Windows Vista (and
possibly later Windows releases) has with the VGA BIOS using a stack
in the e-segment.

Reported-by: Richard Laager <rlaager at wiktel.com>
Signed-off-by: Kevin O'Connor <kevin at koconnor.net>
---

Unfortunately, the previous patch I sent would not work well if a boot
loader issues a VBE mode set call.  (Indeed, SeaBIOS itself will issue
a vbe mode set call when a bootsplash image is in use.)  In that
situation, with the previous patch, the extra stack would be disabled
prematurely.

This patch extends the previous idea - it disables the extra stack if
the last video mode switch was a vbe mode switch.  Thus, if a
bootloader (or seabios itself) switches modes via vbe, but then
switches back using a legacy mode switch, the extra stack will get
reenabled.  I've also thrown in the extra check of only changing the
extra stack setting if the last mode change was not done in vm86 mode.
This wasn't required in my tests with skifree, but I added in to make
sure a mode switch from within a Windows emulation session wouldn't
reenable the extra stack and cause a fault.

Patch is also at:
  https://github.com/KevinOConnor/seabios/tree/testing

-Kevin


---
 src/x86.h         |  5 +++++
 vgasrc/vgabios.c  |  3 +++
 vgasrc/vgabios.h  |  4 +++-
 vgasrc/vgaentry.S | 15 +++++++++++++++
 4 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/src/x86.h b/src/x86.h
index 7798b1c..14ebb7d 100644
--- a/src/x86.h
+++ b/src/x86.h
@@ -83,6 +83,11 @@ static inline u32 getcr0(void) {
 static inline void setcr0(u32 cr0) {
     asm("movl %0, %%cr0" : : "r"(cr0));
 }
+static inline u16 getcr0_vm86(void) {
+    u16 cr0;
+    asm("smsww %0" : "=r"(cr0));
+    return cr0;
+}
 
 static inline u64 rdmsr(u32 index)
 {
diff --git a/vgasrc/vgabios.c b/vgasrc/vgabios.c
index 4aa50e1..ec064fd 100644
--- a/vgasrc/vgabios.c
+++ b/vgasrc/vgabios.c
@@ -304,6 +304,9 @@ vga_set_mode(int mode, int flags)
         SET_BDA(video_mode, 0xff);
     SET_BDA_EXT(vbe_mode, mode | (flags & MF_VBEFLAGS));
     SET_BDA_EXT(vgamode_offset, (u32)vmode_g);
+    if (!(getcr0_vm86() & CR0_PE))
+        MASK_BDA_EXT(flags, BF_LEGACY_MODE
+                     , (flags & MF_LEGACY) ? BF_LEGACY_MODE : 0);
     if (memmodel == MM_TEXT) {
         SET_BDA(video_cols, width);
         SET_BDA(video_rows, height-1);
diff --git a/vgasrc/vgabios.h b/vgasrc/vgabios.h
index fd796f2..b3c5594 100644
--- a/vgasrc/vgabios.h
+++ b/vgasrc/vgabios.h
@@ -62,7 +62,8 @@ struct gfx_op {
 #define GO_MEMSET  3
 #define GO_MEMMOVE 4
 
-// Custom internal storage in BDA
+// Custom internal storage in BDA (don't change here without also
+// updating vgaentry.S)
 #define VGA_CUSTOM_BDA 0xb9
 
 struct vga_bda_s {
@@ -74,6 +75,7 @@ struct vga_bda_s {
 #define BF_PM_MASK      0x0f
 #define BF_EMULATE_TEXT 0x10
 #define BF_SWCURSOR     0x20
+#define BF_LEGACY_MODE  0x40
 
 #define GET_BDA_EXT(var) \
     GET_FARVAR(SEG_BDA, ((struct vga_bda_s *)VGA_CUSTOM_BDA)->var)
diff --git a/vgasrc/vgaentry.S b/vgasrc/vgaentry.S
index f9cf656..a1646a9 100644
--- a/vgasrc/vgaentry.S
+++ b/vgasrc/vgaentry.S
@@ -104,6 +104,9 @@ entry_10:
         ENTRY_ARG_VGA handle_10
         iretw
 
+#define VGA_CUSTOM_BDA_FLAGS 0xb9
+#define BF_LEGACY_MODE 0x40
+
         // Entry point using extra stack
         DECLFUNC entry_10_extrastack
 entry_10_extrastack:
@@ -111,6 +114,13 @@ entry_10_extrastack:
         cld
         pushw %ds               // Set %ds:%eax to space on ExtraStack
         pushl %eax
+
+        movw $SEG_BDA, %ax      // Check if extra stack is enabled
+        movw %ax, %ds
+        movb VGA_CUSTOM_BDA_FLAGS, %al
+        testb $BF_LEGACY_MODE, %al
+        jz 1f
+
         movw %cs:ExtraStackSeg, %ds
         movl $(CONFIG_VGA_EXTRA_STACK_SIZE-PUSHBREGS_size-16), %eax
         SAVEBREGS_POP_DSEAX     // Save registers on extra stack
@@ -134,6 +144,11 @@ entry_10_extrastack:
         RESTOREBREGS_DSEAX
         iretw
 
+1:      // Use regular entry point if the extra stack is disabled
+        popl %eax
+        popw %ds
+        jmp entry_10
+
         // Timer irq handling
         DECLFUNC entry_timer_hook
 entry_timer_hook:
-- 
1.9.3