[SeaBIOS] [PATCH RFC 2/2] support booting with more than 255 CPUs

[Kevin O'Connor]

On Mon, May 09, 2016 at 11:43:54AM +0200, Igor Mammedov wrote:
> SDM[*1] says that if there are CPUs with APIC ID
> greater than 254, BIOS is to pass control to OS
> in x2APIC mode. Use the fact that QEMU passes in
> "etc/max-cpus" max possible "APIC ID + 1" to
> detect need for x2APIC mode. Also instead of
> CMOS_BIOS_SMP_COUNT which is limited to 256 CPUs
> use a new rom file "etc/boot-cpus" that QEMU
> supporting more than 256 CPUs will provide.
> 
> *1) SDM: Volume 3: EXTENDED XAPIC (X2APIC):
>      Initialization by System Software
> 
> Signed-off-by: Igor Mammedov <imammedo at redhat.com>
> ---
>  src/fw/smp.c | 48 +++++++++++++++++++++++++++++++++++++++---------
>  src/x86.h    |  1 +
>  2 files changed, 40 insertions(+), 9 deletions(-)
> 
> diff --git a/src/fw/smp.c b/src/fw/smp.c
> index 579acdb..2bb5e1b 100644
> --- a/src/fw/smp.c
> +++ b/src/fw/smp.c
> @@ -19,6 +19,9 @@
>  #define APIC_LINT1   ((u8*)BUILD_APIC_ADDR + 0x360)
>  
>  #define APIC_ENABLED 0x0100
> +#define MSR_IA32_APIC_BASE 0x01B
> +#define MSR_LOCAL_APIC_ID 0x802
> +#define MSR_IA32_APICBASE_EXTD (1ULL << 10) /* Enable x2APIC mode */
>  
>  static struct { u32 index; u64 val; } smp_mtrr[32];
>  static u32 smp_mtrr_count;
> @@ -46,6 +49,19 @@ int apic_id_is_present(u8 apic_id)
>      return !!(FoundAPICIDs[apic_id/32] & (1ul << (apic_id % 32)));
>  }
>  
> +static void handle_x2apic(u32 has_x2apic)
> +{
> +    if (MaxCountCPUs < 256)
> +        return;
> +
> +    if (!has_x2apic)
> +        return;
> +
> +    // switch to x2APIC mode
> +    u64 apic_base = rdmsr(MSR_IA32_APIC_BASE);
> +    wrmsr(MSR_IA32_APIC_BASE, apic_base | MSR_IA32_APICBASE_EXTD);
> +}
> +
>  void VISIBLE32FLAT
>  handle_smp(void)
>  {
> @@ -55,17 +71,24 @@ handle_smp(void)
>      // Detect apic_id
>      u32 eax, ebx, ecx, cpuid_features;
>      cpuid(1, &eax, &ebx, &ecx, &cpuid_features);
> -    u8 apic_id = ebx>>24;
> -    dprintf(DEBUG_HDL_smp, "handle_smp: apic_id=%d\n", apic_id);

Is it no longer possible to report an identifier for the apic?  Was
the dprintf removed because the log was filled with cpu reports or
because there is no equivalent id?

> +
> +    handle_x2apic(ecx & CPUID_X2APIC);
>  
>      // MTRR setup
>      int i;
>      for (i=0; i<smp_mtrr_count; i++)
>          wrmsr(smp_mtrr[i].index, smp_mtrr[i].val);
>  
> -    // Set bit on FoundAPICIDs
> -    FoundAPICIDs[apic_id/32] |= (1 << (apic_id % 32));
> -
> +    /*
> +     * QEMU that supports APIC ID > 255 provides its own BIOS tables
> +     * so skip filling present APIC map as it's not used.
> +     * (it's used for internal BIOS tables for QEMU older than 1.8)
> +     */
> +    if (MaxCountCPUs < 255) {
> +       u32 apic_id = ebx>>24;
> +       // Set bit on FoundAPICIDs
> +       FoundAPICIDs[apic_id/32] |= (1 << (apic_id % 32));
> +    }
>      CountCPUs++;
>  }
>  
> @@ -91,6 +114,11 @@ smp_setup(void)
>          return;
>      }
>  
> +    /* set max possible APIC ID limit for AP bootstrap to decide
> +     * if it neds to switch into x2APIC mode
> +     */
> +    MaxCountCPUs = romfile_loadint("etc/max-cpus", 1);
> +
>      // mark the BSP initial APIC ID as found, too:
>      u8 apic_id = ebx>>24;
>      FoundAPICIDs[apic_id/32] |= (1 << (apic_id % 32));

This updates FoundAPICIDs even if MaxCountCPUs > 255 which is a little
confusing.

I think this patch would be simpler if the updating of FoundAPICIDs
was refactored first.  Something like the below (totally untested).

-Kevin


--- a/src/fw/smp.c
+++ b/src/fw/smp.c
@@ -46,27 +46,34 @@ int apic_id_is_present(u8 apic_id)
     return !!(FoundAPICIDs[apic_id/32] & (1ul << (apic_id % 32)));
 }
 
+static int
+count_cpu(void)
+{
+    CountCPUs++;
+
+    // Track found apic id for use in legacy internal bios tables
+    u32 eax, ebx, ecx, cpuid_features;
+    cpuid(1, &eax, &ebx, &ecx, &cpuid_features);
+    u8 apic_id = ebx>>24;
+    FoundAPICIDs[apic_id/32] |= 1 << (apic_id % 32);
+
+    return apic_id;
+}
+
 void VISIBLE32FLAT
 handle_smp(void)
 {
     if (!CONFIG_QEMU)
         return;
 
-    // Detect apic_id
-    u32 eax, ebx, ecx, cpuid_features;
-    cpuid(1, &eax, &ebx, &ecx, &cpuid_features);
-    u8 apic_id = ebx>>24;
+    // Track this CPU and detect the apic_id
+    int apic_id = count_cpu();
     dprintf(DEBUG_HDL_smp, "handle_smp: apic_id=%d\n", apic_id);
 
     // MTRR setup
     int i;
     for (i=0; i<smp_mtrr_count; i++)
         wrmsr(smp_mtrr[i].index, smp_mtrr[i].val);
-
-    // Set bit on FoundAPICIDs
-    FoundAPICIDs[apic_id/32] |= (1 << (apic_id % 32));
-
-    CountCPUs++;
 }
 
 // Atomic lock for shared stack across processors.
@@ -91,10 +98,8 @@ smp_setup(void)
         return;
     }
 
-    // mark the BSP initial APIC ID as found, too:
-    u8 apic_id = ebx>>24;
-    FoundAPICIDs[apic_id/32] |= (1 << (apic_id % 32));
-    CountCPUs = 1;
+    // Detect initial boot cpu
+    count_cpu();
 
     // Setup jump trampoline to counter code.
     u64 old = *(u64*)BUILD_AP_BOOT_ADDR;