Re: [SeaBIOS] [PATCH RFC 2/2] support booting with more than 255 CPUs

[Igor Mammedov]

On Mon, 9 May 2016 11:49:36 -0400
"Kevin O'Connor"  wrote:
> On Mon, May 09, 2016 at 11:43:54AM +0200, Igor Mammedov wrote:
> > SDM[*1] says that if there are CPUs with APIC ID
> > greater than 254, BIOS is to pass control to OS
> > in x2APIC mode. Use the fact that QEMU passes in
> > "etc/max-cpus" max possible "APIC ID + 1" to
> > detect need for x2APIC mode. Also instead of
> > CMOS_BIOS_SMP_COUNT which is limited to 256 CPUs
> > use a new rom file "etc/boot-cpus" that QEMU
> > supporting more than 256 CPUs will provide.
> >
> > *1) SDM: Volume 3: EXTENDED XAPIC (X2APIC):
> >      Initialization by System Software
> > Signed-off-by: Igor Mammedov
> >
> > src/fw/smp.c | 48 +++++++++++++++++++++++++++++++++++++++---------
> >  src/x86.h    |  1 +
> >  2 files changed, 40 insertions(+), 9 deletions(-)
> >
> > diff --git a/src/fw/smp.c b/src/fw/smp.c
> > index 579acdb..2bb5e1b 100644
> > --- a/src/fw/smp.c
> > +++ b/src/fw/smp.c
> > @@ -19,6 +19,9 @@
> >  #define APIC_LINT1   ((u8*)BUILD_APIC_ADDR + 0x360)
> >
> > #define APIC_ENABLED 0x0100
> > +#define MSR_IA32_APIC_BASE 0x01B
> > +#define MSR_LOCAL_APIC_ID 0x802
> > +#define MSR_IA32_APICBASE_EXTD (1ULL << 10) /* Enable x2APIC mode */
> >
> > static struct { u32 index; u64 val; } smp_mtrr[32];
> >  static u32 smp_mtrr_count;
> > @@ -46,6 +49,19 @@ int apic_id_is_present(u8 apic_id)
> >      return !!(FoundAPICIDs[apic_id/32] & (1ul << (apic_id % 32)));
> >  }
> >
> > +static void handle_x2apic(u32 has_x2apic)
> > +{
> > if (MaxCountCPUs < 256)
> >    return;
> > if (!has_x2apic)
> >    return;
> > // switch to x2APIC mode
> > u64 apic_base = rdmsr(MSR_IA32_APIC_BASE);
> > wrmsr(MSR_IA32_APIC_BASE, apic_base | MSR_IA32_APICBASE_EXTD);
> >
> > +}
> >
> > void VISIBLE32FLAT
> >  handle_smp(void)
> >  {
> > @@ -55,17 +71,24 @@ handle_smp(void)
> >      // Detect apic_id
> >      u32 eax, ebx, ecx, cpuid_features;
> >      cpuid(1, &eax, &ebx, &ecx, &cpuid_features);
> > u8 apic_id = ebx>>24;
> > dprintf(DEBUG_HDL_smp, "handle_smp: apic_id=%d\n", apic_id);
>
> Is it no longer possible to report an identifier for the apic?  Was
> the dprintf removed because the log was filled with cpu reports or
> because there is no equivalent id?

I've deleted it by accident, it's possible to read x2APIC ID from
from CPUID.0BH.EDX (but that isn't supported by current QEMU, so it
returns bogus 0 value) or from MSR after switching into x2APIC mode.
I'll add it back using value returned from MSR.
> > handle_x2apic(ecx & CPUID_X2APIC);
> >
> > // MTRR setup
> > int i;
> > for (i=0; i<smp_mtrr_count; i++)
> >     wrmsr(smp_mtrr[i].index, smp_mtrr[i].val);
> > // Set bit on FoundAPICIDs
> > FoundAPICIDs[apic_id/32] |= (1 << (apic_id % 32));
> > /*
> > * QEMU that supports APIC ID > 255 provides its own BIOS tables
> > * so skip filling present APIC map as it's not used.
> > * (it's used for internal BIOS tables for QEMU older than 1.8)
> > */
> > if (MaxCountCPUs < 255) {
> >   u32 apic_id = ebx>>24;
> >   // Set bit on FoundAPICIDs
> >   FoundAPICIDs[apic_id/32] |= (1 << (apic_id % 32));
> > }
> > CountCPUs++;
> >
> > }
> >
> > @@ -91,6 +114,11 @@ smp_setup(void)
> >          return;
> >      }
> > /* set max possible APIC ID limit for AP bootstrap to decide
> > * if it neds to switch into x2APIC mode
> > */
> > MaxCountCPUs = romfile_loadint("etc/max-cpus", 1);
> > // mark the BSP initial APIC ID as found, too:
> >    u8 apic_id = ebx>>24;
> >    FoundAPICIDs[apic_id/32] |= (1 << (apic_id % 32));
>
> This updates FoundAPICIDs even if MaxCountCPUs > 255 which is a little
> confusing.
>
> I think this patch would be simpler if the updating of FoundAPICIDs
> was refactored first.  Something like the below (totally untested).
>
> -Kevin
>
> --- a/src/fw/smp.c
> +++ b/src/fw/smp.c
> @@ -46,27 +46,34 @@ int apic_id_is_present(u8 apic_id)
>      return !!(FoundAPICIDs[apic_id/32] & (1ul << (apic_id % 32)));
>  }
>
> +static int
> +count_cpu(void)
> +{
> CountCPUs++;
> // Track found apic id for use in legacy internal bios tables
> u32 eax, ebx, ecx, cpuid_features;
> cpuid(1, &eax, &ebx, &ecx, &cpuid_features);
> u8 apic_id = ebx>>24;
> FoundAPICIDs[apic_id/32] |= 1 << (apic_id % 32);
> return apic_id;
>
> +}
>
> void VISIBLE32FLAT
>  handle_smp(void)
>  {
>      if (!CONFIG_QEMU)
>          return;
> // Detect apic_id
> u32 eax, ebx, ecx, cpuid_features;
> cpuid(1, &eax, &ebx, &ecx, &cpuid_features);
> u8 apic_id = ebx>>24;
>
> // Track this CPU and detect the apic_id
>
> int apic_id = count_cpu();
> dprintf(DEBUG_HDL_smp, "handle_smp: apic_id=%d\n", apic_id);
>
> // MTRR setup
> int i;
> for (i=0; i<smp_mtrr_count; i++)
>     wrmsr(smp_mtrr[i].index, smp_mtrr[i].val);
> // Set bit on FoundAPICIDs
> FoundAPICIDs[apic_id/32] |= (1 << (apic_id % 32));
> CountCPUs++;
>
> }
>
> // Atomic lock for shared stack across processors.
> @@ -91,10 +98,8 @@ smp_setup(void)
>          return;
>      }
> // mark the BSP initial APIC ID as found, too:
> u8 apic_id = ebx>>24;
> FoundAPICIDs[apic_id/32] |= (1 << (apic_id % 32));
> CountCPUs = 1;
>
> // Detect initial boot cpu
>
> count_cpu();
>
> // Setup jump trampoline to counter code.
> u64 old = *(u64*)BUILD_AP_BOOT_ADDR;