Author: rminnich Date: 2008-08-28 07:41:48 +0200 (Thu, 28 Aug 2008) New Revision: 839
Modified: coreboot-v3/arch/x86/amd/model_fxx/dualcore.c coreboot-v3/arch/x86/amd/model_fxx/dualcore_id.c coreboot-v3/arch/x86/amd/model_fxx/init_cpus.c coreboot-v3/mainboard/amd/serengeti/Makefile coreboot-v3/mainboard/amd/serengeti/initram.c Log: Off to bed. I'm done for now.
If anyone wants to review my comments, or maybe even try to fix compile issues, have at it :-)
The more I work with the K8 stuff the more impressed I am with the people who got it all to go 6 years ago. (and at how much I've forgotten but that's another story :=)
If we can get this next step done we're very close to having a working initram.
And, once you have ram and hit stage2, life is just better all around.
Signed-off-by: Ronald G. Minnich rminnich@gmail.com Acked-by: Ronald G. Minnich rminnich@gmail.com
Modified: coreboot-v3/arch/x86/amd/model_fxx/dualcore.c =================================================================== --- coreboot-v3/arch/x86/amd/model_fxx/dualcore.c 2008-08-28 02:55:14 UTC (rev 838) +++ coreboot-v3/arch/x86/amd/model_fxx/dualcore.c 2008-08-28 05:41:48 UTC (rev 839) @@ -34,147 +34,59 @@ }
#if SET_NB_CFG_54 == 1 -static inline uint8_t set_apicid_cpuid_lo(void) +u8 set_apicid_cpuid_lo(void) { -#if K8_REV_F_SUPPORT == 0 - if(is_cpu_pre_e0()) return 0; // pre_e0 can not be set -#endif - - // set the NB_CFG[54]=1; why the OS will be happy with that ??? - msr_t msr; + // set the NB_CFG[54]=1; Why the OS thinks this is a good thing is not yet known. + struct msr msr; msr = rdmsr(NB_CFG_MSR); msr.hi |= (1<<(54-32)); // InitApicIdCpuIdLo wrmsr(NB_CFG_MSR, msr);
return 1; } -#else - -static inline void set_apicid_cpuid_lo(void) { } - #endif
-static inline void real_start_other_core(unsigned nodeid) +/** + * Start the cores on a given node (cores > 0). It is important that MC4 and other accesses + * be directed to core 0, per the BKDG. Note that this is pretty dual core specific. + * @param nodeid Node on which to start the cores + */ +void start_cores(unsigned nodeid) { - uint32_t dword; - // set PCI_DEV(0, 0x18+nodeid, 3), 0x44 bit 27 to redirect all MC4 accesses and error logging to core0 - dword = pci_read_config32(PCI_DEV(0, 0x18+nodeid, 3), 0x44); - dword |= 1<<27; // NbMcaToMstCpuEn bit - pci_write_config32(PCI_DEV(0, 0x18+nodeid, 3), 0x44, dword); + u32 dword; + /* set PCI_DEV(0, 0x18+nodeid, 3), 0x44 bit 27 to redirect all MC4 + * accesses and error logging to core0 + */ + dword = pci_read_config32(PCI_DEV(0, 0x18+nodeid, 3), MCA_NB_CONFIG); + dword |= MNC_NBMCATOMSTCPUEN; // NbMcaToMstCpuEn bit + pci_write_config32(PCI_DEV(0, 0x18+nodeid, 3), MCA_NB_CONFIG, dword); // set PCI_DEV(0, 0x18+nodeid, 0), 0x68 bit 5 to start core1 - dword = pci_read_config32(PCI_DEV(0, 0x18+nodeid, 0), 0x68); - dword |= 1<<5; - pci_write_config32(PCI_DEV(0, 0x18+nodeid, 0), 0x68, dword); + dword = pci_read_config32(PCI_DEV(0, 0x18+nodeid, 0), HT_TRANSACTION_CONTROL); + dword |= HTTC_CPU1_EN; + pci_write_config32(PCI_DEV(0, 0x18+nodeid, 0), HT_TRANSACTION_CONTROL, dword); }
-//it is running on core0 of node0 +/** + * start cores on all nodes including BSP. This is assumed to be running on core 0 of node 0 + */ static inline void start_other_cores(void) { unsigned nodes; unsigned nodeid; + int dual_core = 0;
- if(read_option(CMOS_VSTART_dual_core, CMOS_VLEN_dual_core, 0) != 0) { // disable dual_core - return; - } + get_option(&dual_core, "dual_core");
+ if (! dual_core) + return; + nodes = get_nodes();
for(nodeid=0; nodeid<nodes; nodeid++) { - if( get_core_num_in_bsp(nodeid) > 0) { - real_start_other_core(nodeid); + if( get_core_count(nodeid) > 0) { + start_cores(nodeid); } }
} -#if USE_DCACHE_RAM == 0 -static void do_k8_init_and_stop_secondaries(void) -{ - struct node_core_id id; - device_t dev; - unsigned apicid; - unsigned max_siblings; - msr_t msr; - - /* Skip this if there was a built in self test failure */
- if (is_cpu_pre_e0()) { - id.nodeid = lapicid() & 0x7; - id.coreid = 0; - } else { - /* Which cpu are we on? */ - id = get_node_core_id_x(); - - /* Set NB_CFG_MSR - * Linux expect the core to be in the least signficant bits. - */ - msr = rdmsr(NB_CFG_MSR); - msr.hi |= (1<<(54-32)); // InitApicIdCpuIdLo - wrmsr(NB_CFG_MSR, msr); - } - - /* For now assume all cpus have the same number of siblings */ - max_siblings = (cpuid_ecx(0x80000008) & 0xff) + 1; - - /* Enable extended apic ids */ - device_t dev_f0 = PCI_DEV(0, 0x18+id.nodeid, 0); - unsigned val = pci_read_config32(dev_f0, 0x68); - val |= (1 << 18) | (1 << 17); - pci_write_config32(dev_f0, 0x68, val); - - /* Set the lapicid */ - #if (ENABLE_APIC_EXT_ID == 1) - unsigned initial_apicid = get_initial_apicid(); - #if LIFT_BSP_APIC_ID == 0 - if( initial_apicid != 0 ) // other than bsp - #endif - { - /* use initial apic id to lift it */ - uint32_t dword = lapic_read(LAPIC_ID); - dword &= ~(0xff<<24); - dword |= (((initial_apicid + APIC_ID_OFFSET) & 0xff)<<24); - - lapic_write(LAPIC_ID, dword); - } - - #if LIFT_BSP_APIC_ID == 1 - bsp_apicid += APIC_ID_OFFSET; - #endif - - #endif - - - /* Remember the cpuid */ - if (id.coreid == 0) { - dev = PCI_DEV(0, 0x18 + id.nodeid, 2); - pci_write_config32(dev, 0x9c, cpuid_eax(1)); - } - - /* Maybe call distinguish_cpu_resets only on the last core? */ - distinguish_cpu_resets(id.nodeid); - if (!boot_cpu()) { - stop_this_cpu(); - } -} - -static void k8_init_and_stop_secondaries(void) -{ - /* This doesn't work with Cache As Ram because it messes with - the MTRR state, which breaks the init detection. - do_k8_init_and_stop_secondaries should be usable by CAR code. - */ - - int init_detected; - - init_detected = early_mtrr_init_detected(); - amd_early_mtrr_init(); - - enable_lapic(); - init_timer(); - if (init_detected) { - asm volatile ("jmp __cpu_reset"); - } - - do_k8_init_and_stop_secondaries(); -} - -#endif
Modified: coreboot-v3/arch/x86/amd/model_fxx/dualcore_id.c =================================================================== --- coreboot-v3/arch/x86/amd/model_fxx/dualcore_id.c 2008-08-28 02:55:14 UTC (rev 838) +++ coreboot-v3/arch/x86/amd/model_fxx/dualcore_id.c 2008-08-28 05:41:48 UTC (rev 839) @@ -1,3 +1,15 @@ +#include <mainboard.h> +#include <types.h> +#include <lib.h> +#include <console.h> +#include <globalvars.h> +#include <device/device.h> +#include <device/pci.h> +#include <string.h> +#include <msr.h> +#include <io.h> +#include <cpu.h> +#include <amd/k8/k8.h> /* We need to get a unified Id that works on dual, quad, whatever. We are going to leave * this as-is and fix later, working with AMD to improve it. */ @@ -5,23 +17,41 @@ /* 2004.12 yhlu add dual core support */
//called by bus_cpu_scan too + +/** + * Return the value of Initial APIC ID CPU ID Low (InitApicIdCpuIdLo)—Bit 54. + * "When this bit is set, CpuId and NodeId[2:0] bit field positions are swapped in the APICID. " + * BKDG page 374 + * @returns 1 if the bit is set. + */ + unsigned int read_nb_cfg_54(void) { - msr_t msr; + struct msr msr; msr = rdmsr(NB_CFG_MSR); return ( ( msr.hi >> (54-32)) & 1); }
+/** + * Return the initial CPU APIC ID. + * @return APIC ID as read from cpuid_ebx, bits 24:27 + */ unsigned get_initial_apicid(void) { return ((cpuid_ebx(1) >> 24) & 0xf); }
+/** + * Get the node and core id of the current CPU. This is formatted differently in the CPU as + * determined by the nb_cfg_54 parameter. See the note above for read_nb_cfg_54. + * @returns a struct containing the node and core it. + */ //called by amd_siblings too #define CORE_ID_BIT 1 #define NODE_ID_BIT 3 -struct node_core_id get_node_core_id(unsigned nb_cfg_54) +struct node_core_id get_node_core_id(void) { + int nb_cfg_54 = read_nb_cfg_54(); struct node_core_id id; // get the apicid via cpuid(1) ebx[27:24] if( nb_cfg_54) { @@ -40,13 +70,11 @@ return id; }
-inline unsigned get_core_num(void) +/** + * Return the number of "other" cores in this CPU -- i.e. cores other than core 0 + * @return number of cores in addition to core 0 + */ +unsigned int get_core_count(void) { return (cpuid_ecx(0x80000008) & 0xff); } - -inline struct node_core_id get_node_core_id_x(void) { - - return get_node_core_id( read_nb_cfg_54() ); // for pre_e0() nb_cfg_54 always be 0 -} -
Modified: coreboot-v3/arch/x86/amd/model_fxx/init_cpus.c =================================================================== --- coreboot-v3/arch/x86/amd/model_fxx/init_cpus.c 2008-08-28 02:55:14 UTC (rev 838) +++ coreboot-v3/arch/x86/amd/model_fxx/init_cpus.c 2008-08-28 05:41:48 UTC (rev 839) @@ -334,19 +334,25 @@ #endif
/** - * Init all the CPUs. Part of the process involves setting APIC IDs for all cores on all sockets. The code that is run + * Init all the CPUs. Part of the process involves setting APIC IDs for all cores on all sockets. + * The code that is run * is for the most part the same on all cpus and cores of cpus. * Since we only support F2 and later Opteron CPUs our job is considerably simplified * as compared to v2. The basic process it to set up the cpu 0 core 0, then the other cpus, one by one. - * Complications: BSP, a.k.a. cpu 0, comes up with APIC id 0, the others all come up with APIC id 7, including other cpu 0 cores. - * There is also the question of the need to "lift" the BSP APIC id. For some setups, we want the BSP APIC id to be 0; for others, + * Complications: BSP, a.k.a. cpu 0, comes up with APIC id 0, the others all come up with APIC id 7, + * including other cpu 0 cores. Why? Because the BSP brings them up one by one and assigns their APIC ID. + * There is also the question of the need to "lift" the BSP APIC id. + * For some setups, we want the BSP APIC id to be 0; for others, * a non-zero value is preferred. This means that we have to change the BSP APIC ID on the fly. + * * So here we have it, some of the slickest code you'll ever read. Which cores run this function? * All of them. * Do they communicate through APIC Interrupts or memory? Yes. Both. APICs before * memory is ready, memory afterword. What is the state of the cores at the end of this function? * They are all ready to go, just waiting to be started up. What is the state of memory on all sockets? * It's all working. + * Except that it's not quite that simple. We'll try to comment this well enough to make sense. + * But rest assured, it's complicated! * @param cpu_init_detectedx has this cpu been init'ed before? * @param sysinfo The sys_info pointer * @returns the BSP APIC ID @@ -363,7 +369,8 @@ */
/* that is from initial apicid, we need nodeid and coreid later */ - id = get_node_core_id_x(); + /* this comment still confuses me, but I *think* "that" means the "bsp_apicid. Not sure. */ + id = get_node_core_id(); printk(BIOS_DEBUG, "init_cpus: node %d core %d\n", id.nodeid, id.coreid);
/* The NB_CFG MSR is shared between cores on a given node. @@ -385,10 +392,14 @@ #endif }
+ /* enable the local APIC, which we need to do message passing between sockets. */ enable_lapic(); // init_timer(); // We need TMICT to pass msg for FID/VID change
#if (ENABLE_APIC_EXT_ID == 1) + /* we wish to enable extended APIC IDs. We have an APIC ID already which we can + * use as a "base" for the extended ID. + */ unsigned initial_apicid = get_initial_apicid(); /* We don't always need to lift the BSP APIC ID. * Again, is there harm if we do it anyway? @@ -405,6 +416,9 @@
lapic_write(LAPIC_ID, dword); } + /* Again, the bsp_apicid is a special case and if we changed it + * we need to remember that change. + */ #if LIFT_BSP_APIC_ID == 1 bsp_apicid += APIC_ID_OFFSET; #endif @@ -440,9 +454,16 @@ // start_other_core(id.nodeid); // start second core in first cpu, only allowed for nb_cfg_54 is not set } //Indicate to other CPUs that our CPU is running. + /* and, again, recall that this is running on all sockets at some point, although it runs at + * different times. + */ lapic_write(LAPIC_MSG_REG, (apicid << 24) | 0x33);
- /* non-BSP CPUs are now set up and need to halt. BSP will train memory on all CPUs. */ + /* non-BSP CPUs are now set up and need to halt. There are a few possibilities here. + * BSP may train memory + * AP may train memory + * In v2, both are possible. + */ if (apicid != bsp_apicid) { unsigned timeout = 1; unsigned loop = 100; @@ -454,6 +475,15 @@ #endif
// We need to stop the CACHE as RAM for this CPU, really? + /* Yes we do. What happens here is really interesting. To this point + * we have used APICs to communicate. We're going to use the sysinfo + * struct. But to do that we have to use real memory. So we have to + * disable car, and do it in a way that lets us continue in this function. + * The way we do it for non-node 0 is to never return from this function, + * but to do the work in this function to train RAM. + * Note that serengeti, the SimNow target, does not do this; it lets BSP train AP memory. + */ + /* Wait for the bsp to enter state 44. */ while (timeout && (loop-- > 0)) { timeout = wait_cpu_state(bsp_apicid, 0x44); } @@ -462,13 +492,18 @@ ("while waiting for BSP signal to STOP, timeout in ap ", apicid); } - lapic_write(LAPIC_MSG_REG, (apicid << 24) | 0x44); // bsp can not check it before stop_this_cpu + /* indicate that we are in state 44 as well. We are catching up to the BSP. */ + // old comment follows -- not sure what this means yet. + // bsp can not check it before stop_this_cpu + lapic_write(LAPIC_MSG_REG, (apicid << 24) | 0x44); + /* Now set up so we can use RAM. This will be low memory, i.e. BSP memory, already working. */ set_init_ram_access(); + /* this is not done on Serengeti. */ #if MEM_TRAIN_SEQ == 1 train_ram_on_node(id.nodeid, id.coreid, sysinfo, STOP_CAR_AND_CPU); #endif - + /* this is inline and there is no return. */ STOP_CAR_AND_CPU(); }
@@ -497,7 +532,7 @@ static void wait_all_core0_started(void) { //When core0 is started, it will distingush_cpu_resets. So wait for that - // whatever that means + // whatever that comment means? unsigned i; unsigned nodes = get_nodes();
Modified: coreboot-v3/mainboard/amd/serengeti/Makefile =================================================================== --- coreboot-v3/mainboard/amd/serengeti/Makefile 2008-08-28 02:55:14 UTC (rev 838) +++ coreboot-v3/mainboard/amd/serengeti/Makefile 2008-08-28 05:41:48 UTC (rev 839) @@ -29,13 +29,15 @@ $(src)/northbridge/amd/k8/coherent_ht.c \ $(src)/northbridge/amd/k8/incoherent_ht.c \ $(src)/northbridge/amd/k8/libstage1.c \ -# $(src)/arch/x86/amd/model_fxx/stage1.c \
INITRAM_SRC= $(src)/mainboard/$(MAINBOARDDIR)/initram.c \ $(src)/northbridge/amd/k8/raminit.c \ $(src)/northbridge/amd/k8/dqs.c \ $(src)/arch/x86/pci_ops_conf1.c \ $(src)/southbridge/amd/amd8111/stage1_smbus.c \ + $(src)/arch/x86/amd/model_fxx/init_cpus.c \ + $(src)/arch/x86/amd/model_fxx/dualcore.c \ + $(src)/arch/x86/amd/model_fxx/dualcore_id.c \ $(src)/lib/clog2.c
Modified: coreboot-v3/mainboard/amd/serengeti/initram.c =================================================================== --- coreboot-v3/mainboard/amd/serengeti/initram.c 2008-08-28 02:55:14 UTC (rev 838) +++ coreboot-v3/mainboard/amd/serengeti/initram.c 2008-08-28 05:41:48 UTC (rev 839) @@ -135,7 +135,6 @@ post_code(POST_START_OF_MAIN); sysinfo = &(global_vars()->sys_info);
- init_detected = sysinfo->init_detected; /* well, here we are. For starters, we need to know if this is cpu0 core0. * cpu0 core 0 will do all the DRAM setup. */