Author: rminnich
Date: 2008-08-28 07:41:48 +0200 (Thu, 28 Aug 2008)
New Revision: 839
Modified:
coreboot-v3/arch/x86/amd/model_fxx/dualcore.c
coreboot-v3/arch/x86/amd/model_fxx/dualcore_id.c
coreboot-v3/arch/x86/amd/model_fxx/init_cpus.c
coreboot-v3/mainboard/amd/serengeti/Makefile
coreboot-v3/mainboard/amd/serengeti/initram.c
Log:
Off to bed. I'm done for now.
If anyone wants to review my comments, or maybe even try to fix compile
issues, have at it :-)
The more I work with the K8 stuff the more impressed I am with the
people who got it all to go 6 years ago. (and at how much I've forgotten
but that's another story :=)
If we can get this next step done we're very close to having a working
initram.
And, once you have ram and hit stage2, life is just better all around.
Signed-off-by: Ronald G. Minnich <rminnich(a)gmail.com>
Acked-by: Ronald G. Minnich <rminnich(a)gmail.com>
Modified: coreboot-v3/arch/x86/amd/model_fxx/dualcore.c
===================================================================
--- coreboot-v3/arch/x86/amd/model_fxx/dualcore.c 2008-08-28 02:55:14 UTC (rev 838)
+++ coreboot-v3/arch/x86/amd/model_fxx/dualcore.c 2008-08-28 05:41:48 UTC (rev 839)
@@ -34,147 +34,59 @@
}
#if SET_NB_CFG_54 == 1
-static inline uint8_t set_apicid_cpuid_lo(void)
+u8 set_apicid_cpuid_lo(void)
{
-#if K8_REV_F_SUPPORT == 0
- if(is_cpu_pre_e0()) return 0; // pre_e0 can not be set
-#endif
-
- // set the NB_CFG[54]=1; why the OS will be happy with that ???
- msr_t msr;
+ // set the NB_CFG[54]=1; Why the OS thinks this is a good thing is not yet known.
+ struct msr msr;
msr = rdmsr(NB_CFG_MSR);
msr.hi |= (1<<(54-32)); // InitApicIdCpuIdLo
wrmsr(NB_CFG_MSR, msr);
return 1;
}
-#else
-
-static inline void set_apicid_cpuid_lo(void) { }
-
#endif
-static inline void real_start_other_core(unsigned nodeid)
+/**
+ * Start the cores on a given node (cores > 0). It is important that MC4 and other accesses
+ * be directed to core 0, per the BKDG. Note that this is pretty dual core specific.
+ * @param nodeid Node on which to start the cores
+ */
+void start_cores(unsigned nodeid)
{
- uint32_t dword;
- // set PCI_DEV(0, 0x18+nodeid, 3), 0x44 bit 27 to redirect all MC4 accesses and error logging to core0
- dword = pci_read_config32(PCI_DEV(0, 0x18+nodeid, 3), 0x44);
- dword |= 1<<27; // NbMcaToMstCpuEn bit
- pci_write_config32(PCI_DEV(0, 0x18+nodeid, 3), 0x44, dword);
+ u32 dword;
+ /* set PCI_DEV(0, 0x18+nodeid, 3), 0x44 bit 27 to redirect all MC4
+ * accesses and error logging to core0
+ */
+ dword = pci_read_config32(PCI_DEV(0, 0x18+nodeid, 3), MCA_NB_CONFIG);
+ dword |= MNC_NBMCATOMSTCPUEN; // NbMcaToMstCpuEn bit
+ pci_write_config32(PCI_DEV(0, 0x18+nodeid, 3), MCA_NB_CONFIG, dword);
// set PCI_DEV(0, 0x18+nodeid, 0), 0x68 bit 5 to start core1
- dword = pci_read_config32(PCI_DEV(0, 0x18+nodeid, 0), 0x68);
- dword |= 1<<5;
- pci_write_config32(PCI_DEV(0, 0x18+nodeid, 0), 0x68, dword);
+ dword = pci_read_config32(PCI_DEV(0, 0x18+nodeid, 0), HT_TRANSACTION_CONTROL);
+ dword |= HTTC_CPU1_EN;
+ pci_write_config32(PCI_DEV(0, 0x18+nodeid, 0), HT_TRANSACTION_CONTROL, dword);
}
-//it is running on core0 of node0
+/**
+ * start cores on all nodes including BSP. This is assumed to be running on core 0 of node 0
+ */
static inline void start_other_cores(void)
{
unsigned nodes;
unsigned nodeid;
+ int dual_core = 0;
- if(read_option(CMOS_VSTART_dual_core, CMOS_VLEN_dual_core, 0) != 0) { // disable dual_core
- return;
- }
+ get_option(&dual_core, "dual_core");
+ if (! dual_core)
+ return;
+
nodes = get_nodes();
for(nodeid=0; nodeid<nodes; nodeid++) {
- if( get_core_num_in_bsp(nodeid) > 0) {
- real_start_other_core(nodeid);
+ if( get_core_count(nodeid) > 0) {
+ start_cores(nodeid);
}
}
}
-#if USE_DCACHE_RAM == 0
-static void do_k8_init_and_stop_secondaries(void)
-{
- struct node_core_id id;
- device_t dev;
- unsigned apicid;
- unsigned max_siblings;
- msr_t msr;
-
- /* Skip this if there was a built in self test failure */
- if (is_cpu_pre_e0()) {
- id.nodeid = lapicid() & 0x7;
- id.coreid = 0;
- } else {
- /* Which cpu are we on? */
- id = get_node_core_id_x();
-
- /* Set NB_CFG_MSR
- * Linux expect the core to be in the least signficant bits.
- */
- msr = rdmsr(NB_CFG_MSR);
- msr.hi |= (1<<(54-32)); // InitApicIdCpuIdLo
- wrmsr(NB_CFG_MSR, msr);
- }
-
- /* For now assume all cpus have the same number of siblings */
- max_siblings = (cpuid_ecx(0x80000008) & 0xff) + 1;
-
- /* Enable extended apic ids */
- device_t dev_f0 = PCI_DEV(0, 0x18+id.nodeid, 0);
- unsigned val = pci_read_config32(dev_f0, 0x68);
- val |= (1 << 18) | (1 << 17);
- pci_write_config32(dev_f0, 0x68, val);
-
- /* Set the lapicid */
- #if (ENABLE_APIC_EXT_ID == 1)
- unsigned initial_apicid = get_initial_apicid();
- #if LIFT_BSP_APIC_ID == 0
- if( initial_apicid != 0 ) // other than bsp
- #endif
- {
- /* use initial apic id to lift it */
- uint32_t dword = lapic_read(LAPIC_ID);
- dword &= ~(0xff<<24);
- dword |= (((initial_apicid + APIC_ID_OFFSET) & 0xff)<<24);
-
- lapic_write(LAPIC_ID, dword);
- }
-
- #if LIFT_BSP_APIC_ID == 1
- bsp_apicid += APIC_ID_OFFSET;
- #endif
-
- #endif
-
-
- /* Remember the cpuid */
- if (id.coreid == 0) {
- dev = PCI_DEV(0, 0x18 + id.nodeid, 2);
- pci_write_config32(dev, 0x9c, cpuid_eax(1));
- }
-
- /* Maybe call distinguish_cpu_resets only on the last core? */
- distinguish_cpu_resets(id.nodeid);
- if (!boot_cpu()) {
- stop_this_cpu();
- }
-}
-
-static void k8_init_and_stop_secondaries(void)
-{
- /* This doesn't work with Cache As Ram because it messes with
- the MTRR state, which breaks the init detection.
- do_k8_init_and_stop_secondaries should be usable by CAR code.
- */
-
- int init_detected;
-
- init_detected = early_mtrr_init_detected();
- amd_early_mtrr_init();
-
- enable_lapic();
- init_timer();
- if (init_detected) {
- asm volatile ("jmp __cpu_reset");
- }
-
- do_k8_init_and_stop_secondaries();
-}
-
-#endif
Modified: coreboot-v3/arch/x86/amd/model_fxx/dualcore_id.c
===================================================================
--- coreboot-v3/arch/x86/amd/model_fxx/dualcore_id.c 2008-08-28 02:55:14 UTC (rev 838)
+++ coreboot-v3/arch/x86/amd/model_fxx/dualcore_id.c 2008-08-28 05:41:48 UTC (rev 839)
@@ -1,3 +1,15 @@
+#include <mainboard.h>
+#include <types.h>
+#include <lib.h>
+#include <console.h>
+#include <globalvars.h>
+#include <device/device.h>
+#include <device/pci.h>
+#include <string.h>
+#include <msr.h>
+#include <io.h>
+#include <cpu.h>
+#include <amd/k8/k8.h>
/* We need to get a unified Id that works on dual, quad, whatever. We are going to leave
* this as-is and fix later, working with AMD to improve it.
*/
@@ -5,23 +17,41 @@
/* 2004.12 yhlu add dual core support */
//called by bus_cpu_scan too
+
+/**
+ * Return the value of Initial APIC ID CPU ID Low (InitApicIdCpuIdLo)—Bit 54.
+ * "When this bit is set, CpuId and NodeId[2:0] bit field positions are swapped in the APICID. "
+ * BKDG page 374
+ * @returns 1 if the bit is set.
+ */
+
unsigned int read_nb_cfg_54(void)
{
- msr_t msr;
+ struct msr msr;
msr = rdmsr(NB_CFG_MSR);
return ( ( msr.hi >> (54-32)) & 1);
}
+/**
+ * Return the initial CPU APIC ID.
+ * @return APIC ID as read from cpuid_ebx, bits 24:27
+ */
unsigned get_initial_apicid(void)
{
return ((cpuid_ebx(1) >> 24) & 0xf);
}
+/**
+ * Get the node and core id of the current CPU. This is formatted differently in the CPU as
+ * determined by the nb_cfg_54 parameter. See the note above for read_nb_cfg_54.
+ * @returns a struct containing the node and core it.
+ */
//called by amd_siblings too
#define CORE_ID_BIT 1
#define NODE_ID_BIT 3
-struct node_core_id get_node_core_id(unsigned nb_cfg_54)
+struct node_core_id get_node_core_id(void)
{
+ int nb_cfg_54 = read_nb_cfg_54();
struct node_core_id id;
// get the apicid via cpuid(1) ebx[27:24]
if( nb_cfg_54) {
@@ -40,13 +70,11 @@
return id;
}
-inline unsigned get_core_num(void)
+/**
+ * Return the number of "other" cores in this CPU -- i.e. cores other than core 0
+ * @return number of cores in addition to core 0
+ */
+unsigned int get_core_count(void)
{
return (cpuid_ecx(0x80000008) & 0xff);
}
-
-inline struct node_core_id get_node_core_id_x(void) {
-
- return get_node_core_id( read_nb_cfg_54() ); // for pre_e0() nb_cfg_54 always be 0
-}
-
Modified: coreboot-v3/arch/x86/amd/model_fxx/init_cpus.c
===================================================================
--- coreboot-v3/arch/x86/amd/model_fxx/init_cpus.c 2008-08-28 02:55:14 UTC (rev 838)
+++ coreboot-v3/arch/x86/amd/model_fxx/init_cpus.c 2008-08-28 05:41:48 UTC (rev 839)
@@ -334,19 +334,25 @@
#endif
/**
- * Init all the CPUs. Part of the process involves setting APIC IDs for all cores on all sockets. The code that is run
+ * Init all the CPUs. Part of the process involves setting APIC IDs for all cores on all sockets.
+ * The code that is run
* is for the most part the same on all cpus and cores of cpus.
* Since we only support F2 and later Opteron CPUs our job is considerably simplified
* as compared to v2. The basic process it to set up the cpu 0 core 0, then the other cpus, one by one.
- * Complications: BSP, a.k.a. cpu 0, comes up with APIC id 0, the others all come up with APIC id 7, including other cpu 0 cores.
- * There is also the question of the need to "lift" the BSP APIC id. For some setups, we want the BSP APIC id to be 0; for others,
+ * Complications: BSP, a.k.a. cpu 0, comes up with APIC id 0, the others all come up with APIC id 7,
+ * including other cpu 0 cores. Why? Because the BSP brings them up one by one and assigns their APIC ID.
+ * There is also the question of the need to "lift" the BSP APIC id.
+ * For some setups, we want the BSP APIC id to be 0; for others,
* a non-zero value is preferred. This means that we have to change the BSP APIC ID on the fly.
+ *
* So here we have it, some of the slickest code you'll ever read. Which cores run this function?
* All of them.
* Do they communicate through APIC Interrupts or memory? Yes. Both. APICs before
* memory is ready, memory afterword. What is the state of the cores at the end of this function?
* They are all ready to go, just waiting to be started up. What is the state of memory on all sockets?
* It's all working.
+ * Except that it's not quite that simple. We'll try to comment this well enough to make sense.
+ * But rest assured, it's complicated!
* @param cpu_init_detectedx has this cpu been init'ed before?
* @param sysinfo The sys_info pointer
* @returns the BSP APIC ID
@@ -363,7 +369,8 @@
*/
/* that is from initial apicid, we need nodeid and coreid later */
- id = get_node_core_id_x();
+ /* this comment still confuses me, but I *think* "that" means the "bsp_apicid. Not sure. */
+ id = get_node_core_id();
printk(BIOS_DEBUG, "init_cpus: node %d core %d\n", id.nodeid, id.coreid);
/* The NB_CFG MSR is shared between cores on a given node.
@@ -385,10 +392,14 @@
#endif
}
+ /* enable the local APIC, which we need to do message passing between sockets. */
enable_lapic();
// init_timer(); // We need TMICT to pass msg for FID/VID change
#if (ENABLE_APIC_EXT_ID == 1)
+ /* we wish to enable extended APIC IDs. We have an APIC ID already which we can
+ * use as a "base" for the extended ID.
+ */
unsigned initial_apicid = get_initial_apicid();
/* We don't always need to lift the BSP APIC ID.
* Again, is there harm if we do it anyway?
@@ -405,6 +416,9 @@
lapic_write(LAPIC_ID, dword);
}
+ /* Again, the bsp_apicid is a special case and if we changed it
+ * we need to remember that change.
+ */
#if LIFT_BSP_APIC_ID == 1
bsp_apicid += APIC_ID_OFFSET;
#endif
@@ -440,9 +454,16 @@
// start_other_core(id.nodeid); // start second core in first cpu, only allowed for nb_cfg_54 is not set
}
//Indicate to other CPUs that our CPU is running.
+ /* and, again, recall that this is running on all sockets at some point, although it runs at
+ * different times.
+ */
lapic_write(LAPIC_MSG_REG, (apicid << 24) | 0x33);
- /* non-BSP CPUs are now set up and need to halt. BSP will train memory on all CPUs. */
+ /* non-BSP CPUs are now set up and need to halt. There are a few possibilities here.
+ * BSP may train memory
+ * AP may train memory
+ * In v2, both are possible.
+ */
if (apicid != bsp_apicid) {
unsigned timeout = 1;
unsigned loop = 100;
@@ -454,6 +475,15 @@
#endif
// We need to stop the CACHE as RAM for this CPU, really?
+ /* Yes we do. What happens here is really interesting. To this point
+ * we have used APICs to communicate. We're going to use the sysinfo
+ * struct. But to do that we have to use real memory. So we have to
+ * disable car, and do it in a way that lets us continue in this function.
+ * The way we do it for non-node 0 is to never return from this function,
+ * but to do the work in this function to train RAM.
+ * Note that serengeti, the SimNow target, does not do this; it lets BSP train AP memory.
+ */
+ /* Wait for the bsp to enter state 44. */
while (timeout && (loop-- > 0)) {
timeout = wait_cpu_state(bsp_apicid, 0x44);
}
@@ -462,13 +492,18 @@
("while waiting for BSP signal to STOP, timeout in ap ",
apicid);
}
- lapic_write(LAPIC_MSG_REG, (apicid << 24) | 0x44); // bsp can not check it before stop_this_cpu
+ /* indicate that we are in state 44 as well. We are catching up to the BSP. */
+ // old comment follows -- not sure what this means yet.
+ // bsp can not check it before stop_this_cpu
+ lapic_write(LAPIC_MSG_REG, (apicid << 24) | 0x44);
+ /* Now set up so we can use RAM. This will be low memory, i.e. BSP memory, already working. */
set_init_ram_access();
+ /* this is not done on Serengeti. */
#if MEM_TRAIN_SEQ == 1
train_ram_on_node(id.nodeid, id.coreid, sysinfo,
STOP_CAR_AND_CPU);
#endif
-
+ /* this is inline and there is no return. */
STOP_CAR_AND_CPU();
}
@@ -497,7 +532,7 @@
static void wait_all_core0_started(void)
{
//When core0 is started, it will distingush_cpu_resets. So wait for that
- // whatever that means
+ // whatever that comment means?
unsigned i;
unsigned nodes = get_nodes();
Modified: coreboot-v3/mainboard/amd/serengeti/Makefile
===================================================================
--- coreboot-v3/mainboard/amd/serengeti/Makefile 2008-08-28 02:55:14 UTC (rev 838)
+++ coreboot-v3/mainboard/amd/serengeti/Makefile 2008-08-28 05:41:48 UTC (rev 839)
@@ -29,13 +29,15 @@
$(src)/northbridge/amd/k8/coherent_ht.c \
$(src)/northbridge/amd/k8/incoherent_ht.c \
$(src)/northbridge/amd/k8/libstage1.c \
-# $(src)/arch/x86/amd/model_fxx/stage1.c \
INITRAM_SRC= $(src)/mainboard/$(MAINBOARDDIR)/initram.c \
$(src)/northbridge/amd/k8/raminit.c \
$(src)/northbridge/amd/k8/dqs.c \
$(src)/arch/x86/pci_ops_conf1.c \
$(src)/southbridge/amd/amd8111/stage1_smbus.c \
+ $(src)/arch/x86/amd/model_fxx/init_cpus.c \
+ $(src)/arch/x86/amd/model_fxx/dualcore.c \
+ $(src)/arch/x86/amd/model_fxx/dualcore_id.c \
$(src)/lib/clog2.c
Modified: coreboot-v3/mainboard/amd/serengeti/initram.c
===================================================================
--- coreboot-v3/mainboard/amd/serengeti/initram.c 2008-08-28 02:55:14 UTC (rev 838)
+++ coreboot-v3/mainboard/amd/serengeti/initram.c 2008-08-28 05:41:48 UTC (rev 839)
@@ -135,7 +135,6 @@
post_code(POST_START_OF_MAIN);
sysinfo = &(global_vars()->sys_info);
- init_detected = sysinfo->init_detected;
/* well, here we are. For starters, we need to know if this is cpu0 core0.
* cpu0 core 0 will do all the DRAM setup.
*/
