[PATCH] megasas: Add boot support for LSI MegaRAID SAS

[Hannes Reinecke]

This patch adds support for LSI MegaRAID SAS controllers.
Currently only 8708EM2 is supported.

Signed-off-by: Hannes Reinecke 
Cc: Gerd Hofmann 
Cc: Alexander Graf 
Cc: Paolo Bonzini

diff --git a/Makefile b/Makefile
index 5486f88..b0e2031 100644
--- a/Makefile
+++ b/Makefile
@@ -13,7 +13,7 @@ SRCBOTH=misc.c stacks.c pmm.c output.c util.c block.c floppy.c ata.c mouse.c \
     pnpbios.c pirtable.c vgahooks.c ramdisk.c pcibios.c blockcmd.c \
     usb.c usb-uhci.c usb-ohci.c usb-ehci.c usb-hid.c usb-msc.c \
     virtio-ring.c virtio-pci.c virtio-blk.c virtio-scsi.c apm.c ahci.c \
-    usb-uas.c lsi-scsi.c esp-scsi.c
+    usb-uas.c lsi-scsi.c esp-scsi.c megasas.c
 SRC16=$(SRCBOTH) system.c disk.c font.c
 SRC32FLAT=$(SRCBOTH) post.c shadow.c memmap.c coreboot.c boot.c \
     acpi.c smm.c mptable.c smbios.c pciinit.c optionroms.c mtrr.c \
diff --git a/src/Kconfig b/src/Kconfig
index ee6fdb6..0b112ed 100644
--- a/src/Kconfig
+++ b/src/Kconfig
@@ -141,6 +141,12 @@ menu "Hardware support"
         default y
         help
             Support boot from qemu-emulated lsi53c895a scsi storage.
+    config MEGASAS
+        depends on DRIVES
+        bool "LSI MegaRAID SAS controllers"
+        default y
+        help
+            Support boot from LSI MegaRAID SAS scsi storage.
     config FLOPPY
         depends on DRIVES
         bool "Floppy controller"
diff --git a/src/block.c b/src/block.c
index 243428e..cfe5c33 100644
--- a/src/block.c
+++ b/src/block.c
@@ -336,6 +336,7 @@ process_op(struct disk_op_s *op)
     case DTYPE_VIRTIO_SCSI:
     case DTYPE_LSI_SCSI:
     case DTYPE_ESP_SCSI:
+    case DTYPE_MEGASAS:
         return process_scsi_op(op);
     default:
         op->count = 0;
diff --git a/src/blockcmd.c b/src/blockcmd.c
index 77c690f..81b191b 100644
--- a/src/blockcmd.c
+++ b/src/blockcmd.c
@@ -17,6 +17,7 @@
 #include "virtio-scsi.h" // virtio_scsi_cmd_data
 #include "lsi-scsi.h" // lsi_scsi_cmd_data
 #include "esp-scsi.h" // esp_scsi_cmd_data
+#include "megasas.h" // megasas_cmd_data
 #include "boot.h" // boot_add_hd

// Route command to low-level handler.
@@ -39,6 +40,8 @@ cdb_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize)
         return lsi_scsi_cmd_data(op, cdbcmd, blocksize);
     case DTYPE_ESP_SCSI:
         return esp_scsi_cmd_data(op, cdbcmd, blocksize);
+    case DTYPE_MEGASAS:
+        return megasas_cmd_data(op, cdbcmd, blocksize);
     default:
         op->count = 0;
         return DISK_RET_EPARAM;
diff --git a/src/disk.h b/src/disk.h
index 3d07372..21debec 100644
--- a/src/disk.h
+++ b/src/disk.h
@@ -235,6 +235,7 @@ struct drive_s {
 #define DTYPE_UAS          0x0b
 #define DTYPE_LSI_SCSI     0x0c
 #define DTYPE_ESP_SCSI     0x0d
+#define DTYPE_MEGASAS      0x0e

#define MAXDESCSIZE 80

diff --git a/src/megasas.c b/src/megasas.c
new file mode 100644
index 0000000..4b81334
--- /dev/null
+++ b/src/megasas.c
@@ -0,0 +1,311 @@
+// (qemu-emulated) megaraid_sas boot support.
+//
+// Copyright (C) 2012 Hannes Reinecke, SUSE Linux Products GmbH
+//
+// Authors:
+//  Hannes Reinecke 
+//
+// based on virtio-scsi.c which is written by:
+//  Paolo Bonzini 
+//
+// This file may be distributed under the terms of the GNU LGPLv3 license.
+
+#include "util.h" // dprintf
+#include "pci.h" // foreachpci
+#include "config.h" // CONFIG_*
+#include "biosvar.h" // GET_GLOBAL
+#include "pci_ids.h" // PCI_DEVICE_ID_VIRTIO_BLK
+#include "pci_regs.h" // PCI_VENDOR_ID
+#include "boot.h" // bootprio_find_scsi_device
+#include "blockcmd.h" // scsi_init_drive
+#include "disk.h"
+
+#define MFI_OMSG0 0x18 // Outbound message 0
+#define MFI_ODB 0x2c // Outbound doorbell
+#define MFI_IQP 0x40 // Inbound queue port
+
+#define MFI_STATE_MASK                0xf0000000
+#define MFI_STATE_READY               0xb0000000
+#define MFI_STATE_OPERATIONAL         0xc0000000
+
+/* MFI Commands */
+typedef enum {
+    MFI_CMD_INIT = 0x00,
+    MFI_CMD_LD_READ,
+    MFI_CMD_LD_WRITE,
+    MFI_CMD_LD_SCSI_IO,
+    MFI_CMD_PD_SCSI_IO,
+    MFI_CMD_DCMD,
+    MFI_CMD_ABORT,
+    MFI_CMD_SMP,
+    MFI_CMD_STP
+} mfi_cmd_t;
+
+struct megasas_cmd_frame {
+    u8 cmd;             /*00h */
+    u8 sense_len;       /*01h */
+    u8 cmd_status;      /*02h */
+    u8 scsi_status;     /*03h */
+
+    u8 target_id;       /*04h */
+    u8 lun;             /*05h */
+    u8 cdb_len;         /*06h */
+    u8 sge_count;       /*07h */
+
+    u32 context;        /*08h */
+    u32 context_64;     /*0Ch */
+
+    u16 flags;          /*10h */
+    u16 timeout;        /*12h */
+    u32 data_xfer_len;   /*14h */
+
+    union {
+        struct {
+            u32 opcode;       /*18h */
+            u8 mbox[12];      /*1Ch */
+            u32 sgl_addr;     /*28h */
+            u32 sgl_len;      /*32h */
+            u32 pad;          /*34h */
+        } dcmd;
+        struct {
+            u32 sense_buf_lo; /*18h */
+            u32 sense_buf_hi; /*1Ch */
+            u8 cdb[16];       /*20h */
+            u32 sgl_addr;     /*30h */
+            u32 sgl_len;      /*34h */
+        } pthru;
+        struct {
+            u8 pad[22];       /*18h */
+        } gen;
+    };
+} __attribute__ ((packed));
+
+struct mfi_ld_list_s {
+    u32     count;
+    u32     reserved_0;
+    struct {
+        u8          target;
+        u8          lun;
+        u16         seq;
+        u8          state;
+        u8          reserved_1[3];
+        u64         size;
+    } lds[64];
+} __attribute__ ((packed));
+
+#define MEGASAS_POLL_TIMEOUT 60000 // 60 seconds polling timeout
+
+struct megasas_lun_s {
+    struct drive_s drive;
+    struct pci_device *pci;
+    struct megasas_cmd_frame *frame;
+    u32 iobase;
+    u8 target;
+    u8 lun;
+};
+
+static int megasas_fire_cmd(u32 ioaddr, struct megasas_cmd_frame *frame)
+{
+    u32 frame_addr = (u32)frame;
+    int frame_count = 1;
+    u8 cmd_state;
+    u64 end;
+
+    dprintf(1, "Frame 0x%x\n", frame_addr);
+    outl(frame_addr | frame_count << 1 | 1, ioaddr + MFI_IQP);
+
+    end = calc_future_tsc(MEGASAS_POLL_TIMEOUT);
+    do {
+        for (;;) {
+            cmd_state = GET_LOWFLAT(frame->cmd_status);
+            if (cmd_state != 0xff)
+                break;
+            if (check_tsc(end)) {
+                warn_timeout();
+                return -1;
+            }
+            yield();
+        }
+    } while (cmd_state == 0xff);
+
+    if (cmd_state == 0 || cmd_state == 0x2d)
+        return 0;
+    dprintf(1, "Frame 0x%x, status 0x%x\n", frame_addr, cmd_state);
+    return -1;
+}
+
+int
+megasas_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize)
+{
+    struct megasas_lun_s *mlun =
+        container_of(op->drive_g, struct megasas_lun_s, drive);
+    u8 *cdb = cdbcmd;
+    struct megasas_cmd_frame *frame = GET_GLOBAL(mlun->frame);
+    int i;
+
+    if (!CONFIG_MEGASAS)
+        return DISK_RET_EBADTRACK;
+
+    memset_fl(frame, 0, sizeof(*frame));
+    SET_LOWFLAT(frame->cmd, MFI_CMD_LD_SCSI_IO);
+    SET_LOWFLAT(frame->cmd_status, 0xFF);
+    SET_LOWFLAT(frame->target_id, GET_GLOBAL(mlun->target));
+    SET_LOWFLAT(frame->lun, GET_GLOBAL(mlun->lun));
+    SET_LOWFLAT(frame->flags, 0x0001);
+    SET_LOWFLAT(frame->data_xfer_len, op->count * blocksize);
+    SET_LOWFLAT(frame->cdb_len, 16);
+
+    for (i = 0; i < 16; i++) {
+        SET_LOWFLAT(frame->pthru.cdb[i], cdb[i]);
+    }
+    dprintf(1, "pthru cmd 0x%x count %d bs %d\n",
+            cdb[0], op->count, blocksize);
+
+    if (op->count) {
+        SET_LOWFLAT(frame->pthru.sgl_addr, (u32)op->buf_fl);
+        SET_LOWFLAT(frame->pthru.sgl_len, op->count * blocksize);
+        SET_LOWFLAT(frame->sge_count, 1);
+    }
+    SET_LOWFLAT(frame->context, (u32)frame);
+
+    if (megasas_fire_cmd(GET_GLOBAL(mlun->iobase), frame) == 0)
+        return DISK_RET_SUCCESS;
+
+    dprintf(1, "pthru cmd 0x%x failed\n", cdb[0]);
+    return DISK_RET_EBADTRACK;
+}
+
+static int
+megasas_add_lun(struct pci_device *pci, u32 iobase, u8 target, u8 lun)
+{
+    struct megasas_lun_s *mlun = malloc_fseg(sizeof(*mlun));
+    char *name;
+    int prio, ret = 0;
+
+    if (!mlun) {
+        warn_noalloc();
+        return -1;
+    }
+    memset(mlun, 0, sizeof(*mlun));
+    mlun->drive.type = DTYPE_MEGASAS;
+    mlun->drive.cntl_id = pci->bdf;
+    mlun->pci = pci;
+    mlun->target = target;
+    mlun->lun = lun;
+    mlun->iobase = iobase;
+    mlun->frame = memalign_high(256, sizeof(struct megasas_cmd_frame));
+    if (!mlun->frame) {
+        warn_noalloc();
+        free(mlun);
+        return -1;
+    }
+    name = znprintf(36, "MegaRAID SAS (PCI %02x:%02x.%x) LD %d:%d",
+                    pci_bdf_to_bus(pci->bdf), pci_bdf_to_dev(pci->bdf),
+                    pci_bdf_to_fn(pci->bdf), target, lun);
+    prio = bootprio_find_scsi_device(pci, target, lun);
+    ret = scsi_init_drive(&mlun->drive, name, prio);
+    free(name);
+    if (ret) {
+        free(mlun->frame);
+        free(mlun);
+        ret = -1;
+    }
+
+    return ret;
+}
+
+static void megasas_scan_target(struct pci_device *pci, u32 iobase)
+{
+    struct mfi_ld_list_s ld_list;
+    struct megasas_cmd_frame *frame = memalign_high(256, sizeof(*frame));
+    int i;
+
+    memset(&ld_list, 0, sizeof(ld_list));
+    memset(frame, 0, sizeof(*frame));
+
+    frame->cmd = MFI_CMD_DCMD;
+    frame->cmd_status = 0xFF;
+    frame->sge_count = 1;
+    frame->flags = 0x0011;
+    frame->data_xfer_len = sizeof(ld_list);
+    frame->dcmd.opcode = 0x03010000;
+    frame->dcmd.sgl_addr = (u32)MAKE_FLATPTR(GET_SEG(SS), &ld_list);
+    frame->dcmd.sgl_len = sizeof(ld_list);
+    frame->context = (u32)frame;
+
+    if (megasas_fire_cmd(iobase, frame) != 0)
+        return;
+
+    dprintf(1, "%d LD found\n", ld_list.count);
+    for (i = 0; i < ld_list.count; i++) {
+        dprintf(1, "LD %d:%d state 0x%x\n",
+                ld_list.lds[i].target, ld_list.lds[i].lun,
+                ld_list.lds[i].state);
+        if (ld_list.lds[i].state != 0) {
+            megasas_add_lun(pci, iobase,
+                            ld_list.lds[i].target, ld_list.lds[i].lun);
+        }
+    }
+    free(frame);
+}
+
+static int megasas_transition_to_ready(u32 ioaddr)
+{
+    u32 fw_state, i = 0;
+
+    fw_state = inl(ioaddr + MFI_OMSG0) & MFI_STATE_MASK;
+    while (i < MEGASAS_POLL_TIMEOUT) {
+        if (fw_state == MFI_STATE_OPERATIONAL)
+            outw(ioaddr + MFI_ODB, 0x07);
+        if (fw_state == MFI_STATE_READY)
+            break;
+        msleep(1);
+        fw_state = inl(ioaddr + MFI_OMSG0) & MFI_STATE_MASK;
+        i++;
+    }
+    if (fw_state == MFI_STATE_READY) {
+            dprintf(1, "MegaRAID SAS fw ready\n");
+            return 0;
+    }
+
+    dprintf(1, "ERROR: fw in state %x\n", fw_state & MFI_STATE_MASK);
+    return -1;
+}
+
+static void
+init_megasas(struct pci_device *pci)
+{
+    u16 bdf = pci->bdf;
+    u32 iobase = pci_config_readl(pci->bdf, PCI_BASE_ADDRESS_2)
+        & PCI_BASE_ADDRESS_IO_MASK;
+
+    dprintf(1, "found MegaRAID SAS at %02x:%02x.%x, io @ %x\n",
+            pci_bdf_to_bus(bdf), pci_bdf_to_dev(bdf),
+            pci_bdf_to_fn(bdf), iobase);
+
+    pci_config_maskw(pci->bdf, PCI_COMMAND, 0,
+                     PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER);
+    // reset
+    if (megasas_transition_to_ready(iobase) == 0)
+        megasas_scan_target(pci, iobase);
+
+    return;
+}
+
+void
+megasas_setup(void)
+{
+    ASSERT32FLAT();
+    if (!CONFIG_MEGASAS)
+        return;
+
+    dprintf(3, "init megasas\n");
+
+    struct pci_device *pci;
+    foreachpci(pci) {
+        if (pci->vendor != PCI_VENDOR_ID_LSI_LOGIC
+            || pci->device !=PCI_DEVICE_ID_LSI_SAS1078)
+            continue;
+        init_megasas(pci);
+    }
+}
diff --git a/src/megasas.h b/src/megasas.h
new file mode 100644
index 0000000..124042e
--- /dev/null
+++ b/src/megasas.h
@@ -0,0 +1,8 @@
+#ifndef __MEGASAS_H
+#define __MEGASAS_H
+
+struct disk_op_s;
+int megasas_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize);
+void megasas_setup(void);
+
+#endif /* __MEGASAS_H */
diff --git a/src/post.c b/src/post.c
index 924b311..0133f75 100644
--- a/src/post.c
+++ b/src/post.c
@@ -29,7 +29,7 @@
 #include "virtio-scsi.h" // virtio_scsi_setup
 #include "lsi-scsi.h" // lsi_scsi_setup
 #include "esp-scsi.h" // esp_scsi_setup
-
+#include "megasas.h" // megasas_setup

/****************************************************************
  * BIOS init
@@ -198,6 +198,7 @@ init_hw(void)
     virtio_scsi_setup();
     lsi_scsi_setup();
     esp_scsi_setup();
+    megasas_setup();
 }

// Begin the boot process by invoking an int0x19 in 16bit mode.

[Paolo Bonzini]

Il 12/11/2012 15:45, Hannes Reinecke ha scritto:
> This patch adds support for LSI MegaRAID SAS controllers.
> Currently only 8708EM2 is supported.

Did you test it on real hardware?  For example via PCI passthrough.
> diff --git a/src/megasas.c b/src/megasas.c
> new file mode 100644
> index 0000000..4b81334
> --- /dev/null
> +++ b/src/megasas.c
> @@ -0,0 +1,311 @@
> +// (qemu-emulated) megaraid_sas boot support.

If also "real" MegaSAS is supported, please fix the comment.  Otherwise
make it depend on QEMU too.

I know nil about the adapter, so I have not really reviewed the rest of
the code.  Seems sane though, thanks for this work.

Paolo
> +//
> +// Copyright (C) 2012 Hannes Reinecke, SUSE Linux Products GmbH
> +//
> +// Authors:
> +//  Hannes Reinecke 
> +//
> +// based on virtio-scsi.c which is written by:
> +//  Paolo Bonzini 
> +//
> +// This file may be distributed under the terms of the GNU LGPLv3 license.
>
> +#include "util.h" // dprintf
> +#include "pci.h" // foreachpci
> +#include "config.h" // CONFIG_*
> +#include "biosvar.h" // GET_GLOBAL
> +#include "pci_ids.h" // PCI_DEVICE_ID_VIRTIO_BLK
> +#include "pci_regs.h" // PCI_VENDOR_ID
> +#include "boot.h" // bootprio_find_scsi_device
> +#include "blockcmd.h" // scsi_init_drive
> +#include "disk.h"
>
> +#define MFI_OMSG0 0x18 // Outbound message 0
> +#define MFI_ODB 0x2c // Outbound doorbell
> +#define MFI_IQP 0x40 // Inbound queue port
>
> +#define MFI_STATE_MASK                0xf0000000
> +#define MFI_STATE_READY               0xb0000000
> +#define MFI_STATE_OPERATIONAL         0xc0000000
>
> +/* MFI Commands */
> +typedef enum {
> MFI_CMD_INIT = 0x00,
> MFI_CMD_LD_READ,
> MFI_CMD_LD_WRITE,
> MFI_CMD_LD_SCSI_IO,
> MFI_CMD_PD_SCSI_IO,
> MFI_CMD_DCMD,
> MFI_CMD_ABORT,
> MFI_CMD_SMP,
> MFI_CMD_STP
>
> +} mfi_cmd_t;
>
> +struct megasas_cmd_frame {
> u8 cmd;             /*00h */
> u8 sense_len;       /*01h */
> u8 cmd_status;      /*02h */
> u8 scsi_status;     /*03h */
> u8 target_id;       /*04h */
> u8 lun;             /*05h */
> u8 cdb_len;         /*06h */
> u8 sge_count;       /*07h */
> u32 context;        /*08h */
> u32 context_64;     /*0Ch */
> u16 flags;          /*10h */
> u16 timeout;        /*12h */
> u32 data_xfer_len;   /*14h */
> union {
>    struct {
>        u32 opcode;       /*18h */
>        u8 mbox[12];      /*1Ch */
>        u32 sgl_addr;     /*28h */
>        u32 sgl_len;      /*32h */
>        u32 pad;          /*34h */
>    } dcmd;
>    struct {
>        u32 sense_buf_lo; /*18h */
>        u32 sense_buf_hi; /*1Ch */
>        u8 cdb[16];       /*20h */
>        u32 sgl_addr;     /*30h */
>        u32 sgl_len;      /*34h */
>    } pthru;
>    struct {
>        u8 pad[22];       /*18h */
>    } gen;
> };
>
> +} __attribute__ ((packed));
>
> +struct mfi_ld_list_s {
> u32     count;
> u32     reserved_0;
> struct {
>    u8          target;
>    u8          lun;
>    u16         seq;
>    u8          state;
>    u8          reserved_1[3];
>    u64         size;
> } lds[64];
>
> +} __attribute__ ((packed));
>
> +#define MEGASAS_POLL_TIMEOUT 60000 // 60 seconds polling timeout
>
> +struct megasas_lun_s {
> struct drive_s drive;
> struct pci_device *pci;
> struct megasas_cmd_frame *frame;
> u32 iobase;
> u8 target;
> u8 lun;
>
> +};
>
> +static int megasas_fire_cmd(u32 ioaddr, struct megasas_cmd_frame *frame)
> +{
> u32 frame_addr = (u32)frame;
> int frame_count = 1;
> u8 cmd_state;
> u64 end;
> dprintf(1, "Frame 0x%x\n", frame_addr);
> outl(frame_addr | frame_count << 1 | 1, ioaddr + MFI_IQP);
> end = calc_future_tsc(MEGASAS_POLL_TIMEOUT);
> do {
>    for (;;) {
>        cmd_state = GET_LOWFLAT(frame->cmd_status);
>        if (cmd_state != 0xff)
>            break;
>        if (check_tsc(end)) {
>            warn_timeout();
>            return -1;
>        }
>        yield();
>    }
> } while (cmd_state == 0xff);
> if (cmd_state == 0 || cmd_state == 0x2d)
>    return 0;
> dprintf(1, "Frame 0x%x, status 0x%x\n", frame_addr, cmd_state);
> return -1;
>
> +}
>
> +int
> +megasas_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize)
> +{
> struct megasas_lun_s *mlun =
>    container_of(op->drive_g, struct megasas_lun_s, drive);
> u8 *cdb = cdbcmd;
> struct megasas_cmd_frame *frame = GET_GLOBAL(mlun->frame);
> int i;
> if (!CONFIG_MEGASAS)
>    return DISK_RET_EBADTRACK;
> memset_fl(frame, 0, sizeof(*frame));
> SET_LOWFLAT(frame->cmd, MFI_CMD_LD_SCSI_IO);
> SET_LOWFLAT(frame->cmd_status, 0xFF);
> SET_LOWFLAT(frame->target_id, GET_GLOBAL(mlun->target));
> SET_LOWFLAT(frame->lun, GET_GLOBAL(mlun->lun));
> SET_LOWFLAT(frame->flags, 0x0001);
> SET_LOWFLAT(frame->data_xfer_len, op->count * blocksize);
> SET_LOWFLAT(frame->cdb_len, 16);
> for (i = 0; i < 16; i++) {
>    SET_LOWFLAT(frame->pthru.cdb[i], cdb[i]);
> }
> dprintf(1, "pthru cmd 0x%x count %d bs %d\n",
>        cdb[0], op->count, blocksize);
> if (op->count) {
>    SET_LOWFLAT(frame->pthru.sgl_addr, (u32)op->buf_fl);
>    SET_LOWFLAT(frame->pthru.sgl_len, op->count * blocksize);
>    SET_LOWFLAT(frame->sge_count, 1);
> }
> SET_LOWFLAT(frame->context, (u32)frame);
> if (megasas_fire_cmd(GET_GLOBAL(mlun->iobase), frame) == 0)
>    return DISK_RET_SUCCESS;
> dprintf(1, "pthru cmd 0x%x failed\n", cdb[0]);
> return DISK_RET_EBADTRACK;
>
> +}
>
> +static int
> +megasas_add_lun(struct pci_device *pci, u32 iobase, u8 target, u8 lun)
> +{
> struct megasas_lun_s *mlun = malloc_fseg(sizeof(*mlun));
> char *name;
> int prio, ret = 0;
> if (!mlun) {
>    warn_noalloc();
>    return -1;
> }
> memset(mlun, 0, sizeof(*mlun));
> mlun->drive.type = DTYPE_MEGASAS;
> mlun->drive.cntl_id = pci->bdf;
> mlun->pci = pci;
> mlun->target = target;
> mlun->lun = lun;
> mlun->iobase = iobase;
> mlun->frame = memalign_high(256, sizeof(struct megasas_cmd_frame));
> if (!mlun->frame) {
>    warn_noalloc();
>    free(mlun);
>    return -1;
> }
> name = znprintf(36, "MegaRAID SAS (PCI %02x:%02x.%x) LD %d:%d",
>                pci_bdf_to_bus(pci->bdf), pci_bdf_to_dev(pci->bdf),
>                pci_bdf_to_fn(pci->bdf), target, lun);
> prio = bootprio_find_scsi_device(pci, target, lun);
> ret = scsi_init_drive(&mlun->drive, name, prio);
> free(name);
> if (ret) {
>    free(mlun->frame);
>    free(mlun);
>    ret = -1;
> }
> return ret;
>
> +}
>
> +static void megasas_scan_target(struct pci_device *pci, u32 iobase)
> +{
> struct mfi_ld_list_s ld_list;
> struct megasas_cmd_frame *frame = memalign_high(256, sizeof(*frame));
> int i;
> memset(&ld_list, 0, sizeof(ld_list));
> memset(frame, 0, sizeof(*frame));
> frame->cmd = MFI_CMD_DCMD;
> frame->cmd_status = 0xFF;
> frame->sge_count = 1;
> frame->flags = 0x0011;
> frame->data_xfer_len = sizeof(ld_list);
> frame->dcmd.opcode = 0x03010000;
> frame->dcmd.sgl_addr = (u32)MAKE_FLATPTR(GET_SEG(SS), &ld_list);
> frame->dcmd.sgl_len = sizeof(ld_list);
> frame->context = (u32)frame;
> if (megasas_fire_cmd(iobase, frame) != 0)
>    return;
> dprintf(1, "%d LD found\n", ld_list.count);
> for (i = 0; i < ld_list.count; i++) {
>    dprintf(1, "LD %d:%d state 0x%x\n",
>            ld_list.lds[i].target, ld_list.lds[i].lun,
>            ld_list.lds[i].state);
>    if (ld_list.lds[i].state != 0) {
>        megasas_add_lun(pci, iobase,
>                        ld_list.lds[i].target, ld_list.lds[i].lun);
>    }
> }
> free(frame);
>
> +}
>
> +static int megasas_transition_to_ready(u32 ioaddr)
> +{
> u32 fw_state, i = 0;
> fw_state = inl(ioaddr + MFI_OMSG0) & MFI_STATE_MASK;
> while (i < MEGASAS_POLL_TIMEOUT) {
>    if (fw_state == MFI_STATE_OPERATIONAL)
>        outw(ioaddr + MFI_ODB, 0x07);
>    if (fw_state == MFI_STATE_READY)
>        break;
>    msleep(1);
>    fw_state = inl(ioaddr + MFI_OMSG0) & MFI_STATE_MASK;
>    i++;
> }
> if (fw_state == MFI_STATE_READY) {
>        dprintf(1, "MegaRAID SAS fw ready\n");
>        return 0;
> }
> dprintf(1, "ERROR: fw in state %x\n", fw_state & MFI_STATE_MASK);
> return -1;
>
> +}
>
> +static void
> +init_megasas(struct pci_device *pci)
> +{
> u16 bdf = pci->bdf;
> u32 iobase = pci_config_readl(pci->bdf, PCI_BASE_ADDRESS_2)
>    & PCI_BASE_ADDRESS_IO_MASK;
> dprintf(1, "found MegaRAID SAS at %02x:%02x.%x, io @ %x\n",
>        pci_bdf_to_bus(bdf), pci_bdf_to_dev(bdf),
>        pci_bdf_to_fn(bdf), iobase);
> pci_config_maskw(pci->bdf, PCI_COMMAND, 0,
>                 PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER);
> // reset
> if (megasas_transition_to_ready(iobase) == 0)
>    megasas_scan_target(pci, iobase);
> return;
>
> +}
>
> +void
> +megasas_setup(void)
> +{
> ASSERT32FLAT();
> if (!CONFIG_MEGASAS)
>    return;
> dprintf(3, "init megasas\n");
> struct pci_device *pci;
> foreachpci(pci) {
>    if (pci->vendor != PCI_VENDOR_ID_LSI_LOGIC
>        || pci->device !=PCI_DEVICE_ID_LSI_SAS1078)
>        continue;
>    init_megasas(pci);
> }
>
> +}
> diff --git a/src/megasas.h b/src/megasas.h
> new file mode 100644
> index 0000000..124042e
> --- /dev/null
> +++ b/src/megasas.h
> @@ -0,0 +1,8 @@
> +#ifndef __MEGASAS_H
> +#define __MEGASAS_H
>
> +struct disk_op_s;
> +int megasas_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize);
> +void megasas_setup(void);
>
> +#endif /* __MEGASAS_H */
> diff --git a/src/post.c b/src/post.c
> index 924b311..0133f75 100644
> --- a/src/post.c
> +++ b/src/post.c
> @@ -29,7 +29,7 @@
>  #include "virtio-scsi.h" // virtio_scsi_setup
>  #include "lsi-scsi.h" // lsi_scsi_setup
>  #include "esp-scsi.h" // esp_scsi_setup
>
> +#include "megasas.h" // megasas_setup
>
> /****************************************************************
> BIOS init
>
> @@ -198,6 +198,7 @@ init_hw(void)
>      virtio_scsi_setup();
>      lsi_scsi_setup();
>      esp_scsi_setup();
> megasas_setup();
>
> }
>
> // Begin the boot process by invoking an int0x19 in 16bit mode.

[Kevin O'Connor]

On Mon, Nov 12, 2012 at 03:45:19PM +0100, Hannes Reinecke wrote:
> This patch adds support for LSI MegaRAID SAS controllers.
> Currently only 8708EM2 is supported.

Thanks.  Some comments below.

[...]
> SET_LOWFLAT(frame->cmd, MFI_CMD_LD_SCSI_IO);

[...]
> mlun->frame = memalign_high(256, sizeof(struct megasas_cmd_frame));

Something's not right here.  If the memory is allocated in high ram,
then it isn't accessible via GET/SET_LOW.  Only memalign_low memory is
available with the GET/SET_LOW macros.

[...]
> struct megasas_cmd_frame *frame = memalign_high(256, sizeof(*frame));
> int i;
> memset(&ld_list, 0, sizeof(ld_list));
> memset(frame, 0, sizeof(*frame));
> frame->cmd = MFI_CMD_DCMD;
> frame->cmd_status = 0xFF;
> frame->sge_count = 1;
> frame->flags = 0x0011;
> frame->data_xfer_len = sizeof(ld_list);
> frame->dcmd.opcode = 0x03010000;
> frame->dcmd.sgl_addr = (u32)MAKE_FLATPTR(GET_SEG(SS), &ld_list);
> frame->dcmd.sgl_len = sizeof(ld_list);
> frame->context = (u32)frame;
> if (megasas_fire_cmd(iobase, frame) != 0)
>    return;

Missing free(frame).
> dprintf(1, "%d LD found\n", ld_list.count);
> for (i = 0; i < ld_list.count; i++) {
>    dprintf(1, "LD %d:%d state 0x%x\n",
>            ld_list.lds[i].target, ld_list.lds[i].lun,
>            ld_list.lds[i].state);
>    if (ld_list.lds[i].state != 0) {
>        megasas_add_lun(pci, iobase,
>                        ld_list.lds[i].target, ld_list.lds[i].lun);
>    }
> }
> free(frame);
>
> +}

Although valid, it's not a good idea to use memalign_high for
temporary space.  One should use memalign_tmp for that purpose.

-Kevin

[Hannes Reinecke]

On 11/12/2012 06:28 PM, Paolo Bonzini wrote:
> Il 12/11/2012 15:45, Hannes Reinecke ha scritto:
> > This patch adds support for LSI MegaRAID SAS controllers.
> > Currently only 8708EM2 is supported.
>
> Did you test it on real hardware?  For example via PCI passthrough.

Not yet.
> > diff --git a/src/megasas.c b/src/megasas.c
> > new file mode 100644
> > index 0000000..4b81334
> > --- /dev/null
> > +++ b/src/megasas.c
> > @@ -0,0 +1,311 @@
> > +// (qemu-emulated) megaraid_sas boot support.
>
> If also "real" MegaSAS is supported, please fix the comment.  Otherwise
> make it depend on QEMU too.

The goal here is to have this work on any MegaRAID SAS HBA, as they 
only differ in the low-level interface.
IE only the 'outl' line in megasas_fire_cmd needs to be modified, 
the rest is pretty much the same.

Hence I've omitted the dependency on Qemu, as this should work on 
any MegaRAID SAS HBA.

I'll be updating the patch to include all PCI-IDs, and put in the 
correct low-level functions.

Cheers,

Hannes

[Hannes Reinecke]

On 11/13/2012 01:39 AM, Kevin O'Connor wrote:
> On Mon, Nov 12, 2012 at 03:45:19PM +0100, Hannes Reinecke wrote:
> > This patch adds support for LSI MegaRAID SAS controllers.
> > Currently only 8708EM2 is supported.
>
> Thanks.  Some comments below.
>
> [...]
> > SET_LOWFLAT(frame->cmd, MFI_CMD_LD_SCSI_IO);
>
> [...]
> > mlun->frame = memalign_high(256, sizeof(struct megasas_cmd_frame));
>
> Something's not right here.  If the memory is allocated in high ram,
> then it isn't accessible via GET/SET_LOW.  Only memalign_low memory is
> available with the GET/SET_LOW macros.

I'm not at all surprised. I've still only the haziest of ideas of 
how to use memalign_XX and GET/SET macros.

I actually took the ahci code as a reference on how to use those 
macros, and they are using GET/SET_LOW and memalign_tmp().

So yeah, I'll be converting it to memalign_tmp().
> [...]
> > struct megasas_cmd_frame *frame = memalign_high(256, sizeof(*frame));
> > int i;
> > memset(&ld_list, 0, sizeof(ld_list));
> > memset(frame, 0, sizeof(*frame));
> > frame->cmd = MFI_CMD_DCMD;
> > frame->cmd_status = 0xFF;
> > frame->sge_count = 1;
> > frame->flags = 0x0011;
> > frame->data_xfer_len = sizeof(ld_list);
> > frame->dcmd.opcode = 0x03010000;
> > frame->dcmd.sgl_addr = (u32)MAKE_FLATPTR(GET_SEG(SS), &ld_list);
> > frame->dcmd.sgl_len = sizeof(ld_list);
> > frame->context = (u32)frame;
> > if (megasas_fire_cmd(iobase, frame) != 0)
> >    return;
>
> Missing free(frame).

Yep.
> > dprintf(1, "%d LD found\n", ld_list.count);
> > for (i = 0; i < ld_list.count; i++) {
> >    dprintf(1, "LD %d:%d state 0x%x\n",
> >            ld_list.lds[i].target, ld_list.lds[i].lun,
> >            ld_list.lds[i].state);
> >    if (ld_list.lds[i].state != 0) {
> >        megasas_add_lun(pci, iobase,
> >                        ld_list.lds[i].target, ld_list.lds[i].lun);
> >    }
> > }
> > free(frame);
> >
> > +}
>
> Although valid, it's not a good idea to use memalign_high for
> temporary space.  One should use memalign_tmp for that purpose.

The thing here is that megasas_fire_cmd() is a generic function, so 
it needs to work in both 16 and 32-bit environs. So the frame 
pointer passed to this function needs to reference the same segment.

And this is where things get confused.
I've converted the driver to use memalign_tmp() for allocating space 
for the frame. Which works.

But then you claim that GET/SET_LOW won't work for space located
high memory. However, when using memalign_tmp() it's impossible to 
tell where the space is located at:

static inline void *memalign_tmp(u32 align, u32 size) {
     void *ret = memalign_tmphigh(align, size);
     if (ret)
         return ret;
     return memalign_tmplow(align, size);
}

So which macro do have to use to access this memory?
And why does GET/SET_LOW work?

Cheers,

Hannes

[Gerd Hoffmann]

Hi,
> > Although valid, it's not a good idea to use memalign_high for
> > temporary space.  One should use memalign_tmp for that purpose.
>
> The thing here is that megasas_fire_cmd() is a generic function, so it
> needs to work in both 16 and 32-bit environs. So the frame pointer
> passed to this function needs to reference the same segment.

32-bit mode is relaxed, everything is a flat 32bit ptr and all those
funky segment/offset macros do basically nothing.

For 16-bit mode it is very important to get things right ...
> And this is where things get confused.
> I've converted the driver to use memalign_tmp() for allocating space for
> the frame. Which works.

The strategy ahci uses is to memalign_tmp() the data structures needed
for device probing, so they can easily freed in case the port is unused.
 When a drive is actually found (and registered) the memory for the data
structures is allocated again using memalign_low(), so it is permanently
available, accessible in 16-bit mode and the 16-bit only code paths
don't have to do any memory allocations when kicking transfers.

I think there are code paths where SET_LOWFLAT & friends are used for
memory allocated via memalign_tmp(), but that happens in 32bit mode only
where it doesn't really matter.

See ahci_port_realloc().

HTH,
  Gerd

[Hannes Reinecke]

On 11/13/2012 09:01 AM, Gerd Hoffmann wrote:
> Hi,
> > > Although valid, it's not a good idea to use memalign_high for
> > > temporary space.  One should use memalign_tmp for that purpose.
> >
> > The thing here is that megasas_fire_cmd() is a generic function, so it
> > needs to work in both 16 and 32-bit environs. So the frame pointer
> > passed to this function needs to reference the same segment.
>
> 32-bit mode is relaxed, everything is a flat 32bit ptr and all those
> funky segment/offset macros do basically nothing.
>
> For 16-bit mode it is very important to get things right ...
> > And this is where things get confused.
> > I've converted the driver to use memalign_tmp() for allocating space for
> > the frame. Which works.
>
> The strategy ahci uses is to memalign_tmp() the data structures needed
> for device probing, so they can easily freed in case the port is unused.
>   When a drive is actually found (and registered) the memory for the data
> structures is allocated again using memalign_low(), so it is permanently
> available, accessible in 16-bit mode and the 16-bit only code paths
> don't have to do any memory allocations when kicking transfers.
>
> I think there are code paths where SET_LOWFLAT & friends are used for
> memory allocated via memalign_tmp(), but that happens in 32bit mode only
> where it doesn't really matter.
>
> See ahci_port_realloc().

Ah. Thanks for the explanation.

I've converted it now to use memalign_tmplow(), so everybody should 
be happy.

Cheers,

Hannes

[Kevin O'Connor]

On Tue, Nov 13, 2012 at 09:15:03AM +0100, Hannes Reinecke wrote:
> On 11/13/2012 09:01 AM, Gerd Hoffmann wrote:
> > Hi,
> > > > Although valid, it's not a good idea to use memalign_high for
> > > > temporary space.  One should use memalign_tmp for that purpose.
> > >
> > > The thing here is that megasas_fire_cmd() is a generic function, so it
> > > needs to work in both 16 and 32-bit environs. So the frame pointer
> > > passed to this function needs to reference the same segment.
> >
> > 32-bit mode is relaxed, everything is a flat 32bit ptr and all those
> > funky segment/offset macros do basically nothing.
> >
> > For 16-bit mode it is very important to get things right ...
> > > And this is where things get confused.
> > > I've converted the driver to use memalign_tmp() for allocating space for
> > > the frame. Which works.
> >
> > The strategy ahci uses is to memalign_tmp() the data structures needed
> > for device probing, so they can easily freed in case the port is unused.
> >  When a drive is actually found (and registered) the memory for the data
> > structures is allocated again using memalign_low(), so it is permanently
> > available, accessible in 16-bit mode and the 16-bit only code paths
> > don't have to do any memory allocations when kicking transfers.
> >
> > I think there are code paths where SET_LOWFLAT & friends are used for
> > memory allocated via memalign_tmp(), but that happens in 32bit mode only
> > where it doesn't really matter.
> >
> > See ahci_port_realloc().
>
> Ah. Thanks for the explanation.
>
> I've converted it now to use memalign_tmplow(), so everybody should
> be happy.

The memalign_tmp* variants are implicitly free'd once the init section
of the code is completed.  So, you can't use that for any allocations
used during the 16bit runtime phase.

If the memory is to be used during the 16bit runtime, it must be
allocated with memalign_low().  If the memory is used only during the
32bit probe phase, then you want to use memalign_tmp().

I wrote up a description of all the allocators, but never got a chance
to publish it.  I'll see if I can get it out.

-Kevin