[PATCH v2 0/2] pci: add mmconfig support

[Gerd Hoffmann]

Gerd Hoffmann (2):
  pci: factor out ioconfig_cmd()
  pci: add mmconfig support

src/hw/pci.h     |  1 +
 src/fw/pciinit.c |  1 +
 src/hw/pci.c     | 69 +++++++++++++++++++++++++++++++++++++++---------
 3 files changed, 59 insertions(+), 12 deletions(-)

[PATCH v2 1/2] pci: factor out ioconfig_cmd()

[Gerd Hoffmann]

Add helper function to calculate PORT_PCI_CMD value from bdf + addr.

Signed-off-by: Gerd Hoffmann 
---
 src/hw/pci.c | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/hw/pci.c b/src/hw/pci.c
index 9855badbc7da..7aca1e6bfa97 100644
--- a/src/hw/pci.c
+++ b/src/hw/pci.c
@@ -14,39 +14,44 @@
 #define PORT_PCI_CMD           0x0cf8
 #define PORT_PCI_DATA          0x0cfc

+static u32 ioconfig_cmd(u16 bdf, u32 addr)
+{
+    return 0x80000000 | (bdf << 8) | (addr & 0xfc);
+}
+
 void pci_config_writel(u16 bdf, u32 addr, u32 val)
 {
-    outl(0x80000000 | (bdf << 8) | (addr & 0xfc), PORT_PCI_CMD);
+    outl(ioconfig_cmd(bdf, addr), PORT_PCI_CMD);
     outl(val, PORT_PCI_DATA);
 }

void pci_config_writew(u16 bdf, u32 addr, u16 val)
 {
-    outl(0x80000000 | (bdf << 8) | (addr & 0xfc), PORT_PCI_CMD);
+    outl(ioconfig_cmd(bdf, addr), PORT_PCI_CMD);
     outw(val, PORT_PCI_DATA + (addr & 2));
 }

void pci_config_writeb(u16 bdf, u32 addr, u8 val)
 {
-    outl(0x80000000 | (bdf << 8) | (addr & 0xfc), PORT_PCI_CMD);
+    outl(ioconfig_cmd(bdf, addr), PORT_PCI_CMD);
     outb(val, PORT_PCI_DATA + (addr & 3));
 }

u32 pci_config_readl(u16 bdf, u32 addr)
 {
-    outl(0x80000000 | (bdf << 8) | (addr & 0xfc), PORT_PCI_CMD);
+    outl(ioconfig_cmd(bdf, addr), PORT_PCI_CMD);
     return inl(PORT_PCI_DATA);
 }

u16 pci_config_readw(u16 bdf, u32 addr)
 {
-    outl(0x80000000 | (bdf << 8) | (addr & 0xfc), PORT_PCI_CMD);
+    outl(ioconfig_cmd(bdf, addr), PORT_PCI_CMD);
     return inw(PORT_PCI_DATA + (addr & 2));
 }

u8 pci_config_readb(u16 bdf, u32 addr)
 {
-    outl(0x80000000 | (bdf << 8) | (addr & 0xfc), PORT_PCI_CMD);
+    outl(ioconfig_cmd(bdf, addr), PORT_PCI_CMD);
     return inb(PORT_PCI_DATA + (addr & 3));
 }

[PATCH v2 2/2] pci: add mmconfig support

[Gerd Hoffmann]

Add support for pci config space access via mmconfig bar.  Enable for
qemu q35 chipset.  Main advantage is that we need only one instead of
two io operations per config space access, which translates to one
instead of two vmexits for virtualization.

Signed-off-by: Gerd Hoffmann 
---
 src/hw/pci.h     |  1 +
 src/fw/pciinit.c |  1 +
 src/hw/pci.c     | 64 +++++++++++++++++++++++++++++++++++++++---------
 3 files changed, 54 insertions(+), 12 deletions(-)

diff --git a/src/hw/pci.h b/src/hw/pci.h
index 2e30e28918a0..01c51f705a00 100644
--- a/src/hw/pci.h
+++ b/src/hw/pci.h
@@ -39,6 +39,7 @@ u32 pci_config_readl(u16 bdf, u32 addr);
 u16 pci_config_readw(u16 bdf, u32 addr);
 u8 pci_config_readb(u16 bdf, u32 addr);
 void pci_config_maskw(u16 bdf, u32 addr, u16 off, u16 on);
+void pci_enable_mmconfig(u64 addr, const char *name);
 u8 pci_find_capability(u16 bdf, u8 cap_id, u8 cap);
 int pci_next(int bdf, int bus);
 int pci_probe_host(void);
diff --git a/src/fw/pciinit.c b/src/fw/pciinit.c
index d5e87f00f164..d25931bb0573 100644
--- a/src/fw/pciinit.c
+++ b/src/fw/pciinit.c
@@ -480,6 +480,7 @@ static void mch_mmconfig_setup(u16 bdf)
     pci_config_writel(bdf, Q35_HOST_BRIDGE_PCIEXBAR, 0);
     pci_config_writel(bdf, Q35_HOST_BRIDGE_PCIEXBAR + 4, upper);
     pci_config_writel(bdf, Q35_HOST_BRIDGE_PCIEXBAR, lower);
+    pci_enable_mmconfig(Q35_HOST_BRIDGE_PCIEXBAR_ADDR, "q35");
 }

static void mch_mem_addr_setup(struct pci_device *dev, void *arg)
diff --git a/src/hw/pci.c b/src/hw/pci.c
index 7aca1e6bfa97..d9dbf313965c 100644
--- a/src/hw/pci.c
+++ b/src/hw/pci.c
@@ -14,6 +14,13 @@
 #define PORT_PCI_CMD           0x0cf8
 #define PORT_PCI_DATA          0x0cfc

+static u32 mmconfig;
+
+static void *mmconfig_addr(u16 bdf, u32 addr)
+{
+    return (void*)(mmconfig + ((u32)bdf << 12) + addr);
+}
+
 static u32 ioconfig_cmd(u16 bdf, u32 addr)
 {
     return 0x80000000 | (bdf << 8) | (addr & 0xfc);
@@ -21,38 +28,62 @@ static u32 ioconfig_cmd(u16 bdf, u32 addr)

void pci_config_writel(u16 bdf, u32 addr, u32 val)
 {
-    outl(ioconfig_cmd(bdf, addr), PORT_PCI_CMD);
-    outl(val, PORT_PCI_DATA);
+    if (MODESEGMENT && mmconfig) {
+        writel(mmconfig_addr(bdf, addr), val);
+    } else {
+        outl(ioconfig_cmd(bdf, addr), PORT_PCI_CMD);
+        outl(val, PORT_PCI_DATA);
+    }
 }

void pci_config_writew(u16 bdf, u32 addr, u16 val)
 {
-    outl(ioconfig_cmd(bdf, addr), PORT_PCI_CMD);
-    outw(val, PORT_PCI_DATA + (addr & 2));
+    if (MODESEGMENT && mmconfig) {
+        writew(mmconfig_addr(bdf, addr), val);
+    } else {
+        outl(ioconfig_cmd(bdf, addr), PORT_PCI_CMD);
+        outw(val, PORT_PCI_DATA + (addr & 2));
+    }
 }

void pci_config_writeb(u16 bdf, u32 addr, u8 val)
 {
-    outl(ioconfig_cmd(bdf, addr), PORT_PCI_CMD);
-    outb(val, PORT_PCI_DATA + (addr & 3));
+    if (MODESEGMENT && mmconfig) {
+        writeb(mmconfig_addr(bdf, addr), val);
+    } else {
+        outl(ioconfig_cmd(bdf, addr), PORT_PCI_CMD);
+        outb(val, PORT_PCI_DATA + (addr & 3));
+    }
 }

u32 pci_config_readl(u16 bdf, u32 addr)
 {
-    outl(ioconfig_cmd(bdf, addr), PORT_PCI_CMD);
-    return inl(PORT_PCI_DATA);
+    if (MODESEGMENT && mmconfig) {
+        return readl(mmconfig_addr(bdf, addr));
+    } else {
+        outl(ioconfig_cmd(bdf, addr), PORT_PCI_CMD);
+        return inl(PORT_PCI_DATA);
+    }
 }

u16 pci_config_readw(u16 bdf, u32 addr)
 {
-    outl(ioconfig_cmd(bdf, addr), PORT_PCI_CMD);
-    return inw(PORT_PCI_DATA + (addr & 2));
+    if (MODESEGMENT && mmconfig) {
+        return readw(mmconfig_addr(bdf, addr));
+    } else {
+        outl(ioconfig_cmd(bdf, addr), PORT_PCI_CMD);
+        return inw(PORT_PCI_DATA + (addr & 2));
+    }
 }

u8 pci_config_readb(u16 bdf, u32 addr)
 {
-    outl(ioconfig_cmd(bdf, addr), PORT_PCI_CMD);
-    return inb(PORT_PCI_DATA + (addr & 3));
+    if (MODESEGMENT && mmconfig) {
+        return readb(mmconfig_addr(bdf, addr));
+    } else {
+        outl(ioconfig_cmd(bdf, addr), PORT_PCI_CMD);
+        return inb(PORT_PCI_DATA + (addr & 3));
+    }
 }

void
@@ -63,6 +94,15 @@ pci_config_maskw(u16 bdf, u32 addr, u16 off, u16 on)
     pci_config_writew(bdf, addr, val);
 }

+void
+pci_enable_mmconfig(u64 addr, const char *name)
+{
+    if (addr >= 0x100000000ll)
+        return;
+    dprintf(1, "PCIe: using %s mmconfig at 0x%llx\n", name, addr);
+    mmconfig = addr;
+}
+
 u8 pci_find_capability(u16 bdf, u8 cap_id, u8 cap)
 {
     int i;

[Kevin O'Connor]

On Mon, Mar 23, 2020 at 03:59:09PM +0100, Gerd Hoffmann wrote:
> Gerd Hoffmann (2):
>   pci: factor out ioconfig_cmd()
>   pci: add mmconfig support

Thanks.  Looks good to me.

One question - how would this work with multiple independent PCI
buses?  Or is the intent to only enable this on machines with a single
PCI bus?

-Kevin

[Gerd Hoffmann]

On Mon, Mar 23, 2020 at 11:12:25AM -0400, Kevin O'Connor wrote:
> On Mon, Mar 23, 2020 at 03:59:09PM +0100, Gerd Hoffmann wrote:
> > Gerd Hoffmann (2):
> >   pci: factor out ioconfig_cmd()
> >   pci: add mmconfig support
>
> Thanks.  Looks good to me.
>
> One question - how would this work with multiple independent PCI
> buses?  Or is the intent to only enable this on machines with a single
> PCI bus?

multiple pci busses/roots should work fine.

multiple pci domains probably not (but I think that is the case
for ioconfig too).

cheers,
  Gerd

[Gerd Hoffmann]

On Mon, Mar 23, 2020 at 11:12:25AM -0400, Kevin O'Connor wrote:
> On Mon, Mar 23, 2020 at 03:59:09PM +0100, Gerd Hoffmann wrote:
> > Gerd Hoffmann (2):
> >   pci: factor out ioconfig_cmd()
> >   pci: add mmconfig support
>
> Thanks.  Looks good to me.

Pushed.

cheers,
  Gerd