[PATCH v1 3/3] vpci/msix: move common msix code to common file

[Mykyta Poturai <Mykyta_Poturai@xxxxxxxx>]

From: Stewart Hildebrand <stewart.hildebrand@xxxxxxx>

vPCI MSI-X code dealing with intercepting the MSI-X tables and PBA won't
be used on Arm. Moving this code would result in a large diffstat, so
instead we move the common code to a common msix file to achieve a
smaller diffstat.

Signed-off-by: Stewart Hildebrand <stewart.hildebrand@xxxxxxx>
Signed-off-by: Mykyta Poturai <mykyta_poturai@xxxxxxxx>
---
 xen/drivers/vpci/Makefile      |   6 +-
 xen/drivers/vpci/msix.c        | 160 +-------------------------
 xen/drivers/vpci/msix_common.c | 202 +++++++++++++++++++++++++++++++++
 xen/include/xen/vpci.h         |   5 +
 4 files changed, 215 insertions(+), 158 deletions(-)
 create mode 100644 xen/drivers/vpci/msix_common.c

diff --git a/xen/drivers/vpci/Makefile b/xen/drivers/vpci/Makefile
index 9793a4f9b0..b4026cec81 100644
--- a/xen/drivers/vpci/Makefile
+++ b/xen/drivers/vpci/Makefile
@@ -1,3 +1,7 @@
 obj-y += cap.o
 obj-y += vpci.o header.o rebar.o
-obj-$(CONFIG_HAS_PCI_MSI) += msi.o msix.o
+obj-$(CONFIG_HAS_PCI_MSI) += msi.o
+obj-$(CONFIG_HAS_PCI_MSI) += msix_common.o
+ifeq ($(CONFIG_X86),y)
+obj-$(CONFIG_HAS_PCI_MSI) += msix.o
+endif
diff --git a/xen/drivers/vpci/msix.c b/xen/drivers/vpci/msix.c
index 4a1e7561f6..ab0853f70d 100644
--- a/xen/drivers/vpci/msix.c
+++ b/xen/drivers/vpci/msix.c
@@ -36,16 +36,6 @@
      PFN_DOWN(addr) <= PFN_DOWN(vmsix_table_addr(vpci, nr) +              \
                                 vmsix_table_size(vpci, nr) - 1))
 
-static uint32_t cf_check control_read(
-    const struct pci_dev *pdev, unsigned int reg, void *data)
-{
-    const struct vpci_msix *msix = data;
-
-    return (msix->max_entries - 1) |
-           (msix->enabled ? PCI_MSIX_FLAGS_ENABLE : 0) |
-           (msix->masked ? PCI_MSIX_FLAGS_MASKALL : 0);
-}
-
 static void update_entry(struct vpci_msix_entry *entry,
                          const struct pci_dev *pdev, unsigned int nr)
 {
@@ -74,7 +64,7 @@ static void update_entry(struct vpci_msix_entry *entry,
     entry->updated = false;
 }
 
-static void cf_check control_write(
+void vpci_msix_arch_control_write(
     const struct pci_dev *pdev, unsigned int reg, uint32_t val, void *data)
 {
     struct vpci_msix *msix = data;
@@ -83,9 +73,6 @@ static void cf_check control_write(
     unsigned int i;
     int rc;
 
-    if ( new_masked == msix->masked && new_enabled == msix->enabled )
-        return;
-
     /*
      * According to the PCI 3.0 specification, switching the enable bit to 1
      * or the function mask bit to 0 should cause all the cached addresses
@@ -138,20 +125,6 @@ static void cf_check control_write(
             }
         }
     }
-
-    /* Make sure domU doesn't enable INTx while enabling MSI-X. */
-    if ( new_enabled && !msix->enabled && !is_hardware_domain(pdev->domain) )
-    {
-        pci_intx(pdev, false);
-        pdev->vpci->header.guest_cmd |= PCI_COMMAND_INTX_DISABLE;
-    }
-
-    msix->masked = new_masked;
-    msix->enabled = new_enabled;
-
-    val = control_read(pdev, reg, data);
-    if ( pci_msi_conf_write_intercept(msix->pdev, reg, 2, &val) >= 0 )
-        pci_conf_write16(pdev->sbdf, reg, val);
 }
 
 static struct vpci_msix *msix_find(const struct domain *d, unsigned long addr)
@@ -694,151 +667,24 @@ int vpci_make_msix_hole(const struct pci_dev *pdev)
     return 0;
 }
 
-static int cf_check cleanup_msix(const struct pci_dev *pdev, bool hide)
+void vpci_msix_arch_cleanup(struct vpci *vpci)
 {
-    int rc;
-    struct vpci *vpci = pdev->vpci;
-    const unsigned int msix_pos = pdev->msix_pos;
-
     if ( vpci->msix )
     {
         list_del(&vpci->msix->next);
         for ( unsigned int i = 0; i < ARRAY_SIZE(vpci->msix->table); i++ )
             if ( vpci->msix->table[i] )
                 iounmap(vpci->msix->table[i]);
-
-        XFREE(vpci->msix);
-    }
-
-    if ( !hide )
-        return 0;
-
-    rc = vpci_remove_registers(vpci, msix_control_reg(msix_pos), 2);
-    if ( rc )
-    {
-        printk(XENLOG_ERR "%pd %pp: fail to remove MSIX handlers rc=%d\n",
-               pdev->domain, &pdev->sbdf, rc);
-        ASSERT_UNREACHABLE();
-        return rc;
     }
-
-    /*
-     * Unprivileged domains have a deny by default register access policy, no
-     * need to add any further handlers for them.
-     */
-    if ( !is_hardware_domain(pdev->domain) )
-        return 0;
-
-    /*
-     * The driver may not traverse the capability list and think device
-     * supports MSIX by default. So here let the control register of MSIX
-     * be Read-Only is to ensure MSIX disabled.
-     */
-    rc = vpci_add_register(vpci, vpci_hw_read16, NULL,
-                           msix_control_reg(msix_pos), 2, NULL);
-    if ( rc )
-        printk(XENLOG_ERR "%pd %pp: fail to add MSIX ctrl handler rc=%d\n",
-               pdev->domain, &pdev->sbdf, rc);
-
-    return rc;
 }
 
-static int cf_check init_msix(struct pci_dev *pdev)
+void vpci_msix_arch_register(struct vpci_msix *msix, struct domain *d)
 {
-    struct domain *d = pdev->domain;
-    unsigned int msix_offset, i, max_entries;
-    uint16_t control;
-    struct vpci_msix *msix;
-    int rc;
-
-    msix_offset = pdev->msix_pos;
-    if ( !msix_offset )
-        return 0;
-
-    control = pci_conf_read16(pdev->sbdf, msix_control_reg(msix_offset));
-
-    max_entries = msix_table_size(control);
-
-    msix = xzalloc_flex_struct(struct vpci_msix, entries, max_entries);
-    if ( !msix )
-        return -ENOMEM;
-
-    msix->tables[VPCI_MSIX_TABLE] =
-        pci_conf_read32(pdev->sbdf, msix_table_offset_reg(msix_offset));
-    msix->tables[VPCI_MSIX_PBA] =
-        pci_conf_read32(pdev->sbdf, msix_pba_offset_reg(msix_offset));
-
-    /* Check that the referenced BAR(s) regions are valid. */
-    for ( i = 0; i < ARRAY_SIZE(msix->tables); i++ )
-    {
-        const char *name = (i == VPCI_MSIX_TABLE) ? "vector" : "PBA";
-        const struct vpci_bar *bars = pdev->vpci->header.bars;
-        unsigned int bir = msix->tables[i] & PCI_MSIX_BIRMASK;
-        unsigned int type;
-        unsigned int offset = msix->tables[i] & ~PCI_MSIX_BIRMASK;
-        unsigned int size =
-            (i == VPCI_MSIX_TABLE) ? max_entries * PCI_MSIX_ENTRY_SIZE
-                                   : ROUNDUP(DIV_ROUND_UP(max_entries, 8), 8);
-
-        if ( bir >= ARRAY_SIZE(pdev->vpci->header.bars) )
-        {
-            printk(XENLOG_ERR DEV_BUG_PREFIX
-                   "%pp: MSI-X %s table with out of range BIR %u\n",
-                   &pdev->sbdf, name, bir);
- invalid:
-            xfree(msix);
-            return -ENODEV;
-        }
-
-        type = bars[bir].type;
-        if ( type != VPCI_BAR_MEM32 && type != VPCI_BAR_MEM64_LO )
-        {
-            printk(XENLOG_ERR DEV_BUG_PREFIX
-                   "%pp: MSI-X %s table at invalid BAR%u with type %u\n",
-                   &pdev->sbdf, name, bir, type);
-            goto invalid;
-        }
-
-        if ( (uint64_t)offset + size > bars[bir].size )
-        {
-            printk(XENLOG_ERR DEV_BUG_PREFIX
-                   "%pp: MSI-X %s table offset %#x size %#x outside of BAR%u 
size %#lx\n",
-                   &pdev->sbdf, name, offset, size, bir, bars[bir].size);
-            goto invalid;
-        }
-    }
-
-    rc = vpci_add_register(pdev->vpci, control_read, control_write,
-                           msix_control_reg(msix_offset), 2, msix);
-    if ( rc )
-    {
-        xfree(msix);
-        return rc;
-    }
-
-    msix->max_entries = max_entries;
-    msix->pdev = pdev;
-
-    for ( i = 0; i < max_entries; i++)
-    {
-        msix->entries[i].masked = true;
-        vpci_msix_arch_init_entry(&msix->entries[i]);
-    }
-
     if ( list_empty(&d->arch.hvm.msix_tables) )
         register_mmio_handler(d, &vpci_msix_table_ops);
 
-    pdev->vpci->msix = msix;
     list_add(&msix->next, &d->arch.hvm.msix_tables);
-
-    /*
-     * vPCI header initialization will have mapped the whole BAR into the
-     * p2m, as MSI-X capability was not yet initialized.  Carve a hole for
-     * the MSI-X table here, so that Xen can trap accesses.
-     */
-    return vpci_make_msix_hole(pdev);
 }
-REGISTER_VPCI_CAP(MSIX, init_msix, cleanup_msix);
 
 /*
  * Local variables:
diff --git a/xen/drivers/vpci/msix_common.c b/xen/drivers/vpci/msix_common.c
new file mode 100644
index 0000000000..b38de48579
--- /dev/null
+++ b/xen/drivers/vpci/msix_common.c
@@ -0,0 +1,202 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Handlers for accesses to the MSI-X capability structure and the memory
+ * region.
+ *
+ * Copyright (C) 2017 Citrix Systems R&D
+ */
+
+#include "private.h"
+
+#include <xen/io.h>
+#include <xen/lib.h>
+#include <xen/msi.h>
+#include <xen/sched.h>
+
+#include <asm/p2m.h>
+
+static uint32_t cf_check control_read(
+    const struct pci_dev *pdev, unsigned int reg, void *data)
+{
+    const struct vpci_msix *msix = data;
+
+    return (msix->max_entries - 1) |
+           (msix->enabled ? PCI_MSIX_FLAGS_ENABLE : 0) |
+           (msix->masked ? PCI_MSIX_FLAGS_MASKALL : 0);
+}
+
+static void cf_check control_write(
+    const struct pci_dev *pdev, unsigned int reg, uint32_t val, void *data)
+{
+    struct vpci_msix *msix = data;
+    bool new_masked = val & PCI_MSIX_FLAGS_MASKALL;
+    bool new_enabled = val & PCI_MSIX_FLAGS_ENABLE;
+
+    if ( new_masked == msix->masked && new_enabled == msix->enabled )
+        return;
+
+    vpci_msix_arch_control_write(pdev, reg, val, data);
+
+    /* Make sure domU doesn't enable INTx while enabling MSI-X. */
+    if ( new_enabled && !msix->enabled && !is_hardware_domain(pdev->domain) )
+    {
+        pci_intx(pdev, false);
+        pdev->vpci->header.guest_cmd |= PCI_COMMAND_INTX_DISABLE;
+    }
+
+    msix->masked = new_masked;
+    msix->enabled = new_enabled;
+
+    val = control_read(pdev, reg, data);
+    if ( pci_msi_conf_write_intercept(msix->pdev, reg, 2, &val) >= 0 )
+        pci_conf_write16(pdev->sbdf, reg, val);
+}
+
+static int cf_check cleanup_msix(const struct pci_dev *pdev, bool hide)
+{
+    int rc;
+    struct vpci *vpci = pdev->vpci;
+    const unsigned int msix_pos = pdev->msix_pos;
+
+    vpci_msix_arch_cleanup(vpci);
+
+    if ( vpci->msix )
+        XFREE(vpci->msix);
+
+    if ( !hide )
+        return 0;
+
+    rc = vpci_remove_registers(vpci, msix_control_reg(msix_pos), 2);
+    if ( rc )
+    {
+        printk(XENLOG_ERR "%pd %pp: fail to remove MSIX handlers rc=%d\n",
+               pdev->domain, &pdev->sbdf, rc);
+        ASSERT_UNREACHABLE();
+        return rc;
+    }
+
+    /*
+     * Unprivileged domains have a deny by default register access policy, no
+     * need to add any further handlers for them.
+     */
+    if ( !is_hardware_domain(pdev->domain) )
+        return 0;
+
+    /*
+     * The driver may not traverse the capability list and think device
+     * supports MSIX by default. So here let the control register of MSIX
+     * be Read-Only is to ensure MSIX disabled.
+     */
+    rc = vpci_add_register(vpci, vpci_hw_read16, NULL,
+                           msix_control_reg(msix_pos), 2, NULL);
+    if ( rc )
+        printk(XENLOG_ERR "%pd %pp: fail to add MSIX ctrl handler rc=%d\n",
+               pdev->domain, &pdev->sbdf, rc);
+
+    return rc;
+}
+
+static int cf_check init_msix(struct pci_dev *pdev)
+{
+    struct domain *d = pdev->domain;
+    unsigned int msix_offset, i, max_entries;
+    uint16_t control;
+    struct vpci_msix *msix;
+    int rc;
+
+    msix_offset = pdev->msix_pos;
+    if ( !msix_offset )
+        return 0;
+
+    control = pci_conf_read16(pdev->sbdf, msix_control_reg(msix_offset));
+
+    max_entries = msix_table_size(control);
+
+    msix = xzalloc_flex_struct(struct vpci_msix, entries, max_entries);
+    if ( !msix )
+        return -ENOMEM;
+
+    msix->tables[VPCI_MSIX_TABLE] =
+        pci_conf_read32(pdev->sbdf, msix_table_offset_reg(msix_offset));
+    msix->tables[VPCI_MSIX_PBA] =
+        pci_conf_read32(pdev->sbdf, msix_pba_offset_reg(msix_offset));
+
+    /* Check that the referenced BAR(s) regions are valid. */
+    for ( i = 0; i < ARRAY_SIZE(msix->tables); i++ )
+    {
+        const char *name = (i == VPCI_MSIX_TABLE) ? "vector" : "PBA";
+        const struct vpci_bar *bars = pdev->vpci->header.bars;
+        unsigned int bir = msix->tables[i] & PCI_MSIX_BIRMASK;
+        unsigned int type;
+        unsigned int offset = msix->tables[i] & ~PCI_MSIX_BIRMASK;
+        unsigned int size =
+            (i == VPCI_MSIX_TABLE) ? max_entries * PCI_MSIX_ENTRY_SIZE
+                                   : ROUNDUP(DIV_ROUND_UP(max_entries, 8), 8);
+
+        if ( bir >= ARRAY_SIZE(pdev->vpci->header.bars) )
+        {
+            printk(XENLOG_ERR DEV_BUG_PREFIX
+                   "%pp: MSI-X %s table with out of range BIR %u\n",
+                   &pdev->sbdf, name, bir);
+ invalid:
+            xfree(msix);
+            return -ENODEV;
+        }
+
+        type = bars[bir].type;
+        if ( type != VPCI_BAR_MEM32 && type != VPCI_BAR_MEM64_LO )
+        {
+            printk(XENLOG_ERR DEV_BUG_PREFIX
+                   "%pp: MSI-X %s table at invalid BAR%u with type %u\n",
+                   &pdev->sbdf, name, bir, type);
+            goto invalid;
+        }
+
+        if ( (uint64_t)offset + size > bars[bir].size )
+        {
+            printk(XENLOG_ERR DEV_BUG_PREFIX
+                   "%pp: MSI-X %s table offset %#x size %#x outside of BAR%u 
size %#lx\n",
+                   &pdev->sbdf, name, offset, size, bir, bars[bir].size);
+            goto invalid;
+        }
+    }
+
+    rc = vpci_add_register(pdev->vpci, control_read, control_write,
+                           msix_control_reg(msix_offset), 2, msix);
+    if ( rc )
+    {
+        xfree(msix);
+        return rc;
+    }
+
+    msix->max_entries = max_entries;
+    msix->pdev = pdev;
+
+    for ( i = 0; i < max_entries; i++)
+    {
+        msix->entries[i].masked = true;
+        vpci_msix_arch_init_entry(&msix->entries[i]);
+    }
+
+    pdev->vpci->msix = msix;
+
+    vpci_msix_arch_register(msix, d);
+
+    /*
+     * vPCI header initialization will have mapped the whole BAR into the
+     * p2m, as MSI-X capability was not yet initialized.  Carve a hole for
+     * the MSI-X table here, so that Xen can trap accesses.
+     */
+    return vpci_make_msix_hole(pdev);
+}
+REGISTER_VPCI_CAP(MSIX, init_msix, cleanup_msix);
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/xen/include/xen/vpci.h b/xen/include/xen/vpci.h
index 877aa391d1..07b388a998 100644
--- a/xen/include/xen/vpci.h
+++ b/xen/include/xen/vpci.h
@@ -199,6 +199,11 @@ bool vpci_ecam_write(pci_sbdf_t sbdf, unsigned int reg, 
unsigned int len,
 bool vpci_ecam_read(pci_sbdf_t sbdf, unsigned int reg, unsigned int len,
                     unsigned long *data);
 
+void vpci_msix_arch_control_write(
+    const struct pci_dev *pdev, unsigned int reg, uint32_t val, void *data);
+void vpci_msix_arch_cleanup(struct vpci *vpci);
+void vpci_msix_arch_register(struct vpci_msix *msix, struct domain *d);
+
 #endif /* __XEN__ */
 
 #else /* !CONFIG_HAS_VPCI */
-- 
2.55.0