[PATCH v1 4/5] vpci: use separate rangeset for BAR unmapping

[Stewart Hildebrand <stewart.hildebrand@xxxxxxx>]

Introduce a new per-BAR rangeset, unmap_mem, for p2m unmapping. Rename
existing mem rangeset to map_mem, which is now only used for mapping.
Populate unmap_mem by moving just-mapped ranges from map_mem to
unmap_mem. In modify_bars(), skip recalculating the ranges when
unmapping as they are already stored in unmap_mem.

Signed-off-by: Stewart Hildebrand <stewart.hildebrand@xxxxxxx>
---
 xen/drivers/vpci/header.c | 74 +++++++++++++++++++++++++++++----------
 xen/drivers/vpci/vpci.c   |  5 ++-
 xen/include/xen/vpci.h    |  3 +-
 3 files changed, 62 insertions(+), 20 deletions(-)

diff --git a/xen/drivers/vpci/header.c b/xen/drivers/vpci/header.c
index b09ccc5e6be6..c9519c804d97 100644
--- a/xen/drivers/vpci/header.c
+++ b/xen/drivers/vpci/header.c
@@ -90,6 +90,8 @@ static int cf_check map_range(
         if ( rc == 0 )
         {
             *c += size;
+            if ( map->map )
+                rc = rangeset_add_range(map->bar->unmap_mem, s, e);
             break;
         }
         if ( rc < 0 )
@@ -102,6 +104,13 @@ static int cf_check map_range(
         }
         ASSERT(rc < size);
         *c += rc;
+        if ( map->map )
+        {
+            int rc2 = rangeset_add_range(map->bar->unmap_mem, s, s + rc);
+
+            if ( rc2 )
+                return rc2;
+        }
         s += rc;
         if ( general_preempt_check() )
                 return -ERESTART;
@@ -185,12 +194,13 @@ static int map_bars(struct vpci_header *header, struct 
domain *d, bool map)
             .map = map,
             .bar = bar,
         };
+        struct rangeset *r = map ? bar->map_mem : bar->unmap_mem;
         int rc;
 
-        if ( rangeset_is_empty(bar->mem) )
+        if ( rangeset_is_empty(r) )
             continue;
 
-        rc = rangeset_consume_ranges(bar->mem, map_range, &data);
+        rc = rangeset_consume_ranges(r, map_range, &data);
 
         if ( rc )
             return rc;
@@ -248,8 +258,13 @@ bool vpci_process_pending(struct vcpu *v)
 
     /* Clean all the rangesets */
     for ( i = 0; i < ARRAY_SIZE(header->bars); i++ )
-        if ( !rangeset_is_empty(header->bars[i].mem) )
-             rangeset_purge(header->bars[i].mem);
+    {
+        if ( !rangeset_is_empty(header->bars[i].map_mem) )
+             rangeset_purge(header->bars[i].map_mem);
+
+        if ( !rangeset_is_empty(header->bars[i].unmap_mem) )
+             rangeset_purge(header->bars[i].unmap_mem);
+    }
 
     v->vpci.pdev = NULL;
 
@@ -275,10 +290,10 @@ static int __init apply_map(struct domain *d, const 
struct pci_dev *pdev,
         struct vpci_bar *bar = &header->bars[i];
         struct map_data data = { .d = d, .map = true, .bar = bar };
 
-        if ( rangeset_is_empty(bar->mem) )
+        if ( rangeset_is_empty(bar->map_mem) )
             continue;
 
-        while ( (rc = rangeset_consume_ranges(bar->mem, map_range,
+        while ( (rc = rangeset_consume_ranges(bar->map_mem, map_range,
                                               &data)) == -ERESTART )
         {
             /*
@@ -329,6 +344,13 @@ static int modify_bars(const struct pci_dev *pdev, 
uint16_t cmd, bool rom_only)
 
     ASSERT(rw_is_write_locked(&pdev->domain->pci_lock));
 
+    if ( !(cmd & PCI_COMMAND_MEMORY) )
+    {
+        defer_map(pdev, cmd, rom_only);
+
+        return 0;
+    }
+
     /*
      * Create a rangeset per BAR that represents the current device memory
      * region and compare it against all the currently active BAR memory
@@ -349,7 +371,7 @@ static int modify_bars(const struct pci_dev *pdev, uint16_t 
cmd, bool rom_only)
         unsigned long start_guest = PFN_DOWN(bar->guest_addr);
         unsigned long end_guest = PFN_DOWN(bar->guest_addr + bar->size - 1);
 
-        if ( !bar->mem )
+        if ( !bar->map_mem || !bar->unmap_mem )
             continue;
 
         if ( !MAPPABLE_BAR(bar) ||
@@ -367,7 +389,7 @@ static int modify_bars(const struct pci_dev *pdev, uint16_t 
cmd, bool rom_only)
             continue;
         }
 
-        ASSERT(rangeset_is_empty(bar->mem));
+        ASSERT(rangeset_is_empty(bar->map_mem));
 
         /*
          * Make sure that the guest set address has the same page offset
@@ -382,7 +404,7 @@ static int modify_bars(const struct pci_dev *pdev, uint16_t 
cmd, bool rom_only)
             return -EINVAL;
         }
 
-        rc = rangeset_add_range(bar->mem, start_guest, end_guest);
+        rc = rangeset_add_range(bar->map_mem, start_guest, end_guest);
         if ( rc )
         {
             printk(XENLOG_G_WARNING "Failed to add [%lx, %lx]: %d\n",
@@ -395,10 +417,10 @@ static int modify_bars(const struct pci_dev *pdev, 
uint16_t cmd, bool rom_only)
         {
             struct vpci_bar *prev_bar = &header->bars[j];
 
-            if ( rangeset_is_empty(prev_bar->mem) )
+            if ( rangeset_is_empty(prev_bar->map_mem) )
                 continue;
 
-            rc = rangeset_remove_range(prev_bar->mem, start_guest, end_guest);
+            rc = rangeset_remove_range(prev_bar->map_mem, start_guest, 
end_guest);
             if ( rc )
             {
                 gprintk(XENLOG_WARNING,
@@ -408,7 +430,7 @@ static int modify_bars(const struct pci_dev *pdev, uint16_t 
cmd, bool rom_only)
             }
         }
 
-        rc = pci_sanitize_bar_memory(bar->mem);
+        rc = pci_sanitize_bar_memory(bar->map_mem);
         if ( rc )
         {
             gprintk(XENLOG_WARNING,
@@ -429,10 +451,10 @@ static int modify_bars(const struct pci_dev *pdev, 
uint16_t cmd, bool rom_only)
         {
             const struct vpci_bar *bar = &header->bars[j];
 
-            if ( rangeset_is_empty(bar->mem) )
+            if ( rangeset_is_empty(bar->map_mem) )
                 continue;
 
-            rc = rangeset_remove_range(bar->mem, start, end);
+            rc = rangeset_remove_range(bar->map_mem, start, end);
             if ( rc )
             {
                 gprintk(XENLOG_WARNING,
@@ -486,7 +508,7 @@ static int modify_bars(const struct pci_dev *pdev, uint16_t 
cmd, bool rom_only)
                 {
                     const struct vpci_bar *bar = &header->bars[j];
 
-                    if ( !rangeset_overlaps_range(bar->mem, start, end) ||
+                    if ( !rangeset_overlaps_range(bar->map_mem, start, end) ||
                          /*
                           * If only the ROM enable bit is toggled check against
                           * other BARs in the same device for overlaps, but not
@@ -497,7 +519,7 @@ static int modify_bars(const struct pci_dev *pdev, uint16_t 
cmd, bool rom_only)
                           bar->type == VPCI_BAR_ROM) )
                         continue;
 
-                    rc = rangeset_remove_range(bar->mem, start, end);
+                    rc = rangeset_remove_range(bar->map_mem, start, end);
                     if ( rc )
                     {
                         gprintk(XENLOG_WARNING,
@@ -752,12 +774,28 @@ static int bar_add_rangeset(const struct pci_dev *pdev, 
struct vpci_bar *bar,
                             unsigned int i)
 {
     char str[32];
+    int rc = 0;
 
     snprintf(str, sizeof(str), "%pp:BAR%u", &pdev->sbdf, i);
 
-    bar->mem = rangeset_new(pdev->domain, str, RANGESETF_no_print);
+    bar->map_mem = rangeset_new(pdev->domain, str, RANGESETF_no_print);
+    bar->unmap_mem = rangeset_new(pdev->domain, str, RANGESETF_no_print);
+
+    if ( !bar->map_mem )
+        rc = -ENOMEM;
+
+    if ( !bar->unmap_mem )
+        rc = -ENOMEM;
 
-    return !bar->mem ? -ENOMEM : 0;
+    if ( rc == -ENOMEM )
+    {
+        rangeset_destroy(bar->map_mem);
+        rangeset_destroy(bar->unmap_mem);
+        bar->map_mem = NULL;
+        bar->unmap_mem = NULL;
+    }
+
+    return rc;
 }
 
 static int cf_check init_header(struct pci_dev *pdev)
diff --git a/xen/drivers/vpci/vpci.c b/xen/drivers/vpci/vpci.c
index d2f0f97e0a04..c0198aa1b08d 100644
--- a/xen/drivers/vpci/vpci.c
+++ b/xen/drivers/vpci/vpci.c
@@ -118,7 +118,10 @@ void vpci_deassign_device(struct pci_dev *pdev)
     }
 
     for ( i = 0; i < ARRAY_SIZE(pdev->vpci->header.bars); i++ )
-        rangeset_destroy(pdev->vpci->header.bars[i].mem);
+    {
+        rangeset_destroy(pdev->vpci->header.bars[i].map_mem);
+        rangeset_destroy(pdev->vpci->header.bars[i].unmap_mem);
+    }
 
     xfree(pdev->vpci->msix);
     xfree(pdev->vpci->msi);
diff --git a/xen/include/xen/vpci.h b/xen/include/xen/vpci.h
index 27eebdcef170..e74359848440 100644
--- a/xen/include/xen/vpci.h
+++ b/xen/include/xen/vpci.h
@@ -101,7 +101,8 @@ struct vpci {
             uint64_t guest_addr;
             uint64_t size;
             uint64_t resizable_sizes;
-            struct rangeset *mem;
+            struct rangeset *map_mem;
+            struct rangeset *unmap_mem;
             enum {
                 VPCI_BAR_EMPTY,
                 VPCI_BAR_IO,
-- 
2.49.0