[PATCH v3 30/39] xen/riscv: implement APLIC-IMSIC-hart sync barrier for vCPU migration

[Oleksii Kurochko <oleksii.kurochko@xxxxxxxxx>]

When a vCPU is moved to a different guest interrupt file, MSIs that the
APLIC has already sent towards the old file may still be in flight. They
must land before the old file's state is saved and the switch is done,
otherwise they would be lost.

To wait for them, use the APLIC's genmsi register. Writing it makes the
APLIC itself send an MSI (an "extempore" MSI) with a given interrupt
identity to a given hart. genmsi can only target the hart's hypervisor-
level interrupt file, not a guest one, but the AIA spec guarantees that
all MSIs previously sent by the same APLIC to the same hart become
visible at the hart's IMSIC before the extempore MSI does. So once the
extempore MSI has been delivered, no older MSI from this APLIC to the
hart can still be in flight, whichever interrupt file it targets.

genmsi's Busy bit clearing only tells that the APLIC has sent the
extempore MSI, not that it has arrived, so the sequence described in the
AIA spec under "Synchronizing interactions between a hart and the APLIC"
is split in two. aplic_genmsi_barrier() does the APLIC side, while
imsic_aplic_sync() does the hart side: it clears the pending bit of the
reserved identity before the genmsi write and spins on it afterwards,
which is what tells that the extempore MSI, and hence every older MSI
from this APLIC to this hart, has really arrived. Keeping the hart side
in imsic.c leaves the APLIC code free of the IMSIC local CSR accessors.

The last interrupt identity (nr_ids) is reserved for this purpose.

Migration doesn't use the barrier at this stage, so imsic_migrate_vcpu()
only references imsic_aplic_sync() to keep the compiler from complaining
about it being unused.

Signed-off-by: Oleksii Kurochko <oleksii.kurochko@xxxxxxxxx>
---
Changes in v3:
 - Rename from "xen/riscv: implement APLIC-hart sync barrier for vCPU
   migration", as the barrier waits for the MSIs to reach the hart's IMSIC.
 - Re-word the commit message, describing in it how the sequence is split
   between aplic_genmsi_barrier() and imsic_aplic_sync().
 - Move imsic_aplic_sync(), imsic_local_is_pending() and imsic_csr_read()
   here from "xen/riscv: remap interrupts to new IMSIC VS-file", so that
   both halves of the synchronization sequence are introduced together.
 - s/aplic_hart_field/aplic_hart_index/ following the rename of the helper
   in "xen/riscv: implement virtual APLIC MMIO emulation".
 - Drop masking of sync_id with APLIC_TARGET_EIID, which could silently
   chop off bits; instead add BUILD_BUG_ON() and ASSERT() to make it
   explicit that sync_id always fits into the EIID field.
 - Drop the redundant parentheses around the macro argument in
   imsic_csr_read().
 - Use unsigned int for isel in imsic_local_is_pending() and drop the
   redundant !! as the function returns bool.
 - Rename the unused argument of imsic_aplic_sync() to unused.
---
Changes in v2:
 - New patch.
---
---
 xen/arch/riscv/aplic.c             | 33 +++++++++++++++++++
 xen/arch/riscv/imsic.c             | 52 ++++++++++++++++++++++++++++++
 xen/arch/riscv/include/asm/aplic.h |  3 ++
 xen/arch/riscv/include/asm/imsic.h |  8 +++++
 4 files changed, 96 insertions(+)

diff --git a/xen/arch/riscv/aplic.c b/xen/arch/riscv/aplic.c
index 283e5d6046a6..5f9a894b5a76 100644
--- a/xen/arch/riscv/aplic.c
+++ b/xen/arch/riscv/aplic.c
@@ -27,7 +27,9 @@
 #include <asm/imsic.h>
 #include <asm/intc.h>
 #include <asm/io.h>
+#include <asm/processor.h>
 #include <asm/riscv_encoding.h>
+#include <asm/smp.h>
 
 static struct aplic_priv aplic = {
     .lock = SPIN_LOCK_UNLOCKED,
@@ -183,6 +185,37 @@ void aplic_hw_write_reg(unsigned int offset, uint32_t 
value)
     spin_unlock_irqrestore(&aplic.lock, flags);
 }
 
+/*
+ * As needed, synchronize with all IOMMUs and APLICs to ensure that no
+ * straggler MSIs will arrive at the old interrupt file after this step.
+ */
+void aplic_genmsi_barrier(void)
+{
+    const struct imsic_config *imsic = imsic_get_config();
+    unsigned int cpu = smp_processor_id();
+    unsigned long flags;
+    uint32_t val;
+
+    /*
+     * sync_id is nr_ids, which imsic_parse_node() limits to IMSIC_MAX_ID,
+     * so it always fits into the EIID field.
+     */
+    BUILD_BUG_ON(IMSIC_MAX_ID > MASK_EXTR(~0U, APLIC_TARGET_EIID));
+    ASSERT(imsic->sync_id <= IMSIC_MAX_ID);
+
+    val = MASK_INSR(aplic_hart_index(cpu), APLIC_TARGET_HART_IDX) |
+          imsic->sync_id;
+
+    spin_lock_irqsave(&aplic.lock, flags);
+
+    writel(val, &aplic.regs->genmsi);
+
+    while ( readl(&aplic.regs->genmsi) & APLIC_GENMSI_BUSY )
+        cpu_relax();
+
+    spin_unlock_irqrestore(&aplic.lock, flags);
+}
+
 static void __init aplic_init_hw_interrupts(void)
 {
     unsigned int i;
diff --git a/xen/arch/riscv/imsic.c b/xen/arch/riscv/imsic.c
index f1e44dc56bb8..8e0273eab902 100644
--- a/xen/arch/riscv/imsic.c
+++ b/xen/arch/riscv/imsic.c
@@ -27,6 +27,7 @@
 #include <xen/xvmalloc.h>
 
 #include <asm/aia.h>
+#include <asm/aplic.h>
 #include <asm/imsic.h>
 
 #define IMSIC_HART_SIZE(guest_bits) (BIT(guest_bits, U) * IMSIC_MMIO_PAGE_SZ)
@@ -60,6 +61,12 @@ static unsigned int __ro_after_init guest_num_msis;
 #define IMSIC_DISABLE_EITHRESHOLD   1
 #define IMSIC_ENABLE_EITHRESHOLD    0
 
+#define imsic_csr_read(c)       \
+({                              \
+    csr_write(CSR_SISELECT, c); \
+    csr_read(CSR_SIREG);        \
+})
+
 #define imsic_csr_write(c, v)   \
 do {                            \
     csr_write(CSR_SISELECT, c); \
@@ -203,6 +210,15 @@ void imsic_irq_enable(unsigned int irq)
      */
     ASSERT(!local_irq_is_enabled());
 
+    if ( irq == imsic_cfg.sync_id )
+    {
+        printk(XENLOG_WARNING
+               "irq%u is reserved for APLIC sync so shouldn't be set by %s\n",
+               irq, __func__);
+
+        return;
+    }
+
     spin_lock(&imsic_cfg.lock);
     /*
      * There is no irq - 1 here (look at aplic_set_irq_type()) because:
@@ -241,6 +257,15 @@ void imsic_irq_disable(unsigned int irq)
     spin_unlock(&imsic_cfg.lock);
 }
 
+static bool imsic_local_is_pending(unsigned int id)
+{
+    unsigned int isel =
+        (id / BITS_PER_LONG) * (BITS_PER_LONG / IMSIC_EIPx_BITS) + IMSIC_EIP0;
+    unsigned long bit = BIT(id % BITS_PER_LONG, UL);
+
+    return imsic_csr_read(isel) & bit;
+}
+
 /* Callers aren't intended to changed imsic_cfg so return const. */
 const struct imsic_config *imsic_get_config(void)
 {
@@ -387,6 +412,9 @@ static int __init imsic_parse_node(const struct 
dt_device_node *node,
 
     imsic_cfg.nr_eix = DIV_ROUND_UP(imsic_cfg.nr_ids + 1, 64);
 
+    /* Reserve last identity for APLIC-to-hart synchronization */
+    imsic_cfg.sync_id = imsic_cfg.nr_ids;
+
     /* Compute base address */
     *nr_mmios = 0;
     rc = dt_device_get_address(node, *nr_mmios, &base_addr, NULL);
@@ -497,6 +525,27 @@ static void imsic_call_on_cpu(unsigned int cpu, void 
(*func)(void *),
         on_selected_cpus(cpumask_of(cpu), func, data, 1);
 }
 
+/*
+ * Ensure that all the MSIs the APLIC has already generated for the hart this
+ * runs on have really reached the hart's IMSIC.
+ *
+ * The barrier is the one described by the AIA specification in
+ * "Synchronizing interactions between a hart and the APLIC": ask the APLIC to
+ * send an MSI to the hart itself and wait until it shows up as pending in the
+ * hart's own interrupt file. As it says nothing about MSIs on their way to
+ * any other hart, it has to be executed by the pCPU owning the interrupt file
+ * the MSIs were being sent to.
+ */
+static void cf_check imsic_aplic_sync(void *unused)
+{
+    imsic_local_eix_update(imsic_cfg.sync_id, 1, true, false);
+
+    aplic_genmsi_barrier();
+
+    while ( !imsic_local_is_pending(imsic_cfg.sync_id) )
+        cpu_relax();
+}
+
 static void cf_check imsic_vsfile_local_clear(void *data)
 {
     unsigned int i;
@@ -862,5 +911,8 @@ void imsic_migrate_vcpu(struct vcpu *v)
     /* Zero-out new IMSIC VS-file */
     imsic_call_on_cpu(v->processor, imsic_vsfile_local_clear, &vsfile_data);
 
+    /* Avoid an unused-function error until the barrier has a user. */
+    (void)imsic_aplic_sync;
+
     BUG_ON("unimplemented");
 }
diff --git a/xen/arch/riscv/include/asm/aplic.h 
b/xen/arch/riscv/include/asm/aplic.h
index 9018936ca95a..bfb510b2ae5e 100644
--- a/xen/arch/riscv/include/asm/aplic.h
+++ b/xen/arch/riscv/include/asm/aplic.h
@@ -94,6 +94,7 @@
 #define APLIC_SETIPNUM_LE               0x2000
 
 #define APLIC_GENMSI                    0x3000
+#define APLIC_GENMSI_BUSY               BIT(12, U)
 
 #define APLIC_TARGET_BASE               0x3004
 #define APLIC_TARGET_LAST               0x3ffc
@@ -177,4 +178,6 @@ struct aplic_regs {
 uint32_t aplic_hw_read_reg(unsigned int offset);
 void aplic_hw_write_reg(unsigned int offset, uint32_t value);
 
+void aplic_genmsi_barrier(void);
+
 #endif /* ASM_RISCV_APLIC_H */
diff --git a/xen/arch/riscv/include/asm/imsic.h 
b/xen/arch/riscv/include/asm/imsic.h
index 8130d125f5c3..f86407e21765 100644
--- a/xen/arch/riscv/include/asm/imsic.h
+++ b/xen/arch/riscv/include/asm/imsic.h
@@ -62,6 +62,14 @@ struct imsic_config {
      */
     unsigned int nr_eix;
 
+    /*
+     * Interrupt identity reserved exclusively for APLIC-to-hart
+     * synchronization.
+     *
+     * Must not be allocated to any interrupt source.
+     */
+    unsigned int sync_id;
+
     /* MSI */
     const struct imsic_msi *msi;
 
-- 
2.55.0