[RFC PATCH v2 3/5] x86: Track vcpu context switches and introduce needs_tlb_flush field

[Teddy Astie <teddy.astie@xxxxxxxxxx>]

Introduce needs_tlb_flush that indicate that the vCPU related TLB
needs to be flushed before entering this domain. This is intended
to be used later for using the same ASID for all vCPU of a domain.

Also track vCPU context switch to determine :
 - per vcpu "latest_cpu" that tracks which pCPU last ran this vCPU,
   this is used to know whether our current TLB (of our pCPU) state
   is stale.
   Schedule a TLB flush if the vCPU ran on another pCPU previously.
 - per pCPU "latest_vcpu" (per domain) that tracks which vCPU the
   TLB+ASID/VPID is holding onto.
   Schedule a TLB flush if this pCPU hasn't ran this vCPU previously
   (which can happen if we context-switch multiples vCPUs of a same
   domain on a same pCPU).

If ASID use is disabled, unconditionnaly perform a TLB flush.

Signed-off-by: Teddy Astie <teddy.astie@xxxxxxxxxx>
---
v2: Merge several patches into one, move logic to context_switch(),
    move fields to arch_vcpu/arch_domain, consider !asid_enabled case.
---
 xen/arch/x86/domain.c             | 26 ++++++++++++++++++++++++++
 xen/arch/x86/include/asm/domain.h |  6 ++++++
 2 files changed, 32 insertions(+)

diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c
index 996b50af7a..27155546ca 100644
--- a/xen/arch/x86/domain.c
+++ b/xen/arch/x86/domain.c
@@ -38,6 +38,7 @@
 #include <xen/smp.h>
 #include <xen/softirq.h>
 #include <xen/wait.h>
+#include <xen/xvmalloc.h>
 
 #include <asm/amd.h>
 #include <asm/cpu-policy.h>
@@ -874,6 +875,13 @@ int arch_domain_create(struct domain *d,
 
     spec_ctrl_init_domain(d);
 
+    rc = -ENOMEM;
+    d->arch.latest_vcpu = xvmalloc_array(int, nr_cpu_ids);
+    if ( !d->arch.latest_vcpu )
+        goto fail;
+    for (unsigned int i = 0; i < nr_cpu_ids; i++)
+        d->arch.latest_vcpu[i] = -1;
+
     if ( (rc = paging_domain_init(d)) != 0 )
         goto fail;
     paging_initialised = true;
@@ -964,6 +972,7 @@ void arch_domain_destroy(struct domain *d)
 
     xfree(d->arch.e820);
     XFREE(d->arch.cpu_policy);
+    XFREE(d->arch.latest_vcpu);
 
     free_domain_pirqs(d);
     if ( !is_idle_domain(d) )
@@ -2183,6 +2192,23 @@ void context_switch(struct vcpu *prev, struct vcpu *next)
         ASSERT(!vcpu_cpu_dirty(next));
     }
 
+    /**
+     * Check if we were the latest vCPU of this domain that ran on this pCPU.
+     * Flush the TLB if it is not, as the TLB entries are the ones from the 
previous
+     * vCPU. If we weren't the latest pCPU, always perform a TLB flush as we 
may be
+     * out of sync.
+     */
+    if ( nextd->arch.latest_vcpu )
+    {
+        if ( !asid_enabled ||
+             nextd->arch.latest_vcpu[cpu] != next->vcpu_id ||
+             next->arch.latest_cpu != cpu )
+            next->arch.needs_tlb_flush = true;
+
+        nextd->arch.latest_vcpu[cpu] = next->vcpu_id;
+        next->arch.latest_cpu = cpu;
+    }
+
     _update_runstate_area(prev);
     vpmu_switch_from(prev);
     np2m_schedule(NP2M_SCHEDLE_OUT);
diff --git a/xen/arch/x86/include/asm/domain.h 
b/xen/arch/x86/include/asm/domain.h
index e0ce8b4c39..c67af79da2 100644
--- a/xen/arch/x86/include/asm/domain.h
+++ b/xen/arch/x86/include/asm/domain.h
@@ -345,6 +345,10 @@ struct arch_domain
         struct hvm_domain hvm;
     };
 
+    /* Mapping of the latest vCPU that ran on a specific CPU, thus hold
+     * the current domain TLB state (-1 if the vCPU hasn't ran yet) */
+    int *latest_vcpu;
+
     struct paging_domain paging;
     struct p2m_domain *p2m;
     /* To enforce lock ordering in the pod code wrt the
@@ -647,8 +651,10 @@ struct arch_vcpu
                                         /* former, if any */
     bool old_guest_table_partial;       /* Are we dropping a type ref, or just
                                          * finishing up a partial 
de-validation? */
+    bool needs_tlb_flush;               /* VCPU needs its TLB flushed before 
waking? */
 
     unsigned long cr3;                  /* (MA) value to install in HW CR3 */
+    unsigned int latest_cpu;            /* Latest pCPU that ran this VCPU. */
 
     /*
      * The save area for Processor Extended States and the bitmask of the
-- 
2.54.0



--
Teddy Astie | Vates XCP-ng Developer

XCP-ng & Xen Orchestra - Vates solutions

web: https://vates.tech