[PATCH v2 03/14] x86/pv: use populate_perdomain_mapping() to map the Xen GDT

[George Dunlap <dunlapg@xxxxxxxxx>]

From: Roger Pau Monné <roger.pau@xxxxxxxxxx>

Currently, update_xen_slot_in_full_gdt() uses the stashed direct-map
pointer in d->arch.pv.gdt_ldt_l1tab to update the incoming vcpu's
page tables with Xen's GDT, by writing a stashed per-cpu copy of a
pre-baked L1 entry (either 64-bit or compat version).

Switch this to using populate_perdomain_mapping(), which doesn't rely
on the stashed address of the l1 page in the direct map.  Rather than
also stashing a pre-baked value for the payload, compute the mfn from
the per-cpu GDT pointer at use: the conversion is a handful of cycles
on a path costing thousands, and computing at use removes the
parallel {,compat_}gdt_l1e bookkeeping along with its boot-ordering
constraint (the cached value could only be generated after Xen's
physical relocation, and had to be in place before the first context
switch; a use-time lookup is correct by construction).  The flags on
the final mapping are identical.

Signed-off-by: Roger Pau Monné <roger.pau@xxxxxxxxxx>
Assisted-by: Claude Code:claude-fable-5, Claude Code:claude-opus-4-8
Signed-off-by: George Dunlap <gwd@xxxxxxxxxxxxxx>
---
Changes in v2:
- Drop the {,compat_}gdt_mfn caching entirely (suggested by Andrew
   Cooper): compute virt_to_mfn() from the per-cpu GDT pointer at use.
   The PDX lookup behind it measures ~5-10 cycles warm against a
   ~1,500-cycle context switch, and this removes the double
   bookkeeping and the after-relocation caching constraint.  The
   cached-MFN assertion goes with the cache: a use-time computation
   from a live pointer needs no staleness check.

Changes since the previously posted version:
- populate_perdomain_mapping() introduction split into the previous
   patch; this patch is now just the Xen GDT conversion.
- Retain the "GDT MFN cached" check as ASSERT(mfn_x(mfn)).
---
 xen/arch/x86/domain.c           | 13 ++++++++-----
 xen/arch/x86/include/asm/desc.h |  2 --
 xen/arch/x86/smpboot.c          | 15 ---------------
 xen/arch/x86/traps.c            |  2 --
 4 files changed, 8 insertions(+), 24 deletions(-)

diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c
index 996b50af7a..d8af06e533 100644
--- a/xen/arch/x86/domain.c
+++ b/xen/arch/x86/domain.c
@@ -2062,11 +2062,14 @@ static always_inline bool need_full_gdt(const struct 
domain *d)
 
 static void update_xen_slot_in_full_gdt(const struct vcpu *v, unsigned int cpu)
 {
-    ASSERT(per_cpu(gdt_l1e, cpu).l1); /* Confirm these have been cached. */
-
-    l1e_write(pv_gdt_ptes(v) + FIRST_RESERVED_GDT_PAGE,
-              !is_pv_32bit_vcpu(v) ? per_cpu(gdt_l1e, cpu)
-                                   : per_cpu(compat_gdt_l1e, cpu));
+    mfn_t mfn = _mfn(virt_to_mfn(!is_pv_32bit_vcpu(v)
+                                 ? per_cpu(gdt, cpu)
+                                 : per_cpu(compat_gdt, cpu)));
+
+    populate_perdomain_mapping(v,
+                               GDT_VIRT_START(v) +
+                               (FIRST_RESERVED_GDT_PAGE << PAGE_SHIFT),
+                               &mfn, 1, __PAGE_HYPERVISOR_RW);
 }
 
 static void load_full_gdt(const struct vcpu *v, unsigned int cpu)
diff --git a/xen/arch/x86/include/asm/desc.h b/xen/arch/x86/include/asm/desc.h
index dcbdac3ff7..a860134211 100644
--- a/xen/arch/x86/include/asm/desc.h
+++ b/xen/arch/x86/include/asm/desc.h
@@ -136,10 +136,8 @@ struct __packed desc_ptr {
 
 extern seg_desc_t boot_gdt[];
 DECLARE_PER_CPU(seg_desc_t *, gdt);
-DECLARE_PER_CPU(l1_pgentry_t, gdt_l1e);
 extern seg_desc_t boot_compat_gdt[];
 DECLARE_PER_CPU(seg_desc_t *, compat_gdt);
-DECLARE_PER_CPU(l1_pgentry_t, compat_gdt_l1e);
 DECLARE_PER_CPU(bool, full_gdt_loaded);
 
 static inline void lgdt(const struct desc_ptr *gdtr)
diff --git a/xen/arch/x86/smpboot.c b/xen/arch/x86/smpboot.c
index 84e9e4beed..9b837a1769 100644
--- a/xen/arch/x86/smpboot.c
+++ b/xen/arch/x86/smpboot.c
@@ -1085,8 +1085,6 @@ static int cpu_smpboot_alloc(unsigned int cpu)
     if ( gdt == NULL )
         goto out;
     per_cpu(gdt, cpu) = gdt;
-    per_cpu(gdt_l1e, cpu) =
-        l1e_from_pfn(virt_to_mfn(gdt), __PAGE_HYPERVISOR_RW);
     memcpy(gdt, boot_gdt, NR_RESERVED_GDT_PAGES * PAGE_SIZE);
     BUILD_BUG_ON(NR_CPUS > 0x10000);
     gdt[PER_CPU_GDT_ENTRY - FIRST_RESERVED_GDT_ENTRY].a = cpu;
@@ -1095,8 +1093,6 @@ static int cpu_smpboot_alloc(unsigned int cpu)
     per_cpu(compat_gdt, cpu) = gdt = alloc_xenheap_pages(0, memflags);
     if ( gdt == NULL )
         goto out;
-    per_cpu(compat_gdt_l1e, cpu) =
-        l1e_from_pfn(virt_to_mfn(gdt), __PAGE_HYPERVISOR_RW);
     memcpy(gdt, boot_compat_gdt, NR_RESERVED_GDT_PAGES * PAGE_SIZE);
     gdt[PER_CPU_GDT_ENTRY - FIRST_RESERVED_GDT_ENTRY].a = cpu;
 #endif
@@ -1173,17 +1169,6 @@ void __init smp_prepare_cpus(void)
     initialize_cpu_data(0); /* Final full version of the data */
     print_cpu_info(0);
 
-    /*
-     * Cache {,compat_}gdt_l1e for the BSP now that physically relocation is
-     * done.  It must be after physical relocation of Xen, and before the
-     * first context_switch().
-     */
-    this_cpu(gdt_l1e) =
-        l1e_from_pfn(virt_to_mfn(boot_gdt), __PAGE_HYPERVISOR_RW);
-    if ( IS_ENABLED(CONFIG_PV32) )
-        this_cpu(compat_gdt_l1e) =
-            l1e_from_pfn(virt_to_mfn(boot_compat_gdt), __PAGE_HYPERVISOR_RW);
-
     boot_cpu_physical_apicid = get_apic_id();
     x86_cpu_to_apicid[0] = boot_cpu_physical_apicid;
 
diff --git a/xen/arch/x86/traps.c b/xen/arch/x86/traps.c
index 1774966305..2ab61db167 100644
--- a/xen/arch/x86/traps.c
+++ b/xen/arch/x86/traps.c
@@ -71,10 +71,8 @@ DEFINE_PER_CPU(uint64_t, efer);
 static DEFINE_PER_CPU(unsigned long, last_extable_addr);
 
 DEFINE_PER_CPU_READ_MOSTLY(seg_desc_t *, gdt);
-DEFINE_PER_CPU_READ_MOSTLY(l1_pgentry_t, gdt_l1e);
 #ifdef CONFIG_PV32
 DEFINE_PER_CPU_READ_MOSTLY(seg_desc_t *, compat_gdt);
-DEFINE_PER_CPU_READ_MOSTLY(l1_pgentry_t, compat_gdt_l1e);
 #endif
 
 /*
-- 
2.55.0