[PATCH] xen/x86: Change stub page freeing to fix smt=0

[Jason Andryuk <jason.andryuk@xxxxxxx>]

A single stubs page is initialized with 0xcc and re-used, with multiple
CPUs each using a portion of the shared page.  In cpu_smpboot_free(),
each stubs area is checked against 0xcc.  When all are set to 0xcc, the
page is freed.

Booting a system with smt=0, CPU0 is initially setup, allocating the
stubs page and initializing to 0xcc.  When more CPUs are brought up,
CPU1 is initialized and then immediately brough offline as it is the
sibling of CPU0.  Since the page was initially memset with 0xcc,
cpu_smpboot_free() finds all stubs as 0xcc and frees the page.
However, the page is still assigned to CPU0 and continues to be assigned
to other CPUs.

Meanwhile the page can be reallocated, which can lead to misbehavior.
The particular instance was the stubs page re-used as a page table which
later faulted when the entry was all 0xcc.

Change to initializing the page as 0xd6/STUB_BUF_FREE, and initializing
individual stubs as 0xcc/STUB_BUF_USED.  0xd6 now indicates unused, and
0xcc indicates used/assigned.  When freeing a CPU, the stub is set to
0xd6, and the page is freed if all stubs are 0xd6.  Initializing with
STUB_BUF_FREE lets cpu_smpboot_free() a page that was only ever
partially used.

0xd6/UDB is a 1 byte invalid opcode, which is similar to the existing
use of 0xcc.  0xd6 is used to identify bug frames, but the stub addr
(e.g. 0xffff82d07fffe000) fails the is_active_kernel_text() check.  It
should be okay to use here.

Fixes: 7a66ac8d1633 ("x86: move syscall trampolines off the stack")
Signed-off-by: Jason Andryuk <jason.andryuk@xxxxxxx>
---
It would be nice to use get_page()/put_page() to let count_info handle
reference counting, but they require an owning domain.

The listed Fixes introduced the use of 0xcc, but the smt commit may have
made it more problematic.
Fixes: d8f974f1a646 ("x86: command line option to avoid use of secondary 
hyper-threads")
---
 xen/arch/x86/smpboot.c | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/xen/arch/x86/smpboot.c b/xen/arch/x86/smpboot.c
index ff05955bae..a5d485b732 100644
--- a/xen/arch/x86/smpboot.c
+++ b/xen/arch/x86/smpboot.c
@@ -643,10 +643,16 @@ static int do_boot_cpu(int apicid, int cpu)
 
 #define STUB_BUF_CPU_OFFS(cpu) (((cpu) & (STUBS_PER_PAGE - 1)) * STUB_BUF_SIZE)
 
+/* Fill values indicating state of stub. */
+#define STUB_BUF_USED 0xcc
+#define STUB_BUF_FREE 0xd6
+
 unsigned long alloc_stub_page(unsigned int cpu, unsigned long *mfn)
 {
+    unsigned char *stub_page;
     unsigned long stub_va;
     struct page_info *pg;
+    bool initialize = false;
 
     BUILD_BUG_ON(STUBS_PER_PAGE & (STUBS_PER_PAGE - 1));
 
@@ -661,7 +667,7 @@ unsigned long alloc_stub_page(unsigned int cpu, unsigned 
long *mfn)
         if ( !pg )
             return 0;
 
-        unmap_domain_page(memset(__map_domain_page(pg), 0xcc, PAGE_SIZE));
+        initialize = true;
     }
 
     stub_va = XEN_VIRT_END - FIXADDR_X_SIZE - (cpu + 1) * PAGE_SIZE;
@@ -675,6 +681,14 @@ unsigned long alloc_stub_page(unsigned int cpu, unsigned 
long *mfn)
     else if ( !*mfn )
         *mfn = mfn_x(page_to_mfn(pg));
 
+    stub_page = __map_domain_page(pg);
+    /* Newly allocated page is marked entirely unused. */
+    if ( initialize )
+        memset(stub_page, STUB_BUF_FREE, PAGE_SIZE);
+    /* Specific CPU is marked used. */
+    memset(stub_page + STUB_BUF_CPU_OFFS(cpu), STUB_BUF_USED, STUB_BUF_SIZE);
+    unmap_domain_page(stub_page);
+
     return stub_va;
 }
 
@@ -992,9 +1006,10 @@ static void cpu_smpboot_free(unsigned int cpu, bool 
remove)
         unsigned char *stub_page = map_domain_page(mfn);
         unsigned int i;
 
-        memset(stub_page + STUB_BUF_CPU_OFFS(cpu), 0xcc, STUB_BUF_SIZE);
+        memset(stub_page + STUB_BUF_CPU_OFFS(cpu), STUB_BUF_FREE,
+               STUB_BUF_SIZE);
         for ( i = 0; i < STUBS_PER_PAGE; ++i )
-            if ( stub_page[i * STUB_BUF_SIZE] != 0xcc )
+            if ( stub_page[i * STUB_BUF_SIZE] != STUB_BUF_FREE )
                 break;
         unmap_domain_page(stub_page);
         destroy_xen_mappings(per_cpu(stubs.addr, cpu) & PAGE_MASK,
-- 
2.54.0