[xen staging] x86: introduce "hot" and "cold" page clearing functions

[patchbot@xxxxxxx]

commit 2593def544247477eb3fbe353b56c710d1dab6ca
Author:     Jan Beulich <jbeulich@xxxxxxxx>
AuthorDate: Wed Jul 23 15:19:52 2025 +0200
Commit:     Jan Beulich <jbeulich@xxxxxxxx>
CommitDate: Wed Jul 23 15:19:52 2025 +0200

    x86: introduce "hot" and "cold" page clearing functions
    
    The present clear_page_sse2() is useful in case a page isn't going to
    get touched again soon, or if we want to limit churn on the caches.
    Amend it by alternatively using CLZERO, which has been found to be quite
    a bit faster on Zen2 hardware at least. Note that to use CLZERO, we need
    to know the cache line size, and hence a feature dependency on CLFLUSH
    gets introduced.
    
    For cases where latency is the most important aspect, or when it is
    expected that sufficiently large parts of a page will get accessed again
    soon after the clearing, introduce a "hot" alternative. Again use
    alternatives patching to select between a "legacy" and an ERMS variant.
    
    Don't switch any callers just yet - this will be the subject of
    subsequent changes.
    
    Signed-off-by: Jan Beulich <jbeulich@xxxxxxxx>
    Reviewed-by: Jason Andryuk <jason.andryuk@xxxxxxx>
---
 xen/arch/x86/clear_page.S            | 42 ++++++++++++++++++++++++++++++++++--
 xen/arch/x86/cpu/common.c            | 37 +++++++++++++++++++++++++++++--
 xen/arch/x86/include/asm/asm-defns.h |  4 ++++
 xen/arch/x86/include/asm/page.h      |  5 +++--
 xen/tools/gen-cpuid.py               |  4 ++++
 5 files changed, 86 insertions(+), 6 deletions(-)

diff --git a/xen/arch/x86/clear_page.S b/xen/arch/x86/clear_page.S
index dc3c3c26bf..517685711d 100644
--- a/xen/arch/x86/clear_page.S
+++ b/xen/arch/x86/clear_page.S
@@ -5,7 +5,7 @@
 #include <asm/asm_defns.h>
 #include <asm/page.h>
 
-FUNC(clear_page_sse2)
+        .macro clear_page_sse2
         mov     $PAGE_SIZE/32, %ecx
         xor     %eax,%eax
 
@@ -19,4 +19,42 @@ FUNC(clear_page_sse2)
 
         sfence
         RET
-END(clear_page_sse2)
+        .endm
+
+        .macro clear_page_clzero
+        mov     %rdi, %rax
+        mov     $PAGE_SIZE/64, %ecx
+        .globl clear_page_clzero_post_count
+clear_page_clzero_post_count:
+
+0:      clzero
+        sub     $-64, %rax
+        .globl clear_page_clzero_post_neg_size
+clear_page_clzero_post_neg_size:
+        sub     $1, %ecx
+        jnz     0b
+
+        sfence
+        RET
+        .endm
+
+FUNC(clear_page_cold)
+        ALTERNATIVE clear_page_sse2, clear_page_clzero, X86_FEATURE_CLZERO
+END(clear_page_cold)
+
+        .macro clear_page_stosb
+        mov     $PAGE_SIZE, %ecx
+        xor     %eax,%eax
+        rep stosb
+        .endm
+
+        .macro clear_page_stosq
+        mov     $PAGE_SIZE/8, %ecx
+        xor     %eax, %eax
+        rep stosq
+        .endm
+
+FUNC(clear_page_hot)
+        ALTERNATIVE clear_page_stosq, clear_page_stosb, X86_FEATURE_ERMS
+        RET
+END(clear_page_hot)
diff --git a/xen/arch/x86/cpu/common.c b/xen/arch/x86/cpu/common.c
index 9194ddbd56..e063fe790a 100644
--- a/xen/arch/x86/cpu/common.c
+++ b/xen/arch/x86/cpu/common.c
@@ -60,6 +60,9 @@ DEFINE_PER_CPU(bool, full_gdt_loaded);
 
 DEFINE_PER_CPU(uint32_t, pkrs);
 
+extern uint32_t clear_page_clzero_post_count[];
+extern int8_t clear_page_clzero_post_neg_size[];
+
 void __init setup_clear_cpu_cap(unsigned int cap)
 {
        const uint32_t *dfs;
@@ -357,8 +360,38 @@ void __init early_cpu_init(bool verbose)
 
        edx &= ~cleared_caps[FEATURESET_1d];
        ecx &= ~cleared_caps[FEATURESET_1c];
-       if (edx & cpufeat_mask(X86_FEATURE_CLFLUSH))
-               c->x86_cache_alignment = ((ebx >> 8) & 0xff) * 8;
+       if (edx & cpufeat_mask(X86_FEATURE_CLFLUSH)) {
+               unsigned int size = ((ebx >> 8) & 0xff) * 8;
+
+               c->x86_cache_alignment = size;
+
+               /*
+                * Patch in parameters of clear_page_cold()'s CLZERO
+                * alternative. Note that for now we cap this at 128 bytes.
+                * Larger cache line sizes would still be dealt with
+                * correctly, but would cause redundant work done.
+                */
+               if (size > 128)
+                       size = 128;
+               if (size && !(size & (size - 1))) {
+                       /*
+                        * Need to play some games to keep the compiler from
+                        * recognizing the negative array index as being out
+                        * of bounds. The labels in assembler code really are
+                        * _after_ the locations to be patched, so the
+                        * negative index is intentional.
+                        */
+                       uint32_t *pcount = clear_page_clzero_post_count;
+                       int8_t *neg_size = clear_page_clzero_post_neg_size;
+
+                       OPTIMIZER_HIDE_VAR(pcount);
+                       OPTIMIZER_HIDE_VAR(neg_size);
+                       pcount[-1] = PAGE_SIZE / size;
+                       neg_size[-1] = -size;
+               }
+               else
+                       setup_clear_cpu_cap(X86_FEATURE_CLZERO);
+       }
        /* Leaf 0x1 capabilities filled in early for Xen. */
        c->x86_capability[FEATURESET_1d] = edx;
        c->x86_capability[FEATURESET_1c] = ecx;
diff --git a/xen/arch/x86/include/asm/asm-defns.h 
b/xen/arch/x86/include/asm/asm-defns.h
index 96370dd92a..61a5faf904 100644
--- a/xen/arch/x86/include/asm/asm-defns.h
+++ b/xen/arch/x86/include/asm/asm-defns.h
@@ -1,5 +1,9 @@
 #include <asm/page-bits.h>
 
+.macro clzero
+    .byte 0x0f, 0x01, 0xfc
+.endm
+
 /*
  * Call a noreturn function.  This could be JMP, but CALL results in a more
  * helpful backtrace.  BUG is to catch functions which do decide to return...
diff --git a/xen/arch/x86/include/asm/page.h b/xen/arch/x86/include/asm/page.h
index 8b5a9f77d0..77194c1885 100644
--- a/xen/arch/x86/include/asm/page.h
+++ b/xen/arch/x86/include/asm/page.h
@@ -219,10 +219,11 @@ typedef struct { u64 pfn; } pagetable_t;
 #define pagetable_from_paddr(p) pagetable_from_pfn((p)>>PAGE_SHIFT)
 #define pagetable_null()        pagetable_from_pfn(0)
 
-void clear_page_sse2(void *pg);
+void clear_page_hot(void *pg);
+void clear_page_cold(void *pg);
 void copy_page_sse2(void *to, const void *from);
 
-#define clear_page(_p)      clear_page_sse2(_p)
+#define clear_page(_p)      clear_page_cold(_p)
 #define copy_page(_t, _f)   copy_page_sse2(_t, _f)
 
 /* Convert between Xen-heap virtual addresses and machine addresses. */
diff --git a/xen/tools/gen-cpuid.py b/xen/tools/gen-cpuid.py
index 163b105bc6..6f96d1368e 100755
--- a/xen/tools/gen-cpuid.py
+++ b/xen/tools/gen-cpuid.py
@@ -212,6 +212,10 @@ def crunch_numbers(state):
         # the first place.
         APIC: [X2APIC, TSC_DEADLINE, EXTAPIC],
 
+        # The CLZERO insn requires a means to determine the cache line size,
+        # which is tied to the CLFLUSH insn.
+        CLFLUSH: [CLZERO],
+
         # AMD built MMXExtentions and 3DNow as extentions to MMX.
         MMX: [MMXEXT, _3DNOW],
 
--
generated by git-patchbot for /home/xen/git/xen.git#staging