[PATCH 3/3] x86: correct fencing around CLFLUSH

[Jan Beulich <jbeulich@xxxxxxxx>]

As noted in the context of 3330013e6739 ("VT-d / x86: re-arrange cache
syncing"): While cache_writeback() has the SFENCE on the correct side of
CLFLUSHOPT, flush_area_local() doesn't. While I can't prove it due to
lacking a copy of the old SDM version, I can only assume this placement
was a result of what had been described there originally. In any event
recent versions of the SDM hve been telling us otherwise.

For AMD (and Hygon, albeit there it's benign, since all their CPUs are
expected to support CLFLUSHOPT) the situation is more complex: MFENCE is
needed ahead and/or after CLFLUSH when the CPU doesn't also support
CLFLUSHOPT. (It's "and" in our case, as we cannot know what the caller's
needs are.)

Fixes: 623c720fc8da3 ("x86: use CLFLUSHOPT when available")
Signed-off-by: Jan Beulich <jbeulich@xxxxxxxx>

--- a/xen/arch/x86/cpu/common.c
+++ b/xen/arch/x86/cpu/common.c
@@ -346,9 +346,14 @@ void __init early_cpu_init(void)
               c->x86_model, c->x86_model, c->x86_mask, eax);
 
        if (c->cpuid_level >= 7)
-               cpuid_count(7, 0, &eax, &ebx,
+               cpuid_count(7, 0, &eax,
+                            &c->x86_capability[FEATURESET_7b0],
                             &c->x86_capability[FEATURESET_7c0], &edx);
 
+       if (!(c->x86_vendor & (X86_VENDOR_AMD | X86_VENDOR_HYGON)) ||
+           cpu_has(c, X86_FEATURE_CLFLUSHOPT))
+               setup_force_cpu_cap(X86_FEATURE_CLFLUSH_NO_MFENCE);
+
        eax = cpuid_eax(0x80000000);
        if ((eax >> 16) == 0x8000 && eax >= 0x80000008) {
                ebx = eax >= 0x8000001f ? cpuid_ebx(0x8000001f) : 0;
--- a/xen/arch/x86/flushtlb.c
+++ b/xen/arch/x86/flushtlb.c
@@ -245,12 +245,15 @@ unsigned int flush_area_local(const void
              c->x86_clflush_size && c->x86_cache_size && sz &&
              ((sz >> 10) < c->x86_cache_size) )
         {
-            alternative("", "sfence", X86_FEATURE_CLFLUSHOPT);
+            alternative("mfence", , X86_FEATURE_CLFLUSH_NO_MFENCE);
             for ( i = 0; i < sz; i += c->x86_clflush_size )
                 alternative_input("ds; clflush %0",
                                   "data16 clflush %0",      /* clflushopt */
                                   X86_FEATURE_CLFLUSHOPT,
                                   "m" (((const char *)va)[i]));
+            alternative_2("mfence",
+                          , X86_FEATURE_CLFLUSH_NO_MFENCE,
+                          "sfence", X86_FEATURE_CLFLUSHOPT);
             flags &= ~FLUSH_CACHE;
         }
         else
@@ -274,6 +277,8 @@ void cache_writeback(const void *addr, u
     unsigned int clflush_size = current_cpu_data.x86_clflush_size ?: 16;
     const void *end = addr + size;
 
+    alternative("mfence", , X86_FEATURE_CLFLUSH_NO_MFENCE);
+
     addr -= (unsigned long)addr & (clflush_size - 1);
     for ( ; addr < end; addr += clflush_size )
     {
--- a/xen/arch/x86/include/asm/cpufeatures.h
+++ b/xen/arch/x86/include/asm/cpufeatures.h
@@ -24,7 +24,7 @@ XEN_CPUFEATURE(APERFMPERF,        X86_SY
 XEN_CPUFEATURE(MFENCE_RDTSC,      X86_SYNTH( 9)) /* MFENCE synchronizes RDTSC 
*/
 XEN_CPUFEATURE(XEN_SMEP,          X86_SYNTH(10)) /* SMEP gets used by Xen 
itself */
 XEN_CPUFEATURE(XEN_SMAP,          X86_SYNTH(11)) /* SMAP gets used by Xen 
itself */
-/* Bit 12 - unused. */
+XEN_CPUFEATURE(CLFLUSH_NO_MFENCE, X86_SYNTH(12)) /* No MFENCE needed to 
serialize CLFLUSH */
 XEN_CPUFEATURE(IND_THUNK_LFENCE,  X86_SYNTH(13)) /* Use IND_THUNK_LFENCE */
 XEN_CPUFEATURE(IND_THUNK_JMP,     X86_SYNTH(14)) /* Use IND_THUNK_JMP */
 XEN_CPUFEATURE(SC_NO_BRANCH_HARDEN, X86_SYNTH(15)) /* (Disable) Conditional 
branch hardening */