[Xen-devel] [PATCH v3 7/7] x86/nospec: Optimise array_index_mask_nospec() for power-of-2 arrays

[Andrew Cooper <andrew.cooper3@xxxxxxxxxx>]

When the compiler can determine that an array bound is a power of two, the
array index can be bounded even under speculation with a single and
instruction.

Respecify array_index_mask_nospec() to allow for masks other than ~0 and 0,
and introduce an IS_POWER_OF_2() helper.

Signed-off-by: Andrew Cooper <andrew.cooper3@xxxxxxxxxx>
---
CC: Jan Beulich <JBeulich@xxxxxxxx>
CC: Wei Liu <wl@xxxxxxx>
CC: Roger Pau Monné <roger.pau@xxxxxxxxxx>
CC: Juergen Gross <jgross@xxxxxxxx>

This optimisation is not safe on ARM, because some CPUs do data value
speculation, which is why the CSDB barrer was introduced.
---
 xen/include/asm-x86/nospec.h | 25 +++++++++++++++++++------
 xen/include/xen/config.h     |  1 +
 xen/include/xen/nospec.h     |  3 ++-
 3 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/xen/include/asm-x86/nospec.h b/xen/include/asm-x86/nospec.h
index 0039cd2713..4f36069eac 100644
--- a/xen/include/asm-x86/nospec.h
+++ b/xen/include/asm-x86/nospec.h
@@ -7,13 +7,20 @@
 #include <asm/alternative.h>
 
 /**
- * array_index_mask_nospec() - generate a mask that is ~0UL when the
- *      bounds check succeeds and 0 otherwise
+ * array_index_mask_nospec() - generate a mask to bound an array index
+ * which is safe even under adverse speculation.
  * @index: array element index
  * @size: number of elements in array
  *
- * Returns:
+ * In general, returns:
  *     0 - (index < size)
+ *
+ * This yeild ~0UL in within-bounds case, and 0 in the out-of-bounds
+ * case.
+ *
+ * When the compiler can determine that the array is a power of two, a
+ * lower overhead option is to mask the index with a single and
+ * instruction.
  */
 #define array_index_mask_nospec array_index_mask_nospec
 static inline unsigned long array_index_mask_nospec(unsigned long index,
@@ -21,9 +28,15 @@ static inline unsigned long array_index_mask_nospec(unsigned 
long index,
 {
     unsigned long mask;
 
-    asm volatile ( "cmp %[size], %[index]; sbb %[mask], %[mask];"
-                   : [mask] "=r" (mask)
-                   : [size] "g" (size), [index] "r" (index) );
+    if ( __builtin_constant_p(size) && IS_POWER_OF_2(size) )
+    {
+        mask = size - 1;
+        OPTIMIZER_HIDE_VAR(mask);
+    }
+    else
+        asm volatile ( "cmp %[size], %[index]; sbb %[mask], %[mask];"
+                       : [mask] "=r" (mask)
+                       : [size] "g" (size), [index] "r" (index) );
 
     return mask;
 }
diff --git a/xen/include/xen/config.h b/xen/include/xen/config.h
index a106380a23..21c763617c 100644
--- a/xen/include/xen/config.h
+++ b/xen/include/xen/config.h
@@ -75,6 +75,7 @@
 #define GB(_gb)     (_AC(_gb, ULL) << 30)
 
 #define IS_ALIGNED(val, align) (((val) & ((align) - 1)) == 0)
+#define IS_POWER_OF_2(val) ((val) && IS_ALIGNED(val, val))
 
 #define __STR(...) #__VA_ARGS__
 #define STR(...) __STR(__VA_ARGS__)
diff --git a/xen/include/xen/nospec.h b/xen/include/xen/nospec.h
index 7578210f16..cfc31f11b7 100644
--- a/xen/include/xen/nospec.h
+++ b/xen/include/xen/nospec.h
@@ -12,7 +12,8 @@
 #include <asm/nospec.h>
 
 /**
- * array_index_mask_nospec() - generate a ~0 mask when index < size, 0 
otherwise
+ * array_index_mask_nospec() - generate a mask to bound an array index
+ * which is safe even under adverse speculation.
  * @index: array element index
  * @size: number of elements in array
  *
-- 
2.11.0


_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxxx
https://lists.xenproject.org/mailman/listinfo/xen-devel