|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [PATCH 5/8] pdx: allow optimizing PDX conversion helpers
There are four performance critical PDX conversion helpers that do the PFN
to/from PDX and the physical addresses to/from directmap offsets
translations.
In the absence of an active PDX compression, those functions would still do
the calculations needed, just to return the same input value as no
translation is in place and hence PFN and PDX spaces are identity mapped.
To reduce the overhead of having to do the pointless calculations allow
architectures to short-circuit the logic by providing a macro. Implement
such short-circuiting for x86 using asm goto and alternatives. This
results in the optimized case (when PDX compression is not enabled) doing
an unconditional jump to a label, while the non-optimized case is left
as-is.
Signed-off-by: Roger Pau Monné <roger.pau@xxxxxxxxxx>
---
xen/arch/x86/include/asm/cpufeatures.h | 1 +
xen/arch/x86/srat.c | 6 +++-
xen/common/pdx.c | 8 +++--
xen/include/xen/pdx.h | 46 ++++++++++++++++++++------
4 files changed, 47 insertions(+), 14 deletions(-)
diff --git a/xen/arch/x86/include/asm/cpufeatures.h
b/xen/arch/x86/include/asm/cpufeatures.h
index 9e3ed21c026d..85e1a6f0a055 100644
--- a/xen/arch/x86/include/asm/cpufeatures.h
+++ b/xen/arch/x86/include/asm/cpufeatures.h
@@ -43,6 +43,7 @@ XEN_CPUFEATURE(XEN_IBT, X86_SYNTH(27)) /* Xen uses
CET Indirect Branch
XEN_CPUFEATURE(IBPB_ENTRY_PV, X86_SYNTH(28)) /* MSR_PRED_CMD used by Xen
for PV */
XEN_CPUFEATURE(IBPB_ENTRY_HVM, X86_SYNTH(29)) /* MSR_PRED_CMD used by Xen
for HVM */
XEN_CPUFEATURE(USE_VMCALL, X86_SYNTH(30)) /* Use VMCALL instead of
VMMCALL */
+XEN_CPUFEATURE(PDX_COMPRESSION, X86_SYNTH(31)) /* PDX compression */
/* Bug words follow the synthetic words. */
#define X86_NR_BUG 1
diff --git a/xen/arch/x86/srat.c b/xen/arch/x86/srat.c
index 7042fd3c3d88..96a87bbce35b 100644
--- a/xen/arch/x86/srat.c
+++ b/xen/arch/x86/srat.c
@@ -298,7 +298,8 @@ void __init srat_parse_regions(paddr_t addr)
acpi_table_parse_srat(ACPI_SRAT_TYPE_MEMORY_AFFINITY,
srat_parse_region, 0);
- pfn_pdx_compression_setup(addr);
+ if (!pfn_pdx_compression_setup(addr))
+ return;
/* Ensure all ranges in the e820 are covered. */
for (i = 0; i < e820.nr_map; i++) {
@@ -318,6 +319,9 @@ void __init srat_parse_regions(paddr_t addr)
return;
}
}
+
+ /* If we got this far compression is working as expected. */
+ setup_force_cpu_cap(X86_FEATURE_PDX_COMPRESSION);
}
unsigned int numa_node_to_arch_nid(nodeid_t n)
diff --git a/xen/common/pdx.c b/xen/common/pdx.c
index 65b337860d52..7d14100224fe 100644
--- a/xen/common/pdx.c
+++ b/xen/common/pdx.c
@@ -230,7 +230,7 @@ static uint64_t __init pdx_init_mask(uint64_t base_addr)
(uint64_t)1 << (MAX_ORDER + PAGE_SHIFT)) - 1);
}
-void __init pfn_pdx_compression_setup(paddr_t base)
+bool __init pfn_pdx_compression_setup(paddr_t base)
{
unsigned int i, j, bottom_shift = 0, hole_shift = 0;
unsigned long mask = pdx_init_mask(base);
@@ -240,7 +240,7 @@ void __init pfn_pdx_compression_setup(paddr_t base)
printk(XENLOG_WARNING
"Too many PFN ranges (%u), not attempting PFN compression\n",
nr);
- return;
+ return false;
}
for ( i = 0; i < nr; i++ )
@@ -272,7 +272,7 @@ void __init pfn_pdx_compression_setup(paddr_t base)
}
}
if ( !hole_shift )
- return;
+ return false;
printk(KERN_INFO "PFN compression on bits %u...%u\n",
bottom_shift, bottom_shift + hole_shift - 1);
@@ -283,6 +283,8 @@ void __init pfn_pdx_compression_setup(paddr_t base)
pfn_hole_mask = ((1UL << hole_shift) - 1) << bottom_shift;
pfn_top_mask = ~(pfn_pdx_bottom_mask | pfn_hole_mask);
ma_top_mask = pfn_top_mask << PAGE_SHIFT;
+
+ return true;
}
void __init pfn_pdx_compression_reset(void)
diff --git a/xen/include/xen/pdx.h b/xen/include/xen/pdx.h
index 43ce36fcbb56..6cc0f54cff83 100644
--- a/xen/include/xen/pdx.h
+++ b/xen/include/xen/pdx.h
@@ -67,6 +67,26 @@
* region involved.
*/
+/* Macro defined per-arch to skip PDX logic when there's no compression. */
+#ifdef CONFIG_X86
+# include <asm/alternative.h>
+
+# define OPTIMIZE_PDX(transform, raw) \
+ asm_inline goto ( \
+ ALTERNATIVE( \
+ "", \
+ "jmp %l[skip]", \
+ ALT_NOT(X86_FEATURE_PDX_COMPRESSION)) \
+ : : : : skip ); \
+ return (transform); \
+ \
+ skip: \
+ return (raw)
+#else
+# define OPTIMIZE_PDX(transform, raw) \
+ return (transform)
+#endif
+
extern unsigned long max_pdx;
#define PDX_GROUP_COUNT ((1 << PDX_GROUP_SHIFT) / \
@@ -123,8 +143,9 @@ extern unsigned long pfn_top_mask, ma_top_mask;
*/
static inline unsigned long pfn_to_pdx(unsigned long pfn)
{
- return (pfn & pfn_pdx_bottom_mask) |
- ((pfn & pfn_top_mask) >> pfn_pdx_hole_shift);
+ OPTIMIZE_PDX((pfn & pfn_pdx_bottom_mask) |
+ ((pfn & pfn_top_mask) >> pfn_pdx_hole_shift),
+ pfn);
}
/**
@@ -135,8 +156,9 @@ static inline unsigned long pfn_to_pdx(unsigned long pfn)
*/
static inline unsigned long pdx_to_pfn(unsigned long pdx)
{
- return (pdx & pfn_pdx_bottom_mask) |
- ((pdx << pfn_pdx_hole_shift) & pfn_top_mask);
+ OPTIMIZE_PDX((pdx & pfn_pdx_bottom_mask) |
+ ((pdx << pfn_pdx_hole_shift) & pfn_top_mask),
+ pdx);
}
/**
@@ -148,8 +170,9 @@ static inline unsigned long pdx_to_pfn(unsigned long pdx)
*/
static inline unsigned long maddr_to_directmapoff(paddr_t ma)
{
- return (((ma & ma_top_mask) >> pfn_pdx_hole_shift) |
- (ma & ma_va_bottom_mask));
+ OPTIMIZE_PDX(((ma & ma_top_mask) >> pfn_pdx_hole_shift) |
+ (ma & ma_va_bottom_mask),
+ ma);
}
/**
@@ -160,8 +183,9 @@ static inline unsigned long maddr_to_directmapoff(paddr_t
ma)
*/
static inline paddr_t directmapoff_to_maddr(unsigned long offset)
{
- return ((((paddr_t)offset << pfn_pdx_hole_shift) & ma_top_mask) |
- (offset & ma_va_bottom_mask));
+ OPTIMIZE_PDX(((((paddr_t)offset << pfn_pdx_hole_shift) & ma_top_mask) |
+ (offset & ma_va_bottom_mask)),
+ offset);
}
#endif /* CONFIG_PDX_MASK_COMPRESSION */
@@ -188,8 +212,9 @@ static inline void pfn_pdx_add_region(paddr_t base, paddr_t
size)
{
}
-static inline void pfn_pdx_compression_setup(paddr_t base)
+static inline bool pfn_pdx_compression_setup(paddr_t base)
{
+ return false;
}
static inline void pfn_pdx_compression_reset(void)
@@ -222,8 +247,9 @@ void pfn_pdx_add_region(paddr_t base, paddr_t size);
* range of the current memory regions.
*
* @param base address to start compression from.
+ * @return True if PDX compression has been enabled.
*/
-void pfn_pdx_compression_setup(paddr_t base);
+bool pfn_pdx_compression_setup(paddr_t base);
/**
* Reset the global variables to it's default values, thus disabling PFN
--
2.49.0
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |