|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [XEN][RESEND][PATCH 2/3] xen/arm: optimize stage-1,2 combined TLBI in presence of FEAT_nTLBPA
From: Haseeb Ashraf <haseeb.ashraf@xxxxxxxxxxx>
FEAT_nTLBPA guarantees that intermediate caching of translation table
walks does not retain non-coherent cached copies of previously valid
translation table entries since the last completed applicable TLBI.
When FEAT_nTLBPA is present, a stage-2 invalidation no longer needs to
be paired with an additional stage-1 invalidation solely to discard
such side effects. Use this to avoid redundant stage-1 invalidation in
the combined guest TLBI paths.
Update the stage-1-only guest TLBI helpers so that they become no-ops
when FEAT_nTLBPA is present. This allows existing call sites that
conservatively issue stage-1 invalidation after stage-2 invalidation to
benefit without further restructuring.
Also update the Arm64 IPA-based invalidation helper so that:
- stage-2 invalidation by IPA is performed explicitly,
- stage-1 invalidation is issued separately only when FEAT_nTLBPA is
absent,
- completion of the broadcast TLBI sequence remains explicit.
This makes the architectural intent clearer and avoids redundant
invalidations on systems that implement FEAT_nTLBPA.
Suggested-by: Mohamed Mediouni <mohamed@xxxxxxxxxxxxxxxx>
Signed-off-by: Haseeb Ashraf <haseeb.ashraf@xxxxxxxxxxx>
Changes in v4:
- Reworked the Arm64 IPA-based invalidation helper so that stage-2,
optional stage-1, and final TLBI completion are expressed
separately and explicitly.
- Simplified the FEAT_nTLBPA optimization by reusing the explicit TLBI
completion helper instead of open-coding a special final IPA TLBI
sequence.
- Refined comments and commit message wording.
Changes in v3:
- This commit has no functional change in v3, only rebasing changes
due to updates in commit-1.
Changes in v2:
- This commit is implemented in v2 and is splitted from commit-1 in
v1. This is implemented by using CPU capability.
---
xen/arch/arm/cpufeature.c | 19 ++++++++++
xen/arch/arm/include/asm/arm32/flushtlb.h | 14 ++++----
xen/arch/arm/include/asm/arm64/flushtlb.h | 44 +++++++++++++++++++----
xen/arch/arm/include/asm/cpufeature.h | 24 +++++++++++--
xen/arch/arm/include/asm/processor.h | 7 ++++
5 files changed, 93 insertions(+), 15 deletions(-)
diff --git a/xen/arch/arm/cpufeature.c b/xen/arch/arm/cpufeature.c
index 94d14fb6a9..86720c10a3 100644
--- a/xen/arch/arm/cpufeature.c
+++ b/xen/arch/arm/cpufeature.c
@@ -17,7 +17,19 @@ DECLARE_BITMAP(cpu_hwcaps, ARM_NCAPS);
struct cpuinfo_arm __read_mostly domain_cpuinfo;
+#ifdef CONFIG_ARM_32
+static bool has_ntlbpa(const struct arm_cpu_capabilities *entry)
+{
+ return system_cpuinfo.mm32.ntlbpa == MM32_NTLBPA_SUPPORT_IMP;
+}
+#endif
+
#ifdef CONFIG_ARM_64
+static bool has_ntlbpa(const struct arm_cpu_capabilities *entry)
+{
+ return system_cpuinfo.mm64.ntlbpa == MM64_NTLBPA_SUPPORT_IMP;
+}
+
static bool has_sb_instruction(const struct arm_cpu_capabilities *entry)
{
return system_cpuinfo.isa64.sb;
@@ -25,6 +37,13 @@ static bool has_sb_instruction(const struct
arm_cpu_capabilities *entry)
#endif
static const struct arm_cpu_capabilities arm_features[] = {
+#if defined(CONFIG_ARM_32) || defined(CONFIG_ARM_64)
+ {
+ .desc = "Intermediate caching of translation table walks (nTLBPA)",
+ .capability = ARM_HAS_NTLBPA,
+ .matches = has_ntlbpa,
+ },
+#endif
#ifdef CONFIG_ARM_64
{
.desc = "Speculation barrier instruction (SB)",
diff --git a/xen/arch/arm/include/asm/arm32/flushtlb.h
b/xen/arch/arm/include/asm/arm32/flushtlb.h
index 1497f16ba5..30df161323 100644
--- a/xen/arch/arm/include/asm/arm32/flushtlb.h
+++ b/xen/arch/arm/include/asm/arm32/flushtlb.h
@@ -49,8 +49,8 @@ TLB_HELPER(flush_xen_tlb_local, TLBIALLH, nsh)
* Flush TLB of local processor. Use when flush for only stage-1 is intended.
*
* The following function should be used where intention is to clear only
- * stage-1 TLBs. This would be helpful in future in identifying which stage-1
- * TLB flushes can be skipped such as in present of FEAT_nTLBPA.
+ * stage-1 TLBs. This would be helpful in identifying which stage-1 TLB flushes
+ * can be skipped such as in present of FEAT_nTLBPA.
*/
static inline void flush_guest_tlb_s1_local(void)
{
@@ -60,7 +60,8 @@ static inline void flush_guest_tlb_s1_local(void)
*
* See ARMv8 (DDI 0487L.b): G5-11698 Table G5-23.
*/
- return flush_guest_tlb_local();
+ if ( !cpus_have_const_cap(ARM_HAS_NTLBPA) )
+ flush_guest_tlb_local();
}
/*
@@ -68,8 +69,8 @@ static inline void flush_guest_tlb_s1_local(void)
* stage-1 is intended.
*
* The following function should be used where intention is to clear only
- * stage-1 TLBs. This would be helpful in future in identifying which stage-1
- * TLB flushes can be skipped such as in present of FEAT_nTLBPA.
+ * stage-1 TLBs. This would be helpful in identifying which stage-1 TLB flushes
+ * can be skipped such as in present of FEAT_nTLBPA.
*/
static inline void flush_guest_tlb_s1(void)
{
@@ -79,7 +80,8 @@ static inline void flush_guest_tlb_s1(void)
*
* See ARMv8 (DDI 0487L.b): G5-11698 Table G5-23.
*/
- return flush_guest_tlb();
+ if ( !cpus_have_const_cap(ARM_HAS_NTLBPA) )
+ flush_guest_tlb();
}
/* Flush TLB of local processor for address va. */
diff --git a/xen/arch/arm/include/asm/arm64/flushtlb.h
b/xen/arch/arm/include/asm/arm64/flushtlb.h
index cbe2a0da57..de266b0e11 100644
--- a/xen/arch/arm/include/asm/arm64/flushtlb.h
+++ b/xen/arch/arm/include/asm/arm64/flushtlb.h
@@ -63,6 +63,36 @@ static inline void name(void) \
: : : "memory"); \
}
+#define TLB_HELPER_NTLBPA_LOCAL(name, tlbop) \
+static inline void name(void) \
+{ \
+ if ( !cpus_have_const_cap(ARM_HAS_NTLBPA) ) \
+ asm_inline volatile ( \
+ "dsb nshst;" \
+ "tlbi " # tlbop ";" \
+ "dsb nsh;" \
+ "isb;" \
+ : : : "memory"); \
+}
+
+#define TLB_HELPER_NTLBPA(name, tlbop) \
+static inline void name(void) \
+{ \
+ if ( !cpus_have_const_cap(ARM_HAS_NTLBPA) ) \
+ asm_inline volatile ( \
+ "dsb ishst;" \
+ "tlbi " # tlbop ";" \
+ ALTERNATIVE( \
+ "nop; nop;", \
+ "dsb ish;" \
+ "tlbi vale2is, xzr;", \
+ ARM64_WORKAROUND_REPEAT_TLBI, \
+ CONFIG_ARM64_WORKAROUND_REPEAT_TLBI) \
+ "dsb ish;" \
+ "isb;" \
+ : : : "memory"); \
+}
+
/* Flush local TLBs, current VMID only. */
TLB_HELPER_LOCAL(flush_guest_tlb_local, vmalls12e1)
@@ -70,10 +100,10 @@ TLB_HELPER_LOCAL(flush_guest_tlb_local, vmalls12e1)
TLB_HELPER(flush_guest_tlb, vmalls12e1is)
/* Flush local TLBs, current VMID, stage-1 only */
-TLB_HELPER(flush_guest_tlb_s1_local, vmalle1, nsh)
+TLB_HELPER_NTLBPA_LOCAL(flush_guest_tlb_s1_local, vmalle1)
/* Flush innershareable TLBs, current VMID, stage-1 only */
-TLB_HELPER(flush_guest_tlb_s1, vmalle1is, ish)
+TLB_HELPER_NTLBPA(flush_guest_tlb_s1, vmalle1is)
/* Flush local TLBs, all VMIDs, non-hypervisor mode */
TLB_HELPER_LOCAL(flush_all_guests_tlb_local, alle1)
@@ -86,6 +116,8 @@ TLB_HELPER_LOCAL(flush_xen_tlb_local, alle2)
#undef TLB_HELPER_LOCAL
#undef TLB_HELPER
+#undef TLB_HELPER_NTLBPA_LOCAL
+#undef TLB_HELPER_NTLBPA
/*
* FLush TLB by VA. This will likely be used in a loop, so the caller
@@ -169,11 +201,11 @@ static inline void flush_guest_tlb_range_ipa(paddr_t ipa,
unsigned long size)
}
/*
- * Invalidate stage-1 entries for the current VMID as well. Keep this
- * separate from completion of the broadcast TLBI sequence so that the
- * final barriers/workaround are explicit.
+ * Without FEAT_nTLBPA, invalidate corresponding stage-1 side effects
+ * separately. Completion of the broadcast TLBI sequence remains explicit.
*/
- asm_inline volatile("tlbi vmalle1is" : : : "memory");
+ if ( !cpus_have_const_cap(ARM_HAS_NTLBPA) )
+ asm_inline volatile("tlbi vmalle1is" : : : "memory");
tlbi_complete_broadcast();
}
diff --git a/xen/arch/arm/include/asm/cpufeature.h
b/xen/arch/arm/include/asm/cpufeature.h
index bf902a3970..3e56dd27ef 100644
--- a/xen/arch/arm/include/asm/cpufeature.h
+++ b/xen/arch/arm/include/asm/cpufeature.h
@@ -76,8 +76,9 @@
#define ARM_WORKAROUND_BHB_SMCC_3 15
#define ARM_HAS_SB 16
#define ARM64_WORKAROUND_1508412 17
+#define ARM_HAS_NTLBPA 18
-#define ARM_NCAPS 18
+#define ARM_NCAPS 19
#ifndef __ASSEMBLER__
@@ -276,7 +277,8 @@ struct cpuinfo_arm {
unsigned long ets:4;
unsigned long __res2:4;
unsigned long afp:4;
- unsigned long __res3:12;
+ unsigned long ntlbpa:4;
+ unsigned long __res3:8;
unsigned long ecbhb:4;
/* MMFR2 */
@@ -437,8 +439,24 @@ struct cpuinfo_arm {
register_t bits[1];
} aux32;
- struct {
+ union {
register_t bits[6];
+ struct {
+ /* MMFR0 */
+ unsigned long __res0:32;
+ /* MMFR1 */
+ unsigned long __res1:32;
+ /* MMFR2 */
+ unsigned long __res2:32;
+ /* MMFR3 */
+ unsigned long __res3:32;
+ /* MMFR4 */
+ unsigned long __res4:32;
+ /* MMFR5 */
+ unsigned long __res5:4;
+ unsigned long ntlbpa:4;
+ unsigned long __res6:24;
+ };
} mm32;
struct {
diff --git a/xen/arch/arm/include/asm/processor.h
b/xen/arch/arm/include/asm/processor.h
index a3753c317f..dbf9471a7a 100644
--- a/xen/arch/arm/include/asm/processor.h
+++ b/xen/arch/arm/include/asm/processor.h
@@ -484,9 +484,16 @@
/* FSR long format */
#define FSRL_STATUS_DEBUG (_AC(0x22,UL)<<0)
+#ifdef CONFIG_ARM_32
+#define MM32_NTLBPA_SUPPORT_NI 0x0
+#define MM32_NTLBPA_SUPPORT_IMP 0x1
+#endif
+
#ifdef CONFIG_ARM_64
#define MM64_VMID_8_BITS_SUPPORT 0x0
#define MM64_VMID_16_BITS_SUPPORT 0x2
+#define MM64_NTLBPA_SUPPORT_NI 0x0
+#define MM64_NTLBPA_SUPPORT_IMP 0x1
#endif
#define MM64_MSA_PMSA_SUPPORT 0xf
--
2.43.0
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |