[PATCH Linux v3 6/6] xen: Add new Xen pv-op pte_get_and_clear.

[Kevin Lampis <kevin.lampis@xxxxxxxxxx>]

If Xen supports XENFEAT_mmu_pt_update_swap then for performance reasons
call mmu_update with the new MMU_PT_UPDATE_SWAP flag instead of
native_ptep_get_and_clear().

Signed-off-by: Kevin Lampis <kevin.lampis@xxxxxxxxxx>
---
Changes in v3:
- Add spacing to xen_pte_get_and_clear
- BUG_ON if hypercall fails
- Remove extra blank line
- Add documentation for MMU_PT_UPDATE_SWAP

Changes in v2:
- New patch
---
 arch/x86/include/asm/paravirt.h       |  5 +++++
 arch/x86/include/asm/paravirt_types.h |  1 +
 arch/x86/include/asm/pgtable.h        |  3 ++-
 arch/x86/kernel/paravirt.c            |  1 +
 arch/x86/xen/mmu_pv.c                 | 17 +++++++++++++++++
 include/xen/interface/features.h      |  2 ++
 include/xen/interface/xen.h           |  1 +
 7 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index 0591aa38fd85..0a3a286c46f0 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -373,6 +373,11 @@ static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
        PVOP_VCALL2(pv_ops, mmu.set_pmd, pmdp, native_pmd_val(pmd));
 }
 
+static inline pte_t pte_get_and_clear(pte_t *ptep)
+{
+       return (pte_t){PVOP_CALL1(pte_t, mmu.pte_get_and_clear, ptep)};
+}
+
 static inline pmd_t __pmd(pmdval_t val)
 {
        return (pmd_t) { PVOP_ALT_CALLEE1(pmdval_t, pv_ops, mmu.make_pmd, val,
diff --git a/arch/x86/include/asm/paravirt_types.h 
b/arch/x86/include/asm/paravirt_types.h
index b4c4a23e77a1..1bd4c19450b5 100644
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -135,6 +135,7 @@ struct pv_mmu_ops {
        /* Pagetable manipulation functions */
        void (*set_pte)(pte_t *ptep, pte_t pteval);
        void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval);
+       pte_t (*pte_get_and_clear)(pte_t *ptep);
 
        pte_t (*ptep_modify_prot_start)(struct vm_area_struct *vma, unsigned 
long addr,
                                        pte_t *ptep);
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index ac295ca6c92f..c2a7035b4848 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -58,6 +58,7 @@ extern pmdval_t early_pmd_flags;
 #include <asm/paravirt.h>
 #else  /* !CONFIG_PARAVIRT_XXL */
 #define set_pte(ptep, pte)             native_set_pte(ptep, pte)
+#define pte_get_and_clear(ptep)        native_ptep_get_and_clear(ptep)
 
 #define set_pte_atomic(ptep, pte)                                      \
        native_set_pte_atomic(ptep, pte)
@@ -1243,7 +1244,7 @@ bool ptep_clear_flush_young(struct vm_area_struct *vma,
 static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long 
addr,
                                       pte_t *ptep)
 {
-       pte_t pte = native_ptep_get_and_clear(ptep);
+       pte_t pte = pte_get_and_clear(ptep);
        page_table_check_pte_clear(mm, addr, pte);
        return pte;
 }
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index 00b59d774389..58ecd12b10ee 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -178,6 +178,7 @@ struct paravirt_patch_template pv_ops = {
 
        .mmu.set_pte            = native_set_pte,
        .mmu.set_pmd            = native_set_pmd,
+       .mmu.pte_get_and_clear  = native_ptep_get_and_clear,
 
        .mmu.ptep_modify_prot_start     = __ptep_modify_prot_start,
        .mmu.ptep_modify_prot_commit    = __ptep_modify_prot_commit,
diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c
index 820af6f0aa57..a1e79672a408 100644
--- a/arch/x86/xen/mmu_pv.c
+++ b/arch/x86/xen/mmu_pv.c
@@ -359,6 +359,18 @@ static void xen_set_pte(pte_t *ptep, pte_t pteval)
        __xen_set_pte(ptep, pteval);
 }
 
+static pte_t xen_pte_get_and_clear(pte_t *ptep)
+{
+       struct mmu_update u = {
+               .ptr = virt_to_machine(ptep).maddr | MMU_PT_UPDATE_SWAP,
+               .val = pte_val_ma(native_make_pte(0))
+       };
+
+       BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) != 0);
+
+       return native_make_pte(u.val);
+}
+
 static pte_t xen_ptep_modify_prot_start(struct vm_area_struct *vma,
                                        unsigned long addr, pte_t *ptep)
 {
@@ -2165,6 +2177,11 @@ static void __init xen_post_allocator_init(void)
        pv_ops.mmu.set_pud = xen_set_pud;
        pv_ops.mmu.set_p4d = xen_set_p4d;
 
+       if (xen_feature(XENFEAT_mmu_pt_update_swap))
+               pv_ops.mmu.pte_get_and_clear = xen_pte_get_and_clear;
+       else
+               pv_ops.mmu.pte_get_and_clear = native_ptep_get_and_clear;
+
        /* This will work as long as patching hasn't happened yet
           (which it hasn't) */
        pv_ops.mmu.alloc_pte = xen_alloc_pte;
diff --git a/include/xen/interface/features.h b/include/xen/interface/features.h
index 53f760378e39..b346d58c0a41 100644
--- a/include/xen/interface/features.h
+++ b/include/xen/interface/features.h
@@ -97,6 +97,8 @@
 #define XENFEAT_not_direct_mapped         16
 #define XENFEAT_direct_mapped             17
 
+#define XENFEAT_mmu_pt_update_swap        21
+
 #define XENFEAT_NR_SUBMAPS 1
 
 #endif /* __XEN_PUBLIC_FEATURES_H__ */
diff --git a/include/xen/interface/xen.h b/include/xen/interface/xen.h
index 40c9793e9880..64c120d8a100 100644
--- a/include/xen/interface/xen.h
+++ b/include/xen/interface/xen.h
@@ -252,6 +252,7 @@
 #define MMU_MACHPHYS_UPDATE        1 /* ptr = MA of frame to modify entry for 
*/
 #define MMU_PT_UPDATE_PRESERVE_AD  2 /* atomically: *ptr = val | (*ptr&(A|D)) 
*/
 #define MMU_PT_UPDATE_NO_TRANSLATE 3 /* checked '*ptr = val'. ptr is MA.      
*/
+#define MMU_PT_UPDATE_SWAP         4 /* Update and return the old value. */
 
 /*
  * MMU EXTENDED OPERATIONS
-- 
2.52.0