|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] Re: [RFC PATCH kernel 17/17] x86/sev: Flush IOMMU TLB for trusted devices
On Wed, Sep 16, 2026 at 5:07 AM Alexey Kardashevskiy <aik@xxxxxxx> wrote:
> +static int alloc_iommu_tlb_flush_ghcb_pages(void)
> +{
> + unsigned int cpu;
> + struct page *pg;
> + void *p;
> +
> + /*
> + * Allocate per CPU pages while encrypted DMA is not happening yet
> + * and smashing is cheap.
> + */
> + for_each_possible_cpu(cpu) {
> + if (per_cpu(iommu_tlb_flush_ghcb_page, cpu))
> + continue;
> +
> + pg = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL, 0);
> + if (!pg)
> + return -ENOMEM;
> +
> + p = page_to_virt(pg);
> + /* Trigger psmash in the host os now to avoid psmash race
> later */
> + snp_set_memory_shared((unsigned long)p, 1);
> + snp_set_memory_private((unsigned long)p, 1);
> + per_cpu(iommu_tlb_flush_ghcb_page, cpu) = p;
> + }
> +
> + return 0;
> +}
>
> int sev_tio_op(u32 guest_rid, unsigned int op, u64 *fw_err, u64 *tdi_id)
> {
> @@ -111,6 +142,24 @@ int sev_tio_op(u32 guest_rid, unsigned int op, u64
> *fw_err, u64 *tdi_id)
> struct ghcb *ghcb;
> int ret;
>
> + if (!(sev_hv_features & GHCB_HV_FT_SNP_SEV_TIO))
> + return -EPERM;
> +
> + if (op == SVM_VMGEXIT_SEV_TIO_OP_RUN || op ==
> SVM_VMGEXIT_SEV_TIO_OP_STOP) {
> + if (!(sev_hv_features & GHCB_HV_FT_SNP_IOMMU_TLB_FLUSH))
> + return -EPERM;
> +
> + if (op == SVM_VMGEXIT_SEV_TIO_OP_RUN) {
> + if (atomic_inc_return(&sev_tio_devices_num) == 1) {
> + ret = alloc_iommu_tlb_flush_ghcb_pages();
> + if (ret)
> + return ret;
> + }
> + } else if (atomic_dec_return(&sev_tio_devices_num) == 0) {
> + /* Do cleanup or leave it like this? */
> + }
> + }
Hi Alexey,
When testing this series and accepting a locked TDI in the guest
(echo 1 > /sys/bus/pci/devices/.../tsm/accept), the guest immediately
terminates with 0x1:0xd (SEV_TERM_SET_LINUX, GHCB_TERM_IOMMUTLB_FLUSH).
In sev_tio_op(), sev_tio_devices_num is incremented from 0 to 1 before
alloc_iommu_tlb_flush_ghcb_pages() allocates and initializes the per-CPU
iommu_tlb_flush_ghcb_page buffers:
1. atomic_inc_return(&sev_tio_devices_num) sets sev_tio_devices_num = 1
while iommu_tlb_flush_ghcb_page is still NULL on all CPUs.
2. alloc_iommu_tlb_flush_ghcb_pages() allocates p for cpu = 0 and calls
snp_set_memory_shared((unsigned long)p, 1) to pre-smash the 2M page
before per_cpu(iommu_tlb_flush_ghcb_page, cpu) is assigned (and before
other CPUs' pages are allocated, in case this task is running on cpu > 0).
3. snp_set_memory_shared() -> __set_pages_state() sees
atomic_read(&sev_tio_devices_num) != 0 and calls ghcb_flush_iommu_tlb().
4. ghcb_flush_iommu_tlb() reads this_cpu_read(iommu_tlb_flush_ghcb_page),
gets NULL, and returns -ENOMEM.
5. __set_pages_state() treats the non-zero return as fatal and calls
sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_IOMMUTLB_FLUSH).
Calling alloc_iommu_tlb_flush_ghcb_pages() before incrementing
sev_tio_devices_num avoids triggering ghcb_flush_iommu_tlb() while the
per-CPU pages are still being pre-smashed and initialized:
--- a/arch/x86/coco/sev/core.c
+++ b/arch/x86/coco/sev/core.c
@@ -150,13 +150,12 @@ int sev_tio_op(...)
return -EPERM;
if (op == SVM_VMGEXIT_SEV_TIO_OP_RUN) {
- if (atomic_inc_return(&sev_tio_devices_num) == 1) {
- ret = alloc_iommu_tlb_flush_ghcb_pages();
- if (ret)
- return ret;
- }
- } else if (atomic_dec_return(&sev_tio_devices_num) == 0) {
- /* Do cleanup or leave it like this? */
+ ret = alloc_iommu_tlb_flush_ghcb_pages();
+ if (ret)
+ return ret;
+ atomic_inc(&sev_tio_devices_num);
+ } else {
+ atomic_dec_if_positive(&sev_tio_devices_num);
}
}
On Wed, Sep 16, 2026 at 5:07 AM Alexey Kardashevskiy <aik@xxxxxxx> wrote:
>
> IOMMU performs RMP checks when SNP is enabled, the results are
> cached along with the IOMMU translations. When a VM lowers permission
> of a mapped page (moves to a lower VMPL level or from read+write to
> read-only or private to shared), the cached RMP check results require
> invalidation.
>
> At the moment the only way to invalidate IOMMU cache is the RMPUPDATE
> instruction which flushes all IOMMU TLBs. It is a host privileged
> instruction so a VM needs a way to ensure the host has done it.
> Note that the guest's RMPADJUST/PVALIDATE do not flush IOMMU TLBs.
>
> The host implements a new "IOMMU TLB Flush" VMGEXIT code which is
> advertised via bit#11 in the GHCB Hypervisor capabilities.
>
> Use RMPUPDATE in the following way:
> - allocate a page per VCPU (to allow lockless flushing);
> - When invalidation is needed, copy two patterns (A and B) to the page;
> - invalidate the page so the host can make it shared;
> - use new GHCB call to request RMPUPDATE on the host;
> - the host makes the page shared;
> - the host clears pattern A;
> - the host makes the page private again;
> - the host returns to the guest;
> - check if pattern A has changed and pattern B has not;
> - if the above failed, panic().
>
> The patterns are located far enough to not hit the same cache line to
> work with the cipher text hiding feature.
>
> The host can choose to not execute the request, WARN_ON if this
> is the case. Further patches will attempt to handle this in other way.
>
> Signed-off-by: Alexey Kardashevskiy <aik@xxxxxxx>
> ---
> arch/x86/include/asm/sev-common.h | 2 +
> arch/x86/include/uapi/asm/svm.h | 3 +
> arch/x86/coco/sev/core.c | 92 ++++++++++++++++++++
> 3 files changed, 97 insertions(+)
>
> diff --git a/arch/x86/include/asm/sev-common.h
> b/arch/x86/include/asm/sev-common.h
> index ff763c3c5d63..51abf8d061fa 100644
> --- a/arch/x86/include/asm/sev-common.h
> +++ b/arch/x86/include/asm/sev-common.h
> @@ -138,6 +138,7 @@ enum psc_op {
> #define GHCB_HV_FT_SNP_AP_CREATION BIT_ULL(1)
> #define GHCB_HV_FT_SNP_MULTI_VMPL BIT_ULL(5)
> #define GHCB_HV_FT_SNP_SEV_TIO BIT_ULL(7)
> +#define GHCB_HV_FT_SNP_IOMMU_TLB_FLUSH BIT_ULL(11)
>
> /*
> * SNP Page State Change NAE event
> @@ -210,6 +211,7 @@ struct snp_psc_desc {
> #define GHCB_TERM_SECURE_TSC 10 /* Secure TSC initialization
> failed */
> #define GHCB_TERM_SVSM_CA_REMAP_FAIL 11 /* SVSM is present but CA
> could not be remapped */
> #define GHCB_TERM_SAVIC_FAIL 12 /* Secure AVIC-specific
> failure */
> +#define GHCB_TERM_IOMMUTLB_FLUSH 13 /* IOMMUTLB flush failed for
> SEV-TIO device */
>
> #define GHCB_RESP_CODE(v) ((v) & GHCB_MSR_INFO_MASK)
>
> diff --git a/arch/x86/include/uapi/asm/svm.h b/arch/x86/include/uapi/asm/svm.h
> index 93597ad492bf..269050942c8e 100644
> --- a/arch/x86/include/uapi/asm/svm.h
> +++ b/arch/x86/include/uapi/asm/svm.h
> @@ -160,6 +160,8 @@
> #define SVM_VMGEXIT_SEV_TIO_OP_UNBIND 1
> #define SVM_VMGEXIT_SEV_TIO_OP_RUN 2
> #define SVM_VMGEXIT_SEV_TIO_OP_STOP 3
> +#define SVM_VMGEXIT_IOMMU_TLB_FLUSH 0x80000022ull
> +#define SVM_VMGEXIT_IOMMU_TLB_FLUSH_NO_ACTION 1
> #define SVM_VMGEXIT_HV_FEATURES 0x8000fffdull
> #define SVM_VMGEXIT_TERM_REQUEST 0x8000fffeull
> #define SVM_VMGEXIT_TERM_REASON(reason_set, reason_code) \
> @@ -285,6 +287,7 @@
> { SVM_VMGEXIT_AP_CREATION, "vmgexit_ap_creation" }, \
> { SVM_VMGEXIT_SEV_TIO_GR, "vmgexit_sev_tio_guest_request" }, \
> { SVM_VMGEXIT_SEV_TIO_OP, "vmgexit_sev_tio_op" }, \
> + { SVM_VMGEXIT_IOMMU_TLB_FLUSH, "vmgexit_sev_tio_iommu_tlb_flush" }, \
> { SVM_VMGEXIT_HV_FEATURES, "vmgexit_hypervisor_feature" }, \
> { SVM_EXIT_ERR, "invalid_guest_state" }
>
> diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c
> index ed0e4546d5e5..aa5a3abb4796 100644
> --- a/arch/x86/coco/sev/core.c
> +++ b/arch/x86/coco/sev/core.c
> @@ -44,6 +44,7 @@
> #include <asm/cpuid/api.h>
> #include <asm/cmdline.h>
> #include <asm/msr.h>
> +#include <asm/archrandom.h>
>
> #include "internal.h"
>
> @@ -103,6 +104,36 @@ static unsigned long snp_tsc_freq_khz __ro_after_init;
>
> DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data);
> DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa);
> +DEFINE_PER_CPU(u8 *, iommu_tlb_flush_ghcb_page);
> +static atomic_t sev_tio_devices_num;
> +
> +static int alloc_iommu_tlb_flush_ghcb_pages(void)
> +{
> + unsigned int cpu;
> + struct page *pg;
> + void *p;
> +
> + /*
> + * Allocate per CPU pages while encrypted DMA is not happening yet
> + * and smashing is cheap.
> + */
> + for_each_possible_cpu(cpu) {
> + if (per_cpu(iommu_tlb_flush_ghcb_page, cpu))
> + continue;
> +
> + pg = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL, 0);
> + if (!pg)
> + return -ENOMEM;
> +
> + p = page_to_virt(pg);
> + /* Trigger psmash in the host os now to avoid psmash race
> later */
> + snp_set_memory_shared((unsigned long)p, 1);
> + snp_set_memory_private((unsigned long)p, 1);
> + per_cpu(iommu_tlb_flush_ghcb_page, cpu) = p;
> + }
> +
> + return 0;
> +}
>
> int sev_tio_op(u32 guest_rid, unsigned int op, u64 *fw_err, u64 *tdi_id)
> {
> @@ -111,6 +142,24 @@ int sev_tio_op(u32 guest_rid, unsigned int op, u64
> *fw_err, u64 *tdi_id)
> struct ghcb *ghcb;
> int ret;
>
> + if (!(sev_hv_features & GHCB_HV_FT_SNP_SEV_TIO))
> + return -EPERM;
> +
> + if (op == SVM_VMGEXIT_SEV_TIO_OP_RUN || op ==
> SVM_VMGEXIT_SEV_TIO_OP_STOP) {
> + if (!(sev_hv_features & GHCB_HV_FT_SNP_IOMMU_TLB_FLUSH))
> + return -EPERM;
> +
> + if (op == SVM_VMGEXIT_SEV_TIO_OP_RUN) {
> + if (atomic_inc_return(&sev_tio_devices_num) == 1) {
> + ret = alloc_iommu_tlb_flush_ghcb_pages();
> + if (ret)
> + return ret;
> + }
> + } else if (atomic_dec_return(&sev_tio_devices_num) == 0) {
> + /* Do cleanup or leave it like this? */
> + }
> + }
> +
> /* __sev_get_ghcb() needs IRQs disabled because it uses per-CPU GHCB.
> */
> guard(irqsave)();
>
> @@ -347,6 +396,42 @@ static int vmgexit_psc(struct ghcb *ghcb, struct
> snp_psc_desc *desc)
> return ret;
> }
>
> +static int ghcb_flush_iommu_tlb(struct ghcb *ghcb)
> +{
> + /* AES encrypts with 16 byte blocks */
> + unsigned long s1[BITS_TO_LONGS(128)], s2[BITS_TO_LONGS(128)];
> + void *p = this_cpu_read(iommu_tlb_flush_ghcb_page), *p2;
> + struct es_em_ctxt ctxt;
> + int ret;
> +
> + if (!p)
> + return -ENOMEM;
> +
> + /* Keep patterns apart far enough to not share the same cache line */
> + p2 = (u8 *) p + 2048;
> +
> + vc_ghcb_invalidate(ghcb);
> +
> + BUILD_BUG_ON(ARRAY_SIZE(s1) != 2);
> + if (!rdrand_long(s1) || !rdrand_long(s1 + 1) ||
> + !rdrand_long(s2) || !rdrand_long(s2 + 1))
> + return -EFAULT;
> +
> + memcpy(p, s1, sizeof(s1));
> + memcpy(p2, s2, sizeof(s2));
> +
> + pvalidate((unsigned long) p, RMP_PG_SIZE_4K, false);
> + ret = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_IOMMU_TLB_FLUSH,
> __pa(p), 0);
> + pvalidate((unsigned long) p, RMP_PG_SIZE_4K, true);
> +
> + /* Ensure that the host change is visible */
> + smp_mb();
> +
> + if (!memcmp(p, s1, sizeof(s1)) || memcmp(p2, s2, sizeof(s2)))
> + return -EFAULT;
> +
> + return 0;
> +}
> static unsigned long __set_pages_state(struct snp_psc_desc *data, unsigned
> long vaddr,
> unsigned long vaddr_end, int op)
> {
> @@ -404,6 +489,13 @@ static unsigned long __set_pages_state(struct
> snp_psc_desc *data, unsigned long
> if (!ghcb || vmgexit_psc(ghcb, data))
> sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
>
> + if (atomic_read(&sev_tio_devices_num)) {
> + int ret = ghcb_flush_iommu_tlb(ghcb);
> +
> + if (ret)
> + sev_es_terminate(SEV_TERM_SET_LINUX,
> GHCB_TERM_IOMMUTLB_FLUSH);
> + }
> +
> __sev_put_ghcb(&state);
>
> local_irq_restore(flags);
> --
> 2.55.0
>
>
--
Jianxiong Gao
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |