[PATCH v1 17/28] x86/hvm: avic: route interrupt delivery to vCPU

[dmukhin@xxxxxxxx]

From: Denis Mukhin <dmukhin@xxxxxxxx>

Enabling AVIC implicitly disables the V_IRQ, V_INTR_PRIO, V_IGN_TPR,
and V_INTR_VECTOR fields in the VMCB Control Word.

Introduce new interrupt injection code via AVIC backing page.

Essentially, that is handling of 'IsRunning' bit in physical APIC ID
table entry: it needs to be in sync with the scheduled vCPU. Shadow
the state to work around hardware not perceiving 'IsRunning' bit.

Add new doorbell MSR to accelerate IPI emulation. The MSR can be used
for posting interrupts from hypervisor to the guest (e.g. event
channels).

Signed-off-by: Denis Mukhin <dmukhin@xxxxxxxx>
---
 xen/arch/x86/hvm/svm/avic.c              | 156 +++++++++++++++++++++++
 xen/arch/x86/hvm/svm/avic.h              |  17 +++
 xen/arch/x86/hvm/svm/intr.c              |   9 ++
 xen/arch/x86/hvm/svm/svm.c               |  32 +++++
 xen/arch/x86/include/asm/hvm/svm-types.h |   2 +
 xen/arch/x86/include/asm/msr-index.h     |   1 +
 6 files changed, 217 insertions(+)

diff --git a/xen/arch/x86/hvm/svm/avic.c b/xen/arch/x86/hvm/svm/avic.c
index 18a814871133..0bcab0465227 100644
--- a/xen/arch/x86/hvm/svm/avic.c
+++ b/xen/arch/x86/hvm/svm/avic.c
@@ -33,6 +33,8 @@
 #define AVIC_MAX_PHYSICAL_ID                            (0xFEU)
 
 bool __read_mostly svm_avic_enable;
+unsigned int __read_mostly svm_avic_features = AVIC_HW_ISRUNNING |
+                                               AVIC_DOORBELL_MSR;
 
 static const char __section(".bss.page_aligned.const") __aligned(PAGE_SIZE)
     avic_apic_access_page[PAGE_SIZE];
@@ -50,6 +52,11 @@ static int cf_check avic_parse_cmdline(const char *s)
     {
         svm_avic_enable = true;
     }
+    else if ( !strcmp(s, "nodoorbell") )
+    {
+        svm_avic_enable = true;
+        svm_avic_features &= ~AVIC_DOORBELL_MSR;
+    }
     else
     {
         return -EINVAL;
@@ -228,6 +235,7 @@ int svm_avic_vmcb_create(struct vcpu *v)
     entry |= AVIC_PHYSICAL_ID_ENTRY_VALID_MASK;
 
     ACCESS_ONCE(*sv->avic_physical_id) = entry;
+    ACCESS_ONCE(sv->avic_shadow_id) = entry;
 
     avic_vcpu_enable(v, true);
 
@@ -248,6 +256,102 @@ void svm_avic_vmcb_destroy(struct vcpu *v)
     vmcb->avic_vapic_bar = 0;
 }
 
+static void avic_vcpu_set_cpu(struct vcpu *v)
+{
+    struct svm_vcpu *sv = &v->arch.hvm.svm;
+    unsigned apic_id = cpu_data[v->processor].apicid;
+    uint64_t entry;
+
+    BUG_ON(sv->avic_physical_id == NULL);
+    ASSERT(!(apic_id & ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK));
+
+    entry = ACCESS_ONCE(sv->avic_shadow_id);
+    entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK;
+    entry |= apic_id;
+
+    ACCESS_ONCE(*sv->avic_physical_id) = entry;
+
+    ACCESS_ONCE(sv->avic_shadow_id) = entry;
+}
+
+static void avic_vcpu_set_running(struct vcpu *v)
+{
+    struct svm_vcpu *sv = &v->arch.hvm.svm;
+    uint64_t entry;
+
+    BUG_ON(sv->avic_physical_id == NULL);
+
+    entry = ACCESS_ONCE(sv->avic_shadow_id);
+    entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
+
+    /* Shadow IsRunning state update. */
+    ACCESS_ONCE(sv->avic_shadow_id) = entry;
+
+    /*
+     * If hardware has problems detecting IsRunning (erratum 1235)
+     * keep IsRunning in the shadow entry only.
+     */
+    if ( svm_avic_features & AVIC_HW_ISRUNNING )
+        ACCESS_ONCE(*sv->avic_physical_id) = entry;
+}
+
+static void avic_vcpu_clear_running(struct vcpu *v)
+{
+    struct svm_vcpu *sv = &v->arch.hvm.svm;
+    unsigned apic_id = cpu_data[v->processor].apicid;
+    uint64_t entry;
+
+    BUG_ON(sv->avic_physical_id == NULL);
+    ASSERT(!(apic_id & ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK));
+
+    entry = ACCESS_ONCE(sv->avic_shadow_id);
+    entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
+
+    /*
+     * If hardware has problems detecting IsRunning (erratum 1235)
+     * keep IsRunning in the shadow entry only.
+     */
+    if ( svm_avic_features & AVIC_HW_ISRUNNING )
+        ACCESS_ONCE(*sv->avic_physical_id) = entry;
+
+    /* Shadow IsRunning state update. */
+    ACCESS_ONCE(sv->avic_shadow_id) = entry;
+}
+
+/* vCPU enters blocking state. */
+void svm_avic_vcpu_block(struct vcpu *v)
+{
+    avic_vcpu_clear_running(v);
+}
+
+/* vCPU is preparing to resume guest execution. */
+void svm_avic_vcpu_resume(struct vcpu *v)
+{
+    avic_vcpu_set_running(v);
+}
+
+/* vCPU is being assigned a new pCPU. */
+void svm_avic_vcpu_switch_to(struct vcpu *v)
+{
+    avic_vcpu_set_cpu(v);
+}
+
+/* vCPU is being removed from a pCPU. */
+void svm_avic_vcpu_switch_from(struct vcpu *v)
+{
+    avic_vcpu_clear_running(v);
+}
+
+bool svm_avic_vcpu_enabled(const struct vcpu *v)
+{
+    const struct svm_vcpu *sv = &v->arch.hvm.svm;
+    const struct vmcb_struct *vmcb = sv->vmcb;
+
+    ASSERT(!(vmcb->_vintr.fields.x2avic_enable));
+
+    return vmcb->_vintr.fields.avic_enable;
+}
+
 /*
  * Handle the AVIC_INCOMP_IPI #VMEXIT when AVIC is enabled.
  *
@@ -546,6 +650,58 @@ void svm_avic_vmexit_noaccel(struct cpu_user_regs *regs)
         hvm_emulate_one_vm_event(EMUL_KIND_NORMAL, X86_EXC_UD, 
X86_EVENT_NO_EC);
 }
 
+static void avic_deliver_posted_intr(struct vcpu *v)
+{
+    struct svm_vcpu *sv = &v->arch.hvm.svm;
+    uint64_t entry;
+    unsigned int apicid;
+
+    smp_mb();
+
+    entry = ACCESS_ONCE(sv->avic_shadow_id);
+
+    if ( !(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK) )
+        return;
+
+    /*
+     * Some AMD systems (e.g. Zen5 Family 26 (0x1a), Model 36 (0x24)) have
+     * broken remote notification via doorbell MSR.
+     * Fallback to brute force, if requested.
+     */
+    if ( svm_avic_features & AVIC_DOORBELL_MSR )
+    {
+        apicid = entry & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK;
+
+        /* Ensure the pending interrupt wakes vCPU. */
+        vcpu_unblock(v);
+
+        /*
+         * AMD APM, 5.29.8.2 Doorbell Mechanism
+         * The serializing semantics of WRMSR are relaxed when writing to
+         * the Doorbell Register.
+         */
+        wrmsrl(MSR_AMD_AVIC_DOORBELL, apicid);
+    }
+    else
+    {
+        vcpu_kick(v);
+    }
+}
+
+void cf_check svm_avic_hvm_deliver_posted_intr(struct vcpu *v, uint8_t vector)
+{
+    ASSERT(v != current);
+
+    if ( svm_avic_vcpu_enabled(v) )
+    {
+        avic_deliver_posted_intr(v);
+    }
+    else
+    {
+        vcpu_kick(v);
+    }
+}
+
 /*
  * Local variables:
  * mode: C
diff --git a/xen/arch/x86/hvm/svm/avic.h b/xen/arch/x86/hvm/svm/avic.h
index f9ff3ffc9346..098924925e89 100644
--- a/xen/arch/x86/hvm/svm/avic.h
+++ b/xen/arch/x86/hvm/svm/avic.h
@@ -6,6 +6,15 @@
 #ifndef __SVM_AVIC_H__
 #define __SVM_AVIC_H__
 
+#include <xen/const.h>
+
+/* Known AVIC hardware quirks. */
+enum {
+    AVIC_HW_ISRUNNING       = BIT(0, U),    /* See erratum 1235 */
+    AVIC_DOORBELL_MSR       = BIT(1, U),    /* Remote vCPU notification OK */
+};
+
+extern unsigned int svm_avic_features;
 extern bool svm_avic_enable;
 
 int svm_avic_domain_initialise(struct domain *d);
@@ -17,4 +26,12 @@ void svm_avic_vmcb_destroy(struct vcpu *v);
 void svm_avic_vmexit_incomp_ipi(struct cpu_user_regs *regs);
 void svm_avic_vmexit_noaccel(struct cpu_user_regs *regs);
 
+bool svm_avic_vcpu_enabled(const struct vcpu *v);
+void svm_avic_vcpu_block(struct vcpu *v);
+void svm_avic_vcpu_resume(struct vcpu *v);
+void svm_avic_vcpu_switch_to(struct vcpu *v);
+void svm_avic_vcpu_switch_from(struct vcpu *v);
+
+void cf_check svm_avic_hvm_deliver_posted_intr(struct vcpu *v, uint8_t vector);
+
 #endif /* __SVM_AVIC_H__ */
diff --git a/xen/arch/x86/hvm/svm/intr.c b/xen/arch/x86/hvm/svm/intr.c
index 1d39f25e5209..9cceca704a91 100644
--- a/xen/arch/x86/hvm/svm/intr.c
+++ b/xen/arch/x86/hvm/svm/intr.c
@@ -24,6 +24,7 @@
 #include <public/hvm/ioreq.h>
 #include <xen/domain_page.h>
 
+#include "avic.h"
 #include "nestedhvm.h"
 #include "vmcb.h"
 
@@ -80,6 +81,14 @@ static void svm_enable_intr_window(struct vcpu *v, struct 
hvm_intack intack)
 
     ASSERT(intack.source != hvm_intsrc_none);
 
+    if ( svm_avic_vcpu_enabled(v) )
+        /*
+         * AVIC delivers pending LAPIC interrupts from the backing page
+         * when the guest becomes interruptible, no need to arm the legacy
+         * V_IRQ/VINTR interrupt-window mechanism.
+         */
+        return;
+
     if ( nestedhvm_enabled(v->domain) )
     {
         struct nestedvcpu *nv = &vcpu_nestedhvm(v);
diff --git a/xen/arch/x86/hvm/svm/svm.c b/xen/arch/x86/hvm/svm/svm.c
index d1a95e441f03..2fac76725c4f 100644
--- a/xen/arch/x86/hvm/svm/svm.c
+++ b/xen/arch/x86/hvm/svm/svm.c
@@ -35,6 +35,7 @@
 #include <asm/p2m.h>
 #include <asm/paging.h>
 #include <asm/processor.h>
+#include <asm/pv/shim.h>
 #include <asm/vm_event.h>
 #include <asm/x86_emulate.h>
 
@@ -877,6 +878,10 @@ static void cf_check svm_ctxt_switch_from(struct vcpu *v)
     svm_tsc_ratio_save(v);
 
     svm_sync_vmcb(v, vmcb_needs_vmload);
+
+    if ( svm_avic_vcpu_enabled(v) )
+        svm_avic_vcpu_switch_from(v);
+
     svm_vmload_pa(per_cpu(host_vmcb, cpu));
 
     /* Resume use of ISTs now that the host TR is reinstated. */
@@ -914,6 +919,9 @@ static void cf_check svm_ctxt_switch_to(struct vcpu *v)
 
     svm_tsc_ratio_load(v);
 
+    if ( svm_avic_vcpu_enabled(v) )
+        svm_avic_vcpu_switch_to(v);
+
     if ( cpu_has_msr_tsc_aux )
         wrmsr_tsc_aux(v->arch.msrs->tsc_aux);
 
@@ -935,6 +943,9 @@ static bool vcpu_is_vlapic_tpr_emulated(const struct vcpu 
*v, bool force_noaccel
     if ( vlapic_hw_disabled(vlapic) )
         return false;
 
+    if ( svm_avic_vcpu_enabled(v) )
+        return false;
+
     return true;
 }
 
@@ -982,6 +993,9 @@ static void noreturn cf_check svm_do_resume(void)
             (vlapic_get_reg(vlapic, APIC_TASKPRI) & 0xFF) >> 4;
         vmcb_set_vintr(vmcb, intr);
     }
+    else if ( svm_avic_vcpu_enabled(v) )
+        /* AVIC accelerates TPR accesses. */
+        svm_avic_vcpu_resume(v);
 
     hvm_do_resume(v);
 
@@ -1091,6 +1105,12 @@ static int cf_check acpi_c1e_quirk(
     return X86EMUL_OKAY;
 }
 
+static void cf_check svm_vcpu_block(struct vcpu *v)
+{
+    if ( svm_avic_vcpu_enabled(v) )
+        svm_avic_vcpu_block(v);
+}
+
 static int cf_check svm_domain_initialise(struct domain *d)
 {
     static const struct arch_csw csw = {
@@ -1104,6 +1124,7 @@ static int cf_check svm_domain_initialise(struct domain 
*d)
     if ( rc )
         return rc;
 
+    d->arch.hvm.pi_ops.vcpu_block = svm_vcpu_block;
     d->arch.ctxt_switch = &csw;
 
     svm_guest_osvw_init(d);
@@ -2493,6 +2514,13 @@ static void __init svm_avic_start(void)
 {
     if ( !cpu_has_svm_avic || pv_shim )
         svm_avic_enable = false;
+
+    if ( !svm_avic_enable )
+        return;
+
+    svm_function_table.deliver_posted_intr = svm_avic_hvm_deliver_posted_intr;
+
+    svm_function_table.caps.virtual_intr_delivery = true;
 }
 
 const struct hvm_function_table * __init start_svm(void)
@@ -2814,6 +2842,10 @@ void asmlinkage svm_vmexit_handler(void)
 
     case VMEXIT_VINTR:
         svm_clear_intr_window(v);
+
+        if ( svm_avic_vcpu_enabled(v) )
+            domain_crash(v->domain, "%pv: VMEXIT_VINTR=%#"PRIx64"\n", v, 
intr.bytes);
+
         break;
 
     case VMEXIT_INVD:
diff --git a/xen/arch/x86/include/asm/hvm/svm-types.h 
b/xen/arch/x86/include/asm/hvm/svm-types.h
index 377ad2494e7c..cc40b1b02095 100644
--- a/xen/arch/x86/include/asm/hvm/svm-types.h
+++ b/xen/arch/x86/include/asm/hvm/svm-types.h
@@ -51,6 +51,8 @@ struct svm_vcpu {
      */
     uint32_t avic_ldr;
     uint32_t avic_dfr;
+    uint64_t *avic_physical_id;
+    uint64_t avic_shadow_id;
 };
 
 struct nestedsvm {
diff --git a/xen/arch/x86/include/asm/msr-index.h 
b/xen/arch/x86/include/asm/msr-index.h
index ad1c6c97f8f7..84fd9b6581b6 100644
--- a/xen/arch/x86/include/asm/msr-index.h
+++ b/xen/arch/x86/include/asm/msr-index.h
@@ -406,6 +406,7 @@
 #define MSR_K8_PSTATE7                 0xc001006BU
 #define MSR_K8_ENABLE_C1E              0xc0010055U
 #define MSR_K8_VM_HSAVE_PA             0xc0010117U
+#define MSR_AMD_AVIC_DOORBELL          0xc001011bU
 
 #define MSR_AMD_FAM15H_EVNTSEL0                0xc0010200U
 #define MSR_AMD_FAM15H_PERFCTR0                0xc0010201U
-- 
2.55.0