[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[win-pv-devel] [PATCH v2 4/4] Use per-CPU event channel upcalls if available



A recent patch to Xen introduced a new HVM op to set a per-vcpu event
channel upcall. This patch adds code to make use of the latched interrupts
allocated by the FDO code to enable per-vcpu upcalls and adds an extra
EvtchnBind operation to a new v2 EVTCHN interface so that events can be
steered to a specified CPU.

Signed-off-by: Paul Durrant <paul.durrant@xxxxxxxxxx>
---
 include/evtchn_interface.h |  35 ++++++-
 include/xen.h              |  16 +++
 src/xen/event_channel.c    |  30 ++++++
 src/xen/hvm.c              |  28 ++++++
 src/xenbus/evtchn.c        | 239 ++++++++++++++++++++++++++++++++++++++++++---
 src/xenbus/store.c         |   9 ++
 6 files changed, 341 insertions(+), 16 deletions(-)

diff --git a/include/evtchn_interface.h b/include/evtchn_interface.h
index 5898e08..4ab2b28 100644
--- a/include/evtchn_interface.h
+++ b/include/evtchn_interface.h
@@ -112,6 +112,20 @@ typedef PXENBUS_EVTCHN_CHANNEL
     ...
     );
 
+/*! \typedef XENBUS_EVTCHN_BIND
+    \brief Bind an event channel to a specific CPU
+
+    \param Interface The interface header
+    \param Channel The channel handle
+    \param Cpu The CPU that should handle events
+*/
+typedef NTSTATUS
+(*XENBUS_EVTCHN_BIND)(
+    IN  PINTERFACE              Interface,
+    IN  PXENBUS_EVTCHN_CHANNEL  Channel,
+    IN  ULONG                   Cpu
+    );
+
 /*! \typedef XENBUS_EVTCHN_UNMASK
     \brief Unmask an event channel
 
@@ -196,7 +210,24 @@ struct _XENBUS_EVTCHN_INTERFACE_V1 {
     XENBUS_EVTCHN_CLOSE     EvtchnClose;
 };
 
-typedef struct _XENBUS_EVTCHN_INTERFACE_V1 XENBUS_EVTCHN_INTERFACE, 
*PXENBUS_EVTCHN_INTERFACE;
+/*! \struct _XENBUS_EVTCHN_INTERFACE_V2
+    \brief EVTCHN interface version 2
+    \ingroup interfaces
+*/
+struct _XENBUS_EVTCHN_INTERFACE_V2 {
+    INTERFACE               Interface;
+    XENBUS_EVTCHN_ACQUIRE   EvtchnAcquire;
+    XENBUS_EVTCHN_RELEASE   EvtchnRelease;
+    XENBUS_EVTCHN_OPEN      EvtchnOpen;
+    XENBUS_EVTCHN_BIND      EvtchnBind;
+    XENBUS_EVTCHN_UNMASK    EvtchnUnmask;
+    XENBUS_EVTCHN_SEND      EvtchnSend;
+    XENBUS_EVTCHN_TRIGGER   EvtchnTrigger;
+    XENBUS_EVTCHN_GET_PORT  EvtchnGetPort;
+    XENBUS_EVTCHN_CLOSE     EvtchnClose;
+};
+
+typedef struct _XENBUS_EVTCHN_INTERFACE_V2 XENBUS_EVTCHN_INTERFACE, 
*PXENBUS_EVTCHN_INTERFACE;
 
 /*! \def XENBUS_EVTCHN
     \brief Macro at assist in method invocation
@@ -207,7 +238,7 @@ typedef struct _XENBUS_EVTCHN_INTERFACE_V1 
XENBUS_EVTCHN_INTERFACE, *PXENBUS_EVT
 #endif  // _WINDLL
 
 #define XENBUS_EVTCHN_INTERFACE_VERSION_MIN 1
-#define XENBUS_EVTCHN_INTERFACE_VERSION_MAX 1
+#define XENBUS_EVTCHN_INTERFACE_VERSION_MAX 2
 
 #endif  // _XENBUS_EVTCHN_INTERFACE_H
 
diff --git a/include/xen.h b/include/xen.h
index 0dabc74..84197a5 100644
--- a/include/xen.h
+++ b/include/xen.h
@@ -97,6 +97,14 @@ HvmPagetableDying(
     IN  PHYSICAL_ADDRESS    Address
     );
 
+__checkReturn
+XEN_API
+NTSTATUS
+HvmSetEvtchnUpcallVector(
+    IN  unsigned int    vcpu_id,
+    IN  UCHAR           Vector
+    );
+
 // MEMORY
 
 __checkReturn
@@ -196,6 +204,14 @@ EventChannelReset(
     VOID
     );
 
+__checkReturn
+XEN_API
+NTSTATUS
+EventChannelBindVirtualCpu(
+    IN  ULONG               LocalPort,
+    IN  unsigned int        vcpu_id
+    );
+
 // GRANT TABLE
 
 __checkReturn
diff --git a/src/xen/event_channel.c b/src/xen/event_channel.c
index aa87fd4..94a6b82 100644
--- a/src/xen/event_channel.c
+++ b/src/xen/event_channel.c
@@ -327,3 +327,33 @@ fail1:
 
     return status;
 }
+
+__checkReturn
+XEN_API
+NTSTATUS
+EventChannelBindVirtualCpu(
+    IN  ULONG               LocalPort,
+    IN  unsigned int        vcpu_id
+    )
+{
+    struct evtchn_bind_vcpu op;
+    LONG_PTR                rc;
+    NTSTATUS                status;
+
+    op.port = LocalPort;
+    op.vcpu = vcpu_id;
+
+    rc = EventChannelOp(EVTCHNOP_bind_vcpu, &op);
+
+    if (rc < 0) {
+        ERRNO_TO_STATUS(-rc, status);
+        goto fail1;
+    }
+
+    return STATUS_SUCCESS;
+
+fail1:
+    Error("fail1 (%08x)\n", status);
+
+    return status;
+}
diff --git a/src/xen/hvm.c b/src/xen/hvm.c
index 8135e97..3e3e12c 100644
--- a/src/xen/hvm.c
+++ b/src/xen/hvm.c
@@ -166,3 +166,31 @@ HvmPagetableDying(
 fail1:
     return status;
 }
+
+__checkReturn
+XEN_API
+NTSTATUS
+HvmSetEvtchnUpcallVector(
+    IN  unsigned int                        vcpu_id,
+    IN  UCHAR                               Vector
+    )
+{
+    struct xen_hvm_set_evtchn_upcall_vector op;
+    LONG_PTR                                rc;
+    NTSTATUS                                status;
+
+    op.vcpu = vcpu_id;
+    op.vector = Vector;
+
+    rc = HvmOp(HVMOP_set_evtchn_upcall_vector, &op);
+
+    if (rc < 0) {
+        ERRNO_TO_STATUS(-rc, status);
+        goto fail1;
+    }
+
+    return STATUS_SUCCESS;
+
+fail1:
+    return status;
+}
diff --git a/src/xenbus/evtchn.c b/src/xenbus/evtchn.c
index 44043d4..aa26a31 100644
--- a/src/xenbus/evtchn.c
+++ b/src/xenbus/evtchn.c
@@ -82,13 +82,16 @@ struct _XENBUS_EVTCHN_CHANNEL {
     XENBUS_EVTCHN_PARAMETERS    Parameters;
     BOOLEAN                     Mask;
     ULONG                       LocalPort;
+    PXENBUS_INTERRUPT           Interrupt;
 };
 
 struct _XENBUS_EVTCHN_CONTEXT {
     PXENBUS_FDO                     Fdo;
     KSPIN_LOCK                      Lock;
     LONG                            References;
-    PXENBUS_INTERRUPT               Interrupt;
+    PXENBUS_INTERRUPT               LevelSensitiveInterrupt;
+    PXENBUS_INTERRUPT               LatchedInterrupt[MAXIMUM_PROCESSORS];
+    KAFFINITY                       Affinity;
     BOOLEAN                         Enabled;
     XENBUS_SUSPEND_INTERFACE        SuspendInterface;
     PXENBUS_SUSPEND_CALLBACK        SuspendCallbackEarly;
@@ -127,15 +130,39 @@ EvtchnInterruptEnable(
     IN  PXENBUS_EVTCHN_CONTEXT  Context
     )
 {
+    LONG                        Cpu;
     ULONG                       Line;
     NTSTATUS                    status;
 
-    Trace("<===>\n");
+    Trace("====>\n");
+
+    ASSERT3U(Context->Affinity, ==, 0);
+
+    Cpu = 0;
+    while (Cpu < KeNumberProcessors) {
+        unsigned int    vcpu_id;
+        UCHAR           Vector;
+
+        vcpu_id = SystemVirtualCpuIndex(Cpu);
+        Vector = FdoGetInterruptVector(Context->Fdo,
+                                       Context->LatchedInterrupt[Cpu]);
+
+        status = HvmSetEvtchnUpcallVector(vcpu_id, Vector);
+        if (NT_SUCCESS(status)) {
+            Info("CPU %u\n", Cpu);
+            Context->Affinity |= (KAFFINITY)1 << Cpu;
+        }
+
+        Cpu++;
+    }
 
-    Line = FdoGetInterruptLine(Context->Fdo, Context->Interrupt);
+    Line = FdoGetInterruptLine(Context->Fdo,
+                               Context->LevelSensitiveInterrupt);
 
     status = HvmSetParam(HVM_PARAM_CALLBACK_IRQ, Line);
     ASSERT(NT_SUCCESS(status));
+
+    Trace("<====\n");
 }
 
 static VOID
@@ -143,14 +170,29 @@ EvtchnInterruptDisable(
     IN  PXENBUS_EVTCHN_CONTEXT  Context
     )
 {
+    LONG                        Cpu;
     NTSTATUS                    status;
 
     UNREFERENCED_PARAMETER(Context);
 
-    Trace("<===>\n");
+    Trace("====>\n");
 
     status = HvmSetParam(HVM_PARAM_CALLBACK_IRQ, 0);
     ASSERT(NT_SUCCESS(status));
+
+    Cpu = KeNumberProcessors;
+    while (--Cpu >= 0) {
+        unsigned int    vcpu_id;
+
+        vcpu_id = SystemVirtualCpuIndex(Cpu);
+
+        (VOID) HvmSetEvtchnUpcallVector(vcpu_id, 0);
+        Context->Affinity &= ~((KAFFINITY)1 << Cpu);
+    }
+
+    ASSERT3U(Context->Affinity, ==, 0);
+
+    Trace("<====\n");
 }
 
 static FORCEINLINE
@@ -159,10 +201,11 @@ _IRQL_saves_
 _IRQL_raises_(HIGH_LEVEL)
 KIRQL
 __EvtchnAcquireInterruptLock(
-    IN  PXENBUS_EVTCHN_CONTEXT  Context
+    IN  PXENBUS_EVTCHN_CONTEXT  Context,
+    IN  PXENBUS_EVTCHN_CHANNEL  Channel
     )
 {
-    return FdoAcquireInterruptLock(Context->Fdo, Context->Interrupt);
+    return FdoAcquireInterruptLock(Context->Fdo, Channel->Interrupt);
 }
 
 static FORCEINLINE
@@ -170,10 +213,11 @@ __drv_requiresIRQL(HIGH_LEVEL)
 VOID
 __EvtchnReleaseInterruptLock(
     IN  PXENBUS_EVTCHN_CONTEXT      Context,
+    IN  PXENBUS_EVTCHN_CHANNEL      Channel,
     IN  __drv_restoresIRQL KIRQL    Irql
     )
 {
-    FdoReleaseInterruptLock(Context->Fdo, Context->Interrupt, Irql);
+    FdoReleaseInterruptLock(Context->Fdo, Channel->Interrupt, Irql);
 }
 
 static NTSTATUS
@@ -358,6 +402,10 @@ EvtchnOpen(
 
     LocalPort = Channel->LocalPort;
 
+    Channel->Interrupt = (Context->Affinity != 0) ? // Latched available
+                         Context->LatchedInterrupt[0] :
+                         Context->LevelSensitiveInterrupt;
+
     status = XENBUS_EVTCHN_ABI(PortEnable,
                                &Context->EvtchnAbi,
                                LocalPort);
@@ -422,6 +470,89 @@ fail1:
     return NULL;
 }
 
+#define EVTCHN_SWAP_POINTER(_X, _Y)                             \
+        do {                                                    \
+            (_X) = (PVOID)((ULONG_PTR)(_X) ^ (ULONG_PTR)(_Y));  \
+            (_Y) = (PVOID)((ULONG_PTR)(_X) ^ (ULONG_PTR)(_Y));  \
+            (_X) = (PVOID)((ULONG_PTR)(_X) ^ (ULONG_PTR)(_Y));  \
+        } while (FALSE)
+
+static NTSTATUS
+EvtchnBind(
+    IN  PINTERFACE              Interface,
+    IN  PXENBUS_EVTCHN_CHANNEL  Channel,
+    IN  ULONG                   Cpu
+    )
+{
+    PXENBUS_EVTCHN_CONTEXT      Context = Interface->Context;
+    PXENBUS_INTERRUPT           Interrupt;
+    ULONG                       LocalPort;
+    unsigned int                vcpu_id;
+    KIRQL                       Irql;
+    NTSTATUS                    status;
+
+    status = STATUS_INVALID_PARAMETER;
+    if (Cpu >= (ULONG)KeNumberProcessors)
+        goto fail1;
+
+    ASSERT(Context->Enabled);
+
+    status = STATUS_NOT_SUPPORTED;
+    if (~Context->Affinity & ((KAFFINITY)1 << Cpu))
+        goto fail2;
+
+    Interrupt = Context->LatchedInterrupt[Cpu];
+
+    if (Channel->Interrupt == Interrupt)
+        goto done;
+
+    KeRaiseIrql(HIGH_LEVEL, &Irql);
+
+    // Make sure we always lock in a consistent order
+    if ((ULONG_PTR)Interrupt < (ULONG_PTR)Channel->Interrupt) {
+        (VOID) FdoAcquireInterruptLock(Context->Fdo, Interrupt);
+        (VOID) FdoAcquireInterruptLock(Context->Fdo, Channel->Interrupt);
+    } else {
+        (VOID) FdoAcquireInterruptLock(Context->Fdo, Channel->Interrupt);
+        (VOID) FdoAcquireInterruptLock(Context->Fdo, Interrupt);
+    }
+
+    LocalPort = Channel->LocalPort;
+    vcpu_id = SystemVirtualCpuIndex(Cpu);
+
+    status = EventChannelBindVirtualCpu(LocalPort, vcpu_id);
+    if (!NT_SUCCESS(status))
+        goto fail3;
+
+    EVTCHN_SWAP_POINTER(Channel->Interrupt, Interrupt);
+
+    FdoReleaseInterruptLock(Context->Fdo, Channel->Interrupt, HIGH_LEVEL);
+    FdoReleaseInterruptLock(Context->Fdo, Interrupt, HIGH_LEVEL);
+
+    KeLowerIrql(Irql);
+
+    Info("[%u]: CPU %u\n", LocalPort, Cpu);
+
+done:
+    return STATUS_SUCCESS;
+
+fail3:
+    Error("fail3\n");
+
+    FdoReleaseInterruptLock(Context->Fdo, Channel->Interrupt, HIGH_LEVEL);
+    FdoReleaseInterruptLock(Context->Fdo, Interrupt, HIGH_LEVEL);
+
+    KeLowerIrql(Irql);
+
+fail2:
+    Error("fail2\n");
+
+fail1:
+    Error("fail1 (%08x)\n", status);
+
+    return status;
+}
+
 static BOOLEAN
 EvtchnUnmask(
     IN  PINTERFACE              Interface,
@@ -516,7 +647,7 @@ EvtchnTrigger(
 
     ASSERT3U(Channel->Magic, ==, XENBUS_EVTCHN_CHANNEL_MAGIC);
 
-    Irql = __EvtchnAcquireInterruptLock(Context);
+    Irql = __EvtchnAcquireInterruptLock(Context, Channel);
 
     ASSERT3U(KeGetCurrentIrql(), >=, DISPATCH_LEVEL);
 
@@ -527,7 +658,7 @@ EvtchnTrigger(
         DoneSomething = FALSE;
     }
 
-    __EvtchnReleaseInterruptLock(Context, Irql);
+    __EvtchnReleaseInterruptLock(Context, Channel, Irql);
 
     return DoneSomething;
 }
@@ -575,6 +706,8 @@ EvtchnClose(
         ASSERT(NT_SUCCESS(status));
     }
 
+    Channel->Interrupt = NULL;
+
     Channel->LocalPort = 0;
     RtlZeroMemory(&Channel->Parameters, sizeof (XENBUS_EVTCHN_PARAMETERS));
 
@@ -617,6 +750,8 @@ EvtchnPollCallback(
     BOOLEAN                 DoneSomething;
     NTSTATUS                status;
 
+    DoneSomething = FALSE;
+
     status = HashTableLookup(Context->Table,
                              LocalPort,
                              (PULONG_PTR)&Channel);
@@ -628,10 +763,20 @@ EvtchnPollCallback(
                           &Context->EvtchnAbi,
                           LocalPort);
 
-        DoneSomething = FALSE;
         goto done;
     }
 
+    if (Context->Affinity != 0) {
+        ULONG   Cpu;
+
+        ASSERT3U(KeGetCurrentIrql(), >=, DISPATCH_LEVEL);
+        Cpu = KeGetCurrentProcessorNumber();
+
+        // Only handle events on the correct CPU
+        if (Channel->Interrupt != Context->LatchedInterrupt[Cpu])
+            goto done;
+    }
+
     if (Channel->Mask)
         XENBUS_EVTCHN_ABI(PortMask,
                           &Context->EvtchnAbi,
@@ -767,8 +912,10 @@ EvtchnSuspendCallbackLate(
     status = EvtchnAbiAcquire(Context);
     ASSERT(NT_SUCCESS(status));
 
-    if (Context->Enabled)
+    if (Context->Enabled) {
+        EvtchnInterruptDisable(Context);
         EvtchnInterruptEnable(Context);
+    }
 }
 
 static VOID
@@ -862,6 +1009,7 @@ EvtchnAcquire(
     PXENBUS_EVTCHN_CONTEXT  Context = Interface->Context;
     PXENBUS_FDO             Fdo = Context->Fdo;
     KIRQL                   Irql;
+    LONG                    Cpu;
     NTSTATUS                status;
 
     KeAcquireSpinLock(&Context->Lock, &Irql);
@@ -919,10 +1067,24 @@ EvtchnAcquire(
                                   0,
                                   EvtchnInterruptCallback,
                                   Context,
-                                  &Context->Interrupt);
+                                  &Context->LevelSensitiveInterrupt);
     if (!NT_SUCCESS(status))
         goto fail8;
 
+    Cpu = 0;
+    while (Cpu < KeNumberProcessors) {
+        status = FdoAllocateInterrupt(Fdo,
+                                      Latched,
+                                      Cpu,
+                                      EvtchnInterruptCallback,
+                                      Context,
+                                      &Context->LatchedInterrupt[Cpu]);
+        if (!NT_SUCCESS(status))
+            goto fail9;
+
+        Cpu++;
+    }
+
     Trace("<====\n");
 
 done:
@@ -930,6 +1092,17 @@ done:
 
     return STATUS_SUCCESS;
 
+fail9:
+    Error("fail9\n");
+
+    while (--Cpu >= 0) {
+        FdoFreeInterrupt(Fdo, Context->LatchedInterrupt[Cpu]);
+        Context->LatchedInterrupt[Cpu] = NULL;
+    }
+
+    FdoFreeInterrupt(Fdo, Context->LevelSensitiveInterrupt);
+    Context->LevelSensitiveInterrupt = NULL;
+
 fail8:
     Error("fail8\n");
 
@@ -990,7 +1163,9 @@ EvtchnRelease(
     )
 {
     PXENBUS_EVTCHN_CONTEXT  Context = Interface->Context;
+    PXENBUS_FDO             Fdo = Context->Fdo;
     KIRQL                   Irql;
+    LONG                    Cpu;
 
     KeAcquireSpinLock(&Context->Lock, &Irql);
 
@@ -1002,8 +1177,14 @@ EvtchnRelease(
     if (!IsListEmpty(&Context->List))
         BUG("OUTSTANDING EVENT CHANNELS");
 
-    FdoFreeInterrupt(Context->Fdo, Context->Interrupt);
-    Context->Interrupt = NULL;
+    Cpu = KeNumberProcessors;
+    while (--Cpu >= 0) {
+        FdoFreeInterrupt(Fdo, Context->LatchedInterrupt[Cpu]);
+        Context->LatchedInterrupt[Cpu] = NULL;
+    }
+
+    FdoFreeInterrupt(Fdo, Context->LevelSensitiveInterrupt);
+    Context->LevelSensitiveInterrupt = NULL;
 
     EvtchnAbiRelease(Context);
 
@@ -1046,6 +1227,19 @@ static struct _XENBUS_EVTCHN_INTERFACE_V1 
EvtchnInterfaceVersion1 = {
     EvtchnClose
 };
                      
+static struct _XENBUS_EVTCHN_INTERFACE_V2 EvtchnInterfaceVersion2 = {
+    { sizeof (struct _XENBUS_EVTCHN_INTERFACE_V2), 2, NULL, NULL, NULL },
+    EvtchnAcquire,
+    EvtchnRelease,
+    EvtchnOpen,
+    EvtchnBind,
+    EvtchnUnmask,
+    EvtchnSend,
+    EvtchnTrigger,
+    EvtchnGetPort,
+    EvtchnClose
+};
+
 NTSTATUS
 EvtchnInitialize(
     IN  PXENBUS_FDO             Fdo,
@@ -1171,6 +1365,23 @@ EvtchnGetInterface(
         status = STATUS_SUCCESS;
         break;
     }
+    case 2: {
+        struct _XENBUS_EVTCHN_INTERFACE_V2  *EvtchnInterface;
+
+        EvtchnInterface = (struct _XENBUS_EVTCHN_INTERFACE_V2 *)Interface;
+
+        status = STATUS_BUFFER_OVERFLOW;
+        if (Size < sizeof (struct _XENBUS_EVTCHN_INTERFACE_V2))
+            break;
+
+        *EvtchnInterface = EvtchnInterfaceVersion2;
+
+        ASSERT3U(Interface->Version, ==, Version);
+        Interface->Context = Context;
+
+        status = STATUS_SUCCESS;
+        break;
+    }
     default:
         status = STATUS_NOT_SUPPORTED;
         break;
diff --git a/src/xenbus/store.c b/src/xenbus/store.c
index 22c2a12..04fce3a 100644
--- a/src/xenbus/store.c
+++ b/src/xenbus/store.c
@@ -1848,6 +1848,15 @@ StoreEnable(
                                      FALSE);
     ASSERT(Context->Channel != NULL);
 
+    // 
+    // Attempt to use some CPU other than 0 for events from
+    // xenstored.
+    //
+    (VOID) XENBUS_EVTCHN(Bind,
+                         &Context->EvtchnInterface,
+                         Context->Channel,
+                         KeNumberProcessors - 1);
+
     Pending = XENBUS_EVTCHN(Unmask,
                             &Context->EvtchnInterface,
                             Context->Channel,
-- 
2.1.1


_______________________________________________
win-pv-devel mailing list
win-pv-devel@xxxxxxxxxxxxxxxxxxxx
http://lists.xenproject.org/cgi-bin/mailman/listinfo/win-pv-devel


 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.