Xen project Mailing List

[win-pv-devel] [PATCH xenvif 3/8] Revert complete poller subsystem

From: Paul Durrant <paul.durrant@xxxxxxxxxx>

Date: Fri, 2 Nov 2018 16:54:24 +0000

Cc: Paul Durrant <paul.durrant@xxxxxxxxxx>

Delivery-date: Fri, 02 Nov 2018 16:54:48 +0000

List-id: Developer list for the Windows PV Drivers subproject <win-pv-devel.lists.xenproject.org>

It seems to have a bad effect on performance that various attempts at improvement just don't seem to be able fix, so this patch just gets rid of the whole thing. This patch reverts the following commits: 264bde12 "Introduce a threaded DPC into the receiver code" 129ad516 "Stop using a threaded DPC in the poller" 5932938b "Don't bump the receiver event counter if the poller is going to retry" 16002b8f "poller: fix event channels when backends do not support multi-queue" bc722edd "Don't use KTIMERs in receive path" dfaa68cc "Don't affinitize timer DPC" eac9a95a "Move the Receiver and Transmitter event and DPC processing..." 40be5c12 "Add the boilerplate for a new Poller sub-system" Signed-off-by: Paul Durrant <paul.durrant@xxxxxxxxxx> --- src/xenvif/frontend.c | 133 ++-- src/xenvif/frontend.h | 17 +- src/xenvif/poller.c | 1454 ------------------------------------------ src/xenvif/poller.h | 100 --- src/xenvif/receiver.c | 749 ++++++++++++++-------- src/xenvif/receiver.h | 6 - src/xenvif/transmitter.c | 396 ++++++++++-- src/xenvif/transmitter.h | 6 - src/xenvif/vif.c | 7 - vs2015/xenvif/xenvif.vcxproj | 1 - vs2017/xenvif/xenvif.vcxproj | 1 - 11 files changed, 903 insertions(+), 1967 deletions(-) delete mode 100644 src/xenvif/poller.c delete mode 100644 src/xenvif/poller.h diff --git a/src/xenvif/frontend.c b/src/xenvif/frontend.c index 5810947..3e31654 100644 --- a/src/xenvif/frontend.c +++ b/src/xenvif/frontend.c @@ -47,7 +47,6 @@ #include "tcpip.h" #include "receiver.h" #include "transmitter.h" -#include "poller.h" #include "link.h" #include "dbg_print.h" #include "assert.h" @@ -81,12 +80,12 @@ struct _XENVIF_FRONTEND { USHORT BackendDomain; ULONG MaxQueues; ULONG NumQueues; + BOOLEAN Split; ULONG DisableToeplitz; PXENVIF_MAC Mac; PXENVIF_RECEIVER Receiver; PXENVIF_TRANSMITTER Transmitter; - PXENVIF_POLLER Poller; PXENVIF_CONTROLLER Controller; XENBUS_DEBUG_INTERFACE DebugInterface; @@ -339,7 +338,6 @@ FrontendGet ## _Function( \ DEFINE_FRONTEND_GET_FUNCTION(Mac, PXENVIF_MAC) DEFINE_FRONTEND_GET_FUNCTION(Receiver, PXENVIF_RECEIVER) DEFINE_FRONTEND_GET_FUNCTION(Transmitter, PXENVIF_TRANSMITTER) -DEFINE_FRONTEND_GET_FUNCTION(Poller, PXENVIF_POLLER) DEFINE_FRONTEND_GET_FUNCTION(Controller, PXENVIF_CONTROLLER) static BOOLEAN @@ -1802,6 +1800,50 @@ FrontendGetNumQueues( return __FrontendGetNumQueues(Frontend); } +static VOID +FrontendSetSplit( + IN PXENVIF_FRONTEND Frontend + ) +{ + PCHAR Buffer; + NTSTATUS status; + + status = XENBUS_STORE(Read, + &Frontend->StoreInterface, + NULL, + __FrontendGetBackendPath(Frontend), + "feature-split-event-channels", + &Buffer); + if (NT_SUCCESS(status)) { + Frontend->Split = (BOOLEAN)strtol(Buffer, NULL, 2); + + XENBUS_STORE(Free, + &Frontend->StoreInterface, + Buffer); + } else { + Frontend->Split = FALSE; + } + + Info("%s: %s\n", __FrontendGetPath(Frontend), + (Frontend->Split) ? "TRUE" : "FALSE"); +} + +static FORCEINLINE BOOLEAN +__FrontendIsSplit( + IN PXENVIF_FRONTEND Frontend + ) +{ + return Frontend->Split; +} + +BOOLEAN +FrontendIsSplit( + IN PXENVIF_FRONTEND Frontend + ) +{ + return __FrontendIsSplit(Frontend); +} + static FORCEINLINE NTSTATUS __FrontendUpdateHash( PXENVIF_FRONTEND Frontend, @@ -2166,22 +2208,19 @@ FrontendConnect( goto fail3; FrontendSetNumQueues(Frontend); - - status = PollerConnect(__FrontendGetPoller(Frontend)); - if (!NT_SUCCESS(status)) - goto fail4; + FrontendSetSplit(Frontend); status = ReceiverConnect(__FrontendGetReceiver(Frontend)); if (!NT_SUCCESS(status)) - goto fail5; + goto fail4; status = TransmitterConnect(__FrontendGetTransmitter(Frontend)); if (!NT_SUCCESS(status)) - goto fail6; + goto fail5; status = ControllerConnect(__FrontendGetController(Frontend)); if (!NT_SUCCESS(status)) - goto fail7; + goto fail6; Attempt = 0; do { @@ -2193,11 +2232,6 @@ FrontendConnect( if (!NT_SUCCESS(status)) break; - status = PollerStoreWrite(__FrontendGetPoller(Frontend), - Transaction); - if (!NT_SUCCESS(status)) - goto abort; - status = ReceiverStoreWrite(__FrontendGetReceiver(Frontend), Transaction); if (!NT_SUCCESS(status)) @@ -2241,7 +2275,7 @@ abort: } while (status == STATUS_RETRY); if (!NT_SUCCESS(status)) - goto fail8; + goto fail7; State = XenbusStateUnknown; while (State != XenbusStateConnected) { @@ -2280,7 +2314,7 @@ abort: status = STATUS_UNSUCCESSFUL; if (State != XenbusStateConnected) - goto fail9; + goto fail8; ControllerEnable(__FrontendGetController(Frontend)); @@ -2289,34 +2323,30 @@ abort: Trace("<====\n"); return STATUS_SUCCESS; -fail9: - Error("fail9\n"); - fail8: Error("fail8\n"); - ControllerDisconnect(__FrontendGetController(Frontend)); - fail7: Error("fail7\n"); - TransmitterDisconnect(__FrontendGetTransmitter(Frontend)); + ControllerDisconnect(__FrontendGetController(Frontend)); fail6: Error("fail6\n"); - ReceiverDisconnect(__FrontendGetReceiver(Frontend)); + TransmitterDisconnect(__FrontendGetTransmitter(Frontend)); fail5: Error("fail5\n"); - PollerDisconnect(__FrontendGetPoller(Frontend)); + ReceiverDisconnect(__FrontendGetReceiver(Frontend)); fail4: Error("fail4\n"); MacDisconnect(__FrontendGetMac(Frontend)); + Frontend->Split = FALSE; Frontend->NumQueues = 0; fail3: @@ -2351,9 +2381,9 @@ FrontendDisconnect( ControllerDisconnect(__FrontendGetController(Frontend)); TransmitterDisconnect(__FrontendGetTransmitter(Frontend)); ReceiverDisconnect(__FrontendGetReceiver(Frontend)); - PollerDisconnect(__FrontendGetPoller(Frontend)); MacDisconnect(__FrontendGetMac(Frontend)); + Frontend->Split = FALSE; Frontend->NumQueues = 0; XENBUS_DEBUG(Deregister, @@ -2379,41 +2409,32 @@ FrontendEnable( if (!NT_SUCCESS(status)) goto fail1; - status = PollerEnable(__FrontendGetPoller(Frontend)); - if (!NT_SUCCESS(status)) - goto fail2; - status = ReceiverEnable(__FrontendGetReceiver(Frontend)); if (!NT_SUCCESS(status)) - goto fail3; + goto fail2; status = TransmitterEnable(__FrontendGetTransmitter(Frontend)); if (!NT_SUCCESS(status)) - goto fail4; + goto fail3; status = __FrontendUpdateHash(Frontend, &Frontend->Hash); if (!NT_SUCCESS(status)) - goto fail5; + goto fail4; (VOID) FrontendNotifyMulticastAddresses(Frontend, TRUE); Trace("<====\n"); return STATUS_SUCCESS; -fail5: - Error("fail5\n"); - - TransmitterDisable(__FrontendGetTransmitter(Frontend)); - fail4: Error("fail4\n"); - ReceiverDisable(__FrontendGetReceiver(Frontend)); + TransmitterDisable(__FrontendGetTransmitter(Frontend)); fail3: Error("fail3\n"); - PollerDisable(__FrontendGetPoller(Frontend)); + ReceiverDisable(__FrontendGetReceiver(Frontend)); fail2: Error("fail2\n"); @@ -2437,7 +2458,6 @@ FrontendDisable( TransmitterDisable(__FrontendGetTransmitter(Frontend)); ReceiverDisable(__FrontendGetReceiver(Frontend)); - PollerDisable(__FrontendGetPoller(Frontend)); MacDisable(__FrontendGetMac(Frontend)); Trace("<====\n"); @@ -2849,23 +2869,19 @@ FrontendInitialize( if (!NT_SUCCESS(status)) goto fail8; - status = PollerInitialize(*Frontend, &(*Frontend)->Poller); - if (!NT_SUCCESS(status)) - goto fail9; - status = ControllerInitialize(*Frontend, &(*Frontend)->Controller); if (!NT_SUCCESS(status)) - goto fail10; + goto fail9; KeInitializeEvent(&(*Frontend)->EjectEvent, NotificationEvent, FALSE); status = ThreadCreate(FrontendEject, *Frontend, &(*Frontend)->EjectThread); if (!NT_SUCCESS(status)) - goto fail11; + goto fail10; status = ThreadCreate(FrontendMib, *Frontend, &(*Frontend)->MibThread); if (!NT_SUCCESS(status)) - goto fail12; + goto fail11; (*Frontend)->StatisticsCount = KeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS); (*Frontend)->Statistics = __FrontendAllocate(sizeof (XENVIF_FRONTEND_STATISTICS) * @@ -2873,38 +2889,34 @@ FrontendInitialize( status = STATUS_NO_MEMORY; if ((*Frontend)->Statistics == NULL) - goto fail13; + goto fail12; Trace("<====\n"); return STATUS_SUCCESS; -fail13: - Error("fail13\n"); +fail12: + Error("fail12\n"); ThreadAlert((*Frontend)->MibThread); ThreadJoin((*Frontend)->MibThread); (*Frontend)->MibThread = NULL; -fail12: - Error("fail12\n"); +fail11: + Error("fail11\n"); ThreadAlert((*Frontend)->EjectThread); ThreadJoin((*Frontend)->EjectThread); (*Frontend)->EjectThread = NULL; -fail11: - Error("fail11\n"); +fail10: + Error("fail10\n"); RtlZeroMemory(&(*Frontend)->EjectEvent, sizeof (KEVENT)); ControllerTeardown(__FrontendGetController(*Frontend)); (*Frontend)->Controller = NULL; -fail10: - PollerTeardown(__FrontendGetPoller(*Frontend)); - (*Frontend)->Poller = NULL; - fail9: TransmitterTeardown(__FrontendGetTransmitter(*Frontend)); (*Frontend)->Transmitter = NULL; @@ -3012,9 +3024,6 @@ FrontendTeardown( ControllerTeardown(__FrontendGetController(Frontend)); Frontend->Controller = NULL; - PollerTeardown(__FrontendGetPoller(Frontend)); - Frontend->Poller = NULL; - TransmitterTeardown(__FrontendGetTransmitter(Frontend)); Frontend->Transmitter = NULL; diff --git a/src/xenvif/frontend.h b/src/xenvif/frontend.h index 7f3b7c5..8e5552e 100644 --- a/src/xenvif/frontend.h +++ b/src/xenvif/frontend.h @@ -123,6 +123,16 @@ FrontendGetNumQueues( IN PXENVIF_FRONTEND Frontend ); +extern BOOLEAN +FrontendIsSplit( + IN PXENVIF_FRONTEND Frontend + ); + +extern BOOLEAN +FrontendIsSplit( + IN PXENVIF_FRONTEND Frontend + ); + extern PCHAR FrontendFormatPath( IN PXENVIF_FRONTEND Frontend, @@ -156,13 +166,6 @@ FrontendGetTransmitter( IN PXENVIF_FRONTEND Frontend ); -#include "poller.h" - -extern PXENVIF_POLLER -FrontendGetPoller( - IN PXENVIF_FRONTEND Frontend - ); - #include "controller.h" extern PXENVIF_CONTROLLER diff --git a/src/xenvif/poller.c b/src/xenvif/poller.c deleted file mode 100644 index 7f18d13..0000000 --- a/src/xenvif/poller.c +++ /dev/null @@ -1,1454 +0,0 @@ -/* Copyright (c) Citrix Systems Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted provided - * that the following conditions are met: - * - * * Redistributions of source code must retain the above - * copyright notice, this list of conditions and the - * following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the - * following disclaimer in the documetation and/or other - * materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND - * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include <ntddk.h> -#include <procgrp.h> -#include <ntstrsafe.h> -#include <stdarg.h> -#include <stdlib.h> -#include <xen.h> - -#include <debug_interface.h> -#include <store_interface.h> -#include <evtchn_interface.h> - -#include "pdo.h" -#include "frontend.h" -#include "transmitter.h" -#include "receiver.h" -#include "poller.h" -#include "vif.h" -#include "thread.h" -#include "registry.h" -#include "dbg_print.h" -#include "assert.h" -#include "util.h" - -#define MAXNAMELEN 128 - -typedef struct _XENVIF_POLLER_INSTANCE XENVIF_POLLER_INSTANCE, *PXENVIF_POLLER_INSTANCE; - -typedef enum _XENVIF_POLLER_CHANNEL_TYPE { - XENVIF_POLLER_CHANNEL_RECEIVER, - XENVIF_POLLER_CHANNEL_TRANSMITTER, - XENVIF_POLLER_CHANNEL_COMBINED, - XENVIF_POLLER_CHANNEL_TYPE_COUNT -} XENVIF_POLLER_CHANNEL_TYPE, *PXENVIF_POLLER_CHANNEL_TYPE; - -#define XENVIF_POLLER_CHANNEL_INVALID XENVIF_POLLER_CHANNEL_TYPE_COUNT - -typedef struct _XENVIF_POLLER_CHANNEL { - PXENVIF_POLLER_INSTANCE Instance; - XENVIF_POLLER_CHANNEL_TYPE Type; - const CHAR *Node; - PXENBUS_EVTCHN_CHANNEL Channel; - ULONG Events; -} XENVIF_POLLER_CHANNEL, *PXENVIF_POLLER_CHANNEL; - -struct _XENVIF_POLLER_INSTANCE { - PXENVIF_POLLER Poller; - ULONG Index; - PCHAR Path; - KSPIN_LOCK Lock; - KDPC Dpc; - ULONG Dpcs; - PXENVIF_POLLER_CHANNEL Channel[XENVIF_POLLER_CHANNEL_TYPE_COUNT]; - BOOLEAN Enabled; - LONG Pending; -}; - -struct _XENVIF_POLLER { - PXENVIF_FRONTEND Frontend; - PXENVIF_POLLER_INSTANCE *Instance; - BOOLEAN Split; - XENBUS_STORE_INTERFACE StoreInterface; - XENBUS_EVTCHN_INTERFACE EvtchnInterface; - XENBUS_DEBUG_INTERFACE DebugInterface; - PXENBUS_DEBUG_CALLBACK DebugCallback; -}; - -#define XENVIF_POLLER_TAG 'LLOP' - -static FORCEINLINE PVOID -__PollerAllocate( - IN ULONG Length - ) -{ - return __AllocatePoolWithTag(NonPagedPool, Length, XENVIF_POLLER_TAG); -} - -static FORCEINLINE VOID -__PollerFree( - IN PVOID Buffer - ) -{ - __FreePoolWithTag(Buffer, XENVIF_POLLER_TAG); -} - -static NTSTATUS -PollerChannelInitialize( - IN PXENVIF_POLLER_INSTANCE Instance, - IN ULONG Type, - OUT PXENVIF_POLLER_CHANNEL *Channel - ) -{ - NTSTATUS status; - - *Channel = __PollerAllocate(sizeof (XENVIF_POLLER_CHANNEL)); - - status = STATUS_NO_MEMORY; - if (*Channel == NULL) - goto fail1; - - (*Channel)->Instance = Instance; - (*Channel)->Type = Type; - - switch (Type) { - case XENVIF_POLLER_CHANNEL_RECEIVER: - (*Channel)->Node = "event-channel-rx"; - break; - - case XENVIF_POLLER_CHANNEL_TRANSMITTER: - (*Channel)->Node = "event-channel-tx"; - break; - - case XENVIF_POLLER_CHANNEL_COMBINED: - (*Channel)->Node = "event-channel"; - break; - - default: - ASSERT(FALSE); - break; - } - - return STATUS_SUCCESS; - -fail1: - Error("fail1 (%08x)\n", status); - - return status; -} - -static BOOLEAN -PollerChannelSetPending( - IN PXENVIF_POLLER_CHANNEL Channel - ) -{ - PXENVIF_POLLER_INSTANCE Instance; - ULONG Set; - - Instance = Channel->Instance; - - switch (Channel->Type) - { - case XENVIF_POLLER_CHANNEL_RECEIVER: - Set = InterlockedBitTestAndSet(&Instance->Pending, - XENVIF_POLLER_EVENT_RECEIVE); - break; - - case XENVIF_POLLER_CHANNEL_TRANSMITTER: - Set = InterlockedBitTestAndSet(&Instance->Pending, - XENVIF_POLLER_EVENT_TRANSMIT); - break; - - case XENVIF_POLLER_CHANNEL_COMBINED: - Set = InterlockedBitTestAndSet(&Instance->Pending, - XENVIF_POLLER_EVENT_RECEIVE); - Set |= InterlockedBitTestAndSet(&Instance->Pending, - XENVIF_POLLER_EVENT_TRANSMIT); - break; - - default: - ASSERT(FALSE); - Set = 0; - break; - } - - return (Set != 0) ? FALSE : TRUE; -} - -static FORCEINLINE BOOLEAN -__BitTest( - IN PLONG Mask, - IN LONG Bit - ) -{ - return (*Mask & (1L << Bit)) ? TRUE : FALSE; -} - -static BOOLEAN -PollerChannelTestPending( - IN PXENVIF_POLLER_CHANNEL Channel - ) -{ - PXENVIF_POLLER_INSTANCE Instance; - - Instance = Channel->Instance; - - switch (Channel->Type) - { - case XENVIF_POLLER_CHANNEL_RECEIVER: - if (__BitTest(&Instance->Pending, XENVIF_POLLER_EVENT_RECEIVE)) - return TRUE; - - break; - - case XENVIF_POLLER_CHANNEL_TRANSMITTER: - if (__BitTest(&Instance->Pending, XENVIF_POLLER_EVENT_TRANSMIT)) - return TRUE; - - break; - - case XENVIF_POLLER_CHANNEL_COMBINED: - if (__BitTest(&Instance->Pending, XENVIF_POLLER_EVENT_RECEIVE) || - __BitTest(&Instance->Pending, XENVIF_POLLER_EVENT_TRANSMIT)) - return TRUE; - - break; - - default: - ASSERT(FALSE); - break; - } - - return FALSE; -} - -KSERVICE_ROUTINE PollerChannelEvtchnCallback; - -BOOLEAN -PollerChannelEvtchnCallback( - IN PKINTERRUPT InterruptObject, - IN PVOID Argument - ) -{ - PXENVIF_POLLER_CHANNEL Channel = Argument; - PXENVIF_POLLER_INSTANCE Instance; - - UNREFERENCED_PARAMETER(InterruptObject); - - ASSERT(Channel != NULL); - Instance = Channel->Instance; - - Channel->Events++; - - if (PollerChannelSetPending(Channel) && - KeInsertQueueDpc(&Instance->Dpc, NULL, NULL)) - Instance->Dpcs++; - - return TRUE; -} - -static FORCEINLINE BOOLEAN -__PollerIsSplit( - IN PXENVIF_POLLER Poller - ) -{ - return Poller->Split; -} - -static NTSTATUS -PollerChannelConnect( - IN PXENVIF_POLLER_CHANNEL Channel - ) -{ - PXENVIF_POLLER_INSTANCE Instance; - PXENVIF_POLLER Poller; - PXENVIF_FRONTEND Frontend; - PROCESSOR_NUMBER ProcNumber; - NTSTATUS status; - - Instance = Channel->Instance; - Poller = Instance->Poller; - Frontend = Poller->Frontend; - - switch (Channel->Type) - { - case XENVIF_POLLER_CHANNEL_RECEIVER: - case XENVIF_POLLER_CHANNEL_TRANSMITTER: - if (!__PollerIsSplit(Poller)) - goto done; - - break; - - case XENVIF_POLLER_CHANNEL_COMBINED: - if (__PollerIsSplit(Poller)) - goto done; - - break; - - default: - ASSERT(FALSE); - break; - } - - Channel->Channel = XENBUS_EVTCHN(Open, - &Poller->EvtchnInterface, - XENBUS_EVTCHN_TYPE_UNBOUND, - PollerChannelEvtchnCallback, - Channel, - FrontendGetBackendDomain(Frontend), - TRUE); - - status = STATUS_UNSUCCESSFUL; - if (Channel->Channel == NULL) - goto fail1; - - status = KeGetProcessorNumberFromIndex(Instance->Index, &ProcNumber); - ASSERT(NT_SUCCESS(status)); - - (VOID) XENBUS_EVTCHN(Bind, - &Poller->EvtchnInterface, - Channel->Channel, - ProcNumber.Group, - ProcNumber.Number); - - (VOID) XENBUS_EVTCHN(Unmask, - &Poller->EvtchnInterface, - Channel->Channel, - FALSE, - TRUE); - -done: - return STATUS_SUCCESS; - -fail1: - Error("fail1 (%08x)\n", status); - - return status; -} - -static NTSTATUS -PollerChannelStoreWrite( - IN PXENVIF_POLLER_CHANNEL Channel, - IN PXENBUS_STORE_TRANSACTION Transaction - ) -{ - PXENVIF_POLLER_INSTANCE Instance; - PXENVIF_POLLER Poller; - PCHAR Path; - ULONG Port; - NTSTATUS status; - - Instance = Channel->Instance; - Poller = Instance->Poller; - - if (Channel->Channel == NULL) - goto done; - - Path = (FrontendGetNumQueues(Poller->Frontend) == 1) ? - FrontendGetPath(Poller->Frontend) : - Instance->Path; - - Port = XENBUS_EVTCHN(GetPort, - &Poller->EvtchnInterface, - Channel->Channel); - - status = XENBUS_STORE(Printf, - &Poller->StoreInterface, - Transaction, - Path, - (PCHAR)Channel->Node, - "%u", - Port); - if (!NT_SUCCESS(status)) - goto fail1; - -done: - return STATUS_SUCCESS; - -fail1: - Error("fail1 (%08x)\n", status); - - return status; -} - -static VOID -PollerChannelUnmask( - IN PXENVIF_POLLER_CHANNEL Channel - ) -{ - PXENVIF_POLLER_INSTANCE Instance; - PXENVIF_POLLER Poller; - BOOLEAN Pending; - - Instance = Channel->Instance; - Poller = Instance->Poller; - - if (Channel->Channel == NULL) - return; - - if (PollerChannelTestPending(Channel)) - return; - - Pending = XENBUS_EVTCHN(Unmask, - &Poller->EvtchnInterface, - Channel->Channel, - FALSE, - FALSE); - if (Pending) - (VOID) PollerChannelSetPending(Channel); -} - -static VOID -PollerChannelSend( - IN PXENVIF_POLLER_CHANNEL Channel - ) -{ - PXENVIF_POLLER_INSTANCE Instance; - PXENVIF_POLLER Poller; - - Instance = Channel->Instance; - Poller = Instance->Poller; - - XENBUS_EVTCHN(Send, - &Poller->EvtchnInterface, - Channel->Channel); -} - -static VOID -PollerChannelDebugCallback( - IN PXENVIF_POLLER_CHANNEL Channel - ) -{ - PXENVIF_POLLER_INSTANCE Instance; - PXENVIF_POLLER Poller; - - Instance = Channel->Instance; - Poller = Instance->Poller; - - if (Channel->Channel == NULL) - return; - - XENBUS_DEBUG(Printf, - &Poller->DebugInterface, - "[%s]: Events = %lu\n", - Channel->Node, - Channel->Events); -} - -static VOID -PollerChannelDisconnect( - IN PXENVIF_POLLER_CHANNEL Channel - ) -{ - PXENVIF_POLLER_INSTANCE Instance; - PXENVIF_POLLER Poller; - - Instance = Channel->Instance; - Poller = Instance->Poller; - - if (Channel->Channel == NULL) - return; - - Channel->Events = 0; - - XENBUS_EVTCHN(Close, - &Poller->EvtchnInterface, - Channel->Channel); - Channel->Channel = NULL; -} - -static VOID -PollerChannelTeardown( - IN PXENVIF_POLLER_CHANNEL Channel - ) -{ - Channel->Node = NULL; - - Channel->Type = 0; - Channel->Instance = NULL; - - ASSERT(IsZeroMemory(Channel, sizeof (XENVIF_POLLER_CHANNEL))); - __PollerFree(Channel); -} - -__drv_requiresIRQL(DISPATCH_LEVEL) -static VOID -PollerInstanceUnmask( - IN PXENVIF_POLLER_INSTANCE Instance, - IN XENVIF_POLLER_EVENT_TYPE Event - ) -{ - PXENVIF_POLLER Poller; - XENVIF_POLLER_CHANNEL_TYPE Type; - PXENVIF_POLLER_CHANNEL Channel; - - ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL); - - Poller = Instance->Poller; - - KeAcquireSpinLockAtDpcLevel(&Instance->Lock); - - if (!Instance->Enabled) - goto done; - - if (!__PollerIsSplit(Poller)) { - Type = XENVIF_POLLER_CHANNEL_COMBINED; - } else { - switch (Event) { - case XENVIF_POLLER_EVENT_RECEIVE: - Type = XENVIF_POLLER_CHANNEL_RECEIVER; - break; - - case XENVIF_POLLER_EVENT_TRANSMIT: - Type = XENVIF_POLLER_CHANNEL_TRANSMITTER; - break; - - default: - Type = XENVIF_POLLER_CHANNEL_INVALID; - break; - } - } - - ASSERT(Type != XENVIF_POLLER_CHANNEL_INVALID); - - Channel = Instance->Channel[Type]; - - PollerChannelUnmask(Channel); - -done: - KeReleaseSpinLockFromDpcLevel(&Instance->Lock); -} - -__drv_functionClass(KDEFERRED_ROUTINE) -__drv_maxIRQL(DISPATCH_LEVEL) -__drv_minIRQL(DISPATCH_LEVEL) -__drv_sameIRQL -static VOID -PollerInstanceDpc( - IN PKDPC Dpc, - IN PVOID Context, - IN PVOID Argument1, - IN PVOID Argument2 - ) -{ - PXENVIF_POLLER_INSTANCE Instance = Context; - PXENVIF_POLLER Poller; - PXENVIF_FRONTEND Frontend; - BOOLEAN Enabled; - BOOLEAN ReceiverRetry; - BOOLEAN TransmitterRetry; - - UNREFERENCED_PARAMETER(Dpc); - UNREFERENCED_PARAMETER(Argument1); - UNREFERENCED_PARAMETER(Argument2); - - ASSERT(Instance != NULL); - - Poller = Instance->Poller; - Frontend = Poller->Frontend; - Enabled = FALSE; - ReceiverRetry = FALSE; - TransmitterRetry = FALSE; - - for (;;) { - BOOLEAN NeedReceiverPoll; - BOOLEAN NeedTransmitterPoll; - - KeAcquireSpinLockAtDpcLevel(&Instance->Lock); - Enabled = Instance->Enabled; - KeReleaseSpinLockFromDpcLevel(&Instance->Lock); - - if (!Enabled) - break; - - NeedReceiverPoll = - (InterlockedBitTestAndReset(&Instance->Pending, - XENVIF_POLLER_EVENT_RECEIVE) != 0) ? - TRUE : - FALSE; - - NeedTransmitterPoll = - (InterlockedBitTestAndReset(&Instance->Pending, - XENVIF_POLLER_EVENT_TRANSMIT) != 0) ? - TRUE : - FALSE; - - if (!NeedReceiverPoll && !NeedTransmitterPoll) - break; - - if (NeedReceiverPoll) - { - ReceiverRetry = ReceiverPoll(FrontendGetReceiver(Frontend), - Instance->Index); - - if (!ReceiverRetry) { - PollerInstanceUnmask(Instance, XENVIF_POLLER_EVENT_RECEIVE); - } else { - (VOID) InterlockedBitTestAndSet(&Instance->Pending, - XENVIF_POLLER_EVENT_RECEIVE); - } - } - - if (NeedTransmitterPoll) - { - TransmitterRetry = TransmitterPoll(FrontendGetTransmitter(Frontend), - Instance->Index); - - if (!TransmitterRetry) { - PollerInstanceUnmask(Instance, XENVIF_POLLER_EVENT_TRANSMIT); - } else { - (VOID) InterlockedBitTestAndSet(&Instance->Pending, - XENVIF_POLLER_EVENT_TRANSMIT); - } - } - } - - ASSERT(!Enabled || !ReceiverRetry); - ASSERT(!Enabled || !TransmitterRetry); -} - -static NTSTATUS -PollerInstanceInitialize( - IN PXENVIF_POLLER Poller, - IN LONG Index, - OUT PXENVIF_POLLER_INSTANCE *Instance - ) -{ - PXENVIF_FRONTEND Frontend; - LONG Type; - NTSTATUS status; - - Frontend = Poller->Frontend; - - *Instance = __PollerAllocate(sizeof (XENVIF_POLLER_INSTANCE)); - - status = STATUS_NO_MEMORY; - if (*Instance == NULL) - goto fail1; - - (*Instance)->Poller = Poller; - (*Instance)->Index = Index; - - for (Type = 0; Type < XENVIF_POLLER_CHANNEL_TYPE_COUNT; Type++) - { - PXENVIF_POLLER_CHANNEL Channel; - - status = PollerChannelInitialize(*Instance, Type, &Channel); - if (!NT_SUCCESS(status)) - goto fail2; - - (*Instance)->Channel[Type] = Channel; - } - - (*Instance)->Path = FrontendFormatPath(Frontend, Index); - if ((*Instance)->Path == NULL) - goto fail3; - - KeInitializeSpinLock(&(*Instance)->Lock); - - KeInitializeDpc(&(*Instance)->Dpc, PollerInstanceDpc, *Instance); - - return STATUS_SUCCESS; - -fail3: - Error("fail3\n"); - - Type = XENVIF_POLLER_CHANNEL_TYPE_COUNT; - -fail2: - Error("fail2\n"); - - while (--Type >= 0) - { - PXENVIF_POLLER_CHANNEL Channel = (*Instance)->Channel[Type]; - - (*Instance)->Channel[Type] = NULL; - PollerChannelTeardown(Channel); - } - - (*Instance)->Index = 0; - (*Instance)->Poller = NULL; - - ASSERT(IsZeroMemory(*Instance, sizeof (XENVIF_POLLER_INSTANCE))); - __PollerFree(*Instance); - -fail1: - Error("fail1 (%08x)\n", status); - - return status; -} - -__drv_requiresIRQL(DISPATCH_LEVEL) -static NTSTATUS -PollerInstanceConnect( - IN PXENVIF_POLLER_INSTANCE Instance - ) -{ - PROCESSOR_NUMBER ProcNumber; - LONG Type; - NTSTATUS status; - - ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL); - - status = KeGetProcessorNumberFromIndex(Instance->Index, &ProcNumber); - ASSERT(NT_SUCCESS(status)); - - KeSetTargetProcessorDpcEx(&Instance->Dpc, &ProcNumber); - KeSetImportanceDpc(&Instance->Dpc, MediumHighImportance); - - for (Type = 0; Type < XENVIF_POLLER_CHANNEL_TYPE_COUNT; Type++) - { - PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type]; - - status = PollerChannelConnect(Channel); - if (!NT_SUCCESS(status)) - goto fail1; - } - - return STATUS_SUCCESS; - -fail1: - Error("fail1 (%08x)\n", status); - - while (--Type >= 0) - { - PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type]; - - PollerChannelDisconnect(Channel); - } - - return status; -} - -static NTSTATUS -PollerInstanceStoreWrite( - IN PXENVIF_POLLER_INSTANCE Instance, - IN PXENBUS_STORE_TRANSACTION Transaction - ) -{ - ULONG Type; - NTSTATUS status; - - for (Type = 0; Type < XENVIF_POLLER_CHANNEL_TYPE_COUNT; Type++) - { - PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type]; - - status = PollerChannelStoreWrite(Channel, Transaction); - if (!NT_SUCCESS(status)) - goto fail1; - } - - return STATUS_SUCCESS; - -fail1: - Error("fail1 (%08x)\n", status); - - return status; -} - -__drv_requiresIRQL(DISPATCH_LEVEL) -static NTSTATUS -PollerInstanceEnable( - IN PXENVIF_POLLER_INSTANCE Instance - ) -{ - ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL); - - (VOID) InterlockedBitTestAndSet(&Instance->Pending, - XENVIF_POLLER_EVENT_RECEIVE); - (VOID) InterlockedBitTestAndSet(&Instance->Pending, - XENVIF_POLLER_EVENT_TRANSMIT); - - KeAcquireSpinLockAtDpcLevel(&Instance->Lock); - Instance->Enabled = TRUE; - KeReleaseSpinLockFromDpcLevel(&Instance->Lock); - - (VOID) KeInsertQueueDpc(&Instance->Dpc, NULL, NULL); - - return STATUS_SUCCESS; -} - -__drv_requiresIRQL(DISPATCH_LEVEL) -static NTSTATUS -PollerInstanceSend( - IN PXENVIF_POLLER_INSTANCE Instance, - IN XENVIF_POLLER_EVENT_TYPE Event - ) -{ - PXENVIF_POLLER Poller; - XENVIF_POLLER_CHANNEL_TYPE Type; - PXENVIF_POLLER_CHANNEL Channel; - NTSTATUS status; - - ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL); - - Poller = Instance->Poller; - - KeAcquireSpinLockAtDpcLevel(&Instance->Lock); - - Type = XENVIF_POLLER_CHANNEL_INVALID; - - if (Instance->Enabled) { - if (!__PollerIsSplit(Poller)) { - Type = XENVIF_POLLER_CHANNEL_COMBINED; - } else { - switch (Event) { - case XENVIF_POLLER_EVENT_RECEIVE: - Type = XENVIF_POLLER_CHANNEL_RECEIVER; - break; - - case XENVIF_POLLER_EVENT_TRANSMIT: - Type = XENVIF_POLLER_CHANNEL_TRANSMITTER; - break; - - default: - ASSERT(FALSE); - break; - } - } - } - - KeReleaseSpinLockFromDpcLevel(&Instance->Lock); - - status = STATUS_UNSUCCESSFUL; - if (Type == XENVIF_POLLER_CHANNEL_INVALID) - goto fail1; - - Channel = Instance->Channel[Type]; - - PollerChannelSend(Channel); - - return STATUS_SUCCESS; - -fail1: - Error("fail1 (%08x)\n", status); - - return status; -} - -__drv_requiresIRQL(DISPATCH_LEVEL) -static NTSTATUS -PollerInstanceTrigger( - IN PXENVIF_POLLER_INSTANCE Instance, - IN XENVIF_POLLER_EVENT_TYPE Event - ) -{ - NTSTATUS status; - - status = STATUS_INVALID_PARAMETER; - if (Event >= XENVIF_POLLER_EVENT_TYPE_COUNT) - goto fail1; - - (VOID) InterlockedBitTestAndSet(&Instance->Pending, Event); - - if (KeInsertQueueDpc(&Instance->Dpc, NULL, NULL)) - Instance->Dpcs++; - - return STATUS_SUCCESS; - -fail1: - Error("fail1 (%08x)\n", status); - - return status; -} - -static VOID -PollerInstanceDebugCallback( - IN PXENVIF_POLLER_INSTANCE Instance - ) -{ - PXENVIF_POLLER Poller; - ULONG Type; - - Poller = Instance->Poller; - - XENBUS_DEBUG(Printf, - &Poller->DebugInterface, - "[%d]: Dpcs = %lu\n", - Instance->Index, - Instance->Dpcs); - - for (Type = 0; Type < XENVIF_POLLER_CHANNEL_TYPE_COUNT; Type++) - { - PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type]; - - PollerChannelDebugCallback(Channel); - } -} - -__drv_requiresIRQL(DISPATCH_LEVEL) -static VOID -PollerInstanceDisable( - IN PXENVIF_POLLER_INSTANCE Instance - ) -{ - ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL); - - KeAcquireSpinLockAtDpcLevel(&Instance->Lock); - Instance->Enabled = FALSE; - KeReleaseSpinLockFromDpcLevel(&Instance->Lock); -} - -__drv_requiresIRQL(DISPATCH_LEVEL) -static VOID -PollerInstanceDisconnect( - IN PXENVIF_POLLER_INSTANCE Instance - ) -{ - LONG Type; - - ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL); - - Instance->Dpcs = 0; - Instance->Pending = 0; - - Type = XENVIF_POLLER_CHANNEL_TYPE_COUNT; - - while (--Type >= 0) - { - PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type]; - - PollerChannelDisconnect(Channel); - } -} - -static VOID -PollerInstanceTeardown( - IN PXENVIF_POLLER_INSTANCE Instance - ) -{ - PXENVIF_POLLER Poller; - PXENVIF_FRONTEND Frontend; - LONG Type; - - ASSERT3U(KeGetCurrentIrql(), ==, PASSIVE_LEVEL); - KeFlushQueuedDpcs(); - - Poller = Instance->Poller; - Frontend = Poller->Frontend; - - RtlZeroMemory(&Instance->Dpc, sizeof (KDPC)); - - RtlZeroMemory(&Instance->Lock, sizeof (KSPIN_LOCK)); - - FrontendFreePath(Frontend, Instance->Path); - Instance->Path = NULL; - - Type = XENVIF_POLLER_CHANNEL_TYPE_COUNT; - - while (--Type >= 0) - { - PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type]; - - Instance->Channel[Type] = NULL; - PollerChannelTeardown(Channel); - } - - Instance->Index = 0; - Instance->Poller = NULL; - - ASSERT(IsZeroMemory(Instance, sizeof (XENVIF_POLLER_INSTANCE))); - __PollerFree(Instance); -} - -static VOID -PollerDebugCallback( - IN PVOID Argument, - IN BOOLEAN Crashing - ) -{ - PXENVIF_POLLER Poller = Argument; - PXENVIF_FRONTEND Frontend; - ULONG NumQueues; - ULONG Index; - - UNREFERENCED_PARAMETER(Crashing); - - Frontend = Poller->Frontend; - - NumQueues = FrontendGetNumQueues(Frontend); - - for (Index = 0; Index < NumQueues; Index++) { - PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index]; - - PollerInstanceDebugCallback(Instance); - } -} - -static VOID -PollerSetSplit( - IN PXENVIF_POLLER Poller - ) -{ - PXENVIF_FRONTEND Frontend; - PCHAR Buffer; - NTSTATUS status; - - Frontend = Poller->Frontend; - - status = XENBUS_STORE(Read, - &Poller->StoreInterface, - NULL, - FrontendGetBackendPath(Frontend), - "feature-split-event-channels", - &Buffer); - if (NT_SUCCESS(status)) { - Poller->Split = (BOOLEAN)strtol(Buffer, NULL, 2); - - XENBUS_STORE(Free, - &Poller->StoreInterface, - Buffer); - } else { - Poller->Split = FALSE; - } - - Info("%s: %s\n", FrontendGetPath(Frontend), - (Poller->Split) ? "TRUE" : "FALSE"); -} - -NTSTATUS -PollerInitialize( - IN PXENVIF_FRONTEND Frontend, - OUT PXENVIF_POLLER *Poller - ) -{ - LONG MaxQueues; - LONG Index; - NTSTATUS status; - - *Poller = __PollerAllocate(sizeof (XENVIF_POLLER)); - - status = STATUS_NO_MEMORY; - if (*Poller == NULL) - goto fail1; - - FdoGetEvtchnInterface(PdoGetFdo(FrontendGetPdo(Frontend)), - &(*Poller)->EvtchnInterface); - - FdoGetStoreInterface(PdoGetFdo(FrontendGetPdo(Frontend)), - &(*Poller)->StoreInterface); - - FdoGetDebugInterface(PdoGetFdo(FrontendGetPdo(Frontend)), - &(*Poller)->DebugInterface); - - (*Poller)->Frontend = Frontend; - - MaxQueues = FrontendGetMaxQueues(Frontend); - (*Poller)->Instance = __PollerAllocate(sizeof (PXENVIF_POLLER_INSTANCE) * - MaxQueues); - - status = STATUS_NO_MEMORY; - if ((*Poller)->Instance == NULL) - goto fail2; - - for (Index = 0; Index < MaxQueues; Index++) { - PXENVIF_POLLER_INSTANCE Instance; - - status = PollerInstanceInitialize(*Poller, Index, &Instance); - if (!NT_SUCCESS(status)) - goto fail3; - - (*Poller)->Instance[Index] = Instance; - } - - return STATUS_SUCCESS; - -fail3: - Error("fail3\n"); - - while (--Index >= 0) - { - PXENVIF_POLLER_INSTANCE Instance = (*Poller)->Instance[Index]; - - (*Poller)->Instance[Index] = NULL; - PollerInstanceTeardown(Instance); - } - - ASSERT(IsZeroMemory((*Poller)->Instance, - sizeof (PXENVIF_POLLER_INSTANCE) * MaxQueues)); - __PollerFree((*Poller)->Instance); - (*Poller)->Instance = NULL; - -fail2: - Error("fail2\n"); - - (*Poller)->Frontend = NULL; - - RtlZeroMemory(&(*Poller)->DebugInterface, - sizeof (XENBUS_DEBUG_INTERFACE)); - - RtlZeroMemory(&(*Poller)->StoreInterface, - sizeof (XENBUS_STORE_INTERFACE)); - - RtlZeroMemory(&(*Poller)->EvtchnInterface, - sizeof (XENBUS_EVTCHN_INTERFACE)); - -fail1: - Error("fail1 (%08x)\n", status); - - return status; -} - -NTSTATUS -PollerConnect( - IN PXENVIF_POLLER Poller - ) -{ - PXENVIF_FRONTEND Frontend; - LONG NumQueues; - LONG Index; - NTSTATUS status; - - Trace("====>\n"); - - Frontend = Poller->Frontend; - - status = XENBUS_EVTCHN(Acquire, &Poller->EvtchnInterface); - if (!NT_SUCCESS(status)) - goto fail1; - - status = XENBUS_STORE(Acquire, &Poller->StoreInterface); - if (!NT_SUCCESS(status)) - goto fail2; - - status = XENBUS_DEBUG(Acquire, &Poller->DebugInterface); - if (!NT_SUCCESS(status)) - goto fail3; - - PollerSetSplit(Poller); - - NumQueues = FrontendGetNumQueues(Frontend); - - for (Index = 0; Index < NumQueues; Index++) { - PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index]; - - status = PollerInstanceConnect(Instance); - if (!NT_SUCCESS(status)) - goto fail4; - } - - status = XENBUS_DEBUG(Register, - &Poller->DebugInterface, - __MODULE__ "|POLLER", - PollerDebugCallback, - Poller, - &Poller->DebugCallback); - if (!NT_SUCCESS(status)) - goto fail5; - - Trace("<====\n"); - return STATUS_SUCCESS; - -fail5: - Error("fail5\n"); - - Index = NumQueues; - -fail4: - Error("fail4\n"); - - while (--Index >= 0) - { - PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index]; - - PollerInstanceDisconnect(Instance); - } - - Poller->Split = FALSE; - - XENBUS_DEBUG(Release, &Poller->DebugInterface); - -fail3: - Error("fail3\n"); - - XENBUS_STORE(Release, &Poller->StoreInterface); - -fail2: - Error("fail2\n"); - - XENBUS_EVTCHN(Release, &Poller->EvtchnInterface); - -fail1: - Error("fail1 (%08x)\n", status); - - return status; -} - -NTSTATUS -PollerStoreWrite( - IN PXENVIF_POLLER Poller, - IN PXENBUS_STORE_TRANSACTION Transaction - ) -{ - PXENVIF_FRONTEND Frontend; - LONG NumQueues; - LONG Index; - NTSTATUS status; - - Trace("====>\n"); - - Frontend = Poller->Frontend; - - NumQueues = FrontendGetNumQueues(Frontend); - - for (Index = 0; Index < NumQueues; Index++) { - PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index]; - - status = PollerInstanceStoreWrite(Instance, Transaction); - if (!NT_SUCCESS(status)) - goto fail1; - } - - Trace("<====\n"); - - return STATUS_SUCCESS; - -fail1: - Error("fail1 (%08x)\n", status); - - return status; -} - -NTSTATUS -PollerEnable( - IN PXENVIF_POLLER Poller - ) -{ - PXENVIF_FRONTEND Frontend; - LONG NumQueues; - LONG Index; - NTSTATUS status; - - Trace("====>\n"); - - Frontend = Poller->Frontend; - - NumQueues = FrontendGetNumQueues(Frontend); - - for (Index = 0; Index < NumQueues; Index++) { - PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index]; - - status = PollerInstanceEnable(Instance); - if (!NT_SUCCESS(status)) - goto fail1; - } - - Trace("<====\n"); - - return STATUS_SUCCESS; - -fail1: - Error("fail1 (%08x)\n", status); - - while (--Index >= 0) - { - PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index]; - - PollerInstanceDisable(Instance); - } - - return status; -} - -NTSTATUS -PollerSend( - IN PXENVIF_POLLER Poller, - IN ULONG Index, - IN XENVIF_POLLER_EVENT_TYPE Event - ) -{ - PXENVIF_FRONTEND Frontend; - ULONG NumQueues; - PXENVIF_POLLER_INSTANCE Instance; - NTSTATUS status; - - Frontend = Poller->Frontend; - - NumQueues = FrontendGetNumQueues(Frontend); - - status = STATUS_INVALID_PARAMETER; - if (Index >= NumQueues) - goto fail1; - - Instance = Poller->Instance[Index]; - - status = PollerInstanceSend(Instance, Event); - if (!NT_SUCCESS(status)) - goto fail2; - - return STATUS_SUCCESS; - -fail2: - Error("fail2\n"); - -fail1: - Error("fail1 (%08x)\n", status); - - return status; -} - -NTSTATUS -PollerTrigger( - IN PXENVIF_POLLER Poller, - IN ULONG Index, - IN XENVIF_POLLER_EVENT_TYPE Event - ) -{ - PXENVIF_FRONTEND Frontend; - ULONG NumQueues; - PXENVIF_POLLER_INSTANCE Instance; - NTSTATUS status; - - Frontend = Poller->Frontend; - - NumQueues = FrontendGetNumQueues(Frontend); - - status = STATUS_INVALID_PARAMETER; - if (Index >= NumQueues) - goto fail1; - - Instance = Poller->Instance[Index]; - - status = PollerInstanceTrigger(Instance, Event); - if (!NT_SUCCESS(status)) - goto fail2; - - return STATUS_SUCCESS; - -fail2: - Error("fail2\n"); - -fail1: - Error("fail1 (%08x)\n", status); - - return status; -} - -VOID -PollerDisable( - IN PXENVIF_POLLER Poller - ) -{ - PXENVIF_FRONTEND Frontend; - LONG NumQueues; - LONG Index; - - Trace("====>\n"); - - Frontend = Poller->Frontend; - - NumQueues = FrontendGetNumQueues(Frontend); - Index = NumQueues; - - while (--Index >= 0) - { - PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index]; - - PollerInstanceDisable(Instance); - } - - Trace("<====\n"); -} - -VOID -PollerDisconnect( - IN PXENVIF_POLLER Poller - ) -{ - PXENVIF_FRONTEND Frontend; - LONG NumQueues; - LONG Index; - - Trace("====>\n"); - - Frontend = Poller->Frontend; - - XENBUS_DEBUG(Deregister, - &Poller->DebugInterface, - Poller->DebugCallback); - Poller->DebugCallback = NULL; - - NumQueues = FrontendGetNumQueues(Frontend); - Index = NumQueues; - - while (--Index >= 0) - { - PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index]; - - PollerInstanceDisconnect(Instance); - } - - Poller->Split = FALSE; - - XENBUS_DEBUG(Release, &Poller->DebugInterface); - - XENBUS_STORE(Release, &Poller->StoreInterface); - - XENBUS_EVTCHN(Release, &Poller->EvtchnInterface); - - Trace("<====\n"); -} - -VOID -PollerTeardown( - IN PXENVIF_POLLER Poller - ) -{ - PXENVIF_FRONTEND Frontend; - LONG MaxQueues; - LONG Index; - - ASSERT3U(KeGetCurrentIrql(), ==, PASSIVE_LEVEL); - - Frontend = Poller->Frontend; - - MaxQueues = FrontendGetMaxQueues(Frontend); - Index = MaxQueues; - - while (--Index >= 0) - { - PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index]; - - Poller->Instance[Index] = NULL; - PollerInstanceTeardown(Instance); - } - - ASSERT(IsZeroMemory(Poller->Instance, - sizeof (PXENVIF_POLLER_INSTANCE) * MaxQueues)); - __PollerFree(Poller->Instance); - Poller->Instance = NULL; - - Poller->Frontend = NULL; - - RtlZeroMemory(&Poller->DebugInterface, - sizeof (XENBUS_DEBUG_INTERFACE)); - - RtlZeroMemory(&Poller->StoreInterface, - sizeof (XENBUS_STORE_INTERFACE)); - - RtlZeroMemory(&Poller->EvtchnInterface, - sizeof (XENBUS_EVTCHN_INTERFACE)); - - ASSERT(IsZeroMemory(Poller, sizeof (XENVIF_POLLER))); - __PollerFree(Poller); -} diff --git a/src/xenvif/poller.h b/src/xenvif/poller.h deleted file mode 100644 index a2b32a1..0000000 --- a/src/xenvif/poller.h +++ /dev/null @@ -1,100 +0,0 @@ -/* Copyright (c) Citrix Systems Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted provided - * that the following conditions are met: - * - * * Redistributions of source code must retain the above - * copyright notice, this list of conditions and the - * following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND - * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#ifndef _XENVIF_POLLER_H -#define _XENVIF_POLLER_H - -#include <ntddk.h> - -#include <vif_interface.h> - -#include "frontend.h" - -typedef struct _XENVIF_POLLER XENVIF_POLLER, *PXENVIF_POLLER; - -typedef enum _XENVIF_POLLER_EVENT_TYPE { - XENVIF_POLLER_EVENT_RECEIVE, - XENVIF_POLLER_EVENT_TRANSMIT, - XENVIF_POLLER_EVENT_TYPE_COUNT -} XENVIF_POLLER_EVENT_TYPE, *PXENVIF_POLLER_EVENT_TYPE; - -extern NTSTATUS -PollerInitialize( - IN PXENVIF_FRONTEND Frontend, - OUT PXENVIF_POLLER *Poller - ); - -extern NTSTATUS -PollerConnect( - IN PXENVIF_POLLER Poller - ); - -extern NTSTATUS -PollerStoreWrite( - IN PXENVIF_POLLER Poller, - IN PXENBUS_STORE_TRANSACTION Transaction - ); - -extern NTSTATUS -PollerEnable( - IN PXENVIF_POLLER Poller - ); - -extern NTSTATUS -PollerSend( - IN PXENVIF_POLLER Poller, - IN ULONG Index, - IN XENVIF_POLLER_EVENT_TYPE Event - ); - -extern NTSTATUS -PollerTrigger( - IN PXENVIF_POLLER Poller, - IN ULONG Index, - IN XENVIF_POLLER_EVENT_TYPE Event - ); - -extern VOID -PollerDisable( - IN PXENVIF_POLLER Poller - ); - -extern VOID -PollerDisconnect( - IN PXENVIF_POLLER Poller - ); - -extern VOID -PollerTeardown( - IN PXENVIF_POLLER Poller - ); - -#endif // _XENVIF_POLLER_H diff --git a/src/xenvif/receiver.c b/src/xenvif/receiver.c index 4e8046b..3ed5ee8 100644 --- a/src/xenvif/receiver.c +++ b/src/xenvif/receiver.c @@ -41,6 +41,7 @@ #include <store_interface.h> #include <cache_interface.h> #include <gnttab_interface.h> +#include <evtchn_interface.h> #include "pdo.h" #include "registry.h" @@ -87,6 +88,12 @@ typedef struct _XENVIF_RECEIVER_RING { netif_rx_front_ring_t Front; netif_rx_sring_t *Shared; PXENBUS_GNTTAB_ENTRY Entry; + PXENBUS_EVTCHN_CHANNEL Channel; + KDPC Dpc; + ULONG Dpcs; + KTIMER Timer; + KDPC TimerDpc; + ULONG Events; PXENVIF_RECEIVER_FRAGMENT Pending[XENVIF_RECEIVER_MAXIMUM_FRAGMENT_ID + 1]; ULONG RequestsPosted; ULONG RequestsPushed; @@ -98,10 +105,7 @@ typedef struct _XENVIF_RECEIVER_RING { ULONG BackfillSize; PXENBUS_DEBUG_CALLBACK DebugCallback; PXENVIF_THREAD WatchdogThread; - PLIST_ENTRY PacketQueue; - KDPC Dpc; - ULONG Dpcs; - LIST_ENTRY PacketComplete; + LIST_ENTRY PacketList; XENVIF_RECEIVER_HASH Hash; } XENVIF_RECEIVER_RING, *PXENVIF_RECEIVER_RING; @@ -124,6 +128,7 @@ struct _XENVIF_RECEIVER { PXENVIF_FRONTEND Frontend; XENBUS_CACHE_INTERFACE CacheInterface; XENBUS_GNTTAB_INTERFACE GnttabInterface; + XENBUS_EVTCHN_INTERFACE EvtchnInterface; PXENVIF_RECEIVER_RING *Ring; LONG Loaned; LONG Returned; @@ -908,22 +913,10 @@ fail1: } static VOID -ReceiverRingCompletePacket( - IN PXENVIF_RECEIVER_RING Ring, - IN PXENVIF_RECEIVER_PACKET Packet - ) -{ - ReceiverRingProcessTag(Ring, Packet); - ReceiverRingProcessChecksum(Ring, Packet); - - ASSERT(IsZeroMemory(&Packet->ListEntry, sizeof (LIST_ENTRY))); - InsertTailList(&Ring->PacketComplete, &Packet->ListEntry); -} - -static VOID ReceiverRingProcessLargePacket( IN PXENVIF_RECEIVER_RING Ring, - IN PXENVIF_RECEIVER_PACKET Packet + IN PXENVIF_RECEIVER_PACKET Packet, + OUT PLIST_ENTRY List ) { PXENVIF_RECEIVER Receiver; @@ -1011,7 +1004,8 @@ ReceiverRingProcessLargePacket( ASSERT3U(Length, >=, SegmentSize); Length -= SegmentSize; - ReceiverRingCompletePacket(Ring, Segment); + ASSERT(IsZeroMemory(&Segment->ListEntry, sizeof (LIST_ENTRY))); + InsertTailList(List, &Segment->ListEntry); if (Offload) { ASSERT(Ring->OffloadOptions.NeedLargePacketSplit != 0); @@ -1060,7 +1054,8 @@ ReceiverRingProcessLargePacket( if (Receiver->AlwaysPullup != 0) __ReceiverRingPullupPacket(Ring, Packet); - ReceiverRingCompletePacket(Ring, Packet); + ASSERT(IsZeroMemory(&Packet->ListEntry, sizeof (LIST_ENTRY))); + InsertTailList(List, &Packet->ListEntry); } else { __ReceiverRingPutPacket(Ring, Packet, TRUE); } @@ -1097,7 +1092,8 @@ fail1: static VOID ReceiverRingProcessStandardPacket( IN PXENVIF_RECEIVER_RING Ring, - IN PXENVIF_RECEIVER_PACKET Packet + IN PXENVIF_RECEIVER_PACKET Packet, + OUT PLIST_ENTRY List ) { PXENVIF_RECEIVER Receiver; @@ -1167,7 +1163,9 @@ ReceiverRingProcessStandardPacket( Packet->Mdl.Next = Mdl; } - ReceiverRingCompletePacket(Ring, Packet); + ASSERT(IsZeroMemory(&Packet->ListEntry, sizeof (LIST_ENTRY))); + InsertTailList(List, &Packet->ListEntry); + return; fail2: @@ -1200,7 +1198,8 @@ fail1: static VOID ReceiverRingProcessPacket( IN PXENVIF_RECEIVER_RING Ring, - IN PXENVIF_RECEIVER_PACKET Packet + IN PXENVIF_RECEIVER_PACKET Packet, + OUT PLIST_ENTRY List ) { PXENVIF_RECEIVER Receiver; @@ -1286,9 +1285,9 @@ ReceiverRingProcessPacket( goto fail3; if (Packet->MaximumSegmentSize != 0) - ReceiverRingProcessLargePacket(Ring, Packet); + ReceiverRingProcessLargePacket(Ring, Packet, List); else - ReceiverRingProcessStandardPacket(Ring, Packet); + ReceiverRingProcessStandardPacket(Ring, Packet, List); return; @@ -1321,8 +1320,63 @@ fail1: 1); } +static VOID +ReceiverRingProcessPackets( + IN PXENVIF_RECEIVER_RING Ring, + OUT PLIST_ENTRY List, + OUT PULONG Count + ) +{ + PLIST_ENTRY ListEntry; + + while (!IsListEmpty(&Ring->PacketList)) { + PXENVIF_RECEIVER_PACKET Packet; + + ListEntry = RemoveHeadList(&Ring->PacketList); + ASSERT3P(ListEntry, !=, &Ring->PacketList); + + RtlZeroMemory(ListEntry, sizeof (LIST_ENTRY)); + + Packet = CONTAINING_RECORD(ListEntry, XENVIF_RECEIVER_PACKET, ListEntry); + ReceiverRingProcessPacket(Ring, Packet, List); + } + + for (ListEntry = List->Flink; + ListEntry != List; + ListEntry = ListEntry->Flink) { + PXENVIF_RECEIVER_PACKET Packet; + + Packet = CONTAINING_RECORD(ListEntry, XENVIF_RECEIVER_PACKET, ListEntry); + + ReceiverRingProcessTag(Ring, Packet); + ReceiverRingProcessChecksum(Ring, Packet); + + (*Count)++; + } +} + +static FORCEINLINE VOID +__drv_requiresIRQL(DISPATCH_LEVEL) +__ReceiverRingAcquireLock( + IN PXENVIF_RECEIVER_RING Ring + ) +{ + ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL); + + KeAcquireSpinLockAtDpcLevel(&Ring->Lock); +} + +static DECLSPEC_NOINLINE VOID +ReceiverRingAcquireLock( + IN PXENVIF_RECEIVER_RING Ring + ) +{ + __ReceiverRingAcquireLock(Ring); +} + static FORCEINLINE VOID -__ReceiverRingSwizzle( +__drv_requiresIRQL(DISPATCH_LEVEL) +__ReceiverRingReleaseLock( IN PXENVIF_RECEIVER_RING Ring ) { @@ -1330,44 +1384,33 @@ __ReceiverRingSwizzle( PXENVIF_FRONTEND Frontend; PXENVIF_VIF_CONTEXT Context; LIST_ENTRY List; - PLIST_ENTRY ListEntry; + ULONG Count; + BOOLEAN More; + + ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL); Receiver = Ring->Receiver; Frontend = Receiver->Frontend; Context = PdoGetVifContext(FrontendGetPdo(Frontend)); InitializeListHead(&List); + Count = 0; - ListEntry = InterlockedExchangePointer(&Ring->PacketQueue, NULL); - - // Packets are held in the queue in reverse order so that the most - // recent is always head of the list. This is necessary to allow - // addition to the list to be done atomically. - - while (ListEntry != NULL) { - PLIST_ENTRY NextEntry; - - NextEntry = ListEntry->Blink; - ListEntry->Flink = ListEntry->Blink = ListEntry; + ReceiverRingProcessPackets(Ring, &List, &Count); + ASSERT(EQUIV(IsListEmpty(&List), Count == 0)); + ASSERT(IsListEmpty(&Ring->PacketList)); - InsertHeadList(&List, ListEntry); + // We need to bump Loaned before dropping the lock to avoid VifDisable() + // returning prematurely. + __InterlockedAdd(&Receiver->Loaned, Count); - ListEntry = NextEntry; - } - - while (!IsListEmpty(&List)) { - PXENVIF_RECEIVER_PACKET Packet; - - ListEntry = RemoveHeadList(&List); - ASSERT3P(ListEntry, !=, &List); +#pragma prefast(disable:26110) + KeReleaseSpinLockFromDpcLevel(&Ring->Lock); - RtlZeroMemory(ListEntry, sizeof (LIST_ENTRY)); + More = !IsListEmpty(&List) ? TRUE : FALSE; - Packet = CONTAINING_RECORD(ListEntry, XENVIF_RECEIVER_PACKET, ListEntry); - ReceiverRingProcessPacket(Ring, Packet); - } - - while (!IsListEmpty(&Ring->PacketComplete)) { + while (More) { + PLIST_ENTRY ListEntry; PXENVIF_RECEIVER_PACKET Packet; PXENVIF_PACKET_INFO Info; PUCHAR BaseVa; @@ -1375,11 +1418,14 @@ __ReceiverRingSwizzle( PETHERNET_ADDRESS DestinationAddress; ETHERNET_ADDRESS_TYPE Type; - ListEntry = RemoveHeadList(&Ring->PacketComplete); - ASSERT3P(ListEntry, !=, &Ring->PacketComplete); + ListEntry = RemoveHeadList(&List); + ASSERT3P(ListEntry, !=, &List); RtlZeroMemory(ListEntry, sizeof (LIST_ENTRY)); + ASSERT(More); + More = !IsListEmpty(&List) ? TRUE : FALSE; + Packet = CONTAINING_RECORD(ListEntry, XENVIF_RECEIVER_PACKET, ListEntry); @@ -1468,57 +1514,55 @@ __ReceiverRingSwizzle( XENVIF_RECEIVER_UDP_PACKETS, 1); - if (Packet->MaximumSegmentSize != 0) + if (Packet->MaximumSegmentSize != 0) FrontendIncrementStatistic(Frontend, XENVIF_RECEIVER_GSO_PACKETS, 1); - if (Packet->Flags.IpChecksumSucceeded != 0) - FrontendIncrementStatistic(Frontend, - XENVIF_RECEIVER_IPV4_CHECKSUM_SUCCEEDED, - 1); - - if (Packet->Flags.IpChecksumFailed != 0) - FrontendIncrementStatistic(Frontend, - XENVIF_RECEIVER_IPV4_CHECKSUM_FAILED, - 1); - - if (Packet->Flags.IpChecksumNotValidated != 0) - FrontendIncrementStatistic(Frontend, - XENVIF_RECEIVER_IPV4_CHECKSUM_NOT_VALIDATED, - 1); - - if (Packet->Flags.TcpChecksumSucceeded != 0) - FrontendIncrementStatistic(Frontend, - XENVIF_RECEIVER_TCP_CHECKSUM_SUCCEEDED, - 1); - - if (Packet->Flags.TcpChecksumFailed != 0) - FrontendIncrementStatistic(Frontend, - XENVIF_RECEIVER_TCP_CHECKSUM_FAILED, - 1); - - if (Packet->Flags.TcpChecksumNotValidated != 0) - FrontendIncrementStatistic(Frontend, - XENVIF_RECEIVER_TCP_CHECKSUM_NOT_VALIDATED, - 1); - - if (Packet->Flags.UdpChecksumSucceeded != 0) - FrontendIncrementStatistic(Frontend, - XENVIF_RECEIVER_UDP_CHECKSUM_SUCCEEDED, - 1); - - if (Packet->Flags.UdpChecksumFailed != 0) - FrontendIncrementStatistic(Frontend, - XENVIF_RECEIVER_UDP_CHECKSUM_FAILED, - 1); - - if (Packet->Flags.UdpChecksumNotValidated != 0) - FrontendIncrementStatistic(Frontend, - XENVIF_RECEIVER_UDP_CHECKSUM_NOT_VALIDATED, - 1); - - (VOID) InterlockedIncrement(&Receiver->Loaned); + if (Packet->Flags.IpChecksumSucceeded != 0) + FrontendIncrementStatistic(Frontend, + XENVIF_RECEIVER_IPV4_CHECKSUM_SUCCEEDED, + 1); + + if (Packet->Flags.IpChecksumFailed != 0) + FrontendIncrementStatistic(Frontend, + XENVIF_RECEIVER_IPV4_CHECKSUM_FAILED, + 1); + + if (Packet->Flags.IpChecksumNotValidated != 0) + FrontendIncrementStatistic(Frontend, + XENVIF_RECEIVER_IPV4_CHECKSUM_NOT_VALIDATED, + 1); + + if (Packet->Flags.TcpChecksumSucceeded != 0) + FrontendIncrementStatistic(Frontend, + XENVIF_RECEIVER_TCP_CHECKSUM_SUCCEEDED, + 1); + + if (Packet->Flags.TcpChecksumFailed != 0) + FrontendIncrementStatistic(Frontend, + XENVIF_RECEIVER_TCP_CHECKSUM_FAILED, + 1); + + if (Packet->Flags.TcpChecksumNotValidated != 0) + FrontendIncrementStatistic(Frontend, + XENVIF_RECEIVER_TCP_CHECKSUM_NOT_VALIDATED, + 1); + + if (Packet->Flags.UdpChecksumSucceeded != 0) + FrontendIncrementStatistic(Frontend, + XENVIF_RECEIVER_UDP_CHECKSUM_SUCCEEDED, + 1); + + if (Packet->Flags.UdpChecksumFailed != 0) + FrontendIncrementStatistic(Frontend, + XENVIF_RECEIVER_UDP_CHECKSUM_FAILED, + 1); + + if (Packet->Flags.UdpChecksumNotValidated != 0) + FrontendIncrementStatistic(Frontend, + XENVIF_RECEIVER_UDP_CHECKSUM_NOT_VALIDATED, + 1); VifReceiverQueuePacket(Context, Ring->Index, @@ -1530,95 +1574,89 @@ __ReceiverRingSwizzle( Packet->TagControlInformation, &Packet->Info, &Packet->Hash, - !IsListEmpty(&Ring->PacketComplete) ? TRUE : FALSE, + More, Packet); + + --Count; } + + ASSERT3U(Count, ==, 0); } -static FORCEINLINE VOID -__drv_requiresIRQL(DISPATCH_LEVEL) -__ReceiverRingAcquireLock( +static DECLSPEC_NOINLINE VOID +ReceiverRingReleaseLock( IN PXENVIF_RECEIVER_RING Ring ) { - ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL); - - KeAcquireSpinLockAtDpcLevel(&Ring->Lock); + __ReceiverRingReleaseLock(Ring); } -static DECLSPEC_NOINLINE VOID -ReceiverRingAcquireLock( +static FORCEINLINE VOID +__ReceiverRingStop( IN PXENVIF_RECEIVER_RING Ring ) { - __ReceiverRingAcquireLock(Ring); + Ring->Stopped = TRUE; } static FORCEINLINE VOID -__drv_requiresIRQL(DISPATCH_LEVEL) -__ReceiverRingReleaseLock( +__ReceiverRingStart( IN PXENVIF_RECEIVER_RING Ring ) { - ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL); - -#pragma prefast(disable:26110) - KeReleaseSpinLockFromDpcLevel(&Ring->Lock); + Ring->Stopped = FALSE; } -static DECLSPEC_NOINLINE VOID -ReceiverRingReleaseLock( +static FORCEINLINE BOOLEAN +__ReceiverRingIsStopped( IN PXENVIF_RECEIVER_RING Ring ) { - __ReceiverRingReleaseLock(Ring); + return Ring->Stopped; } -__drv_functionClass(KDEFERRED_ROUTINE) -__drv_maxIRQL(DISPATCH_LEVEL) -__drv_minIRQL(PASSIVE_LEVEL) -__drv_sameIRQL -static VOID -ReceiverRingDpc( - IN PKDPC Dpc, - IN PVOID Context, - IN PVOID Argument1, - IN PVOID Argument2 +static FORCEINLINE VOID +__ReceiverRingTrigger( + IN PXENVIF_RECEIVER_RING Ring, + IN BOOLEAN Locked ) { - PXENVIF_RECEIVER_RING Ring = Context; + PXENVIF_RECEIVER Receiver; - UNREFERENCED_PARAMETER(Dpc); - UNREFERENCED_PARAMETER(Argument1); - UNREFERENCED_PARAMETER(Argument2); + Receiver = Ring->Receiver; - ASSERT(Ring != NULL); + if (!Locked) + __ReceiverRingAcquireLock(Ring); - __ReceiverRingSwizzle(Ring); -} + if (Ring->Connected) + (VOID) XENBUS_EVTCHN(Trigger, + &Receiver->EvtchnInterface, + Ring->Channel); -static FORCEINLINE VOID -__ReceiverRingStop( - IN PXENVIF_RECEIVER_RING Ring - ) -{ - Ring->Stopped = TRUE; + if (!Locked) + __ReceiverRingReleaseLock(Ring); } static FORCEINLINE VOID -__ReceiverRingStart( - IN PXENVIF_RECEIVER_RING Ring +__ReceiverRingSend( + IN PXENVIF_RECEIVER_RING Ring, + IN BOOLEAN Locked ) { - Ring->Stopped = FALSE; -} + PXENVIF_RECEIVER Receiver; -static FORCEINLINE BOOLEAN -__ReceiverRingIsStopped( - IN PXENVIF_RECEIVER_RING Ring - ) -{ - return Ring->Stopped; + Receiver = Ring->Receiver; + + if (!Locked) + __ReceiverRingAcquireLock(Ring); + + if (Ring->Connected) + (VOID) XENBUS_EVTCHN(Send, + &Receiver->EvtchnInterface, + Ring->Channel); + + if (!Locked) + __ReceiverRingReleaseLock(Ring); } static FORCEINLINE VOID @@ -1628,13 +1666,8 @@ __ReceiverRingReturnPacket( IN BOOLEAN Locked ) { - PXENVIF_RECEIVER Receiver; - PXENVIF_FRONTEND Frontend; PMDL Mdl; - Receiver = Ring->Receiver; - Frontend = Receiver->Frontend; - Mdl = &Packet->Mdl; while (Mdl != NULL) { @@ -1658,9 +1691,7 @@ __ReceiverRingReturnPacket( if (__ReceiverRingIsStopped(Ring)) { __ReceiverRingStart(Ring); - PollerTrigger(FrontendGetPoller(Frontend), - Ring->Index, - XENVIF_POLLER_EVENT_RECEIVE); + __ReceiverRingTrigger(Ring, TRUE); } if (!Locked) @@ -1740,17 +1771,8 @@ __ReceiverRingPushRequests( #pragma warning (pop) - if (Notify) { - PXENVIF_RECEIVER Receiver; - PXENVIF_FRONTEND Frontend; - - Receiver = Ring->Receiver; - Frontend = Receiver->Frontend; - - PollerSend(FrontendGetPoller(Frontend), - Ring->Index, - XENVIF_POLLER_EVENT_RECEIVE); - } + if (Notify) + __ReceiverRingSend(Ring, TRUE); Ring->RequestsPushed = Ring->RequestsPosted; } @@ -1882,11 +1904,6 @@ ReceiverRingDebugCallback( (Ring->Enabled) ? "ENABLED" : "DISABLED", (__ReceiverRingIsStopped(Ring)) ? "STOPPED" : "RUNNING"); - XENBUS_DEBUG(Printf, - &Receiver->DebugInterface, - "Dpcs = %lu\n", - Ring->Dpcs); - // Dump front ring XENBUS_DEBUG(Printf, &Receiver->DebugInterface, @@ -1911,26 +1928,14 @@ ReceiverRingDebugCallback( Ring->RequestsPosted, Ring->RequestsPushed, Ring->ResponsesProcessed); -} - -static FORCEINLINE VOID -__ReceiverRingQueuePacket( - IN PXENVIF_RECEIVER_RING Ring, - IN PXENVIF_RECEIVER_PACKET Packet - ) -{ - PLIST_ENTRY ListEntry; - PLIST_ENTRY Old; - PLIST_ENTRY New; - - ListEntry = &Packet->ListEntry; - - do { - Old = Ring->PacketQueue; - ListEntry->Blink = Ring->PacketQueue; - New = ListEntry; - } while (InterlockedCompareExchangePointer(&Ring->PacketQueue, (PVOID)New, (PVOID)Old) != Old); + // Dump event channel + XENBUS_DEBUG(Printf, + &Receiver->DebugInterface, + "[%s]: Events = %lu Dpcs = %lu\n", + FrontendIsSplit(Frontend) ? "RX" : "COMBINED", + Ring->Events, + Ring->Dpcs); } static DECLSPEC_NOINLINE BOOLEAN @@ -1988,8 +1993,10 @@ ReceiverRingPoll( RING_IDX WorkToDo; RING_FINAL_CHECK_FOR_RESPONSES(&Ring->Front, WorkToDo); - if (!WorkToDo) - break; + if (WorkToDo) + continue; + + break; } while (rsp_cons != rsp_prod && !Retry) { @@ -2146,7 +2153,7 @@ ReceiverRingPoll( Packet->Flags.Value = flags; ASSERT(IsZeroMemory(&Packet->ListEntry, sizeof (LIST_ENTRY))); - __ReceiverRingQueuePacket(Ring, Packet); + InsertTailList(&Ring->PacketList, &Packet->ListEntry); } if (rsp_cons - Ring->Front.rsp_cons > XENVIF_RECEIVER_BATCH(Ring)) @@ -2179,21 +2186,137 @@ ReceiverRingPoll( if (!__ReceiverRingIsStopped(Ring)) ReceiverRingFill(Ring); - if (Ring->PacketQueue != NULL && - KeInsertQueueDpc(&Ring->Dpc, NULL, NULL)) - Ring->Dpcs++; - done: return Retry; #undef XENVIF_RECEIVER_BATCH } +static FORCEINLINE VOID +__ReceiverRingUnmask( + IN PXENVIF_RECEIVER_RING Ring + ) +{ + PXENVIF_RECEIVER Receiver; + + if (!Ring->Connected) + return; + + Receiver = Ring->Receiver; + + XENBUS_EVTCHN(Unmask, + &Receiver->EvtchnInterface, + Ring->Channel, + FALSE, + TRUE); +} + +static FORCEINLINE BOOLEAN +__ReceiverRingDpcTimeout( + IN PXENVIF_RECEIVER_RING Ring + ) +{ + KDPC_WATCHDOG_INFORMATION Watchdog; + NTSTATUS status; + + UNREFERENCED_PARAMETER(Ring); + + RtlZeroMemory(&Watchdog, sizeof (Watchdog)); + + status = KeQueryDpcWatchdogInformation(&Watchdog); + ASSERT(NT_SUCCESS(status)); + + if (Watchdog.DpcTimeLimit == 0 || + Watchdog.DpcWatchdogLimit == 0) + return FALSE; + + if (Watchdog.DpcTimeCount > (Watchdog.DpcTimeLimit / 2) && + Watchdog.DpcWatchdogCount > (Watchdog.DpcWatchdogLimit / 2)) + return FALSE; + + return TRUE; +} + #define TIME_US(_us) ((_us) * 10) #define TIME_MS(_ms) (TIME_US((_ms) * 1000)) #define TIME_S(_s) (TIME_MS((_s) * 1000)) #define TIME_RELATIVE(_t) (-(_t)) +__drv_functionClass(KDEFERRED_ROUTINE) +__drv_maxIRQL(DISPATCH_LEVEL) +__drv_minIRQL(DISPATCH_LEVEL) +__drv_requiresIRQL(DISPATCH_LEVEL) +__drv_sameIRQL +static VOID +ReceiverRingDpc( + IN PKDPC Dpc, + IN PVOID Context, + IN PVOID Argument1, + IN PVOID Argument2 + ) +{ + PXENVIF_RECEIVER_RING Ring = Context; + + UNREFERENCED_PARAMETER(Dpc); + UNREFERENCED_PARAMETER(Argument1); + UNREFERENCED_PARAMETER(Argument2); + + ASSERT(Ring != NULL); + + for (;;) { + BOOLEAN Retry; + + __ReceiverRingAcquireLock(Ring); + Retry = ReceiverRingPoll(Ring); + __ReceiverRingReleaseLock(Ring); + + if (!Retry) { + __ReceiverRingUnmask(Ring); + break; + } + + if (__ReceiverRingDpcTimeout(Ring)) { + LARGE_INTEGER Delay; + + Delay.QuadPart = TIME_RELATIVE(TIME_US(100)); + + KeSetTimer(&Ring->Timer, Delay, &Ring->TimerDpc); + break; + } + } +} + +KSERVICE_ROUTINE ReceiverRingEvtchnCallback; + +BOOLEAN +ReceiverRingEvtchnCallback( + IN PKINTERRUPT InterruptObject, + IN PVOID Argument + ) +{ + PXENVIF_RECEIVER_RING Ring = Argument; + PXENVIF_RECEIVER Receiver; + PXENVIF_FRONTEND Frontend; + + UNREFERENCED_PARAMETER(InterruptObject); + + ASSERT(Ring != NULL); + + Ring->Events++; + + if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL)) + Ring->Dpcs++; + + Receiver = Ring->Receiver; + Frontend = Receiver->Frontend; + + if (!FrontendIsSplit(Frontend)) + TransmitterNotify(FrontendGetTransmitter(Frontend), + Ring->Index); + + return TRUE; +} + #define XENVIF_RECEIVER_WATCHDOG_PERIOD 30 static NTSTATUS @@ -2257,22 +2380,16 @@ ReceiverRingWatchdog( if (Ring->Shared->rsp_prod != rsp_prod && Ring->Front.rsp_cons == rsp_cons) { PXENVIF_RECEIVER Receiver; - PXENVIF_FRONTEND Frontend; Receiver = Ring->Receiver; - Frontend = Receiver->Frontend; XENBUS_DEBUG(Trigger, &Receiver->DebugInterface, Ring->DebugCallback); // Try to move things along - PollerTrigger(FrontendGetPoller(Frontend), - Ring->Index, - XENVIF_POLLER_EVENT_RECEIVE); - PollerSend(FrontendGetPoller(Frontend), - Ring->Index, - XENVIF_POLLER_EVENT_RECEIVE); + __ReceiverRingTrigger(Ring, TRUE); + __ReceiverRingSend(Ring, TRUE); } KeMemoryBarrier(); @@ -2318,7 +2435,11 @@ __ReceiverRingInitialize( if ((*Ring)->Path == NULL) goto fail2; - InitializeListHead(&(*Ring)->PacketComplete); + InitializeListHead(&(*Ring)->PacketList); + + KeInitializeDpc(&(*Ring)->Dpc, ReceiverRingDpc, *Ring); + KeInitializeTimer(&(*Ring)->Timer); + KeInitializeDpc(&(*Ring)->TimerDpc, ReceiverRingDpc, *Ring); status = RtlStringCbPrintfA(Name, sizeof (Name), @@ -2376,8 +2497,6 @@ __ReceiverRingInitialize( if (!NT_SUCCESS(status)) goto fail7; - KeInitializeThreadedDpc(&(*Ring)->Dpc, ReceiverRingDpc, *Ring); - return STATUS_SUCCESS; fail7: @@ -2405,7 +2524,11 @@ fail4: fail3: Error("fail3\n"); - RtlZeroMemory(&(*Ring)->PacketComplete, sizeof (LIST_ENTRY)); + RtlZeroMemory(&(*Ring)->TimerDpc, sizeof (KDPC)); + RtlZeroMemory(&(*Ring)->Timer, sizeof (KTIMER)); + RtlZeroMemory(&(*Ring)->Dpc, sizeof (KDPC)); + + RtlZeroMemory(&(*Ring)->PacketList, sizeof (LIST_ENTRY)); FrontendFreePath(Frontend, (*Ring)->Path); (*Ring)->Path = NULL; @@ -2502,6 +2625,36 @@ __ReceiverRingConnect( ASSERT(!Ring->Connected); + Ring->Channel = XENBUS_EVTCHN(Open, + &Receiver->EvtchnInterface, + XENBUS_EVTCHN_TYPE_UNBOUND, + ReceiverRingEvtchnCallback, + Ring, + FrontendGetBackendDomain(Frontend), + TRUE); + + status = STATUS_UNSUCCESSFUL; + if (Ring->Channel == NULL) + goto fail6; + + status = KeGetProcessorNumberFromIndex(Ring->Index, &ProcNumber); + ASSERT(NT_SUCCESS(status)); + + KeSetTargetProcessorDpcEx(&Ring->Dpc, &ProcNumber); + KeSetTargetProcessorDpcEx(&Ring->TimerDpc, &ProcNumber); + + (VOID) XENBUS_EVTCHN(Bind, + &Receiver->EvtchnInterface, + Ring->Channel, + ProcNumber.Group, + ProcNumber.Number); + + (VOID) XENBUS_EVTCHN(Unmask, + &Receiver->EvtchnInterface, + Ring->Channel, + FALSE, + TRUE); + Ring->Connected = TRUE; status = XENBUS_DEBUG(Register, @@ -2511,20 +2664,25 @@ __ReceiverRingConnect( Ring, &Ring->DebugCallback); if (!NT_SUCCESS(status)) - goto fail6; + goto fail7; - status = KeGetProcessorNumberFromIndex(Ring->Index, &ProcNumber); - ASSERT(NT_SUCCESS(status)); + return STATUS_SUCCESS; - KeSetTargetProcessorDpcEx(&Ring->Dpc, &ProcNumber); +fail7: + Error("fail7\n"); - return STATUS_SUCCESS; + Ring->Connected = FALSE; + + XENBUS_EVTCHN(Close, + &Receiver->EvtchnInterface, + Ring->Channel); + Ring->Channel = NULL; + + Ring->Events = 0; fail6: Error("fail6\n"); - Ring->Connected = FALSE; - fail5: Error("fail5\n"); @@ -2570,6 +2728,7 @@ __ReceiverRingStoreWrite( { PXENVIF_RECEIVER Receiver; PXENVIF_FRONTEND Frontend; + ULONG Port; PCHAR Path; NTSTATUS status; @@ -2592,8 +2751,25 @@ __ReceiverRingStoreWrite( if (!NT_SUCCESS(status)) goto fail1; + Port = XENBUS_EVTCHN(GetPort, + &Receiver->EvtchnInterface, + Ring->Channel); + + status = XENBUS_STORE(Printf, + &Receiver->StoreInterface, + Transaction, + Path, + FrontendIsSplit(Frontend) ? "event-channel-rx" : "event-channel", + "%u", + Port); + if (!NT_SUCCESS(status)) + goto fail2; + return STATUS_SUCCESS; +fail2: + Error("fail2\n"); + fail1: Error("fail1 (%08x)\n", status); @@ -2628,6 +2804,8 @@ __ReceiverRingEnable( Ring->Enabled = TRUE; + (VOID) KeInsertQueueDpc(&Ring->Dpc, NULL, NULL); + __ReceiverRingReleaseLock(Ring); Info("%s[%u]: <====\n", @@ -2666,11 +2844,14 @@ __ReceiverRingDisable( Ring->Enabled = FALSE; Ring->Stopped = FALSE; - if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL)) - Ring->Dpcs++; - __ReceiverRingReleaseLock(Ring); + // + // No new timers can be scheduled once Enabled goes to FALSE. + // Cancel any existing ones. + // + (VOID) KeCancelTimer(&Ring->Timer); + Info("%s[%u]: <====\n", FrontendGetPath(Frontend), Ring->Index); @@ -2687,13 +2868,19 @@ __ReceiverRingDisconnect( Receiver = Ring->Receiver; Frontend = Receiver->Frontend; - Ring->Dpcs = 0; - __ReceiverRingEmpty(Ring); ASSERT(Ring->Connected); Ring->Connected = FALSE; + XENBUS_EVTCHN(Close, + &Receiver->EvtchnInterface, + Ring->Channel); + Ring->Channel = NULL; + + Ring->Events = 0; + Ring->Dpcs = 0; + ASSERT3U(Ring->ResponsesProcessed, ==, Ring->RequestsPushed); ASSERT3U(Ring->RequestsPushed, ==, Ring->RequestsPosted); @@ -2738,13 +2925,13 @@ __ReceiverRingTeardown( Frontend = Receiver->Frontend; RtlZeroMemory(&Ring->Hash, sizeof (XENVIF_RECEIVER_HASH)); + RtlZeroMemory(&Ring->TimerDpc, sizeof (KDPC)); + RtlZeroMemory(&Ring->Timer, sizeof (KTIMER)); + RtlZeroMemory(&Ring->Dpc, sizeof (KDPC)); Ring->BackfillSize = 0; Ring->OffloadOptions.Value = 0; - KeFlushQueuedDpcs(); - RtlZeroMemory(&Ring->Dpc, sizeof (KDPC)); - ThreadAlert(Ring->WatchdogThread); ThreadJoin(Ring->WatchdogThread); Ring->WatchdogThread = NULL; @@ -2759,8 +2946,8 @@ __ReceiverRingTeardown( Ring->PacketCache); Ring->PacketCache = NULL; - ASSERT(IsListEmpty(&Ring->PacketComplete)); - RtlZeroMemory(&Ring->PacketComplete, sizeof (LIST_ENTRY)); + ASSERT(IsListEmpty(&Ring->PacketList)); + RtlZeroMemory(&Ring->PacketList, sizeof (LIST_ENTRY)); FrontendFreePath(Frontend, Ring->Path); Ring->Path = NULL; @@ -2910,6 +3097,9 @@ ReceiverInitialize( FdoGetGnttabInterface(PdoGetFdo(FrontendGetPdo(Frontend)), &(*Receiver)->GnttabInterface); + FdoGetEvtchnInterface(PdoGetFdo(FrontendGetPdo(Frontend)), + &(*Receiver)->EvtchnInterface); + (*Receiver)->Frontend = Frontend; status = XENBUS_CACHE(Acquire, &(*Receiver)->CacheInterface); @@ -2962,6 +3152,9 @@ fail2: (*Receiver)->Frontend = NULL; + RtlZeroMemory(&(*Receiver)->EvtchnInterface, + sizeof (XENBUS_EVTCHN_INTERFACE)); + RtlZeroMemory(&(*Receiver)->GnttabInterface, sizeof (XENBUS_GNTTAB_INTERFACE)); @@ -3013,17 +3206,21 @@ ReceiverConnect( if (!NT_SUCCESS(status)) goto fail2; - status = XENBUS_GNTTAB(Acquire, &Receiver->GnttabInterface); + status = XENBUS_EVTCHN(Acquire, &Receiver->EvtchnInterface); if (!NT_SUCCESS(status)) goto fail3; + status = XENBUS_GNTTAB(Acquire, &Receiver->GnttabInterface); + if (!NT_SUCCESS(status)) + goto fail4; + Index = 0; while (Index < (LONG)FrontendGetNumQueues(Frontend)) { PXENVIF_RECEIVER_RING Ring = Receiver->Ring[Index]; status = __ReceiverRingConnect(Ring); if (!NT_SUCCESS(status)) - goto fail4; + goto fail5; Index++; } @@ -3035,18 +3232,18 @@ ReceiverConnect( Receiver, &Receiver->DebugCallback); if (!NT_SUCCESS(status)) - goto fail5; + goto fail6; Trace("<====\n"); return STATUS_SUCCESS; -fail5: - Error("fail5\n"); +fail6: + Error("fail6\n"); Index = FrontendGetNumQueues(Frontend); -fail4: - Error("fail4\n"); +fail5: + Error("fail5\n"); while (--Index >= 0) { PXENVIF_RECEIVER_RING Ring = Receiver->Ring[Index]; @@ -3056,6 +3253,11 @@ fail4: XENBUS_GNTTAB(Release, &Receiver->GnttabInterface); +fail4: + Error("fail4\n"); + + XENBUS_EVTCHN(Release, &Receiver->EvtchnInterface); + fail3: Error("fail3\n"); @@ -3295,34 +3497,6 @@ fail1: return status; } -BOOLEAN -ReceiverPoll( - IN PXENVIF_RECEIVER Receiver, - IN ULONG Index - ) -{ - PXENVIF_FRONTEND Frontend; - ULONG NumQueues; - PXENVIF_RECEIVER_RING Ring; - BOOLEAN Retry; - - ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL); - - Frontend = Receiver->Frontend; - - NumQueues = FrontendGetNumQueues(Frontend); - if (Index >= NumQueues) - return FALSE; - - Ring = Receiver->Ring[Index]; - - __ReceiverRingAcquireLock(Ring); - Retry = ReceiverRingPoll(Ring); - __ReceiverRingReleaseLock(Ring); - - return Retry; -} - VOID ReceiverDisable( IN PXENVIF_RECEIVER Receiver @@ -3371,6 +3545,8 @@ ReceiverDisconnect( XENBUS_GNTTAB(Release, &Receiver->GnttabInterface); + XENBUS_EVTCHN(Release, &Receiver->EvtchnInterface); + XENBUS_STORE(Release, &Receiver->StoreInterface); XENBUS_DEBUG(Release, &Receiver->DebugInterface); @@ -3388,6 +3564,9 @@ ReceiverTeardown( Frontend = Receiver->Frontend; + ASSERT3U(KeGetCurrentIrql(), ==, PASSIVE_LEVEL); + KeFlushQueuedDpcs(); + ASSERT3U(Receiver->Returned, ==, Receiver->Loaned); Receiver->Loaned = 0; Receiver->Returned = 0; @@ -3407,6 +3586,9 @@ ReceiverTeardown( Receiver->Frontend = NULL; + RtlZeroMemory(&Receiver->EvtchnInterface, + sizeof (XENBUS_EVTCHN_INTERFACE)); + RtlZeroMemory(&Receiver->GnttabInterface, sizeof (XENBUS_GNTTAB_INTERFACE)); @@ -3541,13 +3723,16 @@ ReceiverWaitForPackets( LARGE_INTEGER Timeout; ASSERT3U(KeGetCurrentIrql(), <, DISPATCH_LEVEL); - KeFlushQueuedDpcs(); Frontend = Receiver->Frontend; Trace("%s: ====>\n", FrontendGetPath(Frontend)); Returned = Receiver->Returned; + + // Make sure Loaned is not sampled before Returned + KeMemoryBarrier(); + Loaned = Receiver->Loaned; ASSERT3S(Loaned - Returned, >=, 0); @@ -3580,6 +3765,32 @@ ReceiverWaitForPackets( Trace("%s: <====\n", FrontendGetPath(Frontend)); } +VOID +ReceiverTrigger( + IN PXENVIF_RECEIVER Receiver, + IN ULONG Index + ) +{ + PXENVIF_RECEIVER_RING Ring; + + Ring = Receiver->Ring[Index]; + + __ReceiverRingTrigger(Ring, FALSE); +} + +VOID +ReceiverSend( + IN PXENVIF_RECEIVER Receiver, + IN ULONG Index + ) +{ + PXENVIF_RECEIVER_RING Ring; + + Ring = Receiver->Ring[Index]; + + __ReceiverRingSend(Ring, FALSE); +} + NTSTATUS ReceiverSetHashAlgorithm( IN PXENVIF_RECEIVER Receiver, diff --git a/src/xenvif/receiver.h b/src/xenvif/receiver.h index e1b7a88..7846f0b 100644 --- a/src/xenvif/receiver.h +++ b/src/xenvif/receiver.h @@ -62,12 +62,6 @@ ReceiverEnable( IN PXENVIF_RECEIVER Receiver ); -extern BOOLEAN -ReceiverPoll( - IN PXENVIF_RECEIVER Receiver, - IN ULONG Index - ); - extern VOID ReceiverDisable( IN PXENVIF_RECEIVER Receiver diff --git a/src/xenvif/transmitter.c b/src/xenvif/transmitter.c index eed0a0f..770aca9 100644 --- a/src/xenvif/transmitter.c +++ b/src/xenvif/transmitter.c @@ -43,6 +43,7 @@ #include <cache_interface.h> #include <gnttab_interface.h> #include <range_set_interface.h> +#include <evtchn_interface.h> #include "pdo.h" #include "frontend.h" @@ -178,6 +179,12 @@ typedef struct _XENVIF_TRANSMITTER_RING { netif_tx_front_ring_t Front; netif_tx_sring_t *Shared; PXENBUS_GNTTAB_ENTRY Entry; + PXENBUS_EVTCHN_CHANNEL Channel; + KDPC Dpc; + ULONG Dpcs; + KTIMER Timer; + KDPC TimerDpc; + ULONG Events; BOOLEAN Connected; BOOLEAN Enabled; BOOLEAN Stopped; @@ -208,6 +215,7 @@ struct _XENVIF_TRANSMITTER { XENBUS_CACHE_INTERFACE CacheInterface; XENBUS_GNTTAB_INTERFACE GnttabInterface; XENBUS_RANGE_SET_INTERFACE RangeSetInterface; + XENBUS_EVTCHN_INTERFACE EvtchnInterface; PXENVIF_TRANSMITTER_RING *Ring; BOOLEAN MulticastControl; ULONG DisableIpVersion4Gso; @@ -762,6 +770,15 @@ TransmitterRingDebugCallback( Ring->PacketsUnprepared, Ring->PacketsSent, Ring->PacketsCompleted); + + if (FrontendIsSplit(Frontend)) { + // Dump event channel + XENBUS_DEBUG(Printf, + &Transmitter->DebugInterface, + "Events = %lu Dpcs = %lu\n", + Ring->Events, + Ring->Dpcs); + } } static BOOLEAN @@ -2622,6 +2639,9 @@ TransmitterRingPoll( RING_IDX rsp_cons; ULONG Extra; + if (Retry) + break; + KeMemoryBarrier(); rsp_prod = Ring->Shared->rsp_prod; @@ -2629,8 +2649,15 @@ TransmitterRingPoll( KeMemoryBarrier(); - if (rsp_cons == rsp_prod || Retry) + if (rsp_cons == rsp_prod) { + RING_IDX WorkToDo; + + RING_FINAL_CHECK_FOR_RESPONSES(&Ring->Front, WorkToDo); + if (WorkToDo) + continue; + break; + } Extra = 0; while (rsp_cons != rsp_prod && !Retry) { @@ -2769,7 +2796,6 @@ TransmitterRingPoll( KeMemoryBarrier(); Ring->Front.rsp_cons = rsp_cons; - Ring->Shared->rsp_event = rsp_cons + 1; } done: @@ -2779,17 +2805,64 @@ done: } static FORCEINLINE VOID -__TransmitterRingPushRequests( +__TransmitterRingTrigger( + IN PXENVIF_TRANSMITTER_RING Ring + ) +{ + PXENVIF_TRANSMITTER Transmitter; + PXENVIF_FRONTEND Frontend; + + Transmitter = Ring->Transmitter; + Frontend = Transmitter->Frontend; + + if (!Ring->Connected) + return; + + if (FrontendIsSplit(Frontend)) { + ASSERT(Ring->Channel != NULL); + + (VOID) XENBUS_EVTCHN(Trigger, + &Transmitter->EvtchnInterface, + Ring->Channel); + } else { + ReceiverTrigger(FrontendGetReceiver(Frontend), + Ring->Index); + } +} + +static FORCEINLINE VOID +__TransmitterRingSend( IN PXENVIF_TRANSMITTER_RING Ring ) { PXENVIF_TRANSMITTER Transmitter; PXENVIF_FRONTEND Frontend; - BOOLEAN Notify; Transmitter = Ring->Transmitter; Frontend = Transmitter->Frontend; + if (!Ring->Connected) + return; + + if (FrontendIsSplit(Frontend)) { + ASSERT(Ring->Channel != NULL); + + (VOID) XENBUS_EVTCHN(Send, + &Transmitter->EvtchnInterface, + Ring->Channel); + } else { + ReceiverSend(FrontendGetReceiver(Frontend), + Ring->Index); + } +} + +static FORCEINLINE VOID +__TransmitterRingPushRequests( + IN PXENVIF_TRANSMITTER_RING Ring + ) +{ + BOOLEAN Notify; + if (Ring->RequestsPosted == Ring->RequestsPushed) return; @@ -2802,9 +2875,7 @@ __TransmitterRingPushRequests( #pragma warning (pop) if (Notify) - PollerSend(FrontendGetPoller(Frontend), - Ring->Index, - XENVIF_POLLER_EVENT_TRANSMIT); + __TransmitterRingSend(Ring); Ring->RequestsPushed = Ring->RequestsPosted; } @@ -3158,11 +3229,131 @@ TransmitterRingReleaseLock( __TransmitterRingReleaseLock(Ring); } +static FORCEINLINE VOID +__TransmitterRingUnmask( + IN PXENVIF_TRANSMITTER_RING Ring + ) +{ + PXENVIF_TRANSMITTER Transmitter; + PXENVIF_FRONTEND Frontend; + + Transmitter = Ring->Transmitter; + Frontend = Transmitter->Frontend; + + if (!Ring->Connected || !FrontendIsSplit(Frontend)) + return; + + XENBUS_EVTCHN(Unmask, + &Transmitter->EvtchnInterface, + Ring->Channel, + FALSE, + TRUE); +} + +static FORCEINLINE BOOLEAN +__TransmitterRingDpcTimeout( + IN PXENVIF_TRANSMITTER_RING Ring + ) +{ + KDPC_WATCHDOG_INFORMATION Watchdog; + NTSTATUS status; + + UNREFERENCED_PARAMETER(Ring); + + RtlZeroMemory(&Watchdog, sizeof (Watchdog)); + + status = KeQueryDpcWatchdogInformation(&Watchdog); + ASSERT(NT_SUCCESS(status)); + + if (Watchdog.DpcTimeLimit == 0 || + Watchdog.DpcWatchdogLimit == 0) + return FALSE; + + if (Watchdog.DpcTimeCount > (Watchdog.DpcTimeLimit / 2) && + Watchdog.DpcWatchdogCount > (Watchdog.DpcWatchdogLimit / 2)) + return FALSE; + + return TRUE; +} + #define TIME_US(_us) ((_us) * 10) #define TIME_MS(_ms) (TIME_US((_ms) * 1000)) #define TIME_S(_s) (TIME_MS((_s) * 1000)) #define TIME_RELATIVE(_t) (-(_t)) +__drv_functionClass(KDEFERRED_ROUTINE) +__drv_maxIRQL(DISPATCH_LEVEL) +__drv_minIRQL(DISPATCH_LEVEL) +__drv_requiresIRQL(DISPATCH_LEVEL) +__drv_sameIRQL +static VOID +TransmitterRingDpc( + IN PKDPC Dpc, + IN PVOID Context, + IN PVOID Argument1, + IN PVOID Argument2 + ) +{ + PXENVIF_TRANSMITTER_RING Ring = Context; + + UNREFERENCED_PARAMETER(Dpc); + UNREFERENCED_PARAMETER(Argument1); + UNREFERENCED_PARAMETER(Argument2); + + ASSERT(Ring != NULL); + + for (;;) { + BOOLEAN Retry; + + __TransmitterRingAcquireLock(Ring); + Retry = TransmitterRingPoll(Ring); + __TransmitterRingReleaseLock(Ring); + + if (!Retry) { + __TransmitterRingUnmask(Ring); + break; + } + + if (__TransmitterRingDpcTimeout(Ring)) { + LARGE_INTEGER Delay; + + Delay.QuadPart = TIME_RELATIVE(TIME_US(100)); + + KeSetTimer(&Ring->Timer, Delay, &Ring->TimerDpc); + break; + } + } +} + +KSERVICE_ROUTINE TransmitterRingEvtchnCallback; + +BOOLEAN +TransmitterRingEvtchnCallback( + IN PKINTERRUPT InterruptObject, + IN PVOID Argument + ) +{ + PXENVIF_TRANSMITTER_RING Ring = Argument; + PXENVIF_TRANSMITTER Transmitter; + PXENVIF_FRONTEND Frontend; + + UNREFERENCED_PARAMETER(InterruptObject); + + ASSERT(Ring != NULL); + + Transmitter = Ring->Transmitter; + Frontend = Transmitter->Frontend; + + ASSERT(FrontendIsSplit(Frontend)); + + Ring->Events++; + + if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL)) + Ring->Dpcs++; + + return TRUE; +} + #define XENVIF_TRANSMITTER_WATCHDOG_PERIOD 30 static NTSTATUS @@ -3221,22 +3412,16 @@ TransmitterRingWatchdog( if (Ring->PacketsQueued == PacketsQueued && Ring->PacketsCompleted != PacketsQueued) { PXENVIF_TRANSMITTER Transmitter; - PXENVIF_FRONTEND Frontend; Transmitter = Ring->Transmitter; - Frontend = Transmitter->Frontend; XENBUS_DEBUG(Trigger, &Transmitter->DebugInterface, Ring->DebugCallback); // Try to move things along - PollerTrigger(FrontendGetPoller(Frontend), - Ring->Index, - XENVIF_POLLER_EVENT_TRANSMIT); - PollerSend(FrontendGetPoller(Frontend), - Ring->Index, - XENVIF_POLLER_EVENT_TRANSMIT); + __TransmitterRingTrigger(Ring); + __TransmitterRingSend(Ring); } PacketsQueued = Ring->PacketsQueued; @@ -3281,6 +3466,10 @@ __TransmitterRingInitialize( InitializeListHead(&(*Ring)->RequestQueue); InitializeListHead(&(*Ring)->PacketComplete); + KeInitializeDpc(&(*Ring)->Dpc, TransmitterRingDpc, *Ring); + KeInitializeTimer(&(*Ring)->Timer); + KeInitializeDpc(&(*Ring)->TimerDpc, TransmitterRingDpc, *Ring); + status = RtlStringCbPrintfA(Name, sizeof (Name), "%s_transmitter_buffer", @@ -3482,6 +3671,8 @@ fail4: fail3: Error("fail3\n"); + RtlZeroMemory(&(*Ring)->Dpc, sizeof (KDPC)); + RtlZeroMemory(&(*Ring)->PacketComplete, sizeof (LIST_ENTRY)); RtlZeroMemory(&(*Ring)->RequestQueue, sizeof (LIST_ENTRY)); RtlZeroMemory(&(*Ring)->PacketQueue, sizeof (LIST_ENTRY)); @@ -3515,6 +3706,7 @@ __TransmitterRingConnect( PFN_NUMBER Pfn; CHAR Name[MAXNAMELEN]; ULONG Index; + PROCESSOR_NUMBER ProcNumber; NTSTATUS status; ASSERT(!Ring->Connected); @@ -3580,6 +3772,38 @@ __TransmitterRingConnect( ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL); + if (FrontendIsSplit(Frontend)) { + Ring->Channel = XENBUS_EVTCHN(Open, + &Transmitter->EvtchnInterface, + XENBUS_EVTCHN_TYPE_UNBOUND, + TransmitterRingEvtchnCallback, + Ring, + FrontendGetBackendDomain(Frontend), + TRUE); + + status = STATUS_UNSUCCESSFUL; + if (Ring->Channel == NULL) + goto fail6; + + status = KeGetProcessorNumberFromIndex(Ring->Index, &ProcNumber); + ASSERT(NT_SUCCESS(status)); + + KeSetTargetProcessorDpcEx(&Ring->Dpc, &ProcNumber); + KeSetTargetProcessorDpcEx(&Ring->TimerDpc, &ProcNumber); + + (VOID) XENBUS_EVTCHN(Bind, + &Transmitter->EvtchnInterface, + Ring->Channel, + ProcNumber.Group, + ProcNumber.Number); + + (VOID) XENBUS_EVTCHN(Unmask, + &Transmitter->EvtchnInterface, + Ring->Channel, + FALSE, + TRUE); + } + status = XENBUS_DEBUG(Register, &Transmitter->DebugInterface, Name, @@ -3587,12 +3811,22 @@ __TransmitterRingConnect( Ring, &Ring->DebugCallback); if (!NT_SUCCESS(status)) - goto fail6; + goto fail7; Ring->Connected = TRUE; return STATUS_SUCCESS; +fail7: + Error("fail7\n"); + + XENBUS_EVTCHN(Close, + &Transmitter->EvtchnInterface, + Ring->Channel); + Ring->Channel = NULL; + + Ring->Events = 0; + fail6: Error("fail6\n"); @@ -3641,6 +3875,7 @@ __TransmitterRingStoreWrite( { PXENVIF_TRANSMITTER Transmitter; PXENVIF_FRONTEND Frontend; + ULONG Port; PCHAR Path; NTSTATUS status; @@ -3663,8 +3898,29 @@ __TransmitterRingStoreWrite( if (!NT_SUCCESS(status)) goto fail1; + if (!FrontendIsSplit(Frontend)) + goto done; + + Port = XENBUS_EVTCHN(GetPort, + &Transmitter->EvtchnInterface, + Ring->Channel); + + status = XENBUS_STORE(Printf, + &Transmitter->StoreInterface, + Transaction, + Path, + "event-channel-tx", + "%u", + Port); + if (!NT_SUCCESS(status)) + goto fail2; + +done: return STATUS_SUCCESS; +fail2: + Error("fail2\n"); + fail1: Error("fail1 (%08x)\n", status); @@ -3691,6 +3947,8 @@ __TransmitterRingEnable( ASSERT(!Ring->Enabled); Ring->Enabled = TRUE; + KeInsertQueueDpc(&Ring->Dpc, NULL, NULL); + __TransmitterRingReleaseLock(Ring); Info("%s[%u]: <====\n", @@ -3770,9 +4028,7 @@ __TransmitterRingDisable( ASSERT(Attempt < 100); // Try to move things along - PollerSend(FrontendGetPoller(Frontend), - Ring->Index, - XENVIF_POLLER_EVENT_TRANSMIT); + __TransmitterRingSend(Ring); (VOID) TransmitterRingPoll(Ring); if (State != XenbusStateConnected) @@ -3790,6 +4046,12 @@ __TransmitterRingDisable( __TransmitterRingReleaseLock(Ring); + // + // No new timers can be scheduled once Enabled goes to FALSE. + // Cancel any existing ones. + // + (VOID) KeCancelTimer(&Ring->Timer); + Info("%s[%u]: <====\n", FrontendGetPath(Frontend), Ring->Index); @@ -3809,6 +4071,17 @@ __TransmitterRingDisconnect( Transmitter = Ring->Transmitter; Frontend = Transmitter->Frontend; + if (Ring->Channel != NULL) { + XENBUS_EVTCHN(Close, + &Transmitter->EvtchnInterface, + Ring->Channel); + Ring->Channel = NULL; + + Ring->Events = 0; + } + + Ring->Dpcs = 0; + ASSERT3U(Ring->ResponsesProcessed, ==, Ring->RequestsPushed); ASSERT3U(Ring->RequestsPushed, ==, Ring->RequestsPosted); @@ -3852,6 +4125,12 @@ __TransmitterRingTeardown( Transmitter = Ring->Transmitter; Frontend = Transmitter->Frontend; + Ring->Dpcs = 0; + + RtlZeroMemory(&Ring->TimerDpc, sizeof (KDPC)); + RtlZeroMemory(&Ring->Timer, sizeof (KTIMER)); + RtlZeroMemory(&Ring->Dpc, sizeof (KDPC)); + ASSERT3U(Ring->PacketsCompleted, ==, Ring->PacketsSent); ASSERT3U(Ring->PacketsSent, ==, Ring->PacketsPrepared - Ring->PacketsUnprepared); ASSERT3U(Ring->PacketsPrepared, ==, Ring->PacketsCopied + Ring->PacketsGranted + Ring->PacketsFaked); @@ -4277,6 +4556,9 @@ TransmitterInitialize( FdoGetGnttabInterface(PdoGetFdo(FrontendGetPdo(Frontend)), &(*Transmitter)->GnttabInterface); + FdoGetEvtchnInterface(PdoGetFdo(FrontendGetPdo(Frontend)), + &(*Transmitter)->EvtchnInterface); + (*Transmitter)->Frontend = Frontend; KeInitializeSpinLock(&(*Transmitter)->Lock); @@ -4429,10 +4711,14 @@ TransmitterConnect( if (!NT_SUCCESS(status)) goto fail2; - status = XENBUS_GNTTAB(Acquire, &Transmitter->GnttabInterface); + status = XENBUS_EVTCHN(Acquire, &Transmitter->EvtchnInterface); if (!NT_SUCCESS(status)) goto fail3; + status = XENBUS_GNTTAB(Acquire, &Transmitter->GnttabInterface); + if (!NT_SUCCESS(status)) + goto fail4; + if (Transmitter->DisableMulticastControl == 0) { status = XENBUS_STORE(Read, &Transmitter->StoreInterface, @@ -4455,7 +4741,7 @@ TransmitterConnect( status = __TransmitterRingConnect(Ring); if (!NT_SUCCESS(status)) - goto fail4; + goto fail5; Index++; } @@ -4467,18 +4753,18 @@ TransmitterConnect( Transmitter, &Transmitter->DebugCallback); if (!NT_SUCCESS(status)) - goto fail5; + goto fail6; Trace("<====\n"); return STATUS_SUCCESS; -fail5: - Error("fail5\n"); +fail6: + Error("fail6\n"); Index = FrontendGetNumQueues(Frontend); -fail4: - Error("fail4\n"); +fail5: + Error("fail5\n"); while (--Index >= 0) { PXENVIF_TRANSMITTER_RING Ring; @@ -4492,6 +4778,11 @@ fail4: XENBUS_GNTTAB(Release, &Transmitter->GnttabInterface); +fail4: + Error("fail4\n"); + + XENBUS_EVTCHN(Release, &Transmitter->EvtchnInterface); + fail3: Error("fail3\n"); @@ -4628,34 +4919,6 @@ TransmitterEnable( return STATUS_SUCCESS; } -BOOLEAN -TransmitterPoll( - IN PXENVIF_TRANSMITTER Transmitter, - IN ULONG Index - ) -{ - PXENVIF_FRONTEND Frontend; - ULONG NumQueues; - PXENVIF_TRANSMITTER_RING Ring; - BOOLEAN Retry; - - ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL); - - Frontend = Transmitter->Frontend; - - NumQueues = FrontendGetNumQueues(Frontend); - if (Index >= NumQueues) - return FALSE; - - Ring = Transmitter->Ring[Index]; - - __TransmitterRingAcquireLock(Ring); - Retry = TransmitterRingPoll(Ring); - __TransmitterRingReleaseLock(Ring); - - return Retry; -} - VOID TransmitterDisable( IN PXENVIF_TRANSMITTER Transmitter @@ -4706,6 +4969,8 @@ TransmitterDisconnect( XENBUS_GNTTAB(Release, &Transmitter->GnttabInterface); + XENBUS_EVTCHN(Release, &Transmitter->EvtchnInterface); + XENBUS_STORE(Release, &Transmitter->StoreInterface); XENBUS_DEBUG(Release, &Transmitter->DebugInterface); @@ -4724,6 +4989,7 @@ TransmitterTeardown( Frontend = Transmitter->Frontend; ASSERT3U(KeGetCurrentIrql(), ==, PASSIVE_LEVEL); + KeFlushQueuedDpcs(); Index = FrontendGetMaxQueues(Frontend); while (--Index >= 0) { @@ -4765,6 +5031,9 @@ TransmitterTeardown( RtlZeroMemory(&Transmitter->DebugInterface, sizeof (XENBUS_DEBUG_INTERFACE)); + RtlZeroMemory(&Transmitter->EvtchnInterface, + sizeof (XENBUS_EVTCHN_INTERFACE)); + Transmitter->DisableIpVersion4Gso = 0; Transmitter->DisableIpVersion6Gso = 0; Transmitter->AlwaysCopy = 0; @@ -5041,6 +5310,25 @@ TransmitterQueryRingSize( } VOID +TransmitterNotify( + IN PXENVIF_TRANSMITTER Transmitter, + IN ULONG Index + ) +{ + PXENVIF_FRONTEND Frontend; + PXENVIF_TRANSMITTER_RING Ring; + + Frontend = Transmitter->Frontend; + + ASSERT(!FrontendIsSplit(Frontend)); + + Ring = Transmitter->Ring[Index]; + + if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL)) + Ring->Dpcs++; +} + +VOID TransmitterQueryOffloadOptions( IN PXENVIF_TRANSMITTER Transmitter, OUT PXENVIF_VIF_OFFLOAD_OPTIONS Options diff --git a/src/xenvif/transmitter.h b/src/xenvif/transmitter.h index 2446220..d44f77e 100644 --- a/src/xenvif/transmitter.h +++ b/src/xenvif/transmitter.h @@ -64,12 +64,6 @@ TransmitterEnable( IN PXENVIF_TRANSMITTER Transmitter ); -extern BOOLEAN -TransmitterPoll( - IN PXENVIF_TRANSMITTER Transmitter, - IN ULONG Index - ); - extern VOID TransmitterDisable( IN PXENVIF_TRANSMITTER Transmitter diff --git a/src/xenvif/vif.c b/src/xenvif/vif.c index 69ced78..ffdec50 100644 --- a/src/xenvif/vif.c +++ b/src/xenvif/vif.c @@ -1161,7 +1161,6 @@ __VifReceiverQueuePacket( Hash, More, Cookie); - } VOID @@ -1180,10 +1179,6 @@ VifReceiverQueuePacket( IN PVOID Cookie ) { - KIRQL Irql; - - KeRaiseIrql(DISPATCH_LEVEL, &Irql); - switch (Context->Version) { case 6: __VifReceiverQueuePacketVersion6(Context, @@ -1234,8 +1229,6 @@ VifReceiverQueuePacket( ASSERT(FALSE); break; } - - KeLowerIrql(Irql); } VOID diff --git a/vs2015/xenvif/xenvif.vcxproj b/vs2015/xenvif/xenvif.vcxproj index 3be0c18..8db43cb 100644 --- a/vs2015/xenvif/xenvif.vcxproj +++ b/vs2015/xenvif/xenvif.vcxproj @@ -73,7 +73,6 @@ <ClCompile Include="../../src/xenvif/mac.c" /> <ClCompile Include="../../src/xenvif/parse.c" /> <ClCompile Include="../../src/xenvif/pdo.c" /> - <ClCompile Include="../../src/xenvif/poller.c" /> <ClCompile Include="../../src/xenvif/receiver.c" /> <ClCompile Include="../../src/xenvif/registry.c" /> <ClCompile Include="../../src/xenvif/settings.c" /> diff --git a/vs2017/xenvif/xenvif.vcxproj b/vs2017/xenvif/xenvif.vcxproj index f2e5d19..c6d02f9 100644 --- a/vs2017/xenvif/xenvif.vcxproj +++ b/vs2017/xenvif/xenvif.vcxproj @@ -81,7 +81,6 @@ <ClCompile Include="../../src/xenvif/mac.c" /> <ClCompile Include="../../src/xenvif/parse.c" /> <ClCompile Include="../../src/xenvif/pdo.c" /> - <ClCompile Include="../../src/xenvif/poller.c" /> <ClCompile Include="../../src/xenvif/receiver.c" /> <ClCompile Include="../../src/xenvif/registry.c" /> <ClCompile Include="../../src/xenvif/settings.c" /> -- 2.5.3 _______________________________________________ win-pv-devel mailing list win-pv-devel@xxxxxxxxxxxxxxxxxxxx https://lists.xenproject.org/mailman/listinfo/win-pv-devel

©2013 Xen Project, A Linux Foundation Collaborative Project. All Rights Reserved.
Linux Foundation is a registered trademark of The Linux Foundation.
Xen Project is a trademark of The Linux Foundation.