|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] Re: [win-pv-devel] [PATCH 6/9] Implement multi-queues
> -----Original Message----- > From: win-pv-devel [mailto:win-pv-devel-bounces@xxxxxxxxxxxxxxxxxxxx] On > Behalf Of Owen Smith > Sent: 29 May 2018 11:07 > To: win-pv-devel@xxxxxxxxxxxxxxxxxxxx > Cc: Owen Smith <owen.smith@xxxxxxxxxx> > Subject: [win-pv-devel] [PATCH 6/9] Implement multi-queues > > Splits XENVBD_RING into multiple XENVBD_BLKIF_RINGs, one for each > shared > ring. Up-to "multi-queue-max-queues" rings are used to pass > blkif_requests and blkif_responses between frontend and backend. > Reworks > the ring interactions to remove the locks used by XENVBD_QUEUE, > implementing a queue system similar to XenVifs transmitter queues. > > Signed-off-by: Owen Smith <owen.smith@xxxxxxxxxx> Acked-by: Paul Durrant <paul.durrant@xxxxxxxxxx> > --- > src/xenvbd/frontend.c | 18 +- > src/xenvbd/ring.c | 3097 +++++++++++++++++++++++++++++-------------- > ------ > 2 files changed, 1847 insertions(+), 1268 deletions(-) > > diff --git a/src/xenvbd/frontend.c b/src/xenvbd/frontend.c > index 900ad2b..fd2c247 100644 > --- a/src/xenvbd/frontend.c > +++ b/src/xenvbd/frontend.c > @@ -1278,15 +1278,15 @@ FrontendConnect( > if (!NT_SUCCESS(Status)) > goto abort; > > - //status = XENBUS_STORE(Printf, > - // &Frontend->StoreInterface, > - // Transaction, > - // Frontend->FrontendPath, > - // "multi-queue-num-queues", > - // "%u", > - // __FrontendGetNumQueues(Frontend)); > - //if (!NT_SUCCESS(status)) > - // goto abort; > + Status = XENBUS_STORE(Printf, > + &Frontend->StoreInterface, > + Transaction, > + Frontend->FrontendPath, > + "multi-queue-num-queues", > + "%u", > + __FrontendGetNumQueues(Frontend)); > + if (!NT_SUCCESS(Status)) > + goto abort; > > Status = XENBUS_STORE(TransactionEnd, > &Frontend->StoreInterface, > diff --git a/src/xenvbd/ring.c b/src/xenvbd/ring.c > index e41c1b7..cd2cc0e 100644 > --- a/src/xenvbd/ring.c > +++ b/src/xenvbd/ring.c > @@ -47,7 +47,6 @@ > #include "srbext.h" > #include "driver.h" > #include "granter.h" > -#include "queue.h" > > #include "util.h" > #include "debug.h" > @@ -56,47 +55,72 @@ > #define XENVBD_MAX_RING_PAGE_ORDER (4) > #define XENVBD_MAX_RING_PAGES (1 << > XENVBD_MAX_RING_PAGE_ORDER) > > -struct _XENVBD_RING { > - PXENVBD_FRONTEND Frontend; > +#define xen_mb KeMemoryBarrier > +#define xen_wmb KeMemoryBarrier > + > +typedef struct _XENVBD_SRB_STATE { > + LIST_ENTRY List; > + ULONG Count; > +} XENVBD_SRB_STATE, *PXENVBD_SRB_STATE; > + > +typedef struct _XENVBD_BLKIF_RING { > + PXENVBD_RING Ring; > + ULONG Index; > + PCHAR Path; > + PXENBUS_CACHE RequestCache; > + PXENBUS_CACHE SegmentCache; > + PXENBUS_CACHE IndirectCache; > + PMDL Mdl; > + blkif_sring_t *Shared; > + blkif_front_ring_t Front; > + PXENBUS_GNTTAB_ENTRY Grants[XENVBD_MAX_RING_PAGES]; > + PXENBUS_EVTCHN_CHANNEL Channel; > + KDPC Dpc; > + ULONG Dpcs; > + ULONG Events; > BOOLEAN Connected; > BOOLEAN Enabled; > + BOOLEAN Stopped; > + PVOID Lock; > + PKTHREAD LockThread; > + XENVBD_SRB_STATE State; > + LIST_ENTRY SrbQueue; > + LIST_ENTRY SubmittedList; > + LIST_ENTRY ShutdownQueue; > + ULONG SrbsQueued; > + ULONG SrbsCompleted; > + ULONG SrbsFailed; > + ULONG RequestsPosted; > + ULONG RequestsPushed; > + ULONG ResponsesProcessed; > + PXENBUS_DEBUG_CALLBACK DebugCallback; > +} XENVBD_BLKIF_RING, *PXENVBD_BLKIF_RING; > + > +typedef enum _XENVBD_STAT { > + XENVBD_STAT_BLKIF_OP_READ_DIRECT = 0, > + XENVBD_STAT_BLKIF_OP_READ_INDIRECT, > + XENVBD_STAT_BLKIF_OP_WRITE_DIRECT, > + XENVBD_STAT_BLKIF_OP_WRITE_INDIRECT, > + XENVBD_STAT_BLKIF_OP_WRITE_BARRIER, > + XENVBD_STAT_BLKIF_OP_FLUSH_DISKCACHE, > + XENVBD_STAT_BLKIF_OP_DISCARD, > + XENVBD_STAT_BLKIF_OP_UNKNOWN, > + XENVBD_STAT_SEGMENTS_GRANTED, > + XENVBD_STAT_SEGMENTS_BOUNCED, > + > + XENVBD_STAT__MAX > +} XENVBD_STAT, *PXENVBD_STAT; > > +struct _XENVBD_RING { > + PXENVBD_FRONTEND Frontend; > + XENBUS_DEBUG_INTERFACE DebugInterface; > XENBUS_CACHE_INTERFACE CacheInterface; > XENBUS_STORE_INTERFACE StoreInterface; > XENBUS_EVTCHN_INTERFACE EvtchnInterface; > - XENBUS_DEBUG_INTERFACE DebugInterface; > - > PXENBUS_DEBUG_CALLBACK DebugCallback; > - > - KSPIN_LOCK Lock; > - PMDL Mdl; > - blkif_sring_t* Shared; > - blkif_front_ring_t Front; > ULONG Order; > - PVOID Grants[XENVBD_MAX_RING_PAGES]; > - PXENBUS_EVTCHN_CHANNEL Channel; > - KDPC Dpc; > - > - PXENBUS_CACHE RequestCache; > - PXENBUS_CACHE SegmentCache; > - PXENBUS_CACHE IndirectCache; > - XENVBD_QUEUE PreparedReqs; > - XENVBD_QUEUE SubmittedReqs; > - XENVBD_QUEUE ShutdownSrbs; > - > - ULONG Submitted; > - ULONG Received; > - ULONG Events; > - ULONG Dpcs; > - ULONG BlkOpRead; > - ULONG BlkOpWrite; > - ULONG BlkOpIndirectRead; > - ULONG BlkOpIndirectWrite; > - ULONG BlkOpBarrier; > - ULONG BlkOpDiscard; > - ULONG BlkOpFlush; > - ULONG64 SegsGranted; > - ULONG64 SegsBounced; > + PXENVBD_BLKIF_RING *Ring; > + LONG Stats[XENVBD_STAT__MAX]; > }; > > #define MAX_NAME_LEN 64 > @@ -123,138 +147,119 @@ __RingFree( > RING_POOL_TAG); > } > > -static FORCEINLINE VOID > -xen_mb() > +static FORCEINLINE PCHAR > +__BlkifOperationName( > + IN UCHAR Operation > + ) > { > - KeMemoryBarrier(); > - _ReadWriteBarrier(); > + switch (Operation) { > + case BLKIF_OP_READ: return "READ"; > + case BLKIF_OP_WRITE: return "WRITE"; > + case BLKIF_OP_WRITE_BARRIER: return "WRITE_BARRIER"; > + case BLKIF_OP_FLUSH_DISKCACHE: return "FLUSH_DISKCACHE"; > + case BLKIF_OP_RESERVED_1: return "RESERVED_1"; > + case BLKIF_OP_DISCARD: return "DISCARD"; > + case BLKIF_OP_INDIRECT: return "INDIRECT"; > + default: return "<unknown>"; > + } > } > > -static FORCEINLINE VOID > -xen_wmb() > +static FORCEINLINE PCHAR > +__StatName( > + IN XENVBD_STAT Operation > + ) > { > - KeMemoryBarrier(); > - _WriteBarrier(); > + switch (Operation) { > + case XENVBD_STAT_BLKIF_OP_READ_DIRECT: return "BLKIF_OP_READ > (direct)"; > + case XENVBD_STAT_BLKIF_OP_READ_INDIRECT: return > "BLKIF_OP_READ (indirect)"; > + case XENVBD_STAT_BLKIF_OP_WRITE_DIRECT: return > "BLKIF_OP_WRITE (direct)"; > + case XENVBD_STAT_BLKIF_OP_WRITE_INDIRECT: return > "BLKIF_OP_WRITE (indirect)"; > + case XENVBD_STAT_BLKIF_OP_WRITE_BARRIER: return > "BLKIF_OP_WRITE_BARRIER"; > + case XENVBD_STAT_BLKIF_OP_FLUSH_DISKCACHE: return > "BLKIF_OP_FLUSH_DISKCACHE"; > + case XENVBD_STAT_BLKIF_OP_DISCARD: return > "BLKIF_OP_DISCARD"; > + case XENVBD_STAT_SEGMENTS_GRANTED: return > "SegmentsGranted"; > + case XENVBD_STAT_SEGMENTS_BOUNCED: return > "SegmentsBounced"; > + default: return "UNKNOWN"; > + } > } > > -static FORCEINLINE VOID > -__RingInsert( > - IN PXENVBD_RING Ring, > - IN PXENVBD_REQUEST Request, > - IN blkif_request_t* req > +static FORCEINLINE ULONG > +__SectorsPerPage( > + IN ULONG SectorSize > ) > { > - PXENVBD_GRANTER Granter = FrontendGetGranter(Ring->Frontend); > - > - switch (Request->Operation) { > - case BLKIF_OP_READ: > - case BLKIF_OP_WRITE: > - if (Request->NrSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST) { > - // Indirect > - ULONG PageIdx; > - ULONG SegIdx; > - PLIST_ENTRY PageEntry; > - PLIST_ENTRY SegEntry; > - blkif_request_indirect_t* req_indirect; > - > - req_indirect = (blkif_request_indirect_t*)req; > - req_indirect->operation = BLKIF_OP_INDIRECT; > - req_indirect->indirect_op = Request->Operation; > - req_indirect->nr_segments = Request->NrSegments; > - req_indirect->id = (ULONG64)(ULONG_PTR)Request; > - req_indirect->sector_number = Request->FirstSector; > - req_indirect->handle = > (USHORT)FrontendGetDeviceId(Ring- > >Frontend); > - > - for (PageIdx = 0, > - PageEntry = Request->Indirects.Flink, > - SegEntry = Request->Segments.Flink; > - PageIdx < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST && > - PageEntry != &Request->Indirects && > - SegEntry != &Request->Segments; > - ++PageIdx, PageEntry = PageEntry->Flink) { > - PXENVBD_INDIRECT Page = CONTAINING_RECORD(PageEntry, > XENVBD_INDIRECT, ListEntry); > - > - req_indirect->indirect_grefs[PageIdx] = > GranterReference(Granter, > Page->Grant); > - > - for (SegIdx = 0; > - SegIdx < XENVBD_MAX_SEGMENTS_PER_PAGE && > - SegEntry != &Request->Segments; > - ++SegIdx, SegEntry = SegEntry->Flink) { > - PXENVBD_SEGMENT Segment = CONTAINING_RECORD(SegEntry, > XENVBD_SEGMENT, ListEntry); > - > - Page->Page[SegIdx].GrantRef = GranterReference(Granter, > Segment->Grant); > - Page->Page[SegIdx].First = Segment->FirstSector; > - Page->Page[SegIdx].Last = Segment->LastSector; > - } > - } > - } else { > - // Direct > - ULONG Index; > - PLIST_ENTRY Entry; > - > - req->operation = Request->Operation; > - req->nr_segments = (UCHAR)Request->NrSegments; > - req->handle = > (USHORT)FrontendGetDeviceId(Ring- > >Frontend); > - req->id = (ULONG64)(ULONG_PTR)Request; > - req->sector_number = Request->FirstSector; > + ASSERT3U(SectorSize, != , 0); > + return PAGE_SIZE / SectorSize; > +} > > - for (Index = 0, Entry = Request->Segments.Flink; > - Index < BLKIF_MAX_SEGMENTS_PER_REQUEST && > - Entry != &Request->Segments; > - ++Index, Entry = Entry->Flink) { > - PXENVBD_SEGMENT Segment = CONTAINING_RECORD(Entry, > XENVBD_SEGMENT, ListEntry); > - req->seg[Index].gref = GranterReference(Granter, > Segment- > >Grant); > - req->seg[Index].first_sect = Segment->FirstSector; > - req->seg[Index].last_sect = Segment->LastSector; > - } > - } > +static FORCEINLINE VOID > +__Operation( > + IN UCHAR CdbOp, > + OUT PUCHAR RingOp, > + OUT PBOOLEAN ReadOnly > + ) > +{ > + switch (CdbOp) { > + case SCSIOP_READ: > + *RingOp = BLKIF_OP_READ; > + *ReadOnly = FALSE; > break; > - > - case BLKIF_OP_WRITE_BARRIER: > - case BLKIF_OP_FLUSH_DISKCACHE: > - req->operation = Request->Operation; > - req->nr_segments = 0; > - req->handle = (USHORT)FrontendGetDeviceId(Ring- > >Frontend); > - req->id = (ULONG64)(ULONG_PTR)Request; > - req->sector_number = Request->FirstSector; > + case SCSIOP_WRITE: > + *RingOp = BLKIF_OP_WRITE; > + *ReadOnly = TRUE; > break; > - > - case BLKIF_OP_DISCARD: { > - blkif_request_discard_t* req_discard; > - req_discard = (blkif_request_discard_t*)req; > - req_discard->operation = BLKIF_OP_DISCARD; > - req_discard->flag = Request->Flags; > - req_discard->handle = (USHORT)FrontendGetDeviceId(Ring- > >Frontend); > - req_discard->id = (ULONG64)(ULONG_PTR)Request; > - req_discard->sector_number = Request->FirstSector; > - req_discard->nr_sectors = Request->NrSectors; > - } break; > - > default: > ASSERT(FALSE); > - break; > } > - ++Ring->Submitted; > +} > + > +static FORCEINLINE ULONG > +__UseIndirect( > + IN ULONG SectorsPerPage, > + IN ULONG MaxIndirectSegs, > + IN ULONG SectorsLeft > + ) > +{ > + if (MaxIndirectSegs <= BLKIF_MAX_SEGMENTS_PER_REQUEST) > + return BLKIF_MAX_SEGMENTS_PER_REQUEST; // not supported > + > + if (SectorsLeft < BLKIF_MAX_SEGMENTS_PER_REQUEST * > SectorsPerPage) > + return BLKIF_MAX_SEGMENTS_PER_REQUEST; // first into a single > BLKIF_OP_{READ/WRITE} > + > + return MaxIndirectSegs; > +} > + > +static FORCEINLINE MM_PAGE_PRIORITY > +__Priority( > + IN PXENVBD_CAPS Caps > + ) > +{ > + return (Caps->Paging || > + Caps->Hibernation || > + Caps->DumpFile) ? HighPagePriority : > + NormalPagePriority; > } > > static PXENVBD_INDIRECT > -RingGetIndirect( > - IN PXENVBD_RING Ring > +BlkifRingGetIndirect( > + IN PXENVBD_BLKIF_RING BlkifRing > ) > { > - PXENVBD_INDIRECT Indirect; > - NTSTATUS status; > - PXENVBD_GRANTER Granter = FrontendGetGranter(Ring->Frontend); > + PXENVBD_RING Ring = BlkifRing->Ring; > + PXENVBD_FRONTEND Frontend = Ring->Frontend; > + PXENVBD_GRANTER Granter = FrontendGetGranter(Frontend); > + PXENVBD_INDIRECT Indirect; > + NTSTATUS status; > > Indirect = XENBUS_CACHE(Get, > &Ring->CacheInterface, > - Ring->IndirectCache, > - FALSE); > + BlkifRing->IndirectCache, > + TRUE); > if (Indirect == NULL) > goto fail1; > > - ASSERT3P(Indirect->Mdl, !=, NULL); > - ASSERT3P(Indirect->Page, !=, NULL); > + ASSERT3P(Indirect->Mdl, != , NULL); > + ASSERT3P(Indirect->Page, != , NULL); > status = GranterGet(Granter, > MmGetMdlPfnArray(Indirect->Mdl)[0], > TRUE, > @@ -267,20 +272,22 @@ RingGetIndirect( > fail2: > XENBUS_CACHE(Put, > &Ring->CacheInterface, > - Ring->IndirectCache, > + BlkifRing->IndirectCache, > Indirect, > - FALSE); > + TRUE); > fail1: > return NULL; > } > > static VOID > -RingPutIndirect( > - IN PXENVBD_RING Ring, > +BlkifRingPutIndirect( > + IN PXENVBD_BLKIF_RING BlkifRing, > IN PXENVBD_INDIRECT Indirect > ) > { > - PXENVBD_GRANTER Granter = FrontendGetGranter(Ring->Frontend); > + PXENVBD_RING Ring = BlkifRing->Ring; > + PXENVBD_FRONTEND Frontend = Ring->Frontend; > + PXENVBD_GRANTER Granter = FrontendGetGranter(Frontend); > > if (Indirect->Grant) > GranterPut(Granter, Indirect->Grant); > @@ -290,30 +297,34 @@ RingPutIndirect( > > XENBUS_CACHE(Put, > &Ring->CacheInterface, > - Ring->IndirectCache, > + BlkifRing->IndirectCache, > Indirect, > - FALSE); > + TRUE); > } > > static PXENVBD_SEGMENT > -RingGetSegment( > - IN PXENVBD_RING Ring > +BlkifRingGetSegment( > + IN PXENVBD_BLKIF_RING BlkifRing > ) > { > + PXENVBD_RING Ring = BlkifRing->Ring; > + > return XENBUS_CACHE(Get, > &Ring->CacheInterface, > - Ring->SegmentCache, > - FALSE); > + BlkifRing->SegmentCache, > + TRUE); > } > > static VOID > -RingPutSegment( > - IN PXENVBD_RING Ring, > - IN PXENVBD_SEGMENT Segment > +BlkifRingPutSegment( > + IN PXENVBD_BLKIF_RING BlkifRing, > + IN PXENVBD_SEGMENT Segment > ) > { > - PXENVBD_GRANTER Granter = FrontendGetGranter(Ring->Frontend); > - PXENVBD_BOUNCE Bounce = Segment->Bounce; > + PXENVBD_RING Ring = BlkifRing->Ring; > + PXENVBD_FRONTEND Frontend = Ring->Frontend; > + PXENVBD_GRANTER Granter = FrontendGetGranter(Frontend); > + PXENVBD_BOUNCE Bounce = Segment->Bounce; > > if (Segment->Grant) > GranterPut(Granter, Segment->Grant); > @@ -329,7 +340,7 @@ RingPutSegment( > Bounce->SourcePfn[0] = 0; > Bounce->SourcePfn[1] = 0; > > - AdapterPutBounce(TargetGetAdapter(FrontendGetTarget(Ring- > >Frontend)), > + AdapterPutBounce(TargetGetAdapter(FrontendGetTarget(Frontend)), > Bounce); > } > Segment->Bounce = NULL; > @@ -340,29 +351,32 @@ RingPutSegment( > > XENBUS_CACHE(Put, > &Ring->CacheInterface, > - Ring->SegmentCache, > + BlkifRing->SegmentCache, > Segment, > - FALSE); > + TRUE); > } > > static PXENVBD_REQUEST > -RingGetRequest( > - IN PXENVBD_RING Ring > +BlkifRingGetRequest( > + IN PXENVBD_BLKIF_RING BlkifRing > ) > { > + PXENVBD_RING Ring = BlkifRing->Ring; > + > return XENBUS_CACHE(Get, > &Ring->CacheInterface, > - Ring->RequestCache, > - FALSE); > + BlkifRing->RequestCache, > + TRUE); > } > > static VOID > -RingPutRequest( > - IN PXENVBD_RING Ring, > - IN PXENVBD_REQUEST Request > +BlkifRingPutRequest( > + IN PXENVBD_BLKIF_RING BlkifRing, > + IN PXENVBD_REQUEST Request > ) > { > - PLIST_ENTRY ListEntry; > + PXENVBD_RING Ring = BlkifRing->Ring; > + PLIST_ENTRY ListEntry; > > for (;;) { > PXENVBD_SEGMENT Segment; > @@ -371,7 +385,7 @@ RingPutRequest( > if (ListEntry == &Request->Segments) > break; > Segment = CONTAINING_RECORD(ListEntry, XENVBD_SEGMENT, > ListEntry); > - RingPutSegment(Ring, Segment); > + BlkifRingPutSegment(BlkifRing, Segment); > } > > for (;;) { > @@ -381,7 +395,7 @@ RingPutRequest( > if (ListEntry == &Request->Indirects) > break; > Indirect = CONTAINING_RECORD(ListEntry, XENVBD_INDIRECT, > ListEntry); > - RingPutIndirect(Ring, Indirect); > + BlkifRingPutIndirect(BlkifRing, Indirect); > } > > Request->SrbExt = NULL; > @@ -394,189 +408,277 @@ RingPutRequest( > > XENBUS_CACHE(Put, > &Ring->CacheInterface, > - Ring->RequestCache, > + BlkifRing->RequestCache, > Request, > - FALSE); > + TRUE); > } > > -static FORCEINLINE PXENVBD_REQUEST > -RingFindRequest( > - IN PXENVBD_RING Ring, > - IN ULONG64 Id > +static DECLSPEC_NOINLINE NTSTATUS > +BlkifRingRequestCtor( > + IN PVOID Argument, > + IN PVOID Object > ) > { > - KIRQL Irql; > - PLIST_ENTRY ListEntry; > - PXENVBD_REQUEST Request; > - PXENVBD_QUEUE Queue = &Ring->SubmittedReqs; > - > - KeAcquireSpinLock(&Queue->Lock, &Irql); > + PXENVBD_REQUEST Request = Object; > > - for (ListEntry = Queue->List.Flink; > - ListEntry != &Queue->List; > - ListEntry = ListEntry->Flink) { > - Request = CONTAINING_RECORD(ListEntry, XENVBD_REQUEST, > ListEntry); > - if ((ULONG64)(ULONG_PTR)Request == Id) { > - RemoveEntryList(&Request->ListEntry); > - --Queue->Current; > - KeReleaseSpinLock(&Queue->Lock, Irql); > - return Request; > - } > - } > + UNREFERENCED_PARAMETER(Argument); > > - KeReleaseSpinLock(&Queue->Lock, Irql); > - Warning("Target[%d] : Tag %llx not found in submitted list (%u items)\n", > - FrontendGetTargetId(Ring->Frontend), > - Id, > - QueueCount(Queue)); > - return NULL; > + InitializeListHead(&Request->Segments); > + InitializeListHead(&Request->Indirects); > + return STATUS_SUCCESS; > } > > -static FORCEINLINE VOID > -__RingIncBlkifOpCount( > - IN PXENVBD_RING Ring, > - IN PXENVBD_REQUEST Request > +static DECLSPEC_NOINLINE VOID > +BlkifRingRequestDtor( > + IN PVOID Argument, > + IN PVOID Object > ) > { > - switch (Request->Operation) { > - case BLKIF_OP_READ: > - if (Request->NrSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST) > - ++Ring->BlkOpIndirectRead; > - else > - ++Ring->BlkOpRead; > - break; > - case BLKIF_OP_WRITE: > - if (Request->NrSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST) > - ++Ring->BlkOpIndirectWrite; > - else > - ++Ring->BlkOpWrite; > - break; > - case BLKIF_OP_WRITE_BARRIER: > - ++Ring->BlkOpBarrier; > - break; > - case BLKIF_OP_DISCARD: > - ++Ring->BlkOpDiscard; > - break; > - case BLKIF_OP_FLUSH_DISKCACHE: > - ++Ring->BlkOpFlush; > - break; > - default: > - ASSERT(FALSE); > - break; > - } > + UNREFERENCED_PARAMETER(Argument); > + UNREFERENCED_PARAMETER(Object); > } > > -static FORCEINLINE ULONG > -__RingSectorsPerPage( > - IN ULONG SectorSize > +static DECLSPEC_NOINLINE NTSTATUS > +BlkifRingSegmentCtor( > + IN PVOID Argument, > + IN PVOID Object > ) > { > - ASSERT3U(SectorSize, !=, 0); > - return PAGE_SIZE / SectorSize; > + UNREFERENCED_PARAMETER(Argument); > + UNREFERENCED_PARAMETER(Object); > + return STATUS_SUCCESS; > } > > -static FORCEINLINE VOID > -__RingOperation( > - IN UCHAR CdbOp, > - OUT PUCHAR RingOp, > - OUT PBOOLEAN ReadOnly > +static DECLSPEC_NOINLINE VOID > +BlkifRingSegmentDtor( > + IN PVOID Argument, > + IN PVOID Object > ) > { > - switch (CdbOp) { > - case SCSIOP_READ: > - *RingOp = BLKIF_OP_READ; > - *ReadOnly = FALSE; > - break; > - case SCSIOP_WRITE: > - *RingOp = BLKIF_OP_WRITE; > - *ReadOnly = TRUE; > - break; > - default: > - ASSERT(FALSE); > - } > + UNREFERENCED_PARAMETER(Argument); > + UNREFERENCED_PARAMETER(Object); > } > > -static FORCEINLINE MM_PAGE_PRIORITY > -__RingPriority( > - IN PXENVBD_RING Ring > +static DECLSPEC_NOINLINE NTSTATUS > +BlkifRingIndirectCtor( > + IN PVOID Argument, > + IN PVOID Object > ) > { > - PXENVBD_CAPS Caps = FrontendGetCaps(Ring->Frontend); > - if (!(Caps->Paging || > - Caps->Hibernation || > - Caps->DumpFile)) > - return NormalPagePriority; > + PXENVBD_INDIRECT Indirect = Object; > + NTSTATUS status; > + > + UNREFERENCED_PARAMETER(Argument); > + > + status = STATUS_NO_MEMORY; > + Indirect->Mdl = __AllocatePage(); > + if (Indirect->Mdl == NULL) > + goto fail1; > + > + Indirect->Page = MmGetSystemAddressForMdlSafe(Indirect->Mdl, > + NormalPagePriority); > + ASSERT(Indirect->Page); > > - return HighPagePriority; > + return STATUS_SUCCESS; > + > +fail1: > + Error("fail1 %08x\n", status); > + return status; > } > > -static FORCEINLINE VOID > -RingRequestCopyOutput( > - IN PXENVBD_REQUEST Request > +static DECLSPEC_NOINLINE VOID > +BlkifRingIndirectDtor( > + IN PVOID Argument, > + IN PVOID Object > ) > { > - PLIST_ENTRY ListEntry; > - > - if (Request->Operation != BLKIF_OP_READ) > - return; > + PXENVBD_INDIRECT Indirect = Object; > > - for (ListEntry = Request->Segments.Flink; > - ListEntry != &Request->Segments; > - ListEntry = ListEntry->Flink) { > - PXENVBD_SEGMENT Segment = CONTAINING_RECORD(ListEntry, > XENVBD_SEGMENT, ListEntry); > - PXENVBD_BOUNCE Bounce = Segment->Bounce; > + UNREFERENCED_PARAMETER(Argument); > > - if (Bounce) { > - RtlCopyMemory(Bounce->SourcePtr, > - Bounce->BouncePtr, > - MmGetMdlByteCount(&Bounce->SourceMdl)); > - } > - } > + __FreePages(Indirect->Mdl); > + Indirect->Page = NULL; > + Indirect->Mdl = NULL; > } > > -static BOOLEAN > -RingPrepareSegment( > - IN PXENVBD_RING Ring, > - IN PXENVBD_SEGMENT Segment, > - IN PXENVBD_SRBEXT SrbExt, > - IN BOOLEAN ReadOnly, > - IN ULONG SectorsLeft, > - OUT PULONG SectorsNow > - ) > -{ > - PFN_NUMBER Pfn; > - ULONG Offset; > - ULONG Length; > - NTSTATUS Status; > - PXENVBD_GRANTER Granter = FrontendGetGranter(Ring->Frontend); > - const ULONG SectorSize = FrontendGetDiskInfo(Ring->Frontend)- > >SectorSize; > - const ULONG SectorsPerPage = __RingSectorsPerPage(SectorSize); > - PXENVBD_TARGET Target = FrontendGetTarget(Ring->Frontend); > - PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target); > +static VOID > +BlkifRingDebugCallback( > + IN PVOID Argument, > + IN BOOLEAN Crashing > + ) > +{ > + PXENVBD_BLKIF_RING BlkifRing = Argument; > + PXENVBD_RING Ring = BlkifRing->Ring; > > - Pfn = AdapterGetNextSGEntry(Adapter, > - SrbExt, > - 0, > - &Offset, > - &Length); > - if ((Offset & (SectorSize - 1)) == 0 && > + UNREFERENCED_PARAMETER(Crashing); > + > + XENBUS_DEBUG(Printf, > + &Ring->DebugInterface, > + "0x%p [%s]\n", > + BlkifRing, > + (BlkifRing->Enabled) ? "ENABLED" : "DISABLED"); > + > + // Dump front ring > + XENBUS_DEBUG(Printf, > + &Ring->DebugInterface, > + "FRONT: req_prod_pvt = %u rsp_cons = %u nr_ents = %u sring = > %p\n", > + BlkifRing->Front.req_prod_pvt, > + BlkifRing->Front.rsp_cons, > + BlkifRing->Front.nr_ents, > + BlkifRing->Front.sring); > + > + // Dump shared ring > + XENBUS_DEBUG(Printf, > + &Ring->DebugInterface, > + "SHARED: req_prod = %u req_event = %u rsp_prod = %u > rsp_event = %u\n", > + BlkifRing->Shared->req_prod, > + BlkifRing->Shared->req_event, > + BlkifRing->Shared->rsp_prod, > + BlkifRing->Shared->rsp_event); > + > + XENBUS_DEBUG(Printf, > + &Ring->DebugInterface, > + "RequestsPosted = %u RequestsPushed = %u ResponsesProcessed > = %u\n", > + BlkifRing->RequestsPosted, > + BlkifRing->RequestsPushed, > + BlkifRing->ResponsesProcessed); > + > + XENBUS_DEBUG(Printf, > + &Ring->DebugInterface, > + "SrbsQueued = %u, SrbsCompleted = %u, SrbsFailed = %u\n", > + BlkifRing->SrbsQueued, > + BlkifRing->SrbsCompleted, > + BlkifRing->SrbsFailed); > + > + XENBUS_DEBUG(Printf, > + &Ring->DebugInterface, > + "Dpcs = %u, Events = %u\n", > + BlkifRing->Dpcs, > + BlkifRing->Events); > +} > + > +static DECLSPEC_NOINLINE VOID > +__BlkifRingCompleteSrb( > + IN PXENVBD_BLKIF_RING BlkifRing, > + IN PXENVBD_SRBEXT SrbExt > + ) > +{ > + PXENVBD_RING Ring = BlkifRing->Ring; > + PXENVBD_FRONTEND Frontend = Ring->Frontend; > + PXENVBD_TARGET Target = FrontendGetTarget(Frontend); > + PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target); > + PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb; > + > + if (Srb->SrbStatus == SRB_STATUS_PENDING) { > + // SRB has not hit a failure condition (BLKIF_RSP_ERROR | > BLKIF_RSP_EOPNOTSUPP) > + // from any of its responses. SRB must have succeeded > + Srb->SrbStatus = SRB_STATUS_SUCCESS; > + Srb->ScsiStatus = 0x00; // SCSI_GOOD > + ++BlkifRing->SrbsCompleted; > + } else { > + // Srb->SrbStatus has already been set by 1 or more requests with > Status != BLKIF_RSP_OKAY > + Srb->ScsiStatus = 0x40; // SCSI_ABORTED > + ++BlkifRing->SrbsFailed; > + } > + > + AdapterCompleteSrb(Adapter, SrbExt); > +} > + > +static FORCEINLINE VOID > +BlkifRingUnprepareRequest( > + IN PXENVBD_BLKIF_RING BlkifRing, > + IN PLIST_ENTRY List > + ) > +{ > + for (;;) { > + PLIST_ENTRY ListEntry; > + PXENVBD_REQUEST Request; > + > + ListEntry = RemoveHeadList(List); > + if (ListEntry == List) > + break; > + > + Request = CONTAINING_RECORD(ListEntry, > + XENVBD_REQUEST, > + ListEntry); > + > + BlkifRingPutRequest(BlkifRing, Request); > + } > +} > + > +static FORCEINLINE VOID > +BlkifRingQueueRequests( > + IN PXENVBD_BLKIF_RING BlkifRing, > + IN PLIST_ENTRY List > + ) > +{ > + PXENVBD_SRB_STATE State = &BlkifRing->State; > + > + for (;;) { > + PLIST_ENTRY ListEntry; > + PXENVBD_REQUEST Request; > + > + ListEntry = RemoveHeadList(List); > + if (ListEntry == List) > + break; > + > + Request = CONTAINING_RECORD(ListEntry, > + XENVBD_REQUEST, > + ListEntry); > + > + InsertTailList(&State->List, ListEntry); > + State->Count++; > + } > +} > + > +static BOOLEAN > +BlkifRingPrepareSegment( > + IN PXENVBD_BLKIF_RING BlkifRing, > + IN PXENVBD_SEGMENT Segment, > + IN PXENVBD_SRBEXT SrbExt, > + IN BOOLEAN ReadOnly, > + IN ULONG SectorsLeft, > + OUT PULONG SectorsNow > + ) > +{ > + PFN_NUMBER Pfn; > + ULONG Offset; > + ULONG Length; > + NTSTATUS Status; > + PXENVBD_RING Ring = BlkifRing->Ring; > + PXENVBD_GRANTER Granter = FrontendGetGranter(Ring- > >Frontend); > + PXENVBD_TARGET Target = FrontendGetTarget(Ring->Frontend); > + PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target); > + > + const ULONG SectorSize = FrontendGetDiskInfo(Ring->Frontend)- > >SectorSize; > + const ULONG SectorsPerPage = __SectorsPerPage(SectorSize); > + > + Pfn = AdapterGetNextSGEntry(Adapter, > + SrbExt, > + 0, > + &Offset, > + &Length); > + if ((Offset & (SectorSize - 1)) == 0 && > (Length & (SectorSize - 1)) == 0) { > - ++Ring->SegsGranted; > + InterlockedIncrement(&Ring- > >Stats[XENVBD_STAT_SEGMENTS_GRANTED]); > + > // get first sector, last sector and count > - Segment->FirstSector = (UCHAR)((Offset + SectorSize - 1) / > SectorSize); > - *SectorsNow = __min(SectorsLeft, SectorsPerPage - > Segment- > >FirstSector); > - Segment->LastSector = (UCHAR)(Segment->FirstSector + > *SectorsNow - 1); > + Segment->FirstSector = (UCHAR)((Offset + SectorSize - 1) / > SectorSize); > + *SectorsNow = __min(SectorsLeft, SectorsPerPage - Segment- > >FirstSector); > + Segment->LastSector = (UCHAR)(Segment->FirstSector + *SectorsNow > - 1); > > - ASSERT3U((Length / SectorSize), ==, *SectorsNow); > + ASSERT3U((Length / SectorSize), == , *SectorsNow); > } else { > PXENVBD_BOUNCE Bounce; > PMDL Mdl; > + PXENVBD_CAPS Caps = FrontendGetCaps(Ring->Frontend); > + > + InterlockedIncrement(&Ring- > >Stats[XENVBD_STAT_SEGMENTS_BOUNCED]); > > - ++Ring->SegsBounced; > // get first sector, last sector and count > - Segment->FirstSector = 0; > - *SectorsNow = __min(SectorsLeft, SectorsPerPage); > - Segment->LastSector = (UCHAR)(*SectorsNow - 1); > + Segment->FirstSector = 0; > + *SectorsNow = __min(SectorsLeft, SectorsPerPage); > + Segment->LastSector = (UCHAR)(*SectorsNow - 1); > > Bounce = AdapterGetBounce(Adapter); > if (Bounce == NULL) > @@ -586,15 +688,15 @@ RingPrepareSegment( > #pragma warning(push) > #pragma warning(disable:28145) > Mdl = &Bounce->SourceMdl; > - Mdl->Next = NULL; > - Mdl->Size = (SHORT)(sizeof(MDL) + sizeof(PFN_NUMBER)); > - Mdl->MdlFlags = MDL_PAGES_LOCKED; > - Mdl->Process = NULL; > - Mdl->MappedSystemVa = NULL; > - Mdl->StartVa = NULL; > - Mdl->ByteCount = Length; > - Mdl->ByteOffset = Offset; > - Bounce->SourcePfn[0] = Pfn; > + Mdl->Next = NULL; > + Mdl->Size = (SHORT)(sizeof(MDL) + sizeof(PFN_NUMBER)); > + Mdl->MdlFlags = MDL_PAGES_LOCKED; > + Mdl->Process = NULL; > + Mdl->MappedSystemVa = NULL; > + Mdl->StartVa = NULL; > + Mdl->ByteCount = Length; > + Mdl->ByteOffset = Offset; > + Bounce->SourcePfn[0] = Pfn; > > if (Length < *SectorsNow * SectorSize) { > Pfn = AdapterGetNextSGEntry(Adapter, > @@ -602,27 +704,27 @@ RingPrepareSegment( > Length, > &Offset, > &Length); > - Mdl->Size += sizeof(PFN_NUMBER); > - Mdl->ByteCount += Length; > + Mdl->Size += sizeof(PFN_NUMBER); > + Mdl->ByteCount += Length; > Bounce->SourcePfn[1] = Pfn; > } > #pragma warning(pop) > > ASSERT((Mdl->ByteCount & (SectorSize - 1)) == 0); > - ASSERT3U(Mdl->ByteCount, <=, PAGE_SIZE); > - ASSERT3U(*SectorsNow, ==, (Mdl->ByteCount / SectorSize)); > + ASSERT3U(Mdl->ByteCount, <= , PAGE_SIZE); > + ASSERT3U(*SectorsNow, == , (Mdl->ByteCount / SectorSize)); > > Bounce->SourcePtr = MmMapLockedPagesSpecifyCache(Mdl, > KernelMode, > MmCached, > NULL, > FALSE, > - > __RingPriority(Ring)); > + __Priority(Caps)); > if (Bounce->SourcePtr == NULL) > goto fail2; > > - ASSERT3P(MmGetMdlPfnArray(Mdl)[0], ==, Bounce->SourcePfn[0]); > - ASSERT3P(MmGetMdlPfnArray(Mdl)[1], ==, Bounce->SourcePfn[1]); > + ASSERT3P(MmGetMdlPfnArray(Mdl)[0], == , Bounce->SourcePfn[0]); > + ASSERT3P(MmGetMdlPfnArray(Mdl)[1], == , Bounce->SourcePfn[1]); > > // copy contents in > if (ReadOnly) { // Operation == BLKIF_OP_WRITE > @@ -647,523 +749,510 @@ fail1: > return FALSE; > } > > -static BOOLEAN > -RingPrepareBlkifReadWrite( > - IN PXENVBD_RING Ring, > - IN PXENVBD_REQUEST Request, > - IN PXENVBD_SRBEXT SrbExt, > - IN ULONG MaxSegments, > - IN ULONG64 SectorStart, > - IN ULONG SectorsLeft, > - OUT PULONG SectorsDone > +static NTSTATUS > +BlkifRingPrepareReadWrite( > + IN PXENVBD_BLKIF_RING BlkifRing, > + IN PXENVBD_SRBEXT SrbExt > ) > { > - PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb; > - UCHAR Operation; > - BOOLEAN ReadOnly; > - ULONG Index; > - __RingOperation(Cdb_OperationEx(Srb), &Operation, &ReadOnly); > - > - Request->Operation = Operation; > - Request->NrSegments = 0; > - Request->FirstSector = SectorStart; > - > - for (Index = 0; > - Index < MaxSegments && > - SectorsLeft > 0; > - ++Index) { > - PXENVBD_SEGMENT Segment; > - ULONG SectorsNow; > - > - Segment = RingGetSegment(Ring); > - if (Segment == NULL) > - goto fail1; > - > - InsertTailList(&Request->Segments, &Segment->ListEntry); > - ++Request->NrSegments; > + PXENVBD_RING Ring = BlkifRing->Ring; > + PXENVBD_FRONTEND Frontend = Ring->Frontend; > + PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb; > + ULONG64 SectorStart = Cdb_LogicalBlock(Srb); > + ULONG SectorsLeft = Cdb_TransferBlock(Srb); > + UCHAR Operation; > + BOOLEAN ReadOnly; > + LIST_ENTRY List; > > - if (!RingPrepareSegment(Ring, > - Segment, > - SrbExt, > - ReadOnly, > - SectorsLeft, > - &SectorsNow)) > - goto fail2; > + const ULONG SectorSize = FrontendGetDiskInfo(Frontend)- > >SectorSize; > + const ULONG SectorsPerPage = __SectorsPerPage(SectorSize); > + const ULONG MaxIndirect = FrontendGetFeatures(Frontend)- > >Indirect; > > - *SectorsDone += SectorsNow; > - SectorsLeft -= SectorsNow; > - } > - ASSERT3U(Request->NrSegments, >, 0); > - ASSERT3U(Request->NrSegments, <=, MaxSegments); > + InitializeListHead(&List); > > - return TRUE; > + __Operation(Cdb_OperationEx(Srb), &Operation, &ReadOnly); > > -fail2: > -fail1: > - return FALSE; > -} > + Srb->SrbStatus = SRB_STATUS_PENDING; > > -static BOOLEAN > -RingPrepareBlkifIndirect( > - IN PXENVBD_RING Ring, > - IN PXENVBD_REQUEST Request > - ) > -{ > - ULONG Index; > - ULONG NrSegments = 0; > + SrbExt->RequestCount = 0; > > - for (Index = 0; > - Index < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST && > - NrSegments < Request->NrSegments; > - ++Index) { > - PXENVBD_INDIRECT Indirect; > + while (SectorsLeft > 0) { > + ULONG Index; > + ULONG MaxSegments; > + PXENVBD_REQUEST Request; > > - Indirect = RingGetIndirect(Ring); > - if (Indirect == NULL) > + Request = BlkifRingGetRequest(BlkifRing); > + if (Request == NULL) > goto fail1; > - InsertTailList(&Request->Indirects, &Indirect->ListEntry); > - > - NrSegments += XENVBD_MAX_SEGMENTS_PER_PAGE; > - } > - > - return TRUE; > - > -fail1: > - return FALSE; > -} > + InsertTailList(&List, &Request->ListEntry); > + SrbExt->RequestCount++; > > -static FORCEINLINE ULONG > -RingUseIndirect( > - IN PXENVBD_RING Ring, > - IN ULONG SectorsLeft > - ) > -{ > - const ULONG SectorsPerPage = > __RingSectorsPerPage(FrontendGetDiskInfo(Ring->Frontend)->SectorSize); > - const ULONG MaxIndirectSegs = FrontendGetFeatures(Ring->Frontend)- > >Indirect; > + Request->SrbExt = SrbExt; > > - if (MaxIndirectSegs <= BLKIF_MAX_SEGMENTS_PER_REQUEST) > - return BLKIF_MAX_SEGMENTS_PER_REQUEST; // not supported > + MaxSegments = __UseIndirect(SectorsPerPage, > + MaxIndirect, > + SectorsLeft); > + > + Request->Operation = Operation; > + Request->NrSegments = 0; > + Request->FirstSector = SectorStart; > + > + for (Index = 0; > + Index < MaxSegments && > + SectorsLeft > 0; > + ++Index) { > + PXENVBD_SEGMENT Segment; > + ULONG SectorsNow; > + > + Segment = BlkifRingGetSegment(BlkifRing); > + if (Segment == NULL) > + goto fail2; > + > + InsertTailList(&Request->Segments, &Segment->ListEntry); > + ++Request->NrSegments; > + > + if (!BlkifRingPrepareSegment(BlkifRing, > + Segment, > + SrbExt, > + ReadOnly, > + SectorsLeft, > + &SectorsNow)) > + goto fail3; > > - if (SectorsLeft < BLKIF_MAX_SEGMENTS_PER_REQUEST * > SectorsPerPage) > - return BLKIF_MAX_SEGMENTS_PER_REQUEST; // first into a single > BLKIF_OP_{READ/WRITE} > + SectorsLeft -= SectorsNow; > + SectorStart += SectorsNow; > + } > + ASSERT3U(Request->NrSegments, >, 0); > + ASSERT3U(Request->NrSegments, <= , MaxSegments); > > - return MaxIndirectSegs; > -} > + if (MaxSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST) { > + ULONG NrSegments = 0; > > -static FORCEINLINE VOID > -RingQueueRequestList( > - IN PXENVBD_RING Ring, > - IN PLIST_ENTRY List > - ) > -{ > - for (;;) { > - PXENVBD_REQUEST Request; > - PLIST_ENTRY ListEntry; > + for (Index = 0; > + Index < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST && > + NrSegments < Request->NrSegments; > + ++Index) { > + PXENVBD_INDIRECT Indirect; > > - ListEntry = RemoveHeadList(List); > - if (ListEntry == List) > - break; > + Indirect = BlkifRingGetIndirect(BlkifRing); > + if (Indirect == NULL) > + goto fail3; > + InsertTailList(&Request->Indirects, &Indirect->ListEntry); > > - Request = CONTAINING_RECORD(ListEntry, XENVBD_REQUEST, > ListEntry); > - __RingIncBlkifOpCount(Ring, Request); > - QueueAppend(&Ring->PreparedReqs, &Request->ListEntry); > + NrSegments += XENVBD_MAX_SEGMENTS_PER_PAGE; > + } > + } > } > -} > - > -static FORCEINLINE VOID > -RingCancelRequestList( > - IN PXENVBD_RING Ring, > - IN PLIST_ENTRY List > - ) > -{ > - for (;;) { > - PXENVBD_REQUEST Request; > - PLIST_ENTRY ListEntry; > > - ListEntry = RemoveHeadList(List); > - if (ListEntry == List) > - break; > + BlkifRingQueueRequests(BlkifRing, &List); > + return STATUS_SUCCESS; > > - Request = CONTAINING_RECORD(ListEntry, XENVBD_REQUEST, > ListEntry); > - RingPutRequest(Ring, Request); > - } > +fail3: > +fail2: > +fail1: > + BlkifRingUnprepareRequest(BlkifRing, &List); > + SrbExt->RequestCount = 0; > + Srb->SrbStatus = SRB_STATUS_ERROR; > + return STATUS_UNSUCCESSFUL; > } > > -static BOOLEAN > -RingPrepareReadWrite( > - IN PXENVBD_RING Ring, > +static NTSTATUS > +BlkifRingPrepareUnmap( > + IN PXENVBD_BLKIF_RING BlkifRing, > IN PXENVBD_SRBEXT SrbExt > ) > { > PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb; > - ULONG64 SectorStart = Cdb_LogicalBlock(Srb); > - ULONG SectorsLeft = Cdb_TransferBlock(Srb); > + PUNMAP_LIST_HEADER Unmap = Srb->DataBuffer; > + ULONG Count; > + ULONG Index; > LIST_ENTRY List; > > + InitializeListHead(&List); > + > + Count = _byteswap_ushort(*(PUSHORT)Unmap->BlockDescrDataLength) > / sizeof(UNMAP_BLOCK_DESCRIPTOR); > Srb->SrbStatus = SRB_STATUS_PENDING; > > - InitializeListHead(&List); > SrbExt->RequestCount = 0; > > - while (SectorsLeft > 0) { > - ULONG MaxSegments; > - ULONG SectorsDone = 0; > - PXENVBD_REQUEST Request; > + for (Index = 0; Index < Count; ++Index) { > + PUNMAP_BLOCK_DESCRIPTOR Descr = &Unmap->Descriptors[Index]; > + PXENVBD_REQUEST Request; > > - Request = RingGetRequest(Ring); > + Request = BlkifRingGetRequest(BlkifRing); > if (Request == NULL) > goto fail1; > InsertTailList(&List, &Request->ListEntry); > - InterlockedIncrement(&SrbExt->RequestCount); > + SrbExt->RequestCount++; > > Request->SrbExt = SrbExt; > - MaxSegments = RingUseIndirect(Ring, SectorsLeft); > - > - if (!RingPrepareBlkifReadWrite(Ring, > - Request, > - SrbExt, > - MaxSegments, > - SectorStart, > - SectorsLeft, > - &SectorsDone)) > - goto fail2; > - > - if (MaxSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST) { > - if (!RingPrepareBlkifIndirect(Ring, Request)) > - goto fail3; > - } > - > - SectorsLeft -= SectorsDone; > - SectorStart += SectorsDone; > + Request->Operation = BLKIF_OP_DISCARD; > + Request->FirstSector = _byteswap_uint64(*(PULONG64)Descr- > >StartingLba); > + Request->NrSectors = _byteswap_ulong(*(PULONG)Descr- > >LbaCount); > + Request->Flags = 0; > } > > - RingQueueRequestList(Ring, &List); > - return TRUE; > + BlkifRingQueueRequests(BlkifRing, &List); > + return STATUS_SUCCESS; > > -fail3: > -fail2: > fail1: > - RingCancelRequestList(Ring, &List); > + BlkifRingUnprepareRequest(BlkifRing, &List); > SrbExt->RequestCount = 0; > Srb->SrbStatus = SRB_STATUS_ERROR; > - return FALSE; > + return STATUS_UNSUCCESSFUL; > } > > -static BOOLEAN > -RingPrepareSyncCache( > - IN PXENVBD_RING Ring, > +static NTSTATUS > +BlkifRingPrepareSyncCache( > + IN PXENVBD_BLKIF_RING BlkifRing, > IN PXENVBD_SRBEXT SrbExt > ) > { > + PXENVBD_RING Ring = BlkifRing->Ring; > + PXENVBD_FRONTEND Frontend = Ring->Frontend; > PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb; > PXENVBD_REQUEST Request; > - LIST_ENTRY List; > UCHAR Operation; > + LIST_ENTRY List; > > + InitializeListHead(&List); > Srb->SrbStatus = SRB_STATUS_PENDING; > > - if (FrontendGetDiskInfo(Ring->Frontend)->FlushCache) > + if (FrontendGetDiskInfo(Frontend)->FlushCache) > Operation = BLKIF_OP_FLUSH_DISKCACHE; > else > Operation = BLKIF_OP_WRITE_BARRIER; > > - InitializeListHead(&List); > SrbExt->RequestCount = 0; > > - Request = RingGetRequest(Ring); > + Request = BlkifRingGetRequest(BlkifRing); > if (Request == NULL) > goto fail1; > InsertTailList(&List, &Request->ListEntry); > - InterlockedIncrement(&SrbExt->RequestCount); > + SrbExt->RequestCount++; > > - Request->SrbExt = SrbExt; > - Request->Operation = Operation; > + Request->SrbExt = SrbExt; > + Request->Operation = Operation; > Request->FirstSector = Cdb_LogicalBlock(Srb); > > - RingQueueRequestList(Ring, &List); > - return TRUE; > + BlkifRingQueueRequests(BlkifRing, &List); > + return STATUS_SUCCESS; > > fail1: > - RingCancelRequestList(Ring, &List); > + BlkifRingUnprepareRequest(BlkifRing, &List); > SrbExt->RequestCount = 0; > Srb->SrbStatus = SRB_STATUS_ERROR; > - return FALSE; > + return STATUS_UNSUCCESSFUL; > } > > -static BOOLEAN > -RingPrepareUnmap( > - IN PXENVBD_RING Ring, > +static DECLSPEC_NOINLINE NTSTATUS > +__BlkifRingPrepareSrb( > + IN PXENVBD_BLKIF_RING BlkifRing, > IN PXENVBD_SRBEXT SrbExt > ) > -{ > - PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb; > - PUNMAP_LIST_HEADER Unmap = Srb->DataBuffer; > - ULONG Count = _byteswap_ushort(*(PUSHORT)Unmap- > >BlockDescrDataLength) / sizeof(UNMAP_BLOCK_DESCRIPTOR); > - ULONG Index; > - LIST_ENTRY List; > - > - Srb->SrbStatus = SRB_STATUS_PENDING; > - > - InitializeListHead(&List); > - SrbExt->RequestCount = 0; > - > - for (Index = 0; Index < Count; ++Index) { > - PUNMAP_BLOCK_DESCRIPTOR Descr = &Unmap->Descriptors[Index]; > - PXENVBD_REQUEST Request; > - > - Request = RingGetRequest(Ring); > - if (Request == NULL) > - goto fail1; > - InsertTailList(&List, &Request->ListEntry); > - InterlockedIncrement(&SrbExt->RequestCount); > - > - Request->SrbExt = SrbExt; > - Request->Operation = BLKIF_OP_DISCARD; > - Request->FirstSector = _byteswap_uint64(*(PULONG64)Descr- > >StartingLba); > - Request->NrSectors = _byteswap_ulong(*(PULONG)Descr- > >LbaCount); > - Request->Flags = 0; > - } > - > - RingQueueRequestList(Ring, &List); > - return TRUE; > - > -fail1: > - RingCancelRequestList(Ring, &List); > - SrbExt->RequestCount = 0; > - Srb->SrbStatus = SRB_STATUS_ERROR; > - return FALSE; > -} > - > -static FORCEINLINE BOOLEAN > -RingPrepareRequest( > - IN PXENVBD_RING Ring, > - IN PXENVBD_SRBEXT SrbExt > - ) > { > switch (Cdb_OperationEx(SrbExt->Srb)) { > case SCSIOP_READ: > case SCSIOP_WRITE: > - return RingPrepareReadWrite(Ring, SrbExt); > + return BlkifRingPrepareReadWrite(BlkifRing, > + SrbExt); > > case SCSIOP_SYNCHRONIZE_CACHE: > - return RingPrepareSyncCache(Ring, SrbExt); > + return BlkifRingPrepareSyncCache(BlkifRing, > + SrbExt); > > case SCSIOP_UNMAP: > - return RingPrepareUnmap(Ring, SrbExt); > + return BlkifRingPrepareUnmap(BlkifRing, > + SrbExt); > > default: > ASSERT(FALSE); > - return FALSE; > + return STATUS_NOT_SUPPORTED; > } > } > > -static BOOLEAN > -RingSubmit( > - IN PXENVBD_RING Ring, > - IN PXENVBD_REQUEST Request > +static FORCEINLINE VOID > +__BlkifRingInsertRequest( > + IN PXENVBD_BLKIF_RING BlkifRing, > + IN PXENVBD_REQUEST Request, > + IN blkif_request_t *req > ) > { > - KIRQL Irql; > - blkif_request_t* req; > - BOOLEAN Notify; > - > - KeAcquireSpinLock(&Ring->Lock, &Irql); > - if (RING_FULL(&Ring->Front)) { > - KeReleaseSpinLock(&Ring->Lock, Irql); > - return FALSE; > - } > - > - req = RING_GET_REQUEST(&Ring->Front, Ring->Front.req_prod_pvt); > - __RingInsert(Ring, Request, req); > - KeMemoryBarrier(); > - ++Ring->Front.req_prod_pvt; > - > - RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&Ring->Front, Notify); > - KeReleaseSpinLock(&Ring->Lock, Irql); > + PXENVBD_RING Ring = BlkifRing->Ring; > + PXENVBD_FRONTEND Frontend = Ring->Frontend; > + PXENVBD_GRANTER Granter = FrontendGetGranter(Frontend); > > - if (Notify) { > - if (!Ring->Enabled) > - return TRUE; > - > - XENBUS_EVTCHN(Send, > - &Ring->EvtchnInterface, > - Ring->Channel); > - } > + switch (Request->Operation) { > + case BLKIF_OP_READ: > + case BLKIF_OP_WRITE: > + if (Request->NrSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST) { > + // Indirect > + ULONG PageIdx; > + ULONG SegIdx; > + PLIST_ENTRY PageEntry; > + PLIST_ENTRY SegEntry; > + blkif_request_indirect_t* req_indirect; > > - return TRUE; > -} > + req_indirect = (blkif_request_indirect_t*)req; > + req_indirect->operation = BLKIF_OP_INDIRECT; > + req_indirect->indirect_op = Request->Operation; > + req_indirect->nr_segments = Request->NrSegments; > + req_indirect->id = (ULONG64)(ULONG_PTR)Request; > + req_indirect->sector_number = Request->FirstSector; > + req_indirect->handle = (USHORT)FrontendGetDeviceId(Frontend); > > -static FORCEINLINE BOOLEAN > -RingSubmitRequests( > - IN PXENVBD_RING Ring > - ) > -{ > - if (!Ring->Enabled) { > - if (QueueCount(&Ring->PreparedReqs)) > - Warning("Target[%d] : Paused, not submitting new requests > (%u)\n", > - FrontendGetTargetId(Ring->Frontend), > - QueueCount(&Ring->PreparedReqs)); > - return FALSE; > - } > + for (PageIdx = 0, > + PageEntry = Request->Indirects.Flink, > + SegEntry = Request->Segments.Flink; > + PageIdx < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST && > + PageEntry != &Request->Indirects && > + SegEntry != &Request->Segments; > + ++PageIdx, PageEntry = PageEntry->Flink) { > + PXENVBD_INDIRECT Page = CONTAINING_RECORD(PageEntry, > XENVBD_INDIRECT, ListEntry); > > - for (;;) { > - PXENVBD_REQUEST Request; > - PLIST_ENTRY ListEntry; > + req_indirect->indirect_grefs[PageIdx] = > GranterReference(Granter, Page->Grant); > > - ListEntry = QueuePop(&Ring->PreparedReqs); > - if (ListEntry == NULL) > - break; > + for (SegIdx = 0; > + SegIdx < XENVBD_MAX_SEGMENTS_PER_PAGE && > + SegEntry != &Request->Segments; > + ++SegIdx, SegEntry = SegEntry->Flink) { > + PXENVBD_SEGMENT Segment = > CONTAINING_RECORD(SegEntry, XENVBD_SEGMENT, ListEntry); > + > + Page->Page[SegIdx].GrantRef = GranterReference(Granter, > Segment->Grant); > + Page->Page[SegIdx].First = Segment->FirstSector; > + Page->Page[SegIdx].Last = Segment->LastSector; > + } > + } > + InterlockedIncrement(&Ring->Stats[(Request->Operation == > BLKIF_OP_READ) ? > + XENVBD_STAT_BLKIF_OP_READ_INDIRECT : > + XENVBD_STAT_BLKIF_OP_WRITE_INDIRECT]); > + } else { > + // Direct > + ULONG Index; > + PLIST_ENTRY Entry; > > - Request = CONTAINING_RECORD(ListEntry, XENVBD_REQUEST, > ListEntry); > + req->operation = Request->Operation; > + req->nr_segments = (UCHAR)Request->NrSegments; > + req->handle = (USHORT)FrontendGetDeviceId(Frontend); > + req->id = (ULONG64)(ULONG_PTR)Request; > + req->sector_number = Request->FirstSector; > > - QueueAppend(&Ring->SubmittedReqs, &Request->ListEntry); > - KeMemoryBarrier(); > + for (Index = 0, Entry = Request->Segments.Flink; > + Index < BLKIF_MAX_SEGMENTS_PER_REQUEST && > + Entry != &Request->Segments; > + ++Index, Entry = Entry->Flink) { > + PXENVBD_SEGMENT Segment = CONTAINING_RECORD(Entry, > XENVBD_SEGMENT, ListEntry); > + req->seg[Index].gref = GranterReference(Granter, Segment- > >Grant); > + req->seg[Index].first_sect = Segment->FirstSector; > + req->seg[Index].last_sect = Segment->LastSector; > + } > + InterlockedIncrement(&Ring->Stats[(Request->Operation == > BLKIF_OP_READ) ? > + XENVBD_STAT_BLKIF_OP_READ_DIRECT : > + XENVBD_STAT_BLKIF_OP_WRITE_DIRECT]); > + } > + break; > > - if (RingSubmit(Ring, Request)) > - continue; > + case BLKIF_OP_WRITE_BARRIER: > + case BLKIF_OP_FLUSH_DISKCACHE: > + req->operation = Request->Operation; > + req->nr_segments = 0; > + req->handle = (USHORT)FrontendGetDeviceId(Ring->Frontend); > + req->id = (ULONG64)(ULONG_PTR)Request; > + req->sector_number = Request->FirstSector; > + InterlockedIncrement(&Ring->Stats[(Request->Operation == > BLKIF_OP_WRITE_BARRIER) ? > + XENVBD_STAT_BLKIF_OP_WRITE_BARRIER : > + XENVBD_STAT_BLKIF_OP_FLUSH_DISKCACHE]); > + break; > + > + case BLKIF_OP_DISCARD: > + { > + blkif_request_discard_t* req_discard; > + req_discard = (blkif_request_discard_t*)req; > + req_discard->operation = BLKIF_OP_DISCARD; > + req_discard->flag = Request->Flags; > + req_discard->handle = (USHORT)FrontendGetDeviceId(Frontend); > + req_discard->id = (ULONG64)(ULONG_PTR)Request; > + req_discard->sector_number = Request->FirstSector; > + req_discard->nr_sectors = Request->NrSectors; > + InterlockedIncrement(&Ring- > >Stats[XENVBD_STAT_BLKIF_OP_DISCARD]); > + } break; > > - QueueRemove(&Ring->SubmittedReqs, &Request->ListEntry); > - QueueUnPop(&Ring->PreparedReqs, &Request->ListEntry); > + default: > + ASSERT(FALSE); > break; > } > - > - return QueueCount(&Ring->PreparedReqs) != 0; > } > > -static FORCEINLINE VOID > -RingCompleteShutdown( > - IN PXENVBD_RING Ring > +static FORCEINLINE NTSTATUS > +__BlkifRingPostRequests( > + IN PXENVBD_BLKIF_RING BlkifRing > ) > { > - PXENVBD_TARGET Target; > - PXENVBD_ADAPTER Adapter; > +#define RING_SLOTS_AVAILABLE(_Front, _req_prod, _rsp_cons) \ > + (RING_SIZE(_Front) - ((_req_prod) - (_rsp_cons))) > > - if (QueueCount(&Ring->ShutdownSrbs) == 0) > - return; > + PXENVBD_SRB_STATE State; > + RING_IDX req_prod; > + RING_IDX rsp_cons; > + NTSTATUS status; > > - if (QueueCount(&Ring->PreparedReqs) || > - QueueCount(&Ring->SubmittedReqs)) > - return; > + State = &BlkifRing->State; > > - Target = FrontendGetTarget(Ring->Frontend); > - Adapter = TargetGetAdapter(Target); > - for (;;) { > - PXENVBD_SRBEXT SrbExt; > - PSCSI_REQUEST_BLOCK Srb; > + req_prod = BlkifRing->Front.req_prod_pvt; > + rsp_cons = BlkifRing->Front.rsp_cons; > + > + status = STATUS_ALLOTTED_SPACE_EXCEEDED; > + if (RING_SLOTS_AVAILABLE(&BlkifRing->Front, req_prod, rsp_cons) <= 1) > + goto fail1; > + > + while (State->Count != 0) { > + blkif_request_t *req; > + PXENVBD_REQUEST Request; > PLIST_ENTRY ListEntry; > > - ListEntry = QueuePop(&Ring->ShutdownSrbs); > - if (ListEntry == NULL) > + --State->Count; > + > + ListEntry = RemoveHeadList(&State->List); > + ASSERT3P(ListEntry, != , &State->List); > + > + RtlZeroMemory(ListEntry, sizeof(LIST_ENTRY)); > + > + Request = CONTAINING_RECORD(ListEntry, > + XENVBD_REQUEST, > + ListEntry); > + > + req = RING_GET_REQUEST(&BlkifRing->Front, req_prod); > + req_prod++; > + BlkifRing->RequestsPosted++; > + > + __BlkifRingInsertRequest(BlkifRing, > + Request, > + req); > + > + InsertTailList(&BlkifRing->SubmittedList, ListEntry); > + > + if (RING_SLOTS_AVAILABLE(&BlkifRing->Front, req_prod, rsp_cons) <= > 1) > break; > - SrbExt = CONTAINING_RECORD(ListEntry, XENVBD_SRBEXT, ListEntry); > - Srb = SrbExt->Srb; > - > - Srb->SrbStatus = SRB_STATUS_SUCCESS; > - AdapterCompleteSrb(Adapter, SrbExt); > } > + > + BlkifRing->Front.req_prod_pvt = req_prod; > + > + return STATUS_SUCCESS; > + > +fail1: > + return status; > + > +#undef RING_SLOTS_AVAILABLE > } > > -static FORCEINLINE PCHAR > -__BlkifOperationName( > - IN UCHAR Operation > +static FORCEINLINE PXENVBD_REQUEST > +__BlkifRingGetSubmittedRequest( > + IN PXENVBD_BLKIF_RING BlkifRing, > + IN ULONG64 Id > ) > { > - switch (Operation) { > - case BLKIF_OP_READ: return "READ"; > - case BLKIF_OP_WRITE: return "WRITE"; > - case BLKIF_OP_WRITE_BARRIER: return "WRITE_BARRIER"; > - case BLKIF_OP_FLUSH_DISKCACHE: return "FLUSH_DISKCACHE"; > - case BLKIF_OP_RESERVED_1: return "RESERVED_1"; > - case BLKIF_OP_DISCARD: return "DISCARD"; > - case BLKIF_OP_INDIRECT: return "INDIRECT"; > - default: return "<unknown>"; > + PLIST_ENTRY ListEntry; > + PXENVBD_REQUEST Request; > + > + for (ListEntry = BlkifRing->SubmittedList.Flink; > + ListEntry != &BlkifRing->SubmittedList; > + ListEntry = ListEntry->Flink) { > + Request = CONTAINING_RECORD(ListEntry, > + XENVBD_REQUEST, > + ListEntry); > + if ((ULONG64)(ULONG_PTR)Request != Id) > + continue; > + > + RemoveEntryList(ListEntry); > + return Request; > } > + return NULL; > } > > -static VOID > -RingCompleteResponse( > - IN PXENVBD_RING Ring, > - IN ULONG64 Id, > - IN SHORT Status > +static FORCEINLINE VOID > +__BlkifRingCompleteResponse( > + IN PXENVBD_BLKIF_RING BlkifRing, > + IN PXENVBD_REQUEST Request, > + IN SHORT Status > ) > { > - PXENVBD_REQUEST Request; > - PSCSI_REQUEST_BLOCK Srb; > - PXENVBD_SRBEXT SrbExt; > + PXENVBD_RING Ring = BlkifRing->Ring; > + PXENVBD_FRONTEND Frontend = Ring->Frontend; > + PSCSI_REQUEST_BLOCK Srb; > + PXENVBD_SRBEXT SrbExt; > + PLIST_ENTRY ListEntry; > > - Request = RingFindRequest(Ring, Id); > - if (Request == NULL) > - return; > - > - SrbExt = Request->SrbExt; > - Srb = SrbExt->Srb; > + SrbExt = Request->SrbExt; > + Srb = SrbExt->Srb; > > switch (Status) { > case BLKIF_RSP_OKAY: > - RingRequestCopyOutput(Request); > + if (Request->Operation != BLKIF_OP_READ) > + break; > + > + for (ListEntry = Request->Segments.Flink; > + ListEntry != &Request->Segments; > + ListEntry = ListEntry->Flink) { > + PXENVBD_SEGMENT Segment = CONTAINING_RECORD(ListEntry, > XENVBD_SEGMENT, ListEntry); > + PXENVBD_BOUNCE Bounce = Segment->Bounce; > + > + if (Bounce) { > + RtlCopyMemory(Bounce->SourcePtr, > + Bounce->BouncePtr, > + MmGetMdlByteCount(&Bounce->SourceMdl)); > + } > + } > break; > > case BLKIF_RSP_EOPNOTSUPP: > // Remove appropriate feature support > - FrontendRemoveFeature(Ring->Frontend, Request->Operation); > + FrontendRemoveFeature(Frontend, Request->Operation); > // Succeed this SRB, subsiquent SRBs will be succeeded instead of > being > passed to the backend. > Srb->SrbStatus = SRB_STATUS_SUCCESS; > break; > > case BLKIF_RSP_ERROR: > default: > - Warning("Target[%d] : %s BLKIF_RSP_ERROR (Tag %llx)\n", > - FrontendGetTargetId(Ring->Frontend), > - __BlkifOperationName(Request->Operation), > - Id); > + Warning("Target[%u][%u] : %s BLKIF_RSP_ERROR\n", > + FrontendGetTargetId(Frontend), > + BlkifRing->Index, > + __BlkifOperationName(Request->Operation)); > Srb->SrbStatus = SRB_STATUS_ERROR; > break; > } > > - RingPutRequest(Ring, Request); > + BlkifRingPutRequest(BlkifRing, Request); > > // complete srb > if (InterlockedDecrement(&SrbExt->RequestCount) == 0) { > - PXENVBD_TARGET Target = FrontendGetTarget(Ring->Frontend); > - PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target); > - > - if (Srb->SrbStatus == SRB_STATUS_PENDING) { > - // SRB has not hit a failure condition (BLKIF_RSP_ERROR | > BLKIF_RSP_EOPNOTSUPP) > - // from any of its responses. SRB must have succeeded > - Srb->SrbStatus = SRB_STATUS_SUCCESS; > - Srb->ScsiStatus = 0x00; // SCSI_GOOD > - } else { > - // Srb->SrbStatus has already been set by 1 or more requests with > Status != BLKIF_RSP_OKAY > - Srb->ScsiStatus = 0x40; // SCSI_ABORTED > - } > - > - AdapterCompleteSrb(Adapter, SrbExt); > + __BlkifRingCompleteSrb(BlkifRing, SrbExt); > } > } > > -static BOOLEAN > -RingPoll( > - IN PXENVBD_RING Ring > +static FORCEINLINE BOOLEAN > +BlkifRingPoll( > + IN PXENVBD_BLKIF_RING BlkifRing > ) > { > - BOOLEAN Retry = FALSE; > +#define XENVBD_BATCH(_Ring) (RING_SIZE(&(_Ring)->Front) / 4) > > - ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL); > - KeAcquireSpinLockAtDpcLevel(&Ring->Lock); > + PXENVBD_RING Ring; > + BOOLEAN Retry; > > - // Guard against this locked region being called after the > - // lock on FrontendSetState > - if (Ring->Enabled == FALSE) > + Ring = BlkifRing->Ring; > + Retry = FALSE; > + > + if (!BlkifRing->Enabled) > goto done; > > for (;;) { > - ULONG rsp_prod; > - ULONG rsp_cons; > + RING_IDX rsp_prod; > + RING_IDX rsp_cons; > > KeMemoryBarrier(); > > - rsp_prod = Ring->Shared->rsp_prod; > - rsp_cons = Ring->Front.rsp_cons; > + rsp_prod = BlkifRing->Shared->rsp_prod; > + rsp_cons = BlkifRing->Front.rsp_cons; > > KeMemoryBarrier(); > > @@ -1171,318 +1260,984 @@ RingPoll( > break; > > while (rsp_cons != rsp_prod && !Retry) { > - blkif_response_t* rsp; > + blkif_response_t *rsp; > + PXENVBD_REQUEST Request; > + > + rsp = RING_GET_RESPONSE(&BlkifRing->Front, rsp_cons); > + rsp_cons++; > + BlkifRing->ResponsesProcessed++; > > - rsp = RING_GET_RESPONSE(&Ring->Front, rsp_cons); > - ++rsp_cons; > - ++Ring->Received; > + BlkifRing->Stopped = FALSE; > > - RingCompleteResponse(Ring, rsp->id, rsp->status); > - RtlZeroMemory(rsp, sizeof(union blkif_sring_entry)); > + Request = __BlkifRingGetSubmittedRequest(BlkifRing, > + rsp->id); > + ASSERT3P(Request, != , NULL); > > - if (rsp_cons - Ring->Front.rsp_cons > RING_SIZE(&Ring->Front) / > 4) > + __BlkifRingCompleteResponse(BlkifRing, > + Request, > + rsp->status); > + > + if (rsp_cons - BlkifRing->Front.rsp_cons > > XENVBD_BATCH(BlkifRing)) > Retry = TRUE; > } > > KeMemoryBarrier(); > > - Ring->Front.rsp_cons = rsp_cons; > - Ring->Shared->rsp_event = rsp_cons + 1; > + BlkifRing->Front.rsp_cons = rsp_cons; > } > > done: > - KeReleaseSpinLockFromDpcLevel(&Ring->Lock); > - > return Retry; > + > +#undef XENVBD_BATCH > } > > -__drv_requiresIRQL(DISPATCH_LEVEL) > -static BOOLEAN > -RingNotifyResponses( > - IN PXENVBD_RING Ring > +static FORCEINLINE VOID > +__BlkifRingSend( > + IN PXENVBD_BLKIF_RING BlkifRing > ) > { > - BOOLEAN Retry = FALSE; > - > - if (!Ring->Enabled) > - return FALSE; > + PXENVBD_RING Ring = BlkifRing->Ring; > > - Retry |= RingPoll(Ring); > - Retry |= RingSubmitRequests(Ring); > - > - RingCompleteShutdown(Ring); > - return Retry; > + XENBUS_EVTCHN(Send, > + &Ring->EvtchnInterface, > + BlkifRing->Channel); > } > > -KSERVICE_ROUTINE RingInterrupt; > - > -BOOLEAN > -RingInterrupt( > - IN PKINTERRUPT Interrupt, > - IN PVOID Context > +static FORCEINLINE VOID > +__BlkifRingPushRequests( > + IN PXENVBD_BLKIF_RING BlkifRing > ) > { > - PXENVBD_RING Ring = Context; > + BOOLEAN Notify; > > - UNREFERENCED_PARAMETER(Interrupt); > + if (BlkifRing->RequestsPosted == BlkifRing->RequestsPushed) > + return; > > - ASSERT(Ring != NULL); > +#pragma warning (push) > +#pragma warning (disable:4244) > > - ++Ring->Events; > - if (!Ring->Connected) > - return TRUE; > + BlkifRing->Shared->rsp_event = BlkifRing->Front.req_prod_pvt; > > - if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL)) > - ++Ring->Dpcs; > + // Make the requests visible to the backend > + RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&BlkifRing->Front, Notify); > > - return TRUE; > +#pragma warning (pop) > + > + if (Notify) > + __BlkifRingSend(BlkifRing); > + > + BlkifRing->RequestsPushed = BlkifRing->RequestsPosted; > } > > -KDEFERRED_ROUTINE RingDpc; > +#define XENVBD_LOCK_BIT ((ULONG_PTR)1) > > -VOID > -RingDpc( > - __in PKDPC Dpc, > - __in_opt PVOID Context, > - __in_opt PVOID Arg1, > - __in_opt PVOID Arg2 > +static DECLSPEC_NOINLINE VOID > +BlkifRingSwizzle( > + IN PXENVBD_BLKIF_RING BlkifRing > ) > { > - PXENVBD_RING Ring = Context; > + ULONG_PTR Old; > + ULONG_PTR New; > + PLIST_ENTRY ListEntry; > + LIST_ENTRY List; > + ULONG Count; > > - UNREFERENCED_PARAMETER(Dpc); > - UNREFERENCED_PARAMETER(Arg1); > - UNREFERENCED_PARAMETER(Arg2); > + ASSERT3P(BlkifRing->LockThread, == , KeGetCurrentThread()); > > - ASSERT(Ring != NULL); > + InitializeListHead(&List); > > - for (;;) { > - KIRQL Irql; > - BOOLEAN Retry; > + New = XENVBD_LOCK_BIT; > + Old = (ULONG_PTR)InterlockedExchangePointer(&BlkifRing->Lock, > (PVOID)New); > > - KeRaiseIrql(DISPATCH_LEVEL, &Irql); > - Retry = RingNotifyResponses(Ring); > - KeLowerIrql(Irql); > + ASSERT(Old & XENVBD_LOCK_BIT); > + ListEntry = (PVOID)(Old & ~XENVBD_LOCK_BIT); > > - if (!Retry) > - break; > + if (ListEntry == NULL) > + return; > + > + // Packets are held in the atomic packet list in reverse order > + // so that the most recent is always head of the list. This is > + // necessary to allow addition to the list to be done atomically. > + > + for (Count = 0; ListEntry != NULL; ++Count) { > + PLIST_ENTRY NextEntry; > + > + NextEntry = ListEntry->Blink; > + ListEntry->Flink = ListEntry->Blink = ListEntry; > + > + InsertHeadList(&List, ListEntry); > + > + ListEntry = NextEntry; > } > > - XENBUS_EVTCHN(Unmask, > - &Ring->EvtchnInterface, > - Ring->Channel, > - FALSE); > + if (!IsListEmpty(&List)) { > + ListEntry = List.Flink; > + > + RemoveEntryList(&List); > + AppendTailList(&BlkifRing->SrbQueue, ListEntry); > + > + BlkifRing->SrbsQueued += Count; > + } > } > > static DECLSPEC_NOINLINE VOID > -RingDebugCallback( > - IN PVOID Argument, > - IN BOOLEAN Crashing > +BlkifRingSchedule( > + IN PXENVBD_BLKIF_RING BlkifRing > ) > { > - PXENVBD_RING Ring = Argument; > - PXENVBD_GRANTER Granter = FrontendGetGranter(Ring->Frontend); > - ULONG Index; > + PXENVBD_SRB_STATE State; > + BOOLEAN Polled; > > - UNREFERENCED_PARAMETER(Crashing); > + if (!BlkifRing->Enabled) > + return; > > - XENBUS_DEBUG(Printf, > - &Ring->DebugInterface, > - "Submitted: %u Received: %u\n", > - Ring->Submitted, > - Ring->Received); > + State = &BlkifRing->State; > + Polled = FALSE; > > - XENBUS_DEBUG(Printf, > - &Ring->DebugInterface, > - "Events: %u Dpcs: %u\n", > - Ring->Events, > - Ring->Dpcs); > + while (!BlkifRing->Stopped) { > + PLIST_ENTRY ListEntry; > + PXENVBD_SRBEXT SrbExt; > + NTSTATUS status; > > - XENBUS_DEBUG(Printf, > - &Ring->DebugInterface, > - "Shared : 0x%p\n", > - Ring->Shared); > + if (State->Count != 0) { > + status = __BlkifRingPostRequests(BlkifRing); > + if (!NT_SUCCESS(status)) > + BlkifRing->Stopped = TRUE; > + } > > - if (Ring->Shared) { > - XENBUS_DEBUG(Printf, > - &Ring->DebugInterface, > - "Shared: %d / %d - %d / %d\n", > - Ring->Shared->req_prod, > - Ring->Shared->req_event, > - Ring->Shared->rsp_prod, > - Ring->Shared->rsp_event); > - } > + if (BlkifRing->Stopped) { > + if (!Polled) { > + (VOID)BlkifRingPoll(BlkifRing); > + Polled = TRUE; > + } > > - XENBUS_DEBUG(Printf, > - &Ring->DebugInterface, > - "Front: %d / %d (%d)\n", > - Ring->Front.req_prod_pvt, > - Ring->Front.rsp_cons, > - Ring->Front.nr_ents); > + continue; > + } > > - XENBUS_DEBUG(Printf, > - &Ring->DebugInterface, > - "Order: %d\n", > - Ring->Order); > + if (BlkifRing->RequestsPosted - BlkifRing->RequestsPushed >= > + RING_SIZE(&BlkifRing->Front) / 4) > + __BlkifRingPushRequests(BlkifRing); > > - for (Index = 0; Index < (1ul << Ring->Order); ++Index) { > - XENBUS_DEBUG(Printf, > - &Ring->DebugInterface, > - "Grants[%-2d]: 0x%p (%u)\n", > - Index, > - Ring->Grants[Index], > - GranterReference(Granter, Ring->Grants[Index])); > + ASSERT3U(State->Count, == , 0); > + > + if (IsListEmpty(&BlkifRing->SrbQueue)) > + break; > + > + ListEntry = RemoveHeadList(&BlkifRing->SrbQueue); > + ASSERT3P(ListEntry, != , &BlkifRing->SrbQueue); > + > + RtlZeroMemory(ListEntry, sizeof(LIST_ENTRY)); > + > + SrbExt = CONTAINING_RECORD(ListEntry, > + XENVBD_SRBEXT, > + ListEntry); > + > + status = __BlkifRingPrepareSrb(BlkifRing, SrbExt); > + if (!NT_SUCCESS(status)) { > + PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb; > + Srb->SrbStatus = SRB_STATUS_BUSY; > + __BlkifRingCompleteSrb(BlkifRing, SrbExt); > + } > } > > - if (Ring->Channel) { > - ULONG Port = XENBUS_EVTCHN(GetPort, > - &Ring->EvtchnInterface, > - Ring->Channel); > + __BlkifRingPushRequests(BlkifRing); > > - XENBUS_DEBUG(Printf, > - &Ring->DebugInterface, > - "Channel : %p (%d)\n", > - Ring->Channel, > - Port); > + if (IsListEmpty(&BlkifRing->ShutdownQueue)) > + return; > + > + if (!IsListEmpty(&BlkifRing->SrbQueue) || > + !IsListEmpty(&BlkifRing->SubmittedList)) > + return; > + > + for (;;) { > + PLIST_ENTRY ListEntry; > + PXENVBD_SRBEXT SrbExt; > + PSCSI_REQUEST_BLOCK Srb; > + > + ListEntry = RemoveHeadList(&BlkifRing->ShutdownQueue); > + if (ListEntry == &BlkifRing->ShutdownQueue) > + break; > + > + SrbExt = CONTAINING_RECORD(ListEntry, > + XENVBD_SRBEXT, > + ListEntry); > + > + Srb = SrbExt->Srb; > + Srb->SrbStatus = SRB_STATUS_SUCCESS; > + __BlkifRingCompleteSrb(BlkifRing, SrbExt); > } > +} > > - XENBUS_DEBUG(Printf, > - &Ring->DebugInterface, > - "BLKIF_OPs: READ=%u WRITE=%u\n", > - Ring->BlkOpRead, > - Ring->BlkOpWrite); > - XENBUS_DEBUG(Printf, > - &Ring->DebugInterface, > - "BLKIF_OPs: INDIRECT_READ=%u INDIRECT_WRITE=%u\n", > - Ring->BlkOpIndirectRead, > - Ring->BlkOpIndirectWrite); > - XENBUS_DEBUG(Printf, > - &Ring->DebugInterface, > - "BLKIF_OPs: BARRIER=%u DISCARD=%u FLUSH=%u\n", > - Ring->BlkOpBarrier, > - Ring->BlkOpDiscard, > - Ring->BlkOpFlush); > - XENBUS_DEBUG(Printf, > - &Ring->DebugInterface, > - "Segments Granted=%llu Bounced=%llu\n", > - Ring->SegsGranted, > - Ring->SegsBounced); > +static FORCEINLINE BOOLEAN > +__drv_requiresIRQL(DISPATCH_LEVEL) > +__BlkifRingTryAcquireLock( > + IN PXENVBD_BLKIF_RING BlkifRing > + ) > +{ > + ULONG_PTR Old; > + ULONG_PTR New; > + BOOLEAN Acquired; > + > + ASSERT3U(KeGetCurrentIrql(), == , DISPATCH_LEVEL); > + > + KeMemoryBarrier(); > + > + Old = (ULONG_PTR)BlkifRing->Lock & ~XENVBD_LOCK_BIT; > + New = Old | XENVBD_LOCK_BIT; > + > + Acquired = > ((ULONG_PTR)InterlockedCompareExchangePointer(&BlkifRing->Lock, > + (PVOID)New, > + (PVOID)Old) == > Old) ? TRUE : FALSE; > > - QueueDebugCallback(&Ring->PreparedReqs, > - "Prepared ", > - &Ring->DebugInterface); > - QueueDebugCallback(&Ring->SubmittedReqs, > - "Submitted", > - &Ring->DebugInterface); > - QueueDebugCallback(&Ring->ShutdownSrbs, > - "Shutdown ", > - &Ring->DebugInterface); > + KeMemoryBarrier(); > + > + if (Acquired) { > + ASSERT3P(BlkifRing->LockThread, == , NULL); > + BlkifRing->LockThread = KeGetCurrentThread(); > + KeMemoryBarrier(); > + } > + > + return Acquired; > } > > -static DECLSPEC_NOINLINE VOID > -RingAcquireLock( > - IN PVOID Argument > +static FORCEINLINE VOID > +__drv_requiresIRQL(DISPATCH_LEVEL) > +__BlkifRingAcquireLock( > + IN PXENVBD_BLKIF_RING BlkifRing > ) > { > - PXENVBD_RING Ring = Argument; > - KeAcquireSpinLockAtDpcLevel(&Ring->Lock); > + ASSERT3U(KeGetCurrentIrql(), == , DISPATCH_LEVEL); > + > + for (;;) { > + if (__BlkifRingTryAcquireLock(BlkifRing)) > + break; > + > + _mm_pause(); > + } > } > > -static DECLSPEC_NOINLINE VOID > -RingReleaseLock( > - IN PVOID Argument > +static VOID > +BlkifRingAcquireLock( > + IN PXENVBD_BLKIF_RING BlkifRing > ) > { > - PXENVBD_RING Ring = Argument; > - KeReleaseSpinLockFromDpcLevel(&Ring->Lock); > + __BlkifRingAcquireLock(BlkifRing); > } > > -static DECLSPEC_NOINLINE NTSTATUS > -RingRequestCtor( > - IN PVOID Argument, > - IN PVOID Object > +static FORCEINLINE BOOLEAN > +__drv_requiresIRQL(DISPATCH_LEVEL) > +__BlkifRingTryReleaseLock( > + IN PXENVBD_BLKIF_RING BlkifRing > ) > { > - PXENVBD_REQUEST Request = Object; > + ULONG_PTR Old; > + ULONG_PTR New; > + BOOLEAN Released; > > - UNREFERENCED_PARAMETER(Argument); > + ASSERT3U(KeGetCurrentIrql(), == , DISPATCH_LEVEL); > + ASSERT3P(KeGetCurrentThread(), == , BlkifRing->LockThread); > > - InitializeListHead(&Request->Segments); > - InitializeListHead(&Request->Indirects); > - return STATUS_SUCCESS; > + Old = XENVBD_LOCK_BIT; > + New = 0; > + > + BlkifRing->LockThread = NULL; > + > + KeMemoryBarrier(); > + > + Released = > ((ULONG_PTR)InterlockedCompareExchangePointer(&BlkifRing->Lock, > + (PVOID)New, > + (PVOID)Old) == > Old) ? TRUE : FALSE; > + > + KeMemoryBarrier(); > + > + if (!Released) { > + ASSERT3P(BlkifRing->LockThread, == , NULL); > + BlkifRing->LockThread = KeGetCurrentThread(); > + KeMemoryBarrier(); > + } > + > + return Released; > } > > -static DECLSPEC_NOINLINE VOID > -RingRequestDtor( > - IN PVOID Argument, > - IN PVOID Object > +static FORCEINLINE VOID > +__drv_requiresIRQL(DISPATCH_LEVEL) > +__BlkifRingReleaseLock( > + IN PXENVBD_BLKIF_RING BlkifRing > ) > { > - UNREFERENCED_PARAMETER(Argument); > - UNREFERENCED_PARAMETER(Object); > + ASSERT3U(KeGetCurrentIrql(), == , DISPATCH_LEVEL); > + > + // As lock holder it is our responsibility to drain the atomic > + // packet list into the transmit queue before we actually drop the > + // lock. This may, of course, take a few attempts as another > + // thread could be simuntaneously adding to the list. > + > + do { > + BlkifRingSwizzle(BlkifRing); > + BlkifRingSchedule(BlkifRing); > + } while (!__BlkifRingTryReleaseLock(BlkifRing)); > } > > -static DECLSPEC_NOINLINE NTSTATUS > -RingSegmentCtor( > - IN PVOID Argument, > - IN PVOID Object > +static VOID > +BlkifRingReleaseLock( > + IN PXENVBD_BLKIF_RING BlkifRing > ) > { > - UNREFERENCED_PARAMETER(Argument); > - UNREFERENCED_PARAMETER(Object); > + __BlkifRingReleaseLock(BlkifRing); > +} > + > +KSERVICE_ROUTINE BlkifRingInterrupt; > + > +BOOLEAN > +BlkifRingInterrupt( > + IN PKINTERRUPT InterruptObject, > + IN PVOID Argument > + ) > +{ > + PXENVBD_BLKIF_RING BlkifRing = Argument; > + > + UNREFERENCED_PARAMETER(InterruptObject); > + > + ASSERT(BlkifRing != NULL); > + > + BlkifRing->Events++; > + > + if (KeInsertQueueDpc(&BlkifRing->Dpc, NULL, NULL)) > + BlkifRing->Dpcs++; > + > + return TRUE; > +} > + > +__drv_functionClass(KDEFERRED_ROUTINE) > +__drv_maxIRQL(DISPATCH_LEVEL) > +__drv_minIRQL(PASSIVE_LEVEL) > +__drv_sameIRQL > +static VOID > +BlkifRingDpc( > + IN PKDPC Dpc, > + IN PVOID Context, > + IN PVOID Argument1, > + IN PVOID Argument2 > + ) > +{ > + PXENVBD_BLKIF_RING BlkifRing = Context; > + PXENVBD_RING Ring; > + > + UNREFERENCED_PARAMETER(Dpc); > + UNREFERENCED_PARAMETER(Argument1); > + UNREFERENCED_PARAMETER(Argument2); > + > + ASSERT(BlkifRing != NULL); > + > + Ring = BlkifRing->Ring; > + > + for (;;) { > + BOOLEAN Retry; > + KIRQL Irql; > + > + KeRaiseIrql(DISPATCH_LEVEL, &Irql); > + __BlkifRingAcquireLock(BlkifRing); > + Retry = BlkifRingPoll(BlkifRing); > + __BlkifRingReleaseLock(BlkifRing); > + KeLowerIrql(Irql); > + > + if (!Retry) > + break; > + } > + > + XENBUS_EVTCHN(Unmask, > + &Ring->EvtchnInterface, > + BlkifRing->Channel, > + FALSE); > +} > + > +static NTSTATUS > +BlkifRingCreate( > + IN PXENVBD_RING Ring, > + IN ULONG Index, > + OUT PXENVBD_BLKIF_RING* BlkifRing > + ) > +{ > + PXENVBD_FRONTEND Frontend; > + ULONG Length; > + PCHAR Path; > + CHAR Name[MAX_NAME_LEN]; > + NTSTATUS status; > + > + Frontend = Ring->Frontend; > + > + Length = (ULONG)strlen(FrontendGetFrontendPath(Frontend)) + > + (ULONG)strlen("/queue-xxx"); > + > + Path = __RingAllocate(Length + 1); > + > + status = STATUS_NO_MEMORY; > + if (Path == NULL) > + goto fail1; > + > + status = RtlStringCchPrintfA(Path, > + Length, > + "%s/queue-%u", > + FrontendGetFrontendPath(Frontend), > + Index); > + if (!NT_SUCCESS(status)) > + goto fail2; > + > + *BlkifRing = __RingAllocate(sizeof(XENVBD_BLKIF_RING)); > + > + status = STATUS_NO_MEMORY; > + if (*BlkifRing == NULL) > + goto fail3; > + > + (*BlkifRing)->Ring = Ring; > + (*BlkifRing)->Index = Index; > + (*BlkifRing)->Path = Path; > + Path = NULL; > + > + InitializeListHead(&(*BlkifRing)->SrbQueue); > + InitializeListHead(&(*BlkifRing)->ShutdownQueue); > + InitializeListHead(&(*BlkifRing)->SubmittedList); > + InitializeListHead(&(*BlkifRing)->State.List); > + > + KeInitializeThreadedDpc(&(*BlkifRing)->Dpc, BlkifRingDpc, *BlkifRing); > + > + status = RtlStringCbPrintfA(Name, > + sizeof(Name), > + "vbd_%u_queue_%u_request", > + FrontendGetTargetId(Frontend), > + Index); > + if (!NT_SUCCESS(status)) > + goto fail4; > + > + status = XENBUS_CACHE(Create, > + &Ring->CacheInterface, > + Name, > + sizeof(XENVBD_REQUEST), > + 0, > + BlkifRingRequestCtor, > + BlkifRingRequestDtor, > + BlkifRingAcquireLock, > + BlkifRingReleaseLock, > + *BlkifRing, > + &(*BlkifRing)->RequestCache); > + if (!NT_SUCCESS(status)) > + goto fail5; > + > + status = RtlStringCbPrintfA(Name, > + sizeof(Name), > + "vbd_%u_queue_%u_segment", > + FrontendGetTargetId(Frontend), > + Index); > + if (!NT_SUCCESS(status)) > + goto fail6; > + > + status = XENBUS_CACHE(Create, > + &Ring->CacheInterface, > + Name, > + sizeof(XENVBD_SEGMENT), > + 0, > + BlkifRingSegmentCtor, > + BlkifRingSegmentDtor, > + BlkifRingAcquireLock, > + BlkifRingReleaseLock, > + *BlkifRing, > + &(*BlkifRing)->SegmentCache); > + if (!NT_SUCCESS(status)) > + goto fail7; > + > + status = RtlStringCbPrintfA(Name, > + sizeof(Name), > + "vbd_%u_queue_%u_indirect", > + FrontendGetTargetId(Frontend), > + Index); > + if (!NT_SUCCESS(status)) > + goto fail8; > + > + status = XENBUS_CACHE(Create, > + &Ring->CacheInterface, > + Name, > + sizeof(XENVBD_INDIRECT), > + 0, > + BlkifRingIndirectCtor, > + BlkifRingIndirectDtor, > + BlkifRingAcquireLock, > + BlkifRingReleaseLock, > + *BlkifRing, > + &(*BlkifRing)->IndirectCache); > + if (!NT_SUCCESS(status)) > + goto fail9; > + > return STATUS_SUCCESS; > + > +fail9: > + Error("fail9\n"); > +fail8: > + Error("fail8\n"); > + XENBUS_CACHE(Destroy, > + &Ring->CacheInterface, > + (*BlkifRing)->SegmentCache); > + (*BlkifRing)->SegmentCache = NULL; > +fail7: > + Error("fail7\n"); > +fail6: > + Error("fail6\n"); > + XENBUS_CACHE(Destroy, > + &Ring->CacheInterface, > + (*BlkifRing)->RequestCache); > + (*BlkifRing)->RequestCache = NULL; > +fail5: > + Error("fail5\n"); > +fail4: > + Error("fail4\n"); > + > + RtlZeroMemory(&(*BlkifRing)->Dpc, sizeof(KDPC)); > + > + RtlZeroMemory(&(*BlkifRing)->State.List, sizeof(LIST_ENTRY)); > + RtlZeroMemory(&(*BlkifRing)->SubmittedList, sizeof(LIST_ENTRY)); > + RtlZeroMemory(&(*BlkifRing)->ShutdownQueue, sizeof(LIST_ENTRY)); > + RtlZeroMemory(&(*BlkifRing)->SrbQueue, sizeof(LIST_ENTRY)); > + > + __RingFree((*BlkifRing)->Path); > + (*BlkifRing)->Path; > + (*BlkifRing)->Index = 0; > + (*BlkifRing)->Ring = NULL; > + > + ASSERT(IsZeroMemory(*BlkifRing, sizeof(XENVBD_BLKIF_RING))); > + __RingFree(*BlkifRing); > + *BlkifRing = NULL; > +fail3: > + Error("fail3\n"); > +fail2: > + Error("fail2\n"); > + __RingFree(Path); > +fail1: > + Error("fail1 (%08x)\n", status); > + return status; > } > > -static DECLSPEC_NOINLINE VOID > -RingSegmentDtor( > - IN PVOID Argument, > - IN PVOID Object > +static VOID > +BlkifRingDestroy( > + IN PXENVBD_BLKIF_RING BlkifRing > ) > { > - UNREFERENCED_PARAMETER(Argument); > - UNREFERENCED_PARAMETER(Object); > + PXENVBD_RING Ring = BlkifRing->Ring; > + > + XENBUS_CACHE(Destroy, > + &Ring->CacheInterface, > + BlkifRing->IndirectCache); > + BlkifRing->IndirectCache = NULL; > + > + XENBUS_CACHE(Destroy, > + &Ring->CacheInterface, > + BlkifRing->SegmentCache); > + BlkifRing->SegmentCache = NULL; > + > + XENBUS_CACHE(Destroy, > + &Ring->CacheInterface, > + BlkifRing->RequestCache); > + BlkifRing->RequestCache = NULL; > + > + RtlZeroMemory(&BlkifRing->Dpc, sizeof(KDPC)); > + > + ASSERT3U(BlkifRing->State.Count, == , 0); > + ASSERT(IsListEmpty(&BlkifRing->State.List)); > + RtlZeroMemory(&BlkifRing->State.List, sizeof(LIST_ENTRY)); > + > + RtlZeroMemory(&BlkifRing->SubmittedList, sizeof(LIST_ENTRY)); > + RtlZeroMemory(&BlkifRing->SrbQueue, sizeof(LIST_ENTRY)); > + RtlZeroMemory(&BlkifRing->ShutdownQueue, sizeof(LIST_ENTRY)); > + > + __RingFree(BlkifRing->Path); > + BlkifRing->Path; > + BlkifRing->Index = 0; > + BlkifRing->Ring = NULL; > + > + ASSERT(IsZeroMemory(BlkifRing, sizeof(XENVBD_BLKIF_RING))); > + __RingFree(BlkifRing); > } > > -static DECLSPEC_NOINLINE NTSTATUS > -RingIndirectCtor( > - IN PVOID Argument, > - IN PVOID Object > +static NTSTATUS > +BlkifRingConnect( > + IN PXENVBD_BLKIF_RING BlkifRing > ) > { > - PXENVBD_INDIRECT Indirect = Object; > - NTSTATUS status; > + PXENVBD_RING Ring = BlkifRing->Ring; > + PXENVBD_FRONTEND Frontend = Ring->Frontend; > + PXENVBD_GRANTER Granter = FrontendGetGranter(Frontend); > + CHAR Name[MAX_NAME_LEN]; > + ULONG Index; > + NTSTATUS status; > > - UNREFERENCED_PARAMETER(Argument); > + Trace("====> %u\n", BlkifRing->Index); > + ASSERT(!BlkifRing->Connected); > + > + if (FrontendGetNumQueues(Frontend) != 1) { > + PROCESSOR_NUMBER ProcNumber; > + > + status = KeGetProcessorNumberFromIndex(BlkifRing->Index, > &ProcNumber); > + ASSERT(NT_SUCCESS(status)); > + > + KeSetTargetProcessorDpcEx(&BlkifRing->Dpc, &ProcNumber); > + } > + KeSetImportanceDpc(&BlkifRing->Dpc, MediumHighImportance); > + > + BlkifRing->Mdl = __AllocatePages(1 << Ring->Order); > > status = STATUS_NO_MEMORY; > - Indirect->Mdl = __AllocatePage(); > - if (Indirect->Mdl == NULL) > + if (BlkifRing->Mdl == NULL) > goto fail1; > > - Indirect->Page = MmGetSystemAddressForMdlSafe(Indirect->Mdl, > - NormalPagePriority); > - ASSERT(Indirect->Page); > + BlkifRing->Shared = MmGetSystemAddressForMdlSafe(BlkifRing->Mdl, > + NormalPagePriority); > + ASSERT(BlkifRing->Shared != NULL); > + > +#pragma warning(push) > +#pragma warning(disable: 4305) > +#pragma warning(disable: 4311) // 'type cast' pointer truncation from > 'blkif_sring_entry[1]' to 'long' > + SHARED_RING_INIT(BlkifRing->Shared); > + FRONT_RING_INIT(&BlkifRing->Front, BlkifRing->Shared, PAGE_SIZE << > Ring->Order); > +#pragma warning(pop) > + > + for (Index = 0; Index < (1ul << Ring->Order); ++Index) { > + status = GranterGet(Granter, > + MmGetMdlPfnArray(BlkifRing->Mdl)[Index], > + FALSE, > + &BlkifRing->Grants[Index]); > + if (!NT_SUCCESS(status)) > + goto fail2; > + } > + > + BlkifRing->Channel = XENBUS_EVTCHN(Open, > + &Ring->EvtchnInterface, > + XENBUS_EVTCHN_TYPE_UNBOUND, > + BlkifRingInterrupt, > + BlkifRing, > + > FrontendGetBackendDomain(Ring->Frontend), > + TRUE); > + status = STATUS_NO_MEMORY; > + if (BlkifRing->Channel == NULL) > + goto fail3; > + > + XENBUS_EVTCHN(Unmask, > + &Ring->EvtchnInterface, > + BlkifRing->Channel, > + FALSE); > + > + status = RtlStringCchPrintfA(Name, > + MAX_NAME_LEN, > + __MODULE__"|RING[%u]", > + Index); > + if (!NT_SUCCESS(status)) > + goto fail4; > + > + status = XENBUS_DEBUG(Register, > + &Ring->DebugInterface, > + Name, > + BlkifRingDebugCallback, > + BlkifRing, > + &BlkifRing->DebugCallback); > + if (!NT_SUCCESS(status)) > + goto fail5; > + > + BlkifRing->Connected = TRUE; > + Trace("<==== %u\n", BlkifRing->Index); > > return STATUS_SUCCESS; > > +fail5: > + Error("fail5\n"); > +fail4: > + Error("fail4\n"); > + XENBUS_EVTCHN(Close, > + &Ring->EvtchnInterface, > + BlkifRing->Channel); > + BlkifRing->Channel = NULL; > +fail3: > + Error("fail3\n"); > +fail2: > + Error("fail2\n"); > + for (Index = 0; Index < (1ul << Ring->Order); ++Index) { > + if (BlkifRing->Grants[Index] == NULL) > + continue; > + > + GranterPut(Granter, BlkifRing->Grants[Index]); > + BlkifRing->Grants[Index] = NULL; > + } > + > + RtlZeroMemory(&BlkifRing->Front, sizeof(blkif_front_ring_t)); > + > + __FreePages(BlkifRing->Mdl); > + BlkifRing->Shared = NULL; > + BlkifRing->Mdl = NULL; > fail1: > Error("fail1 %08x\n", status); > return status; > } > > +static NTSTATUS > +BlkifRingStoreWrite( > + IN PXENVBD_BLKIF_RING BlkifRing, > + IN PVOID Transaction > + ) > +{ > + PXENVBD_RING Ring = BlkifRing->Ring; > + PXENVBD_FRONTEND Frontend = Ring->Frontend; > + PXENVBD_GRANTER Granter = FrontendGetGranter(Frontend); > + PCHAR Path; > + NTSTATUS status; > + > + Path = (FrontendGetNumQueues(Frontend) == 1) ? > + FrontendGetFrontendPath(Frontend) : > + BlkifRing->Path; > + > + status = XENBUS_STORE(Printf, > + &Ring->StoreInterface, > + Transaction, > + Path, > + "event-channel", > + "%u", > + XENBUS_EVTCHN(GetPort, > + &Ring->EvtchnInterface, > + BlkifRing->Channel)); > + if (!NT_SUCCESS(status)) > + goto fail1; > + > + if (Ring->Order == 0) { > + status = XENBUS_STORE(Printf, > + &Ring->StoreInterface, > + Transaction, > + Path, > + "ring-ref", > + "%u", > + GranterReference(Granter, > BlkifRing->Grants[0])); > + if (!NT_SUCCESS(status)) > + goto fail2; > + } else { > + ULONG Index; > + > + for (Index = 0; Index < (1ul << Ring->Order); ++Index) { > + CHAR Name[MAX_NAME_LEN + 1]; > + > + status = RtlStringCchPrintfA(Name, > + MAX_NAME_LEN, > + "ring-ref%u", > + Index); > + if (!NT_SUCCESS(status)) > + goto fail3; > + > + status = XENBUS_STORE(Printf, > + &Ring->StoreInterface, > + Transaction, > + Path, > + Name, > + "%u", > + GranterReference(Granter, > BlkifRing->Grants[Index])); > + if (!NT_SUCCESS(status)) > + goto fail4; > + } > + } > + > + return STATUS_SUCCESS; > + > +fail4: > +fail3: > +fail2: > +fail1: > + return status; > +} > + > +static VOID > +BlkifRingEnable( > + IN PXENVBD_BLKIF_RING BlkifRing > + ) > +{ > + Trace("====> %u\n", BlkifRing->Index); > + > + __BlkifRingAcquireLock(BlkifRing); > + ASSERT(!BlkifRing->Enabled); > + BlkifRing->Enabled = TRUE; > + __BlkifRingReleaseLock(BlkifRing); > + > + Trace("<==== %u\n", BlkifRing->Index); > +} > + > +static VOID > +BlkifRingDisable( > + IN PXENVBD_BLKIF_RING BlkifRing > + ) > +{ > + PXENVBD_RING Ring = BlkifRing->Ring; > + ULONG Attempt; > + > + Trace("====> %u\n", BlkifRing->Index); > + > + __BlkifRingAcquireLock(BlkifRing); > + ASSERT(BlkifRing->Enabled); > + > + // Discard any pending requests > + while (!IsListEmpty(&BlkifRing->State.List)) { > + PLIST_ENTRY ListEntry; > + PXENVBD_REQUEST Request; > + PXENVBD_SRBEXT SrbExt; > + PSCSI_REQUEST_BLOCK Srb; > + > + ListEntry = RemoveHeadList(&BlkifRing->State.List); > + ASSERT3P(ListEntry, != , &BlkifRing->State.List); > + > + Request = CONTAINING_RECORD(ListEntry, > + XENVBD_REQUEST, > + ListEntry); > + SrbExt = Request->SrbExt; > + Srb = SrbExt->Srb; > + Srb->SrbStatus = SRB_STATUS_ABORTED; > + Srb->ScsiStatus = 0x40; // SCSI_ABORTED > + > + BlkifRingPutRequest(BlkifRing, Request); > + > + if (InterlockedDecrement(&SrbExt->RequestCount) == 0) > + __BlkifRingCompleteSrb(BlkifRing, SrbExt); > + } > + > + ASSERT3U(BlkifRing->State.Count, == , 0); > + > + Attempt = 0; > + ASSERT3U(BlkifRing->RequestsPushed, == , BlkifRing->RequestsPosted); > + while (BlkifRing->ResponsesProcessed != BlkifRing->RequestsPushed) { > + Attempt++; > + ASSERT(Attempt < 100); > + > + // Try to move things along > + __BlkifRingSend(BlkifRing); > + (VOID)BlkifRingPoll(BlkifRing); > + > + // We are waiting for a watch event at DISPATCH_LEVEL so > + // it is our responsibility to poll the store ring. > + XENBUS_STORE(Poll, > + &Ring->StoreInterface); > + > + KeStallExecutionProcessor(1000); // 1ms > + } > + > + BlkifRing->Enabled = FALSE; > + __BlkifRingReleaseLock(BlkifRing); > + > + Trace("<==== %u\n", BlkifRing->Index); > +} > + > +static VOID > +BlkifRingDisconnect( > + IN PXENVBD_BLKIF_RING BlkifRing > + ) > +{ > + PXENVBD_RING Ring = BlkifRing->Ring; > + PXENVBD_GRANTER Granter = FrontendGetGranter(Ring- > >Frontend); > + ULONG Index; > + > + Trace("====> %u\n", BlkifRing->Index); > + ASSERT(BlkifRing->Connected); > + > + XENBUS_DEBUG(Deregister, > + &Ring->DebugInterface, > + BlkifRing->DebugCallback); > + BlkifRing->DebugCallback = NULL; > + > + XENBUS_EVTCHN(Close, > + &Ring->EvtchnInterface, > + BlkifRing->Channel); > + BlkifRing->Channel = NULL; > + > + for (Index = 0; Index < (1ul << Ring->Order); ++Index) { > + if (BlkifRing->Grants[Index] == NULL) > + continue; > + > + GranterPut(Granter, BlkifRing->Grants[Index]); > + BlkifRing->Grants[Index] = NULL; > + } > + > + RtlZeroMemory(&BlkifRing->Front, sizeof(blkif_front_ring_t)); > + > + __FreePages(BlkifRing->Mdl); > + BlkifRing->Shared = NULL; > + BlkifRing->Mdl = NULL; > + > + BlkifRing->Events = 0; > + BlkifRing->Dpcs = 0; > + BlkifRing->RequestsPosted = 0; > + BlkifRing->RequestsPushed = 0; > + BlkifRing->ResponsesProcessed = 0; > + > + BlkifRing->Connected = FALSE; > + > + Trace("<==== %u\n", BlkifRing->Index); > +} > + > +static VOID > +__BlkifRingQueueSrb( > + IN PXENVBD_BLKIF_RING BlkifRing, > + IN PXENVBD_SRBEXT SrbExt > + ) > +{ > + PLIST_ENTRY ListEntry; > + ULONG_PTR Old; > + ULONG_PTR LockBit; > + ULONG_PTR New; > + > + ListEntry = &SrbExt->ListEntry; > + > + do { > + Old = (ULONG_PTR)BlkifRing->Lock; > + LockBit = Old & XENVBD_LOCK_BIT; > + > + ListEntry->Blink = (PVOID)(Old & ~XENVBD_LOCK_BIT); > + New = (ULONG_PTR)ListEntry; > + ASSERT((New & XENVBD_LOCK_BIT) == 0); > + New |= LockBit; > + } while ((ULONG_PTR)InterlockedCompareExchangePointer(&BlkifRing- > >Lock, (PVOID)New, (PVOID)Old) != Old); > + > + // __BlkifRingReleaseLock() drains the atomic SRB list into the queue > therefore, > + // after adding to the list we need to attempt to grab and release the > lock. > If we can't > + // grab it then that's ok because whichever thread is holding it will > have to > call > + // __BlkifRingReleaseLock() and will therefore drain the atomic packet > list. > + > + if (__BlkifRingTryAcquireLock(BlkifRing)) > + __BlkifRingReleaseLock(BlkifRing); > +} > + > +static VOID > +__BlkifRingQueueShutdown( > + IN PXENVBD_BLKIF_RING BlkifRing, > + IN PXENVBD_SRBEXT SrbExt > + ) > +{ > + __BlkifRingAcquireLock(BlkifRing); > + InsertTailList(&BlkifRing->ShutdownQueue, &SrbExt->ListEntry); > + __BlkifRingReleaseLock(BlkifRing); > +} > + > static DECLSPEC_NOINLINE VOID > -RingIndirectDtor( > +RingDebugCallback( > IN PVOID Argument, > - IN PVOID Object > + IN BOOLEAN Crashing > ) > { > - PXENVBD_INDIRECT Indirect = Object; > + PXENVBD_RING Ring = Argument; > + XENVBD_STAT Index; > > - UNREFERENCED_PARAMETER(Argument); > + UNREFERENCED_PARAMETER(Crashing); > > - __FreePages(Indirect->Mdl); > - Indirect->Page = NULL; > - Indirect->Mdl = NULL; > + XENBUS_DEBUG(Printf, > + &Ring->DebugInterface, > + "Order: %d\n", > + Ring->Order); > + > + for (Index = 0; Index < XENVBD_STAT__MAX; ++Index) { > + XENBUS_DEBUG(Printf, > + &Ring->DebugInterface, > + "%s: %u\n", > + __StatName(Index), > + Ring->Stats[Index]); > + } > } > > NTSTATUS > @@ -1493,7 +2248,8 @@ RingCreate( > { > PXENVBD_TARGET Target = FrontendGetTarget(Frontend); > PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target); > - CHAR Name[MAX_NAME_LEN]; > + ULONG MaxQueues; > + ULONG Index; > NTSTATUS status; > > *Ring = __RingAllocate(sizeof(XENVBD_RING)); > @@ -1502,127 +2258,74 @@ RingCreate( > if (*Ring == NULL) > goto fail1; > > - (*Ring)->Frontend = Frontend; > - KeInitializeSpinLock(&(*Ring)->Lock); > - KeInitializeThreadedDpc(&(*Ring)->Dpc, RingDpc, *Ring); > - KeSetImportanceDpc(&(*Ring)->Dpc, MediumHighImportance); > - > - QueueInit(&(*Ring)->PreparedReqs); > - QueueInit(&(*Ring)->SubmittedReqs); > - QueueInit(&(*Ring)->ShutdownSrbs); > - > - AdapterGetCacheInterface(Adapter, &(*Ring)->CacheInterface); > - > - status = XENBUS_CACHE(Acquire, &(*Ring)->CacheInterface); > - if (!NT_SUCCESS(status)) > - goto fail2; > + AdapterGetDebugInterface(Adapter, > + &(*Ring)->DebugInterface); > + AdapterGetStoreInterface(Adapter, > + &(*Ring)->StoreInterface); > + AdapterGetCacheInterface(Adapter, > + &(*Ring)->CacheInterface); > + AdapterGetEvtchnInterface(Adapter, > + &(*Ring)->EvtchnInterface); > > - status = RtlStringCbPrintfA(Name, > - sizeof(Name), > - "vbd_%u_req", > - FrontendGetTargetId(Frontend)); > - if (!NT_SUCCESS(status)) > - goto fail3; > + (*Ring)->Frontend = Frontend; > > - status = XENBUS_CACHE(Create, > - &(*Ring)->CacheInterface, > - Name, > - sizeof(XENVBD_REQUEST), > - 32, > - RingRequestCtor, > - RingRequestDtor, > - RingAcquireLock, > - RingReleaseLock, > - *Ring, > - &(*Ring)->RequestCache); > - if (!NT_SUCCESS(status)) > - goto fail4; > + MaxQueues = FrontendGetMaxQueues(Frontend); > + (*Ring)->Ring = __RingAllocate(sizeof(PXENVBD_BLKIF_RING) * > + MaxQueues); > > - status = RtlStringCbPrintfA(Name, > - sizeof(Name), > - "vbd_%u_seg", > - FrontendGetTargetId(Frontend)); > - if (!NT_SUCCESS(status)) > - goto fail5; > + status = STATUS_NO_MEMORY; > + if ((*Ring)->Ring == NULL) > + goto fail2; > > - status = XENBUS_CACHE(Create, > - &(*Ring)->CacheInterface, > - Name, > - sizeof(XENVBD_SEGMENT), > - 32, > - RingSegmentCtor, > - RingSegmentDtor, > - RingAcquireLock, > - RingReleaseLock, > - *Ring, > - &(*Ring)->SegmentCache); > - if (!NT_SUCCESS(status)) > - goto fail6; > + Index = 0; > + while (Index < MaxQueues) { > + PXENVBD_BLKIF_RING BlkifRing; > > - status = RtlStringCbPrintfA(Name, > - sizeof(Name), > - "vbd_%u_ind", > - FrontendGetTargetId(Frontend)); > - if (!NT_SUCCESS(status)) > - goto fail7; > + status = BlkifRingCreate(*Ring, Index, &BlkifRing); > + if (!NT_SUCCESS(status)) > + goto fail3; > > - status = XENBUS_CACHE(Create, > - &(*Ring)->CacheInterface, > - Name, > - sizeof(XENVBD_INDIRECT), > - 1, > - RingIndirectCtor, > - RingIndirectDtor, > - RingAcquireLock, > - RingReleaseLock, > - *Ring, > - &(*Ring)->IndirectCache); > - if (!NT_SUCCESS(status)) > - goto fail8; > + (*Ring)->Ring[Index] = BlkifRing; > + Index++; > + } > > return STATUS_SUCCESS; > > -fail8: > - Error("fail8\n"); > -fail7: > - Error("fail7\n"); > - XENBUS_CACHE(Destroy, > - &(*Ring)->CacheInterface, > - (*Ring)->SegmentCache); > - (*Ring)->SegmentCache = NULL; > -fail6: > - Error("fail6\n"); > -fail5: > - Error("fail5\n"); > - XENBUS_CACHE(Destroy, > - &(*Ring)->CacheInterface, > - (*Ring)->RequestCache); > - (*Ring)->RequestCache = NULL; > -fail4: > - Error("fail4\n"); > fail3: > Error("fail3\n"); > - XENBUS_CACHE(Release, > - &(*Ring)->CacheInterface); > + > + while (--Index > 0) { > + PXENVBD_BLKIF_RING BlkifRing = (*Ring)->Ring[Index]; > + > + (*Ring)->Ring[Index] = NULL; > + BlkifRingDestroy(BlkifRing); > + } > + > + __RingFree((*Ring)->Ring); > + (*Ring)->Ring = NULL; > + > fail2: > Error("fail2\n"); > > + (*Ring)->Frontend = NULL; > + > RtlZeroMemory(&(*Ring)->CacheInterface, > - sizeof (XENBUS_CACHE_INTERFACE)); > + sizeof(XENBUS_CACHE_INTERFACE)); > + > + RtlZeroMemory(&(*Ring)->EvtchnInterface, > + sizeof(XENBUS_EVTCHN_INTERFACE)); > > - RtlZeroMemory(&(*Ring)->PreparedReqs, sizeof(XENVBD_QUEUE)); > - RtlZeroMemory(&(*Ring)->SubmittedReqs, sizeof(XENVBD_QUEUE)); > - RtlZeroMemory(&(*Ring)->ShutdownSrbs, sizeof(XENVBD_QUEUE)); > + RtlZeroMemory(&(*Ring)->StoreInterface, > + sizeof(XENBUS_STORE_INTERFACE)); > > - RtlZeroMemory(&(*Ring)->Dpc, sizeof(KDPC)); > - RtlZeroMemory(&(*Ring)->Lock, sizeof(KSPIN_LOCK)); > - (*Ring)->Frontend = NULL; > + RtlZeroMemory(&(*Ring)->DebugInterface, > + sizeof(XENBUS_DEBUG_INTERFACE)); > > ASSERT(IsZeroMemory(*Ring, sizeof(XENVBD_RING))); > __RingFree(*Ring); > - *Ring = NULL; > + > fail1: > - Error("fail1 %08x\n", status); > + Error("fail1 (%08x)\n", status); > return status; > } > > @@ -1631,44 +2334,35 @@ RingDestroy( > IN PXENVBD_RING Ring > ) > { > - XENBUS_CACHE(Destroy, > - &Ring->CacheInterface, > - Ring->IndirectCache); > - Ring->IndirectCache = NULL; > + ULONG Index; > > - XENBUS_CACHE(Destroy, > - &Ring->CacheInterface, > - Ring->SegmentCache); > - Ring->SegmentCache = NULL; > + Index = FrontendGetMaxQueues(Ring->Frontend); > > - XENBUS_CACHE(Destroy, > - &Ring->CacheInterface, > - Ring->RequestCache); > - Ring->RequestCache = NULL; > + while (--Index > 0) { > + PXENVBD_BLKIF_RING BlkifRing = Ring->Ring[Index]; > + > + Ring->Ring[Index] = NULL; > + BlkifRingDestroy(BlkifRing); > + } > + > + __RingFree(Ring->Ring); > + Ring->Ring = NULL; > > - XENBUS_CACHE(Release, > - &Ring->CacheInterface); > + Ring->Frontend = NULL; > > RtlZeroMemory(&Ring->CacheInterface, > - sizeof (XENBUS_CACHE_INTERFACE)); > + sizeof(XENBUS_CACHE_INTERFACE)); > > - RtlZeroMemory(&Ring->PreparedReqs, sizeof(XENVBD_QUEUE)); > - RtlZeroMemory(&Ring->SubmittedReqs, sizeof(XENVBD_QUEUE)); > - RtlZeroMemory(&Ring->ShutdownSrbs, sizeof(XENVBD_QUEUE)); > + RtlZeroMemory(&Ring->EvtchnInterface, > + sizeof(XENBUS_EVTCHN_INTERFACE)); > > - RtlZeroMemory(&Ring->Dpc, sizeof(KDPC)); > - RtlZeroMemory(&Ring->Lock, sizeof(KSPIN_LOCK)); > - Ring->Frontend = NULL; > + RtlZeroMemory(&Ring->StoreInterface, > + sizeof(XENBUS_STORE_INTERFACE)); > + > + RtlZeroMemory(&Ring->DebugInterface, > + sizeof(XENBUS_DEBUG_INTERFACE)); > > - Ring->BlkOpRead = 0; > - Ring->BlkOpWrite = 0; > - Ring->BlkOpIndirectRead = 0; > - Ring->BlkOpIndirectWrite = 0; > - Ring->BlkOpBarrier = 0; > - Ring->BlkOpDiscard = 0; > - Ring->BlkOpFlush = 0; > - Ring->SegsGranted = 0; > - Ring->SegsBounced = 0; > + RtlZeroMemory(Ring->Stats, sizeof(Ring->Stats)); > > ASSERT(IsZeroMemory(Ring, sizeof(XENVBD_RING))); > __RingFree(Ring); > @@ -1679,31 +2373,27 @@ RingConnect( > IN PXENVBD_RING Ring > ) > { > - PXENVBD_TARGET Target = FrontendGetTarget(Ring->Frontend); > - PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target); > - PXENVBD_GRANTER Granter = FrontendGetGranter(Ring->Frontend); > - PCHAR Buffer; > + ULONG MaxQueues; > ULONG Index; > + PCHAR Buffer; > NTSTATUS status; > > - ASSERT(Ring->Connected == FALSE); > - > - AdapterGetStoreInterface(Adapter, &Ring->StoreInterface); > - AdapterGetEvtchnInterface(Adapter, &Ring->EvtchnInterface); > - AdapterGetDebugInterface(Adapter, &Ring->DebugInterface); > - > - status = XENBUS_STORE(Acquire, &Ring->StoreInterface); > + status = XENBUS_DEBUG(Acquire, &Ring->DebugInterface); > if (!NT_SUCCESS(status)) > goto fail1; > > - status = XENBUS_EVTCHN(Acquire, &Ring->EvtchnInterface); > + status = XENBUS_STORE(Acquire, &Ring->StoreInterface); > if (!NT_SUCCESS(status)) > goto fail2; > > - status = XENBUS_DEBUG(Acquire, &Ring->DebugInterface); > + status = XENBUS_CACHE(Acquire, &Ring->CacheInterface); > if (!NT_SUCCESS(status)) > goto fail3; > > + status = XENBUS_EVTCHN(Acquire, &Ring->EvtchnInterface); > + if (!NT_SUCCESS(status)) > + goto fail4; > + > status = XENBUS_STORE(Read, > &Ring->StoreInterface, > NULL, > @@ -1730,47 +2420,17 @@ RingConnect( > Ring->Order = 0; > } > > - Ring->Mdl = __AllocatePages(1 << Ring->Order); > - > - status = STATUS_NO_MEMORY; > - if (Ring->Mdl == NULL) > - goto fail4; > - > - Ring->Shared = MmGetSystemAddressForMdlSafe(Ring->Mdl, > - NormalPagePriority); > - ASSERT(Ring->Shared != NULL); > - > -#pragma warning(push) > -#pragma warning(disable: 4305) > -#pragma warning(disable: 4311) // 'type cast' pointer truncation from > 'blkif_sring_entry[1]' to 'long' > - SHARED_RING_INIT(Ring->Shared); > - FRONT_RING_INIT(&Ring->Front, Ring->Shared, PAGE_SIZE << Ring- > >Order); > -#pragma warning(pop) > + MaxQueues = FrontendGetNumQueues(Ring->Frontend); > + Index = 0; > + while (Index < MaxQueues) { > + PXENVBD_BLKIF_RING BlkifRing = Ring->Ring[Index]; > > - for (Index = 0; Index < (1ul << Ring->Order); ++Index) { > - status = GranterGet(Granter, > - MmGetMdlPfnArray(Ring->Mdl)[Index], > - FALSE, > - &Ring->Grants[Index]); > + status = BlkifRingConnect(BlkifRing); > if (!NT_SUCCESS(status)) > goto fail5; > - } > - > - Ring->Channel = XENBUS_EVTCHN(Open, > - &Ring->EvtchnInterface, > - XENBUS_EVTCHN_TYPE_UNBOUND, > - RingInterrupt, > - Ring, > - FrontendGetBackendDomain(Ring->Frontend), > - TRUE); > - status = STATUS_NO_MEMORY; > - if (Ring->Channel == NULL) > - goto fail6; > > - XENBUS_EVTCHN(Unmask, > - &Ring->EvtchnInterface, > - Ring->Channel, > - FALSE); > + ++Index; > + } > > status = XENBUS_DEBUG(Register, > &Ring->DebugInterface, > @@ -1779,55 +2439,44 @@ RingConnect( > Ring, > &Ring->DebugCallback); > if (!NT_SUCCESS(status)) > - goto fail7; > + goto fail6; > > - Ring->Connected = TRUE; > return STATUS_SUCCESS; > > -fail7: > - Error("fail7\n"); > - XENBUS_EVTCHN(Close, > - &Ring->EvtchnInterface, > - Ring->Channel); > - Ring->Channel = NULL; > fail6: > Error("fail6\n"); > + Index = FrontendGetNumQueues(Ring->Frontend); > fail5: > Error("fail5\n"); > - for (Index = 0; Index < (1ul << Ring->Order); ++Index) { > - if (Ring->Grants[Index] == NULL) > - continue; > > - GranterPut(Granter, Ring->Grants[Index]); > - Ring->Grants[Index] = NULL; > - } > + while (Index != 0) { > + PXENVBD_BLKIF_RING BlkifRing; > + > + --Index; > + BlkifRing = Ring->Ring[Index]; > > - RtlZeroMemory(&Ring->Front, sizeof(blkif_front_ring_t)); > + BlkifRingDisconnect(BlkifRing); > + } > > - __FreePages(Ring->Mdl); > - Ring->Shared = NULL; > - Ring->Mdl = NULL; > + XENBUS_EVTCHN(Release, &Ring->EvtchnInterface); > > - Ring->Order = 0; > fail4: > Error("fail4\n"); > - XENBUS_DEBUG(Release, &Ring->DebugInterface); > + > + XENBUS_CACHE(Release, &Ring->CacheInterface); > + > fail3: > Error("fail3\n"); > - XENBUS_EVTCHN(Release, &Ring->EvtchnInterface); > + > + XENBUS_STORE(Release, &Ring->StoreInterface); > + > fail2: > Error("fail2\n"); > - XENBUS_STORE(Release, &Ring->StoreInterface); > -fail1: > - Error("fail1 %08x\n", status); > > - RtlZeroMemory(&Ring->DebugInterface, > - sizeof(XENBUS_DEBUG_INTERFACE)); > - RtlZeroMemory(&Ring->EvtchnInterface, > - sizeof(XENBUS_EVTCHN_INTERFACE)); > - RtlZeroMemory(&Ring->StoreInterface, > - sizeof(XENBUS_STORE_INTERFACE)); > + XENBUS_DEBUG(Release, &Ring->DebugInterface); > > +fail1: > + Error("fail1 (%08x)\n", status); > return status; > } > > @@ -1837,23 +2486,33 @@ RingStoreWrite( > IN PVOID Transaction > ) > { > - PXENVBD_GRANTER Granter = FrontendGetGranter(Ring->Frontend); > - ULONG Port; > + ULONG NumQueues; > + ULONG Index; > NTSTATUS status; > > - if (Ring->Order == 0) { > - status = XENBUS_STORE(Printf, > - &Ring->StoreInterface, > - Transaction, > - FrontendGetFrontendPath(Ring->Frontend), > - "ring-ref", > - "%u", > - GranterReference(Granter, Ring->Grants[0])); > + NumQueues = FrontendGetNumQueues(Ring->Frontend); > + Index = 0; > + while (Index < NumQueues) { > + PXENVBD_BLKIF_RING BlkifRing = Ring->Ring[Index]; > + > + status = BlkifRingStoreWrite(BlkifRing, Transaction); > if (!NT_SUCCESS(status)) > - return status; > - } else { > - ULONG Index; > + goto fail1; > + > + ++Index; > + } > + > + status = XENBUS_STORE(Printf, > + &Ring->StoreInterface, > + Transaction, > + FrontendGetFrontendPath(Ring->Frontend), > + "multi-queue-num-queues", > + "%u", > + NumQueues); > + if (!NT_SUCCESS(status)) > + goto fail2; > > + if (Ring->Order != 0) { > status = XENBUS_STORE(Printf, > &Ring->StoreInterface, > Transaction, > @@ -1862,28 +2521,7 @@ RingStoreWrite( > "%u", > Ring->Order); > if (!NT_SUCCESS(status)) > - return status; > - > - for (Index = 0; Index < (1ul << Ring->Order); ++Index) { > - CHAR Name[MAX_NAME_LEN+1]; > - > - status = RtlStringCchPrintfA(Name, > - MAX_NAME_LEN, > - "ring-ref%u", > - Index); > - if (!NT_SUCCESS(status)) > - return status; > - > - status = XENBUS_STORE(Printf, > - &Ring->StoreInterface, > - Transaction, > - FrontendGetFrontendPath(Ring->Frontend), > - Name, > - "%u", > - GranterReference(Granter, > Ring->Grants[Index])); > - if (!NT_SUCCESS(status)) > - return status; > - } > + goto fail3; > } > > status = XENBUS_STORE(Printf, > @@ -1893,23 +2531,15 @@ RingStoreWrite( > "protocol", > XEN_IO_PROTO_ABI); > if (!NT_SUCCESS(status)) > - return status; > - > - Port = XENBUS_EVTCHN(GetPort, > - &Ring->EvtchnInterface, > - Ring->Channel); > - > - status = XENBUS_STORE(Printf, > - &Ring->StoreInterface, > - Transaction, > - FrontendGetFrontendPath(Ring->Frontend), > - "event-channel", > - "%u", > - Port); > - if (!NT_SUCCESS(status)) > - return status; > + goto fail4; > > return STATUS_SUCCESS; > + > +fail4: > +fail3: > +fail2: > +fail1: > + return status; > } > > VOID > @@ -1917,12 +2547,18 @@ RingEnable( > IN PXENVBD_RING Ring > ) > { > - ASSERT(Ring->Enabled == FALSE); > - Ring->Enabled = TRUE; > + ULONG NumQueues; > + ULONG Index; > > - XENBUS_EVTCHN(Trigger, > - &Ring->EvtchnInterface, > - Ring->Channel); > + NumQueues = FrontendGetNumQueues(Ring->Frontend); > + Index = 0; > + while (Index < NumQueues) { > + PXENVBD_BLKIF_RING BlkifRing = Ring->Ring[Index]; > + > + BlkifRingEnable(BlkifRing); > + > + ++Index; > + } > } > > VOID > @@ -1930,55 +2566,15 @@ RingDisable( > IN PXENVBD_RING Ring > ) > { > - ULONG Count; > - KIRQL Irql; > - PXENVBD_TARGET Target = FrontendGetTarget(Ring->Frontend); > - PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target); > - > - ASSERT(Ring->Enabled == TRUE); > - Ring->Enabled = FALSE; > - > - // poll ring and send event channel notification every 1ms (for up to 3 > minutes) > - Count = 0; > - while (QueueCount(&Ring->SubmittedReqs)) { > - if (Count > 180000) > - break; > - KeRaiseIrql(DISPATCH_LEVEL, &Irql); > - RingPoll(Ring); > - KeLowerIrql(Irql); > - XENBUS_EVTCHN(Send, > - &Ring->EvtchnInterface, > - Ring->Channel); > - StorPortStallExecution(1000); // 1000 micro-seconds > - ++Count; > - } > - > - Verbose("Target[%d] : %u Submitted requests left (%u iterrations)\n", > - FrontendGetTargetId(Ring->Frontend), > - QueueCount(&Ring->SubmittedReqs), > - Count); > - > - // Fail PreparedReqs > - for (;;) { > - PXENVBD_SRBEXT SrbExt; > - PSCSI_REQUEST_BLOCK Srb; > - PXENVBD_REQUEST Request; > - PLIST_ENTRY ListEntry; > - > - ListEntry = QueuePop(&Ring->PreparedReqs); > - if (ListEntry == NULL) > - break; > - Request = CONTAINING_RECORD(ListEntry, XENVBD_REQUEST, > ListEntry); > - SrbExt = Request->SrbExt; > - Srb = SrbExt->Srb; > - > - Srb->SrbStatus = SRB_STATUS_ABORTED; > - Srb->ScsiStatus = 0x40; // SCSI_ABORTED > + ULONG Index; > > - RingPutRequest(Ring, Request); > + Index = FrontendGetNumQueues(Ring->Frontend); > + while (Index != 0) { > + PXENVBD_BLKIF_RING BlkifRing; > > - if (InterlockedDecrement(&SrbExt->RequestCount) == 0) > - AdapterCompleteSrb(Adapter, SrbExt); > + --Index; > + BlkifRing = Ring->Ring[Index]; > + BlkifRingDisable(BlkifRing); > } > } > > @@ -1987,54 +2583,49 @@ RingDisconnect( > IN PXENVBD_RING Ring > ) > { > - PXENVBD_GRANTER Granter = FrontendGetGranter(Ring->Frontend); > ULONG Index; > > - ASSERT3U(Ring->Submitted, ==, Ring->Received); > - ASSERT(Ring->Connected); > - Ring->Connected = FALSE; > - > XENBUS_DEBUG(Deregister, > &Ring->DebugInterface, > Ring->DebugCallback); > Ring->DebugCallback = NULL; > > - XENBUS_EVTCHN(Close, > - &Ring->EvtchnInterface, > - Ring->Channel); > - Ring->Channel = NULL; > - > - for (Index = 0; Index < (1ul << Ring->Order); ++Index) { > - if (Ring->Grants[Index] == NULL) > - continue; > + Index = FrontendGetNumQueues(Ring->Frontend); > > - GranterPut(Granter, Ring->Grants[Index]); > - Ring->Grants[Index] = NULL; > - } > + while (Index != 0) { > + PXENVBD_BLKIF_RING BlkifRing; > > - RtlZeroMemory(&Ring->Front, sizeof(blkif_front_ring_t)); > + --Index; > + BlkifRing = Ring->Ring[Index]; > > - __FreePages(Ring->Mdl); > - Ring->Shared = NULL; > - Ring->Mdl = NULL; > + BlkifRingDisconnect(BlkifRing); > + } > > Ring->Order = 0; > > - XENBUS_DEBUG(Release, &Ring->DebugInterface); > XENBUS_EVTCHN(Release, &Ring->EvtchnInterface); > + XENBUS_CACHE(Release, &Ring->CacheInterface); > XENBUS_STORE(Release, &Ring->StoreInterface); > + XENBUS_DEBUG(Release, &Ring->DebugInterface); > +} > > - RtlZeroMemory(&Ring->DebugInterface, > - sizeof(XENBUS_DEBUG_INTERFACE)); > - RtlZeroMemory(&Ring->EvtchnInterface, > - sizeof(XENBUS_EVTCHN_INTERFACE)); > - RtlZeroMemory(&Ring->StoreInterface, > - sizeof(XENBUS_STORE_INTERFACE)); > +static FORCEINLINE PXENVBD_BLKIF_RING > +__RingGetBlkifRing( > + IN PXENVBD_RING Ring, > + IN ULONG Tag > + ) > +{ > + ULONG Value; > + ULONG Index; > + > + if (Tag == 0) > + Value = KeGetCurrentProcessorNumberEx(NULL); > + else > + Value = Tag; > + > + Index = Value % FrontendGetNumQueues(Ring->Frontend); > > - Ring->Events = 0; > - Ring->Dpcs = 0; > - Ring->Submitted = 0; > - Ring->Received = 0; > + return Ring->Ring[Index]; > } > > BOOLEAN > @@ -2044,25 +2635,14 @@ RingQueueRequest( > ) > { > PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb; > + PXENVBD_BLKIF_RING BlkifRing; > > - if (!Ring->Enabled) > - goto fail1; > - > - if (!RingPrepareRequest(Ring, SrbExt)) > - goto fail2; > + BlkifRing = __RingGetBlkifRing(Ring, Srb->QueueTag); > + ASSERT(BlkifRing != NULL); > > - if (RingSubmitRequests(Ring)) { > - // more prepared-reqs to submit > - if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL)) > - ++Ring->Dpcs; > - } > + __BlkifRingQueueSrb(BlkifRing, SrbExt); > > return TRUE; > - > -fail2: > -fail1: > - Srb->SrbStatus = SRB_STATUS_BUSY; > - return FALSE; > } > > VOID > @@ -2071,12 +2651,11 @@ RingQueueShutdown( > IN PXENVBD_SRBEXT SrbExt > ) > { > - QueueAppend(&Ring->ShutdownSrbs, > - &SrbExt->ListEntry); > + PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb; > + PXENVBD_BLKIF_RING BlkifRing; > > - if (!Ring->Enabled) > - return; > + BlkifRing = __RingGetBlkifRing(Ring, Srb->QueueTag); > + ASSERT(BlkifRing != NULL); > > - if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL)) > - ++Ring->Dpcs; > + __BlkifRingQueueShutdown(BlkifRing, SrbExt); > } > -- > 2.16.2.windows.1 > > > _______________________________________________ > win-pv-devel mailing list > win-pv-devel@xxxxxxxxxxxxxxxxxxxx > https://lists.xenproject.org/mailman/listinfo/win-pv-devel _______________________________________________ win-pv-devel mailing list win-pv-devel@xxxxxxxxxxxxxxxxxxxx https://lists.xenproject.org/mailman/listinfo/win-pv-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |