|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] Re: [win-pv-devel] [RFC PATCH 4/6] Add initial multi-queue support
> -----Original Message----- > From: win-pv-devel [mailto:win-pv-devel-bounces@xxxxxxxxxxxxxxxxxxxx] On > Behalf Of owensm > Sent: 03 May 2018 12:03 > To: win-pv-devel@xxxxxxxxxxxxxxxxxxxx > Cc: Owen Smith <owen.smith@xxxxxxxxxx> > Subject: [win-pv-devel] [RFC PATCH 4/6] Add initial multi-queue support > > Create a queue per CPU, up-to the lower limit specified by xenstore and > registry override. Requests are queued on the same processor's DPC as > StartIo runs on. It is assumed that any request dispatched to the > backend on a particular queue will always be completed on the same > queue. > We need to expose the fact we have multiple underlying queues to the stack otherwise we're going to have to maintain a total ordering on SRBs to avoid the possibility of writes to a block getting re-ordered ahead of reads from the same block, or vice versa. Paul > Signed-off-by: owensm <owen.smith@xxxxxxxxxx> > --- > src/xenvbd/protocol.c | 2100 ++++++++++++++++++++++++++++------------ > --------- > 1 file changed, 1222 insertions(+), 878 deletions(-) > > diff --git a/src/xenvbd/protocol.c b/src/xenvbd/protocol.c > index 39767eb..8aa6ec8 100644 > --- a/src/xenvbd/protocol.c > +++ b/src/xenvbd/protocol.c > @@ -56,15 +56,10 @@ > #define XENVBD_MAX_PROTOCOL_PAGE_ORDER (4) > #define XENVBD_MAX_PROTOCOL_PAGES (1 << > XENVBD_MAX_PROTOCOL_PAGE_ORDER) > > -struct _XENVBD_PROTOCOL { > - PXENVBD_FRONTEND Frontend; > - BOOLEAN Connected; > - BOOLEAN Enabled; > - > - XENBUS_CACHE_INTERFACE CacheInterface; > - XENBUS_STORE_INTERFACE StoreInterface; > - XENBUS_EVTCHN_INTERFACE EvtchnInterface; > - XENBUS_DEBUG_INTERFACE DebugInterface; > +typedef struct _XENVBD_RING { > + PXENVBD_PROTOCOL Protocol; > + ULONG Index; > + PCHAR Path; > > PXENBUS_DEBUG_CALLBACK DebugCallback; > > @@ -72,14 +67,10 @@ struct _XENVBD_PROTOCOL { > PMDL Mdl; > blkif_sring_t* Shared; > blkif_front_ring_t Front; > - ULONG Order; > PVOID Grants[XENVBD_MAX_PROTOCOL_PAGES]; > PXENBUS_EVTCHN_CHANNEL Channel; > KDPC Dpc; > > - PXENBUS_CACHE RequestCache; > - PXENBUS_CACHE SegmentCache; > - PXENBUS_CACHE IndirectCache; > XENVBD_QUEUE PreparedReqs; > XENVBD_QUEUE SubmittedReqs; > XENVBD_QUEUE ShutdownSrbs; > @@ -95,6 +86,29 @@ struct _XENVBD_PROTOCOL { > ULONG BlkOpBarrier; > ULONG BlkOpDiscard; > ULONG BlkOpFlush; > +} XENVBD_RING, *PXENVBD_RING; > + > +struct _XENVBD_PROTOCOL { > + PXENVBD_FRONTEND Frontend; > + BOOLEAN Connected; > + BOOLEAN Enabled; > + > + XENBUS_CACHE_INTERFACE CacheInterface; > + XENBUS_STORE_INTERFACE StoreInterface; > + XENBUS_EVTCHN_INTERFACE EvtchnInterface; > + XENBUS_DEBUG_INTERFACE DebugInterface; > + > + PXENBUS_DEBUG_CALLBACK DebugCallback; > + KSPIN_LOCK Lock; > + > + PXENBUS_CACHE RequestCache; > + PXENBUS_CACHE SegmentCache; > + PXENBUS_CACHE IndirectCache; > + > + ULONG Order; > + ULONG NumQueues; > + PXENVBD_RING *Rings; > + > ULONG64 SegsGranted; > ULONG64 SegsBounced; > }; > @@ -138,12 +152,13 @@ xen_wmb() > } > > static FORCEINLINE VOID > -__ProtocolInsert( > - IN PXENVBD_PROTOCOL Protocol, > +__RingInsert( > + IN PXENVBD_RING Ring, > IN PXENVBD_REQUEST Request, > IN blkif_request_t* req > ) > { > + PXENVBD_PROTOCOL Protocol = Ring->Protocol; > PXENVBD_GRANTER Granter = FrontendGetGranter(Protocol- > >Frontend); > > switch (Request->Operation) { > @@ -234,7 +249,7 @@ __ProtocolInsert( > ASSERT(FALSE); > break; > } > - ++Protocol->Submitted; > + ++Ring->Submitted; > } > > static PXENVBD_INDIRECT > @@ -400,15 +415,16 @@ ProtocolPutRequest( > } > > static FORCEINLINE PXENVBD_REQUEST > -ProtocolFindRequest( > - IN PXENVBD_PROTOCOL Protocol, > +RingFindRequest( > + IN PXENVBD_RING Ring, > IN ULONG64 Id > ) > { > KIRQL Irql; > PLIST_ENTRY ListEntry; > PXENVBD_REQUEST Request; > - PXENVBD_QUEUE Queue = &Protocol->SubmittedReqs; > + PXENVBD_QUEUE Queue = &Ring->SubmittedReqs; > + PXENVBD_PROTOCOL Protocol = Ring->Protocol; > > KeAcquireSpinLock(&Queue->Lock, &Irql); > > @@ -425,40 +441,41 @@ ProtocolFindRequest( > } > > KeReleaseSpinLock(&Queue->Lock, Irql); > - Warning("Target[%d] : Tag %llx not found in submitted list (%u items)\n", > + Warning("Target[%d][%u] : Tag %llx not found in submitted list (%u > items)\n", > FrontendGetTargetId(Protocol->Frontend), > + Ring->Index, > Id, > QueueCount(Queue)); > return NULL; > } > > static FORCEINLINE VOID > -__ProtocolIncBlkifOpCount( > - IN PXENVBD_PROTOCOL Protocol, > +__RingIncBlkifOpCount( > + IN PXENVBD_RING Ring, > IN PXENVBD_REQUEST Request > ) > { > switch (Request->Operation) { > case BLKIF_OP_READ: > if (Request->NrSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST) > - ++Protocol->BlkOpIndirectRead; > + ++Ring->BlkOpIndirectRead; > else > - ++Protocol->BlkOpRead; > + ++Ring->BlkOpRead; > break; > case BLKIF_OP_WRITE: > if (Request->NrSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST) > - ++Protocol->BlkOpIndirectWrite; > + ++Ring->BlkOpIndirectWrite; > else > - ++Protocol->BlkOpWrite; > + ++Ring->BlkOpWrite; > break; > case BLKIF_OP_WRITE_BARRIER: > - ++Protocol->BlkOpBarrier; > + ++Ring->BlkOpBarrier; > break; > case BLKIF_OP_DISCARD: > - ++Protocol->BlkOpDiscard; > + ++Ring->BlkOpDiscard; > break; > case BLKIF_OP_FLUSH_DISKCACHE: > - ++Protocol->BlkOpFlush; > + ++Ring->BlkOpFlush; > break; > default: > ASSERT(FALSE); > @@ -467,7 +484,7 @@ __ProtocolIncBlkifOpCount( > } > > static FORCEINLINE ULONG > -__ProtocolSectorsPerPage( > +__SectorsPerPage( > IN ULONG SectorSize > ) > { > @@ -476,7 +493,7 @@ __ProtocolSectorsPerPage( > } > > static FORCEINLINE VOID > -__ProtocolOperation( > +__Operation( > IN UCHAR CdbOp, > OUT PUCHAR ProtocolOp, > OUT PBOOLEAN ReadOnly > @@ -496,22 +513,8 @@ __ProtocolOperation( > } > } > > -static FORCEINLINE MM_PAGE_PRIORITY > -__ProtocolPriority( > - IN PXENVBD_PROTOCOL Protocol > - ) > -{ > - PXENVBD_CAPS Caps = FrontendGetCaps(Protocol->Frontend); > - if (!(Caps->Paging || > - Caps->Hibernation || > - Caps->DumpFile)) > - return NormalPagePriority; > - > - return HighPagePriority; > -} > - > static FORCEINLINE VOID > -ProtocolRequestCopyOutput( > +__RequestCopyOutput( > IN PXENVBD_REQUEST Request > ) > { > @@ -534,532 +537,136 @@ ProtocolRequestCopyOutput( > } > } > > -static BOOLEAN > -ProtocolPrepareSegment( > - IN PXENVBD_PROTOCOL Protocol, > - IN PXENVBD_SEGMENT Segment, > - IN PXENVBD_SRBEXT SrbExt, > - IN BOOLEAN ReadOnly, > - IN ULONG SectorsLeft, > - OUT PULONG SectorsNow > +static FORCEINLINE VOID > +RingQueueRequestList( > + IN PXENVBD_RING Ring, > + IN PLIST_ENTRY List > ) > { > - PFN_NUMBER Pfn; > - ULONG Offset; > - ULONG Length; > - NTSTATUS Status; > - PXENVBD_GRANTER Granter = FrontendGetGranter(Protocol- > >Frontend); > - const ULONG SectorSize = FrontendGetDiskInfo(Protocol->Frontend)- > >SectorSize; > - const ULONG SectorsPerPage = > __ProtocolSectorsPerPage(SectorSize); > - PXENVBD_TARGET Target = FrontendGetTarget(Protocol->Frontend); > - PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target); > - > - Pfn = AdapterGetNextSGEntry(Adapter, > - SrbExt, > - 0, > - &Offset, > - &Length); > - if ((Offset & (SectorSize - 1)) == 0 && > - (Length & (SectorSize - 1)) == 0) { > - ++Protocol->SegsGranted; > - // get first sector, last sector and count > - Segment->FirstSector = (UCHAR)((Offset + SectorSize - 1) / > SectorSize); > - *SectorsNow = __min(SectorsLeft, SectorsPerPage - > Segment- > >FirstSector); > - Segment->LastSector = (UCHAR)(Segment->FirstSector + > *SectorsNow - 1); > - > - ASSERT3U((Length / SectorSize), ==, *SectorsNow); > - } else { > - PXENVBD_BOUNCE Bounce; > - PMDL Mdl; > - > - ++Protocol->SegsBounced; > - // get first sector, last sector and count > - Segment->FirstSector = 0; > - *SectorsNow = __min(SectorsLeft, SectorsPerPage); > - Segment->LastSector = (UCHAR)(*SectorsNow - 1); > - > - Bounce = AdapterGetBounce(Adapter); > - if (Bounce == NULL) > - goto fail1; > - Segment->Bounce = Bounce; > - > -#pragma warning(push) > -#pragma warning(disable:28145) > - Mdl = &Bounce->SourceMdl; > - Mdl->Next = NULL; > - Mdl->Size = (SHORT)(sizeof(MDL) + sizeof(PFN_NUMBER)); > - Mdl->MdlFlags = MDL_PAGES_LOCKED; > - Mdl->Process = NULL; > - Mdl->MappedSystemVa = NULL; > - Mdl->StartVa = NULL; > - Mdl->ByteCount = Length; > - Mdl->ByteOffset = Offset; > - Bounce->SourcePfn[0] = Pfn; > - > - if (Length < *SectorsNow * SectorSize) { > - Pfn = AdapterGetNextSGEntry(Adapter, > - SrbExt, > - Length, > - &Offset, > - &Length); > - Mdl->Size += sizeof(PFN_NUMBER); > - Mdl->ByteCount += Length; > - Bounce->SourcePfn[1] = Pfn; > - } > -#pragma warning(pop) > - > - ASSERT((Mdl->ByteCount & (SectorSize - 1)) == 0); > - ASSERT3U(Mdl->ByteCount, <=, PAGE_SIZE); > - ASSERT3U(*SectorsNow, ==, (Mdl->ByteCount / SectorSize)); > - > - Bounce->SourcePtr = MmMapLockedPagesSpecifyCache(Mdl, > - KernelMode, > - MmCached, > - NULL, > - FALSE, > - > __ProtocolPriority(Protocol)); > - if (Bounce->SourcePtr == NULL) > - goto fail2; > - > - ASSERT3P(MmGetMdlPfnArray(Mdl)[0], ==, Bounce->SourcePfn[0]); > - ASSERT3P(MmGetMdlPfnArray(Mdl)[1], ==, Bounce->SourcePfn[1]); > + for (;;) { > + PXENVBD_REQUEST Request; > + PLIST_ENTRY ListEntry; > > - // copy contents in > - if (ReadOnly) { // Operation == BLKIF_OP_WRITE > - RtlCopyMemory(Bounce->BouncePtr, > - Bounce->SourcePtr, > - MmGetMdlByteCount(&Bounce->SourceMdl)); > - } > + ListEntry = RemoveHeadList(List); > + if (ListEntry == List) > + break; > > - Pfn = MmGetMdlPfnArray(Bounce->BounceMdl)[0]; > + Request = CONTAINING_RECORD(ListEntry, XENVBD_REQUEST, > ListEntry); > + __RingIncBlkifOpCount(Ring, Request); > + QueueAppend(&Ring->PreparedReqs, &Request->ListEntry); > } > - > - // Grant segment's page > - Status = GranterGet(Granter, Pfn, ReadOnly, &Segment->Grant); > - if (!NT_SUCCESS(Status)) > - goto fail3; > - > - return TRUE; > - > -fail3: > -fail2: > -fail1: > - return FALSE; > } > > static BOOLEAN > -ProtocolPrepareBlkifReadWrite( > - IN PXENVBD_PROTOCOL Protocol, > - IN PXENVBD_REQUEST Request, > - IN PXENVBD_SRBEXT SrbExt, > - IN ULONG MaxSegments, > - IN ULONG64 SectorStart, > - IN ULONG SectorsLeft, > - OUT PULONG SectorsDone > +RingSubmit( > + IN PXENVBD_RING Ring, > + IN PXENVBD_REQUEST Request > ) > { > - PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb; > - UCHAR Operation; > - BOOLEAN ReadOnly; > - ULONG Index; > - __ProtocolOperation(Cdb_OperationEx(Srb), &Operation, &ReadOnly); > - > - Request->Operation = Operation; > - Request->NrSegments = 0; > - Request->FirstSector = SectorStart; > - > - for (Index = 0; > - Index < MaxSegments && > - SectorsLeft > 0; > - ++Index) { > - PXENVBD_SEGMENT Segment; > - ULONG SectorsNow; > - > - Segment = ProtocolGetSegment(Protocol); > - if (Segment == NULL) > - goto fail1; > - > - InsertTailList(&Request->Segments, &Segment->ListEntry); > - ++Request->NrSegments; > - > - if (!ProtocolPrepareSegment(Protocol, > - Segment, > - SrbExt, > - ReadOnly, > - SectorsLeft, > - &SectorsNow)) > - goto fail2; > + KIRQL Irql; > + blkif_request_t* req; > + BOOLEAN Notify; > + PXENVBD_PROTOCOL Protocol; > > - *SectorsDone += SectorsNow; > - SectorsLeft -= SectorsNow; > + KeAcquireSpinLock(&Ring->Lock, &Irql); > + if (RING_FULL(&Ring->Front)) { > + KeReleaseSpinLock(&Ring->Lock, Irql); > + return FALSE; > } > - ASSERT3U(Request->NrSegments, >, 0); > - ASSERT3U(Request->NrSegments, <=, MaxSegments); > - > - return TRUE; > - > -fail2: > -fail1: > - return FALSE; > -} > > -static BOOLEAN > -ProtocolPrepareBlkifIndirect( > - IN PXENVBD_PROTOCOL Protocol, > - IN PXENVBD_REQUEST Request > - ) > -{ > - ULONG Index; > - ULONG NrSegments = 0; > + req = RING_GET_REQUEST(&Ring->Front, Ring->Front.req_prod_pvt); > + __RingInsert(Ring, Request, req); > + KeMemoryBarrier(); > + ++Ring->Front.req_prod_pvt; > > - for (Index = 0; > - Index < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST && > - NrSegments < Request->NrSegments; > - ++Index) { > - PXENVBD_INDIRECT Indirect; > + RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&Ring->Front, Notify); > + KeReleaseSpinLock(&Ring->Lock, Irql); > > - Indirect = ProtocolGetIndirect(Protocol); > - if (Indirect == NULL) > - goto fail1; > - InsertTailList(&Request->Indirects, &Indirect->ListEntry); > + if (Notify) { > + Protocol = Ring->Protocol; > + if (!Protocol->Enabled) > + return TRUE; > > - NrSegments += XENVBD_MAX_SEGMENTS_PER_PAGE; > + XENBUS_EVTCHN(Send, > + &Protocol->EvtchnInterface, > + Ring->Channel); > } > > return TRUE; > - > -fail1: > - return FALSE; > } > > -static FORCEINLINE ULONG > -ProtocolUseIndirect( > - IN PXENVBD_PROTOCOL Protocol, > - IN ULONG SectorsLeft > +static FORCEINLINE BOOLEAN > +RingSubmitRequests( > + IN PXENVBD_RING Ring > ) > { > - const ULONG SectorsPerPage = > __ProtocolSectorsPerPage(FrontendGetDiskInfo(Protocol->Frontend)- > >SectorSize); > - const ULONG MaxIndirectSegs = FrontendGetFeatures(Protocol- > >Frontend)->Indirect; > - > - if (MaxIndirectSegs <= BLKIF_MAX_SEGMENTS_PER_REQUEST) > - return BLKIF_MAX_SEGMENTS_PER_REQUEST; // not supported > - > - if (SectorsLeft < BLKIF_MAX_SEGMENTS_PER_REQUEST * > SectorsPerPage) > - return BLKIF_MAX_SEGMENTS_PER_REQUEST; // first into a single > BLKIF_OP_{READ/WRITE} > + PXENVBD_PROTOCOL Protocol = Ring->Protocol; > > - return MaxIndirectSegs; > -} > + if (!Protocol->Enabled) { > + if (QueueCount(&Ring->PreparedReqs)) > + Warning("Target[%d][%u] : Paused, not submitting new requests > (%u)\n", > + FrontendGetTargetId(Protocol->Frontend), > + Ring->Index, > + QueueCount(&Ring->PreparedReqs)); > + return FALSE; > + } > > -static FORCEINLINE VOID > -ProtocolQueueRequestList( > - IN PXENVBD_PROTOCOL Protocol, > - IN PLIST_ENTRY List > - ) > -{ > for (;;) { > PXENVBD_REQUEST Request; > PLIST_ENTRY ListEntry; > > - ListEntry = RemoveHeadList(List); > - if (ListEntry == List) > + ListEntry = QueuePop(&Ring->PreparedReqs); > + if (ListEntry == NULL) > break; > > Request = CONTAINING_RECORD(ListEntry, XENVBD_REQUEST, > ListEntry); > - __ProtocolIncBlkifOpCount(Protocol, Request); > - QueueAppend(&Protocol->PreparedReqs, &Request->ListEntry); > + > + QueueAppend(&Ring->SubmittedReqs, &Request->ListEntry); > + KeMemoryBarrier(); > + > + if (RingSubmit(Ring, Request)) > + continue; > + > + QueueRemove(&Ring->SubmittedReqs, &Request->ListEntry); > + QueueUnPop(&Ring->PreparedReqs, &Request->ListEntry); > + break; > } > + > + return QueueCount(&Ring->PreparedReqs) != 0; > } > > static FORCEINLINE VOID > -ProtocolCancelRequestList( > - IN PXENVBD_PROTOCOL Protocol, > - IN PLIST_ENTRY List > +RingCompleteShutdown( > + IN PXENVBD_RING Ring > ) > { > + PXENVBD_PROTOCOL Protocol; > + PXENVBD_TARGET Target; > + PXENVBD_ADAPTER Adapter; > + > + if (QueueCount(&Ring->ShutdownSrbs) == 0) > + return; > + > + if (QueueCount(&Ring->PreparedReqs) || > + QueueCount(&Ring->SubmittedReqs)) > + return; > + > + Protocol = Ring->Protocol; > + Target = FrontendGetTarget(Protocol->Frontend); > + Adapter = TargetGetAdapter(Target); > for (;;) { > - PXENVBD_REQUEST Request; > - PLIST_ENTRY ListEntry; > + PXENVBD_SRBEXT SrbExt; > + PSCSI_REQUEST_BLOCK Srb; > + PLIST_ENTRY ListEntry; > > - ListEntry = RemoveHeadList(List); > - if (ListEntry == List) > + ListEntry = QueuePop(&Ring->ShutdownSrbs); > + if (ListEntry == NULL) > break; > - > - Request = CONTAINING_RECORD(ListEntry, XENVBD_REQUEST, > ListEntry); > - ProtocolPutRequest(Protocol, Request); > - } > -} > - > -static BOOLEAN > -ProtocolPrepareReadWrite( > - IN PXENVBD_PROTOCOL Protocol, > - IN PXENVBD_SRBEXT SrbExt > - ) > -{ > - PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb; > - ULONG64 SectorStart = Cdb_LogicalBlock(Srb); > - ULONG SectorsLeft = Cdb_TransferBlock(Srb); > - LIST_ENTRY List; > - > - Srb->SrbStatus = SRB_STATUS_PENDING; > - > - InitializeListHead(&List); > - SrbExt->RequestCount = 0; > - > - while (SectorsLeft > 0) { > - ULONG MaxSegments; > - ULONG SectorsDone = 0; > - PXENVBD_REQUEST Request; > - > - Request = ProtocolGetRequest(Protocol); > - if (Request == NULL) > - goto fail1; > - InsertTailList(&List, &Request->ListEntry); > - InterlockedIncrement(&SrbExt->RequestCount); > - > - Request->SrbExt = SrbExt; > - MaxSegments = ProtocolUseIndirect(Protocol, SectorsLeft); > - > - if (!ProtocolPrepareBlkifReadWrite(Protocol, > - Request, > - SrbExt, > - MaxSegments, > - SectorStart, > - SectorsLeft, > - &SectorsDone)) > - goto fail2; > - > - if (MaxSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST) { > - if (!ProtocolPrepareBlkifIndirect(Protocol, Request)) > - goto fail3; > - } > - > - SectorsLeft -= SectorsDone; > - SectorStart += SectorsDone; > - } > - > - ProtocolQueueRequestList(Protocol, &List); > - return TRUE; > - > -fail3: > -fail2: > -fail1: > - ProtocolCancelRequestList(Protocol, &List); > - SrbExt->RequestCount = 0; > - Srb->SrbStatus = SRB_STATUS_ERROR; > - return FALSE; > -} > - > -static BOOLEAN > -ProtocolPrepareSyncCache( > - IN PXENVBD_PROTOCOL Protocol, > - IN PXENVBD_SRBEXT SrbExt > - ) > -{ > - PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb; > - PXENVBD_REQUEST Request; > - LIST_ENTRY List; > - UCHAR Operation; > - > - Srb->SrbStatus = SRB_STATUS_PENDING; > - > - if (FrontendGetDiskInfo(Protocol->Frontend)->FlushCache) > - Operation = BLKIF_OP_FLUSH_DISKCACHE; > - else > - Operation = BLKIF_OP_WRITE_BARRIER; > - > - InitializeListHead(&List); > - SrbExt->RequestCount = 0; > - > - Request = ProtocolGetRequest(Protocol); > - if (Request == NULL) > - goto fail1; > - InsertTailList(&List, &Request->ListEntry); > - InterlockedIncrement(&SrbExt->RequestCount); > - > - Request->SrbExt = SrbExt; > - Request->Operation = Operation; > - Request->FirstSector = Cdb_LogicalBlock(Srb); > - > - ProtocolQueueRequestList(Protocol, &List); > - return TRUE; > - > -fail1: > - ProtocolCancelRequestList(Protocol, &List); > - SrbExt->RequestCount = 0; > - Srb->SrbStatus = SRB_STATUS_ERROR; > - return FALSE; > -} > - > -static BOOLEAN > -ProtocolPrepareUnmap( > - IN PXENVBD_PROTOCOL Protocol, > - IN PXENVBD_SRBEXT SrbExt > - ) > -{ > - PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb; > - PUNMAP_LIST_HEADER Unmap = Srb->DataBuffer; > - ULONG Count = _byteswap_ushort(*(PUSHORT)Unmap- > >BlockDescrDataLength) / sizeof(UNMAP_BLOCK_DESCRIPTOR); > - ULONG Index; > - LIST_ENTRY List; > - > - Srb->SrbStatus = SRB_STATUS_PENDING; > - > - InitializeListHead(&List); > - SrbExt->RequestCount = 0; > - > - for (Index = 0; Index < Count; ++Index) { > - PUNMAP_BLOCK_DESCRIPTOR Descr = &Unmap->Descriptors[Index]; > - PXENVBD_REQUEST Request; > - > - Request = ProtocolGetRequest(Protocol); > - if (Request == NULL) > - goto fail1; > - InsertTailList(&List, &Request->ListEntry); > - InterlockedIncrement(&SrbExt->RequestCount); > - > - Request->SrbExt = SrbExt; > - Request->Operation = BLKIF_OP_DISCARD; > - Request->FirstSector = _byteswap_uint64(*(PULONG64)Descr- > >StartingLba); > - Request->NrSectors = _byteswap_ulong(*(PULONG)Descr- > >LbaCount); > - Request->Flags = 0; > - } > - > - ProtocolQueueRequestList(Protocol, &List); > - return TRUE; > - > -fail1: > - ProtocolCancelRequestList(Protocol, &List); > - SrbExt->RequestCount = 0; > - Srb->SrbStatus = SRB_STATUS_ERROR; > - return FALSE; > -} > - > -static FORCEINLINE BOOLEAN > -ProtocolPrepareRequest( > - IN PXENVBD_PROTOCOL Protocol, > - IN PXENVBD_SRBEXT SrbExt > - ) > -{ > - switch (Cdb_OperationEx(SrbExt->Srb)) { > - case SCSIOP_READ: > - case SCSIOP_WRITE: > - return ProtocolPrepareReadWrite(Protocol, SrbExt); > - > - case SCSIOP_SYNCHRONIZE_CACHE: > - return ProtocolPrepareSyncCache(Protocol, SrbExt); > - > - case SCSIOP_UNMAP: > - return ProtocolPrepareUnmap(Protocol, SrbExt); > - > - default: > - ASSERT(FALSE); > - return FALSE; > - } > -} > - > -static BOOLEAN > -ProtocolSubmit( > - IN PXENVBD_PROTOCOL Protocol, > - IN PXENVBD_REQUEST Request > - ) > -{ > - KIRQL Irql; > - blkif_request_t* req; > - BOOLEAN Notify; > - > - KeAcquireSpinLock(&Protocol->Lock, &Irql); > - if (RING_FULL(&Protocol->Front)) { > - KeReleaseSpinLock(&Protocol->Lock, Irql); > - return FALSE; > - } > - > - req = RING_GET_REQUEST(&Protocol->Front, Protocol- > >Front.req_prod_pvt); > - __ProtocolInsert(Protocol, Request, req); > - KeMemoryBarrier(); > - ++Protocol->Front.req_prod_pvt; > - > - RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&Protocol->Front, Notify); > - KeReleaseSpinLock(&Protocol->Lock, Irql); > - > - if (Notify) { > - if (!Protocol->Enabled) > - return TRUE; > - > - XENBUS_EVTCHN(Send, > - &Protocol->EvtchnInterface, > - Protocol->Channel); > - } > - > - return TRUE; > -} > - > -static FORCEINLINE BOOLEAN > -ProtocolSubmitRequests( > - IN PXENVBD_PROTOCOL Protocol > - ) > -{ > - if (!Protocol->Enabled) { > - if (QueueCount(&Protocol->PreparedReqs)) > - Warning("Target[%d] : Paused, not submitting new requests > (%u)\n", > - FrontendGetTargetId(Protocol->Frontend), > - QueueCount(&Protocol->PreparedReqs)); > - return FALSE; > - } > - > - for (;;) { > - PXENVBD_REQUEST Request; > - PLIST_ENTRY ListEntry; > - > - ListEntry = QueuePop(&Protocol->PreparedReqs); > - if (ListEntry == NULL) > - break; > - > - Request = CONTAINING_RECORD(ListEntry, XENVBD_REQUEST, > ListEntry); > - > - QueueAppend(&Protocol->SubmittedReqs, &Request->ListEntry); > - KeMemoryBarrier(); > - > - if (ProtocolSubmit(Protocol, Request)) > - continue; > - > - QueueRemove(&Protocol->SubmittedReqs, &Request->ListEntry); > - QueueUnPop(&Protocol->PreparedReqs, &Request->ListEntry); > - break; > - } > - > - return QueueCount(&Protocol->PreparedReqs) != 0; > -} > - > -static FORCEINLINE VOID > -ProtocolCompleteShutdown( > - IN PXENVBD_PROTOCOL Protocol > - ) > -{ > - PXENVBD_TARGET Target; > - PXENVBD_ADAPTER Adapter; > - > - if (QueueCount(&Protocol->ShutdownSrbs) == 0) > - return; > - > - if (QueueCount(&Protocol->PreparedReqs) || > - QueueCount(&Protocol->SubmittedReqs)) > - return; > - > - Target = FrontendGetTarget(Protocol->Frontend); > - Adapter = TargetGetAdapter(Target); > - for (;;) { > - PXENVBD_SRBEXT SrbExt; > - PSCSI_REQUEST_BLOCK Srb; > - PLIST_ENTRY ListEntry; > - > - ListEntry = QueuePop(&Protocol->ShutdownSrbs); > - if (ListEntry == NULL) > - break; > - SrbExt = CONTAINING_RECORD(ListEntry, XENVBD_SRBEXT, ListEntry); > - Srb = SrbExt->Srb; > - > - Srb->SrbStatus = SRB_STATUS_SUCCESS; > - AdapterCompleteSrb(Adapter, SrbExt); > + SrbExt = CONTAINING_RECORD(ListEntry, XENVBD_SRBEXT, ListEntry); > + Srb = SrbExt->Srb; > + > + Srb->SrbStatus = SRB_STATUS_SUCCESS; > + AdapterCompleteSrb(Adapter, SrbExt); > } > } > > @@ -1081,8 +688,8 @@ __BlkifOperationName( > } > > static VOID > -ProtocolCompleteResponse( > - IN PXENVBD_PROTOCOL Protocol, > +RingCompleteResponse( > + IN PXENVBD_RING Ring, > IN ULONG64 Id, > IN SHORT Status > ) > @@ -1090,8 +697,9 @@ ProtocolCompleteResponse( > PXENVBD_REQUEST Request; > PSCSI_REQUEST_BLOCK Srb; > PXENVBD_SRBEXT SrbExt; > + PXENVBD_PROTOCOL Protocol = Ring->Protocol; > > - Request = ProtocolFindRequest(Protocol, Id); > + Request = RingFindRequest(Ring, Id); > if (Request == NULL) > return; > > @@ -1100,7 +708,7 @@ ProtocolCompleteResponse( > > switch (Status) { > case BLKIF_RSP_OKAY: > - ProtocolRequestCopyOutput(Request); > + __RequestCopyOutput(Request); > break; > > case BLKIF_RSP_EOPNOTSUPP: > @@ -1112,8 +720,9 @@ ProtocolCompleteResponse( > > case BLKIF_RSP_ERROR: > default: > - Warning("Target[%d] : %s BLKIF_RSP_ERROR (Tag %llx)\n", > + Warning("Target[%d][%u] : %s BLKIF_RSP_ERROR (Tag %llx)\n", > FrontendGetTargetId(Protocol->Frontend), > + Ring->Index, > __BlkifOperationName(Request->Operation), > Id); > Srb->SrbStatus = SRB_STATUS_ERROR; > @@ -1142,14 +751,15 @@ ProtocolCompleteResponse( > } > > static BOOLEAN > -ProtocolPoll( > - IN PXENVBD_PROTOCOL Protocol > +RingPoll( > + IN PXENVBD_RING Ring > ) > { > + PXENVBD_PROTOCOL Protocol = Ring->Protocol; > BOOLEAN Retry = FALSE; > > ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL); > - KeAcquireSpinLockAtDpcLevel(&Protocol->Lock); > + KeAcquireSpinLockAtDpcLevel(&Ring->Lock); > > // Guard against this locked region being called after the > // lock on FrontendSetState > @@ -1162,8 +772,8 @@ ProtocolPoll( > > KeMemoryBarrier(); > > - rsp_prod = Protocol->Shared->rsp_prod; > - rsp_cons = Protocol->Front.rsp_cons; > + rsp_prod = Ring->Shared->rsp_prod; > + rsp_cons = Ring->Front.rsp_cons; > > KeMemoryBarrier(); > > @@ -1173,105 +783,1018 @@ ProtocolPoll( > while (rsp_cons != rsp_prod && !Retry) { > blkif_response_t* rsp; > > - rsp = RING_GET_RESPONSE(&Protocol->Front, rsp_cons); > + rsp = RING_GET_RESPONSE(&Ring->Front, rsp_cons); > ++rsp_cons; > - ++Protocol->Received; > + ++Ring->Received; > > - ProtocolCompleteResponse(Protocol, rsp->id, rsp->status); > + RingCompleteResponse(Ring, rsp->id, rsp->status); > RtlZeroMemory(rsp, sizeof(union blkif_sring_entry)); > > - if (rsp_cons - Protocol->Front.rsp_cons > RING_SIZE(&Protocol- > >Front) / 4) > + if (rsp_cons - Ring->Front.rsp_cons > RING_SIZE(&Ring->Front) / > 4) > Retry = TRUE; > } > > KeMemoryBarrier(); > > - Protocol->Front.rsp_cons = rsp_cons; > - Protocol->Shared->rsp_event = rsp_cons + 1; > + Ring->Front.rsp_cons = rsp_cons; > + Ring->Shared->rsp_event = rsp_cons + 1; > } > > done: > - KeReleaseSpinLockFromDpcLevel(&Protocol->Lock); > + KeReleaseSpinLockFromDpcLevel(&Ring->Lock); > > return Retry; > } > > __drv_requiresIRQL(DISPATCH_LEVEL) > static BOOLEAN > -ProtocolNotifyResponses( > - IN PXENVBD_PROTOCOL Protocol > +RingNotifyResponses( > + IN PXENVBD_RING Ring > ) > { > + PXENVBD_PROTOCOL Protocol = Ring->Protocol; > BOOLEAN Retry = FALSE; > > if (!Protocol->Enabled) > return FALSE; > > - Retry |= ProtocolPoll(Protocol); > - Retry |= ProtocolSubmitRequests(Protocol); > + Retry |= RingPoll(Ring); > + Retry |= RingSubmitRequests(Ring); > > - ProtocolCompleteShutdown(Protocol); > + RingCompleteShutdown(Ring); > return Retry; > } > > -KSERVICE_ROUTINE ProtocolInterrupt; > +KSERVICE_ROUTINE RingInterrupt; > > BOOLEAN > -ProtocolInterrupt( > +RingInterrupt( > IN PKINTERRUPT Interrupt, > IN PVOID Context > ) > { > - PXENVBD_PROTOCOL Protocol = Context; > + PXENVBD_RING Ring = Context; > + PXENVBD_PROTOCOL Protocol; > > UNREFERENCED_PARAMETER(Interrupt); > > + ASSERT(Ring != NULL); > + Protocol = Ring->Protocol; > ASSERT(Protocol != NULL); > > - ++Protocol->Events; > + ++Ring->Events; > if (!Protocol->Connected) > return TRUE; > > - if (KeInsertQueueDpc(&Protocol->Dpc, NULL, NULL)) > - ++Protocol->Dpcs; > + if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL)) > + ++Ring->Dpcs; > > return TRUE; > } > > -KDEFERRED_ROUTINE ProtocolDpc; > +KDEFERRED_ROUTINE RingDpc; > > VOID > -ProtocolDpc( > +RingDpc( > __in PKDPC Dpc, > __in_opt PVOID Context, > __in_opt PVOID Arg1, > __in_opt PVOID Arg2 > ) > { > - PXENVBD_PROTOCOL Protocol = Context; > + PXENVBD_RING Ring = Context; > + PXENVBD_PROTOCOL Protocol; > > UNREFERENCED_PARAMETER(Dpc); > UNREFERENCED_PARAMETER(Arg1); > UNREFERENCED_PARAMETER(Arg2); > > + ASSERT(Ring != NULL); > + Protocol = Ring->Protocol; > ASSERT(Protocol != NULL); > > for (;;) { > KIRQL Irql; > BOOLEAN Retry; > > - KeRaiseIrql(DISPATCH_LEVEL, &Irql); > - Retry = ProtocolNotifyResponses(Protocol); > - KeLowerIrql(Irql); > + KeRaiseIrql(DISPATCH_LEVEL, &Irql); > + Retry = RingNotifyResponses(Ring); > + KeLowerIrql(Irql); > + > + if (!Retry) > + break; > + } > + > + XENBUS_EVTCHN(Unmask, > + &Protocol->EvtchnInterface, > + Ring->Channel, > + FALSE); > +} > + > +static DECLSPEC_NOINLINE VOID > +RingDebugCallback( > + IN PVOID Argument, > + IN BOOLEAN Crashing > + ) > +{ > + PXENVBD_RING Ring = Argument; > + PXENVBD_PROTOCOL Protocol = Ring->Protocol; > + PXENVBD_GRANTER Granter = FrontendGetGranter(Protocol- > >Frontend); > + ULONG Index; > + > + UNREFERENCED_PARAMETER(Crashing); > + > + XENBUS_DEBUG(Printf, > + &Protocol->DebugInterface, > + "Submitted: %u Received: %u\n", > + Ring->Submitted, > + Ring->Received); > + > + XENBUS_DEBUG(Printf, > + &Protocol->DebugInterface, > + "Events: %u Dpcs: %u\n", > + Ring->Events, > + Ring->Dpcs); > + > + XENBUS_DEBUG(Printf, > + &Protocol->DebugInterface, > + "Shared : 0x%p\n", > + Ring->Shared); > + > + if (Ring->Shared) { > + XENBUS_DEBUG(Printf, > + &Protocol->DebugInterface, > + "Shared: %d / %d - %d / %d\n", > + Ring->Shared->req_prod, > + Ring->Shared->req_event, > + Ring->Shared->rsp_prod, > + Ring->Shared->rsp_event); > + } > + > + XENBUS_DEBUG(Printf, > + &Protocol->DebugInterface, > + "Front: %d / %d (%d)\n", > + Ring->Front.req_prod_pvt, > + Ring->Front.rsp_cons, > + Ring->Front.nr_ents); > + > + for (Index = 0; Index < (1ul << Protocol->Order); ++Index) { > + XENBUS_DEBUG(Printf, > + &Protocol->DebugInterface, > + "Grants[%-2d]: 0x%p (%u)\n", > + Index, > + Ring->Grants[Index], > + GranterReference(Granter, Ring->Grants[Index])); > + } > + > + if (Ring->Channel) { > + ULONG Port = XENBUS_EVTCHN(GetPort, > + &Protocol->EvtchnInterface, > + Ring->Channel); > + > + XENBUS_DEBUG(Printf, > + &Protocol->DebugInterface, > + "Channel : %p (%d)\n", > + Ring->Channel, > + Port); > + } > + > + XENBUS_DEBUG(Printf, > + &Protocol->DebugInterface, > + "BLKIF_OPs: READ=%u WRITE=%u\n", > + Ring->BlkOpRead, > + Ring->BlkOpWrite); > + XENBUS_DEBUG(Printf, > + &Protocol->DebugInterface, > + "BLKIF_OPs: INDIRECT_READ=%u INDIRECT_WRITE=%u\n", > + Ring->BlkOpIndirectRead, > + Ring->BlkOpIndirectWrite); > + XENBUS_DEBUG(Printf, > + &Protocol->DebugInterface, > + "BLKIF_OPs: BARRIER=%u DISCARD=%u FLUSH=%u\n", > + Ring->BlkOpBarrier, > + Ring->BlkOpDiscard, > + Ring->BlkOpFlush); > + > + QueueDebugCallback(&Ring->PreparedReqs, > + "Prepared ", > + &Protocol->DebugInterface); > + QueueDebugCallback(&Ring->SubmittedReqs, > + "Submitted", > + &Protocol->DebugInterface); > + QueueDebugCallback(&Ring->ShutdownSrbs, > + "Shutdown ", > + &Protocol->DebugInterface); > +} > + > +static NTSTATUS > +RingCreate( > + IN PXENVBD_PROTOCOL Protocol, > + IN ULONG Index, > + OUT PXENVBD_RING *Ring > + ) > +{ > + PXENVBD_FRONTEND Frontend = Protocol->Frontend; > + PROCESSOR_NUMBER ProcNumber; > + ULONG Length; > + NTSTATUS status; > + > + *Ring = __ProtocolAllocate(sizeof(XENVBD_RING)); > + > + status = STATUS_NO_MEMORY; > + if (*Ring == NULL) > + goto fail1; > + > + (*Ring)->Protocol = Protocol; > + (*Ring)->Index = Index; > + > + Length = 1 + (ULONG)strlen(FrontendGetFrontendPath(Frontend)) + > (ULONG)strlen("queue-xx"); > + > + (*Ring)->Path = __ProtocolAllocate(sizeof(CHAR) * Length); > + > + status = STATUS_NO_MEMORY; > + if ((*Ring)->Path == NULL) > + goto fail2; > + > + status = RtlStringCchPrintfA((*Ring)->Path, > + Length, > + "%s/queue-%u", > + FrontendGetFrontendPath(Frontend), > + Index); > + if (!NT_SUCCESS(status)) > + goto fail3; > + > + KeInitializeSpinLock(&(*Ring)->Lock); > + > + KeInitializeThreadedDpc(&(*Ring)->Dpc, RingDpc, *Ring); > + KeSetImportanceDpc(&(*Ring)->Dpc, MediumHighImportance); > + > + status = KeGetProcessorNumberFromIndex(Index, &ProcNumber); > + ASSERT(NT_SUCCESS(status)); > + > + KeSetTargetProcessorDpcEx(&(*Ring)->Dpc, &ProcNumber); > + > + QueueInit(&(*Ring)->PreparedReqs); > + QueueInit(&(*Ring)->SubmittedReqs); > + QueueInit(&(*Ring)->ShutdownSrbs); > + > + return STATUS_SUCCESS; > + > +fail3: > + Error("fail3\n"); > + __ProtocolFree((*Ring)->Path); > + (*Ring)->Path = NULL; > +fail2: > + Error("fail2\n"); > + (*Ring)->Index = 0; > + (*Ring)->Protocol = NULL; > + > + ASSERT(IsZeroMemory(*Ring, sizeof(XENVBD_RING))); > + __ProtocolFree(*Ring); > +fail1: > + Error("fail1 (%08x)\n", status); > + *Ring = NULL; > + return status; > +} > + > +static VOID > +RingDestroy( > + IN PXENVBD_RING Ring > + ) > +{ > + RtlZeroMemory(&Ring->PreparedReqs, sizeof(XENVBD_QUEUE)); > + RtlZeroMemory(&Ring->SubmittedReqs, sizeof(XENVBD_QUEUE)); > + RtlZeroMemory(&Ring->ShutdownSrbs, sizeof(XENVBD_QUEUE)); > + > + RtlZeroMemory(&Ring->Dpc, sizeof(KDPC)); > + RtlZeroMemory(&Ring->Lock, sizeof(KSPIN_LOCK)); > + > + Ring->BlkOpRead = 0; > + Ring->BlkOpWrite = 0; > + Ring->BlkOpIndirectRead = 0; > + Ring->BlkOpIndirectWrite = 0; > + Ring->BlkOpBarrier = 0; > + Ring->BlkOpDiscard = 0; > + Ring->BlkOpFlush = 0; > + > + __ProtocolFree(Ring->Path); > + Ring->Path = NULL; > + > + Ring->Index = 0; > + Ring->Protocol = NULL; > + > + ASSERT(IsZeroMemory(Ring, sizeof(XENVBD_RING))); > + __ProtocolFree(Ring); > +} > + > +static NTSTATUS > +RingConnect( > + IN PXENVBD_RING Ring > + ) > +{ > + PXENVBD_PROTOCOL Protocol = Ring->Protocol; > + PXENVBD_GRANTER Granter = FrontendGetGranter(Protocol- > >Frontend); > + CHAR Name[MAX_NAME_LEN + 1]; > + ULONG Index; > + NTSTATUS status; > + > + Ring->Mdl = __AllocatePages(1 << Protocol->Order); > + > + status = STATUS_NO_MEMORY; > + if (Ring->Mdl == NULL) > + goto fail1; > + > + Ring->Shared = MmGetSystemAddressForMdlSafe(Ring->Mdl, > + NormalPagePriority); > + ASSERT(Ring->Shared != NULL); > + > +#pragma warning(push) > +#pragma warning(disable: 4305) > +#pragma warning(disable: 4311) // 'type cast' pointer truncation from > 'blkif_sring_entry[1]' to 'long' > + SHARED_RING_INIT(Ring->Shared); > + FRONT_RING_INIT(&Ring->Front, Ring->Shared, PAGE_SIZE << Protocol- > >Order); > +#pragma warning(pop) > + > + for (Index = 0; Index < (1ul << Protocol->Order); ++Index) { > + status = GranterGet(Granter, > + MmGetMdlPfnArray(Ring->Mdl)[Index], > + FALSE, > + &Ring->Grants[Index]); > + if (!NT_SUCCESS(status)) > + goto fail2; > + } > + > + Ring->Channel = XENBUS_EVTCHN(Open, > + &Protocol->EvtchnInterface, > + XENBUS_EVTCHN_TYPE_UNBOUND, > + RingInterrupt, > + Ring, > + > FrontendGetBackendDomain(Protocol->Frontend), > + TRUE); > + status = STATUS_NO_MEMORY; > + if (Ring->Channel == NULL) > + goto fail3; > + > + XENBUS_EVTCHN(Unmask, > + &Protocol->EvtchnInterface, > + Ring->Channel, > + FALSE); > + > + status = RtlStringCchPrintfA(Name, > + MAX_NAME_LEN, > + __MODULE__"%s", > + Ring->Path); > + if (!NT_SUCCESS(status)) > + goto fail4; > + > + status = XENBUS_DEBUG(Register, > + &Protocol->DebugInterface, > + Name, > + RingDebugCallback, > + Ring, > + &Ring->DebugCallback); > + if (!NT_SUCCESS(status)) > + goto fail5; > + > + return STATUS_SUCCESS; > + > +fail5: > + Error("fail5\n"); > +fail4: > + Error("fail4\n"); > + XENBUS_EVTCHN(Close, > + &Protocol->EvtchnInterface, > + Ring->Channel); > + Ring->Channel = NULL; > +fail3: > + Error("fail3\n"); > +fail2: > + Error("fail2\n"); > + for (Index = 0; Index < (1ul << Protocol->Order); ++Index) { > + if (Ring->Grants[Index] == NULL) > + continue; > + > + GranterPut(Granter, Ring->Grants[Index]); > + Ring->Grants[Index] = NULL; > + } > + > + RtlZeroMemory(&Ring->Front, sizeof(blkif_front_ring_t)); > + > + __FreePages(Ring->Mdl); > + Ring->Shared = NULL; > + Ring->Mdl = NULL; > +fail1: > + Error("fail1 (%08x)\n", status); > + return status; > +} > + > +static NTSTATUS > +RingStoreWrite( > + IN PXENVBD_RING Ring, > + IN PVOID Transaction > + ) > +{ > + PXENVBD_PROTOCOL Protocol = Ring->Protocol; > + PXENVBD_GRANTER Granter = FrontendGetGranter(Protocol- > >Frontend); > + PCHAR Path; > + ULONG Port; > + NTSTATUS status; > + > + if (FrontendGetMultiQueueMaxQueues(Protocol->Frontend) > 1) > + Path = Ring->Path; > + else > + Path = FrontendGetFrontendPath(Protocol->Frontend); > + > + if (Protocol->Order == 0) { > + status = XENBUS_STORE(Printf, > + &Protocol->StoreInterface, > + Transaction, > + Path, > + "ring-ref", > + "%u", > + GranterReference(Granter, Ring->Grants[0])); > + if (!NT_SUCCESS(status)) > + return status; > + } > + else { > + ULONG Index; > + > + for (Index = 0; Index < (1ul << Protocol->Order); ++Index) { > + CHAR Name[MAX_NAME_LEN + 1]; > + > + status = RtlStringCchPrintfA(Name, > + MAX_NAME_LEN, > + "ring-ref%u", > + Index); > + if (!NT_SUCCESS(status)) > + return status; > + > + status = XENBUS_STORE(Printf, > + &Protocol->StoreInterface, > + Transaction, > + Path, > + Name, > + "%u", > + GranterReference(Granter, > Ring->Grants[Index])); > + if (!NT_SUCCESS(status)) > + return status; > + } > + } > + > + Port = XENBUS_EVTCHN(GetPort, > + &Protocol->EvtchnInterface, > + Ring->Channel); > + > + status = XENBUS_STORE(Printf, > + &Protocol->StoreInterface, > + Transaction, > + Path, > + "event-channel", > + "%u", > + Port); > + if (!NT_SUCCESS(status)) > + return status; > + > + return STATUS_SUCCESS; > +} > + > +static VOID > +RingDisconnect( > + IN PXENVBD_RING Ring > + ) > +{ > + PXENVBD_PROTOCOL Protocol = Ring->Protocol; > + PXENVBD_GRANTER Granter = FrontendGetGranter(Protocol- > >Frontend); > + ULONG Index; > + > + ASSERT3U(Ring->Submitted, == , Ring->Received); > + > + XENBUS_DEBUG(Deregister, > + &Protocol->DebugInterface, > + Ring->DebugCallback); > + Ring->DebugCallback = NULL; > + > + XENBUS_EVTCHN(Close, > + &Protocol->EvtchnInterface, > + Ring->Channel); > + Ring->Channel = NULL; > + > + for (Index = 0; Index < (1ul << Protocol->Order); ++Index) { > + if (Ring->Grants[Index] == NULL) > + continue; > + > + GranterPut(Granter, Ring->Grants[Index]); > + Ring->Grants[Index] = NULL; > + } > + > + RtlZeroMemory(&Ring->Front, sizeof(blkif_front_ring_t)); > + > + __FreePages(Ring->Mdl); > + Ring->Shared = NULL; > + Ring->Mdl = NULL; > + Ring->Events = 0; > + Ring->Dpcs = 0; > + Ring->Submitted = 0; > + Ring->Received = 0; > +} > + > +static VOID > +RingEnable( > + IN PXENVBD_RING Ring > + ) > +{ > + PXENVBD_PROTOCOL Protocol = Ring->Protocol; > + > + XENBUS_EVTCHN(Trigger, > + &Protocol->EvtchnInterface, > + Ring->Channel); > +} > + > +static VOID > +RingDisable( > + IN PXENVBD_RING Ring > + ) > +{ > + PXENVBD_PROTOCOL Protocol = Ring->Protocol; > + ULONG Count; > + KIRQL Irql; > + PXENVBD_TARGET Target = FrontendGetTarget(Protocol->Frontend); > + PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target); > + > + // poll ring and send event channel notification every 1ms (for up to 3 > minutes) > + Count = 0; > + while (QueueCount(&Ring->SubmittedReqs)) { > + if (Count > 180000) > + break; > + KeRaiseIrql(DISPATCH_LEVEL, &Irql); > + RingPoll(Ring); > + KeLowerIrql(Irql); > + XENBUS_EVTCHN(Send, > + &Protocol->EvtchnInterface, > + Ring->Channel); > + StorPortStallExecution(1000); // 1000 micro-seconds > + ++Count; > + } > + > + Verbose("Target[%d][%u] : %u Submitted requests left (%u > iterrations)\n", > + FrontendGetTargetId(Protocol->Frontend), > + Ring->Index, > + QueueCount(&Ring->SubmittedReqs), > + Count); > + > + // Fail PreparedReqs > + for (;;) { > + PXENVBD_SRBEXT SrbExt; > + PSCSI_REQUEST_BLOCK Srb; > + PXENVBD_REQUEST Request; > + PLIST_ENTRY ListEntry; > + > + ListEntry = QueuePop(&Ring->PreparedReqs); > + if (ListEntry == NULL) > + break; > + Request = CONTAINING_RECORD(ListEntry, XENVBD_REQUEST, > ListEntry); > + SrbExt = Request->SrbExt; > + Srb = SrbExt->Srb; > + > + Srb->SrbStatus = SRB_STATUS_ABORTED; > + Srb->ScsiStatus = 0x40; // SCSI_ABORTED > + > + ProtocolPutRequest(Protocol, Request); > + > + if (InterlockedDecrement(&SrbExt->RequestCount) == 0) > + AdapterCompleteSrb(Adapter, SrbExt); > + } > +} > + > +static VOID > +RingTrigger( > + IN PXENVBD_RING Ring > + ) > +{ > + PXENVBD_PROTOCOL Protocol = Ring->Protocol; > + > + XENBUS_EVTCHN(Trigger, > + &Protocol->EvtchnInterface, > + Ring->Channel); > +} > + > +static FORCEINLINE MM_PAGE_PRIORITY > +__ProtocolPriority( > + IN PXENVBD_PROTOCOL Protocol > + ) > +{ > + PXENVBD_CAPS Caps = FrontendGetCaps(Protocol->Frontend); > + if (!(Caps->Paging || > + Caps->Hibernation || > + Caps->DumpFile)) > + return NormalPagePriority; > + > + return HighPagePriority; > +} > + > +static BOOLEAN > +ProtocolPrepareSegment( > + IN PXENVBD_PROTOCOL Protocol, > + IN PXENVBD_SEGMENT Segment, > + IN PXENVBD_SRBEXT SrbExt, > + IN BOOLEAN ReadOnly, > + IN ULONG SectorsLeft, > + OUT PULONG SectorsNow > + ) > +{ > + PFN_NUMBER Pfn; > + ULONG Offset; > + ULONG Length; > + NTSTATUS Status; > + PXENVBD_GRANTER Granter = FrontendGetGranter(Protocol- > >Frontend); > + const ULONG SectorSize = FrontendGetDiskInfo(Protocol- > >Frontend)->SectorSize; > + const ULONG SectorsPerPage = __SectorsPerPage(SectorSize); > + PXENVBD_TARGET Target = FrontendGetTarget(Protocol- > >Frontend); > + PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target); > + > + Pfn = AdapterGetNextSGEntry(Adapter, > + SrbExt, > + 0, > + &Offset, > + &Length); > + if ((Offset & (SectorSize - 1)) == 0 && > + (Length & (SectorSize - 1)) == 0) { > + ++Protocol->SegsGranted; > + // get first sector, last sector and count > + Segment->FirstSector = (UCHAR)((Offset + SectorSize - 1) / > SectorSize); > + *SectorsNow = __min(SectorsLeft, SectorsPerPage - Segment- > >FirstSector); > + Segment->LastSector = (UCHAR)(Segment->FirstSector + *SectorsNow > - 1); > + > + ASSERT3U((Length / SectorSize), == , *SectorsNow); > + } > + else { > + PXENVBD_BOUNCE Bounce; > + PMDL Mdl; > + > + ++Protocol->SegsBounced; > + // get first sector, last sector and count > + Segment->FirstSector = 0; > + *SectorsNow = __min(SectorsLeft, SectorsPerPage); > + Segment->LastSector = (UCHAR)(*SectorsNow - 1); > + > + Bounce = AdapterGetBounce(Adapter); > + if (Bounce == NULL) > + goto fail1; > + Segment->Bounce = Bounce; > + > +#pragma warning(push) > +#pragma warning(disable:28145) > + Mdl = &Bounce->SourceMdl; > + Mdl->Next = NULL; > + Mdl->Size = (SHORT)(sizeof(MDL) + sizeof(PFN_NUMBER)); > + Mdl->MdlFlags = MDL_PAGES_LOCKED; > + Mdl->Process = NULL; > + Mdl->MappedSystemVa = NULL; > + Mdl->StartVa = NULL; > + Mdl->ByteCount = Length; > + Mdl->ByteOffset = Offset; > + Bounce->SourcePfn[0] = Pfn; > + > + if (Length < *SectorsNow * SectorSize) { > + Pfn = AdapterGetNextSGEntry(Adapter, > + SrbExt, > + Length, > + &Offset, > + &Length); > + Mdl->Size += sizeof(PFN_NUMBER); > + Mdl->ByteCount += Length; > + Bounce->SourcePfn[1] = Pfn; > + } > +#pragma warning(pop) > + > + ASSERT((Mdl->ByteCount & (SectorSize - 1)) == 0); > + ASSERT3U(Mdl->ByteCount, <= , PAGE_SIZE); > + ASSERT3U(*SectorsNow, == , (Mdl->ByteCount / SectorSize)); > + > + Bounce->SourcePtr = MmMapLockedPagesSpecifyCache(Mdl, > + KernelMode, > + MmCached, > + NULL, > + FALSE, > + > __ProtocolPriority(Protocol)); > + if (Bounce->SourcePtr == NULL) > + goto fail2; > + > + ASSERT3P(MmGetMdlPfnArray(Mdl)[0], == , Bounce->SourcePfn[0]); > + ASSERT3P(MmGetMdlPfnArray(Mdl)[1], == , Bounce->SourcePfn[1]); > + > + // copy contents in > + if (ReadOnly) { // Operation == BLKIF_OP_WRITE > + RtlCopyMemory(Bounce->BouncePtr, > + Bounce->SourcePtr, > + MmGetMdlByteCount(&Bounce->SourceMdl)); > + } > + > + Pfn = MmGetMdlPfnArray(Bounce->BounceMdl)[0]; > + } > + > + // Grant segment's page > + Status = GranterGet(Granter, Pfn, ReadOnly, &Segment->Grant); > + if (!NT_SUCCESS(Status)) > + goto fail3; > + > + return TRUE; > + > +fail3: > +fail2: > +fail1: > + return FALSE; > +} > + > +static BOOLEAN > +ProtocolPrepareBlkifReadWrite( > + IN PXENVBD_PROTOCOL Protocol, > + IN PXENVBD_REQUEST Request, > + IN PXENVBD_SRBEXT SrbExt, > + IN ULONG MaxSegments, > + IN ULONG64 SectorStart, > + IN ULONG SectorsLeft, > + OUT PULONG SectorsDone > + ) > +{ > + PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb; > + UCHAR Operation; > + BOOLEAN ReadOnly; > + ULONG Index; > + __Operation(Cdb_OperationEx(Srb), &Operation, &ReadOnly); > + > + Request->Operation = Operation; > + Request->NrSegments = 0; > + Request->FirstSector = SectorStart; > + > + for (Index = 0; > + Index < MaxSegments && > + SectorsLeft > 0; > + ++Index) { > + PXENVBD_SEGMENT Segment; > + ULONG SectorsNow; > + > + Segment = ProtocolGetSegment(Protocol); > + if (Segment == NULL) > + goto fail1; > + > + InsertTailList(&Request->Segments, &Segment->ListEntry); > + ++Request->NrSegments; > + > + if (!ProtocolPrepareSegment(Protocol, > + Segment, > + SrbExt, > + ReadOnly, > + SectorsLeft, > + &SectorsNow)) > + goto fail2; > + > + *SectorsDone += SectorsNow; > + SectorsLeft -= SectorsNow; > + } > + ASSERT3U(Request->NrSegments, >, 0); > + ASSERT3U(Request->NrSegments, <= , MaxSegments); > + > + return TRUE; > + > +fail2: > +fail1: > + return FALSE; > +} > + > +static BOOLEAN > +ProtocolPrepareBlkifIndirect( > + IN PXENVBD_PROTOCOL Protocol, > + IN PXENVBD_REQUEST Request > + ) > +{ > + ULONG Index; > + ULONG NrSegments = 0; > + > + for (Index = 0; > + Index < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST && > + NrSegments < Request->NrSegments; > + ++Index) { > + PXENVBD_INDIRECT Indirect; > + > + Indirect = ProtocolGetIndirect(Protocol); > + if (Indirect == NULL) > + goto fail1; > + InsertTailList(&Request->Indirects, &Indirect->ListEntry); > + > + NrSegments += XENVBD_MAX_SEGMENTS_PER_PAGE; > + } > + > + return TRUE; > + > +fail1: > + return FALSE; > +} > + > +static FORCEINLINE ULONG > +ProtocolUseIndirect( > + IN PXENVBD_PROTOCOL Protocol, > + IN ULONG SectorsLeft > + ) > +{ > + const ULONG SectorsPerPage = > __SectorsPerPage(FrontendGetDiskInfo(Protocol->Frontend)->SectorSize); > + const ULONG MaxIndirectSegs = FrontendGetFeatures(Protocol- > >Frontend)->Indirect; > + > + if (MaxIndirectSegs <= BLKIF_MAX_SEGMENTS_PER_REQUEST) > + return BLKIF_MAX_SEGMENTS_PER_REQUEST; // not supported > + > + if (SectorsLeft < BLKIF_MAX_SEGMENTS_PER_REQUEST * > SectorsPerPage) > + return BLKIF_MAX_SEGMENTS_PER_REQUEST; // first into a single > BLKIF_OP_{READ/WRITE} > + > + return MaxIndirectSegs; > +} > + > +static FORCEINLINE VOID > +ProtocolCancelRequestList( > + IN PXENVBD_PROTOCOL Protocol, > + IN PLIST_ENTRY List > + ) > +{ > + for (;;) { > + PXENVBD_REQUEST Request; > + PLIST_ENTRY ListEntry; > + > + ListEntry = RemoveHeadList(List); > + if (ListEntry == List) > + break; > + > + Request = CONTAINING_RECORD(ListEntry, XENVBD_REQUEST, > ListEntry); > + ProtocolPutRequest(Protocol, Request); > + } > +} > + > +static BOOLEAN > +ProtocolPrepareReadWrite( > + IN PXENVBD_PROTOCOL Protocol, > + IN PXENVBD_SRBEXT SrbExt, > + IN PLIST_ENTRY List > + ) > +{ > + PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb; > + ULONG64 SectorStart = Cdb_LogicalBlock(Srb); > + ULONG SectorsLeft = Cdb_TransferBlock(Srb); > + > + Srb->SrbStatus = SRB_STATUS_PENDING; > + > + SrbExt->RequestCount = 0; > + > + while (SectorsLeft > 0) { > + ULONG MaxSegments; > + ULONG SectorsDone = 0; > + PXENVBD_REQUEST Request; > + > + Request = ProtocolGetRequest(Protocol); > + if (Request == NULL) > + goto fail1; > + InsertTailList(List, &Request->ListEntry); > + InterlockedIncrement(&SrbExt->RequestCount); > + > + Request->SrbExt = SrbExt; > + MaxSegments = ProtocolUseIndirect(Protocol, SectorsLeft); > + > + if (!ProtocolPrepareBlkifReadWrite(Protocol, > + Request, > + SrbExt, > + MaxSegments, > + SectorStart, > + SectorsLeft, > + &SectorsDone)) > + goto fail2; > + > + if (MaxSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST) { > + if (!ProtocolPrepareBlkifIndirect(Protocol, Request)) > + goto fail3; > + } > + > + SectorsLeft -= SectorsDone; > + SectorStart += SectorsDone; > + } > + > + return TRUE; > + > +fail3: > +fail2: > +fail1: > + ProtocolCancelRequestList(Protocol, List); > + SrbExt->RequestCount = 0; > + Srb->SrbStatus = SRB_STATUS_ERROR; > + return FALSE; > +} > + > +static BOOLEAN > +ProtocolPrepareSyncCache( > + IN PXENVBD_PROTOCOL Protocol, > + IN PXENVBD_SRBEXT SrbExt, > + IN PLIST_ENTRY List > + ) > +{ > + PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb; > + PXENVBD_REQUEST Request; > + UCHAR Operation; > + > + Srb->SrbStatus = SRB_STATUS_PENDING; > + > + if (FrontendGetDiskInfo(Protocol->Frontend)->FlushCache) > + Operation = BLKIF_OP_FLUSH_DISKCACHE; > + else > + Operation = BLKIF_OP_WRITE_BARRIER; > + > + SrbExt->RequestCount = 0; > + > + Request = ProtocolGetRequest(Protocol); > + if (Request == NULL) > + goto fail1; > + InsertTailList(List, &Request->ListEntry); > + InterlockedIncrement(&SrbExt->RequestCount); > + > + Request->SrbExt = SrbExt; > + Request->Operation = Operation; > + Request->FirstSector = Cdb_LogicalBlock(Srb); > + > + return TRUE; > + > +fail1: > + ProtocolCancelRequestList(Protocol, List); > + SrbExt->RequestCount = 0; > + Srb->SrbStatus = SRB_STATUS_ERROR; > + return FALSE; > +} > + > +static BOOLEAN > +ProtocolPrepareUnmap( > + IN PXENVBD_PROTOCOL Protocol, > + IN PXENVBD_SRBEXT SrbExt, > + IN PLIST_ENTRY List > + ) > +{ > + PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb; > + PUNMAP_LIST_HEADER Unmap = Srb->DataBuffer; > + ULONG Count = _byteswap_ushort(*(PUSHORT)Unmap- > >BlockDescrDataLength) / sizeof(UNMAP_BLOCK_DESCRIPTOR); > + ULONG Index; > + > + Srb->SrbStatus = SRB_STATUS_PENDING; > + > + SrbExt->RequestCount = 0; > + > + for (Index = 0; Index < Count; ++Index) { > + PUNMAP_BLOCK_DESCRIPTOR Descr = &Unmap->Descriptors[Index]; > + PXENVBD_REQUEST Request; > + > + Request = ProtocolGetRequest(Protocol); > + if (Request == NULL) > + goto fail1; > + InsertTailList(List, &Request->ListEntry); > + InterlockedIncrement(&SrbExt->RequestCount); > + > + Request->SrbExt = SrbExt; > + Request->Operation = BLKIF_OP_DISCARD; > + Request->FirstSector = _byteswap_uint64(*(PULONG64)Descr- > >StartingLba); > + Request->NrSectors = _byteswap_ulong(*(PULONG)Descr- > >LbaCount); > + Request->Flags = 0; > + } > + > + return TRUE; > + > +fail1: > + ProtocolCancelRequestList(Protocol, List); > + SrbExt->RequestCount = 0; > + Srb->SrbStatus = SRB_STATUS_ERROR; > + return FALSE; > +} > + > +static FORCEINLINE BOOLEAN > +ProtocolPrepareRequest( > + IN PXENVBD_PROTOCOL Protocol, > + IN PXENVBD_SRBEXT SrbExt, > + IN PLIST_ENTRY List > + ) > +{ > + switch (Cdb_OperationEx(SrbExt->Srb)) { > + case SCSIOP_READ: > + case SCSIOP_WRITE: > + return ProtocolPrepareReadWrite(Protocol, > + SrbExt, > + List); > + > + case SCSIOP_SYNCHRONIZE_CACHE: > + return ProtocolPrepareSyncCache(Protocol, > + SrbExt, > + List); > + > + case SCSIOP_UNMAP: > + return ProtocolPrepareUnmap(Protocol, > + SrbExt, > + List); > > - if (!Retry) > - break; > + default: > + ASSERT(FALSE); > + return FALSE; > } > - > - XENBUS_EVTCHN(Unmask, > - &Protocol->EvtchnInterface, > - Protocol->Channel, > - FALSE); > } > > static DECLSPEC_NOINLINE VOID > @@ -1281,102 +1804,19 @@ ProtocolDebugCallback( > ) > { > PXENVBD_PROTOCOL Protocol = Argument; > - PXENVBD_GRANTER Granter = FrontendGetGranter(Protocol- > >Frontend); > - ULONG Index; > > UNREFERENCED_PARAMETER(Crashing); > > - XENBUS_DEBUG(Printf, > - &Protocol->DebugInterface, > - "Submitted: %u Received: %u\n", > - Protocol->Submitted, > - Protocol->Received); > - > - XENBUS_DEBUG(Printf, > - &Protocol->DebugInterface, > - "Events: %u Dpcs: %u\n", > - Protocol->Events, > - Protocol->Dpcs); > - > - XENBUS_DEBUG(Printf, > - &Protocol->DebugInterface, > - "Shared : 0x%p\n", > - Protocol->Shared); > - > - if (Protocol->Shared) { > - XENBUS_DEBUG(Printf, > - &Protocol->DebugInterface, > - "Shared: %d / %d - %d / %d\n", > - Protocol->Shared->req_prod, > - Protocol->Shared->req_event, > - Protocol->Shared->rsp_prod, > - Protocol->Shared->rsp_event); > - } > - > - XENBUS_DEBUG(Printf, > - &Protocol->DebugInterface, > - "Front: %d / %d (%d)\n", > - Protocol->Front.req_prod_pvt, > - Protocol->Front.rsp_cons, > - Protocol->Front.nr_ents); > - > XENBUS_DEBUG(Printf, > &Protocol->DebugInterface, > "Order: %d\n", > Protocol->Order); > > - for (Index = 0; Index < (1ul << Protocol->Order); ++Index) { > - XENBUS_DEBUG(Printf, > - &Protocol->DebugInterface, > - "Grants[%-2d]: 0x%p (%u)\n", > - Index, > - Protocol->Grants[Index], > - GranterReference(Granter, Protocol->Grants[Index])); > - } > - > - if (Protocol->Channel) { > - ULONG Port = XENBUS_EVTCHN(GetPort, > - &Protocol->EvtchnInterface, > - Protocol->Channel); > - > - XENBUS_DEBUG(Printf, > - &Protocol->DebugInterface, > - "Channel : %p (%d)\n", > - Protocol->Channel, > - Port); > - } > - > - XENBUS_DEBUG(Printf, > - &Protocol->DebugInterface, > - "BLKIF_OPs: READ=%u WRITE=%u\n", > - Protocol->BlkOpRead, > - Protocol->BlkOpWrite); > - XENBUS_DEBUG(Printf, > - &Protocol->DebugInterface, > - "BLKIF_OPs: INDIRECT_READ=%u INDIRECT_WRITE=%u\n", > - Protocol->BlkOpIndirectRead, > - Protocol->BlkOpIndirectWrite); > - XENBUS_DEBUG(Printf, > - &Protocol->DebugInterface, > - "BLKIF_OPs: BARRIER=%u DISCARD=%u FLUSH=%u\n", > - Protocol->BlkOpBarrier, > - Protocol->BlkOpDiscard, > - Protocol->BlkOpFlush); > XENBUS_DEBUG(Printf, > &Protocol->DebugInterface, > "Segments Granted=%llu Bounced=%llu\n", > Protocol->SegsGranted, > Protocol->SegsBounced); > - > - QueueDebugCallback(&Protocol->PreparedReqs, > - "Prepared ", > - &Protocol->DebugInterface); > - QueueDebugCallback(&Protocol->SubmittedReqs, > - "Submitted", > - &Protocol->DebugInterface); > - QueueDebugCallback(&Protocol->ShutdownSrbs, > - "Shutdown ", > - &Protocol->DebugInterface); > } > > static DECLSPEC_NOINLINE VOID > @@ -1391,7 +1831,7 @@ ProtocolAcquireLock( > static DECLSPEC_NOINLINE VOID > ProtocolReleaseLock( > IN PVOID Argument > - ) > +) > { > PXENVBD_PROTOCOL Protocol = Argument; > KeReleaseSpinLockFromDpcLevel(&Protocol->Lock); > @@ -1426,7 +1866,7 @@ static DECLSPEC_NOINLINE NTSTATUS > ProtocolSegmentCtor( > IN PVOID Argument, > IN PVOID Object > - ) > +) > { > UNREFERENCED_PARAMETER(Argument); > UNREFERENCED_PARAMETER(Object); > @@ -1488,12 +1928,13 @@ ProtocolIndirectDtor( > NTSTATUS > ProtocolCreate( > IN PXENVBD_FRONTEND Frontend, > - OUT PXENVBD_PROTOCOL* Protocol > + OUT PXENVBD_PROTOCOL* Protocol > ) > { > PXENVBD_TARGET Target = FrontendGetTarget(Frontend); > PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target); > CHAR Name[MAX_NAME_LEN]; > + ULONG Index; > NTSTATUS status; > > *Protocol = __ProtocolAllocate(sizeof(XENVBD_PROTOCOL)); > @@ -1504,12 +1945,6 @@ ProtocolCreate( > > (*Protocol)->Frontend = Frontend; > KeInitializeSpinLock(&(*Protocol)->Lock); > - KeInitializeThreadedDpc(&(*Protocol)->Dpc, ProtocolDpc, *Protocol); > - KeSetImportanceDpc(&(*Protocol)->Dpc, MediumHighImportance); > - > - QueueInit(&(*Protocol)->PreparedReqs); > - QueueInit(&(*Protocol)->SubmittedReqs); > - QueueInit(&(*Protocol)->ShutdownSrbs); > > AdapterGetCacheInterface(Adapter, &(*Protocol)->CacheInterface); > > @@ -1580,8 +2015,41 @@ ProtocolCreate( > if (!NT_SUCCESS(status)) > goto fail8; > > + (*Protocol)->NumQueues = > FrontendGetMultiQueueMaxQueues(Frontend); > + if ((*Protocol)->NumQueues == 0) > + (*Protocol)->NumQueues = 1; > + (*Protocol)->Rings = __ProtocolAllocate(sizeof(PXENVBD_RING) * > (*Protocol)->NumQueues); > + > + status = STATUS_NO_MEMORY; > + if ((*Protocol)->Rings == NULL) > + goto fail9; > + > + for (Index = 0; Index < (*Protocol)->NumQueues; ++Index) { > + status = RingCreate(*Protocol, > + Index, > + &(*Protocol)->Rings[Index]); > + if (!NT_SUCCESS(status)) > + goto fail10; > + } > + > return STATUS_SUCCESS; > > +fail10: > + Error("fail10\n"); > + for (Index = 0; Index < (*Protocol)->NumQueues; ++Index) { > + if ((*Protocol)->Rings[Index]) > + RingDestroy((*Protocol)->Rings[Index]); > + (*Protocol)->Rings[Index] = NULL; > + } > + __ProtocolFree((*Protocol)->Rings); > + (*Protocol)->Rings = NULL; > +fail9: > + Error("fail9\n"); > + (*Protocol)->NumQueues = 0; > + XENBUS_CACHE(Destroy, > + &(*Protocol)->CacheInterface, > + (*Protocol)->IndirectCache); > + (*Protocol)->IndirectCache = NULL; > fail8: > Error("fail8\n"); > fail7: > @@ -1608,13 +2076,8 @@ fail2: > Error("fail2\n"); > > RtlZeroMemory(&(*Protocol)->CacheInterface, > - sizeof (XENBUS_CACHE_INTERFACE)); > + sizeof(XENBUS_CACHE_INTERFACE)); > > - RtlZeroMemory(&(*Protocol)->PreparedReqs, sizeof(XENVBD_QUEUE)); > - RtlZeroMemory(&(*Protocol)->SubmittedReqs, sizeof(XENVBD_QUEUE)); > - RtlZeroMemory(&(*Protocol)->ShutdownSrbs, sizeof(XENVBD_QUEUE)); > - > - RtlZeroMemory(&(*Protocol)->Dpc, sizeof(KDPC)); > RtlZeroMemory(&(*Protocol)->Lock, sizeof(KSPIN_LOCK)); > (*Protocol)->Frontend = NULL; > > @@ -1631,6 +2094,19 @@ ProtocolDestroy( > IN PXENVBD_PROTOCOL Protocol > ) > { > + ULONG Index; > + > + for (Index = 0; Index < Protocol->NumQueues; ++Index) { > + if (Protocol->Rings[Index]) > + RingDestroy(Protocol->Rings[Index]); > + Protocol->Rings[Index] = 0; > + } > + > + __ProtocolFree(Protocol->Rings); > + Protocol->Rings = NULL; > + > + Protocol->NumQueues = 0; > + > XENBUS_CACHE(Destroy, > &Protocol->CacheInterface, > Protocol->IndirectCache); > @@ -1650,26 +2126,14 @@ ProtocolDestroy( > &Protocol->CacheInterface); > > RtlZeroMemory(&Protocol->CacheInterface, > - sizeof (XENBUS_CACHE_INTERFACE)); > + sizeof(XENBUS_CACHE_INTERFACE)); > > - RtlZeroMemory(&Protocol->PreparedReqs, sizeof(XENVBD_QUEUE)); > - RtlZeroMemory(&Protocol->SubmittedReqs, sizeof(XENVBD_QUEUE)); > - RtlZeroMemory(&Protocol->ShutdownSrbs, sizeof(XENVBD_QUEUE)); > + Protocol->SegsGranted = 0; > + Protocol->SegsBounced = 0; > > - RtlZeroMemory(&Protocol->Dpc, sizeof(KDPC)); > RtlZeroMemory(&Protocol->Lock, sizeof(KSPIN_LOCK)); > Protocol->Frontend = NULL; > > - Protocol->BlkOpRead = 0; > - Protocol->BlkOpWrite = 0; > - Protocol->BlkOpIndirectRead = 0; > - Protocol->BlkOpIndirectWrite = 0; > - Protocol->BlkOpBarrier = 0; > - Protocol->BlkOpDiscard = 0; > - Protocol->BlkOpFlush = 0; > - Protocol->SegsGranted = 0; > - Protocol->SegsBounced = 0; > - > ASSERT(IsZeroMemory(Protocol, sizeof(XENVBD_PROTOCOL))); > __ProtocolFree(Protocol); > } > @@ -1681,7 +2145,6 @@ ProtocolConnect( > { > PXENVBD_TARGET Target = FrontendGetTarget(Protocol->Frontend); > PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target); > - PXENVBD_GRANTER Granter = FrontendGetGranter(Protocol- > >Frontend); > PCHAR Buffer; > ULONG Index; > NTSTATUS status; > @@ -1730,48 +2193,12 @@ ProtocolConnect( > Protocol->Order = 0; > } > > - Protocol->Mdl = __AllocatePages(1 << Protocol->Order); > - > - status = STATUS_NO_MEMORY; > - if (Protocol->Mdl == NULL) > - goto fail4; > - > - Protocol->Shared = MmGetSystemAddressForMdlSafe(Protocol->Mdl, > - NormalPagePriority); > - ASSERT(Protocol->Shared != NULL); > - > -#pragma warning(push) > -#pragma warning(disable: 4305) > -#pragma warning(disable: 4311) // 'type cast' pointer truncation from > 'blkif_sring_entry[1]' to 'long' > - SHARED_RING_INIT(Protocol->Shared); > - FRONT_RING_INIT(&Protocol->Front, Protocol->Shared, PAGE_SIZE << > Protocol->Order); > -#pragma warning(pop) > - > - for (Index = 0; Index < (1ul << Protocol->Order); ++Index) { > - status = GranterGet(Granter, > - MmGetMdlPfnArray(Protocol->Mdl)[Index], > - FALSE, > - &Protocol->Grants[Index]); > + for (Index = 0; Index < Protocol->NumQueues; ++Index) { > + status = RingConnect(Protocol->Rings[Index]); > if (!NT_SUCCESS(status)) > - goto fail5; > + goto fail4; > } > > - Protocol->Channel = XENBUS_EVTCHN(Open, > - &Protocol->EvtchnInterface, > - XENBUS_EVTCHN_TYPE_UNBOUND, > - ProtocolInterrupt, > - Protocol, > - > FrontendGetBackendDomain(Protocol->Frontend), > - TRUE); > - status = STATUS_NO_MEMORY; > - if (Protocol->Channel == NULL) > - goto fail6; > - > - XENBUS_EVTCHN(Unmask, > - &Protocol->EvtchnInterface, > - Protocol->Channel, > - FALSE); > - > status = XENBUS_DEBUG(Register, > &Protocol->DebugInterface, > __MODULE__"|PROTOCOL", > @@ -1779,38 +2206,20 @@ ProtocolConnect( > Protocol, > &Protocol->DebugCallback); > if (!NT_SUCCESS(status)) > - goto fail7; > + goto fail5; > > Protocol->Connected = TRUE; > return STATUS_SUCCESS; > > -fail7: > - Error("fail7\n"); > - XENBUS_EVTCHN(Close, > - &Protocol->EvtchnInterface, > - Protocol->Channel); > - Protocol->Channel = NULL; > -fail6: > - Error("fail6\n"); > fail5: > Error("fail5\n"); > - for (Index = 0; Index < (1ul << Protocol->Order); ++Index) { > - if (Protocol->Grants[Index] == NULL) > - continue; > - > - GranterPut(Granter, Protocol->Grants[Index]); > - Protocol->Grants[Index] = NULL; > - } > - > - RtlZeroMemory(&Protocol->Front, sizeof(blkif_front_ring_t)); > - > - __FreePages(Protocol->Mdl); > - Protocol->Shared = NULL; > - Protocol->Mdl = NULL; > - > - Protocol->Order = 0; > + Index = Protocol->NumQueues; > fail4: > Error("fail4\n"); > + while (Index-- != 0) { > + RingDisconnect(Protocol->Rings[Index]); > + } > + Protocol->Order = 0; > XENBUS_DEBUG(Release, &Protocol->DebugInterface); > fail3: > Error("fail3\n"); > @@ -1834,26 +2243,20 @@ fail1: > NTSTATUS > ProtocolStoreWrite( > IN PXENVBD_PROTOCOL Protocol, > - IN PVOID Transaction > + IN PVOID Transaction > ) > { > - PXENVBD_GRANTER Granter = FrontendGetGranter(Protocol- > >Frontend); > - ULONG Port; > - NTSTATUS status; > + ULONG Index; > + NTSTATUS status; > > - if (Protocol->Order == 0) { > - status = XENBUS_STORE(Printf, > - &Protocol->StoreInterface, > - Transaction, > - FrontendGetFrontendPath(Protocol->Frontend), > - "ring-ref", > - "%u", > - GranterReference(Granter, > Protocol->Grants[0])); > + for (Index = 0; Index < Protocol->NumQueues; ++Index) { > + status = RingStoreWrite(Protocol->Rings[Index], > + Transaction); > if (!NT_SUCCESS(status)) > return status; > - } else { > - ULONG Index; > + } > > + if (Protocol->Order != 0) { > status = XENBUS_STORE(Printf, > &Protocol->StoreInterface, > Transaction, > @@ -1863,49 +2266,24 @@ ProtocolStoreWrite( > Protocol->Order); > if (!NT_SUCCESS(status)) > return status; > - > - for (Index = 0; Index < (1ul << Protocol->Order); ++Index) { > - CHAR Name[MAX_NAME_LEN+1]; > - > - status = RtlStringCchPrintfA(Name, > - MAX_NAME_LEN, > - "ring-ref%u", > - Index); > - if (!NT_SUCCESS(status)) > - return status; > - > - status = XENBUS_STORE(Printf, > - &Protocol->StoreInterface, > - Transaction, > - > FrontendGetFrontendPath(Protocol->Frontend), > - Name, > - "%u", > - GranterReference(Granter, > Protocol->Grants[Index])); > - if (!NT_SUCCESS(status)) > - return status; > - } > } > > status = XENBUS_STORE(Printf, > &Protocol->StoreInterface, > Transaction, > FrontendGetFrontendPath(Protocol->Frontend), > - "protocol", > - XEN_IO_PROTO_ABI); > + "multi-queue-num-queues", > + "%u", > + Protocol->NumQueues); > if (!NT_SUCCESS(status)) > return status; > > - Port = XENBUS_EVTCHN(GetPort, > - &Protocol->EvtchnInterface, > - Protocol->Channel); > - > status = XENBUS_STORE(Printf, > &Protocol->StoreInterface, > Transaction, > FrontendGetFrontendPath(Protocol->Frontend), > - "event-channel", > - "%u", > - Port); > + "protocol", > + XEN_IO_PROTO_ABI); > if (!NT_SUCCESS(status)) > return status; > > @@ -1917,12 +2295,13 @@ ProtocolEnable( > IN PXENVBD_PROTOCOL Protocol > ) > { > + ULONG Index; > + > ASSERT(Protocol->Enabled == FALSE); > Protocol->Enabled = TRUE; > > - XENBUS_EVTCHN(Trigger, > - &Protocol->EvtchnInterface, > - Protocol->Channel); > + for (Index = 0; Index < Protocol->NumQueues; ++Index) > + RingEnable(Protocol->Rings[Index]); > } > > VOID > @@ -1930,56 +2309,13 @@ ProtocolDisable( > IN PXENVBD_PROTOCOL Protocol > ) > { > - ULONG Count; > - KIRQL Irql; > - PXENVBD_TARGET Target = FrontendGetTarget(Protocol->Frontend); > - PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target); > + ULONG Index; > > ASSERT(Protocol->Enabled == TRUE); > Protocol->Enabled = FALSE; > > - // poll ring and send event channel notification every 1ms (for up to 3 > minutes) > - Count = 0; > - while (QueueCount(&Protocol->SubmittedReqs)) { > - if (Count > 180000) > - break; > - KeRaiseIrql(DISPATCH_LEVEL, &Irql); > - ProtocolPoll(Protocol); > - KeLowerIrql(Irql); > - XENBUS_EVTCHN(Send, > - &Protocol->EvtchnInterface, > - Protocol->Channel); > - StorPortStallExecution(1000); // 1000 micro-seconds > - ++Count; > - } > - > - Verbose("Target[%d] : %u Submitted requests left (%u iterrations)\n", > - FrontendGetTargetId(Protocol->Frontend), > - QueueCount(&Protocol->SubmittedReqs), > - Count); > - > - // Fail PreparedReqs > - for (;;) { > - PXENVBD_SRBEXT SrbExt; > - PSCSI_REQUEST_BLOCK Srb; > - PXENVBD_REQUEST Request; > - PLIST_ENTRY ListEntry; > - > - ListEntry = QueuePop(&Protocol->PreparedReqs); > - if (ListEntry == NULL) > - break; > - Request = CONTAINING_RECORD(ListEntry, XENVBD_REQUEST, > ListEntry); > - SrbExt = Request->SrbExt; > - Srb = SrbExt->Srb; > - > - Srb->SrbStatus = SRB_STATUS_ABORTED; > - Srb->ScsiStatus = 0x40; // SCSI_ABORTED > - > - ProtocolPutRequest(Protocol, Request); > - > - if (InterlockedDecrement(&SrbExt->RequestCount) == 0) > - AdapterCompleteSrb(Adapter, SrbExt); > - } > + for (Index = 0; Index < Protocol->NumQueues; ++Index) > + RingDisable(Protocol->Rings[Index]); > } > > VOID > @@ -1987,10 +2323,8 @@ ProtocolDisconnect( > IN PXENVBD_PROTOCOL Protocol > ) > { > - PXENVBD_GRANTER Granter = FrontendGetGranter(Protocol- > >Frontend); > ULONG Index; > > - ASSERT3U(Protocol->Submitted, ==, Protocol->Received); > ASSERT(Protocol->Connected); > Protocol->Connected = FALSE; > > @@ -1999,24 +2333,8 @@ ProtocolDisconnect( > Protocol->DebugCallback); > Protocol->DebugCallback = NULL; > > - XENBUS_EVTCHN(Close, > - &Protocol->EvtchnInterface, > - Protocol->Channel); > - Protocol->Channel = NULL; > - > - for (Index = 0; Index < (1ul << Protocol->Order); ++Index) { > - if (Protocol->Grants[Index] == NULL) > - continue; > - > - GranterPut(Granter, Protocol->Grants[Index]); > - Protocol->Grants[Index] = NULL; > - } > - > - RtlZeroMemory(&Protocol->Front, sizeof(blkif_front_ring_t)); > - > - __FreePages(Protocol->Mdl); > - Protocol->Shared = NULL; > - Protocol->Mdl = NULL; > + for (Index = 0; Index < Protocol->NumQueues; ++Index) > + RingDisconnect(Protocol->Rings[Index]); > > Protocol->Order = 0; > > @@ -2030,11 +2348,6 @@ ProtocolDisconnect( > sizeof(XENBUS_EVTCHN_INTERFACE)); > RtlZeroMemory(&Protocol->StoreInterface, > sizeof(XENBUS_STORE_INTERFACE)); > - > - Protocol->Events = 0; > - Protocol->Dpcs = 0; > - Protocol->Submitted = 0; > - Protocol->Received = 0; > } > > VOID > @@ -2042,12 +2355,29 @@ ProtocolTrigger( > IN PXENVBD_PROTOCOL Protocol > ) > { > + ULONG Index; > + > if (!Protocol->Enabled) > return; > > - XENBUS_EVTCHN(Trigger, > - &Protocol->EvtchnInterface, > - Protocol->Channel); > + for (Index = 0; Index < Protocol->NumQueues; ++Index) > + RingTrigger(Protocol->Rings[Index]); > +} > + > +static FORCEINLINE PXENVBD_RING > +__ProtocolGetRing( > + IN PXENVBD_PROTOCOL Protocol > + ) > +{ > + ULONG Index; > + > + if (Protocol->NumQueues == 0) > + return Protocol->Rings[0]; > + > + Index = KeGetCurrentProcessorNumberEx(NULL) % Protocol- > >NumQueues; > + ASSERT(Index < Protocol->NumQueues); > + > + return Protocol->Rings[Index]; > } > > BOOLEAN > @@ -2057,17 +2387,26 @@ ProtocolQueueRequest( > ) > { > PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb; > + PXENVBD_RING Ring; > + LIST_ENTRY List; > + > + InitializeListHead(&List); > > if (!Protocol->Enabled) > goto fail1; > > - if (!ProtocolPrepareRequest(Protocol, SrbExt)) > + if (!ProtocolPrepareRequest(Protocol, SrbExt, &List)) > goto fail2; > > - if (ProtocolSubmitRequests(Protocol)) { > + Ring = __ProtocolGetRing(Protocol); > + ASSERT(Ring != NULL); > + > + RingQueueRequestList(Ring, &List); > + > + if (RingSubmitRequests(Ring)) { > // more prepared-reqs to submit > - if (KeInsertQueueDpc(&Protocol->Dpc, NULL, NULL)) > - ++Protocol->Dpcs; > + if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL)) > + ++Ring->Dpcs; > } > > return TRUE; > @@ -2081,15 +2420,20 @@ fail1: > VOID > ProtocolQueueShutdown( > IN PXENVBD_PROTOCOL Protocol, > - IN PXENVBD_SRBEXT SrbExt > + IN PXENVBD_SRBEXT SrbExt > ) > { > - QueueAppend(&Protocol->ShutdownSrbs, > + PXENVBD_RING Ring; > + > + Ring = __ProtocolGetRing(Protocol); > + ASSERT(Ring != NULL); > + > + QueueAppend(&Ring->ShutdownSrbs, > &SrbExt->ListEntry); > > if (!Protocol->Enabled) > return; > > - if (KeInsertQueueDpc(&Protocol->Dpc, NULL, NULL)) > - ++Protocol->Dpcs; > + if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL)) > + ++Ring->Dpcs; > } > -- > 2.16.2.windows.1 > > > _______________________________________________ > win-pv-devel mailing list > win-pv-devel@xxxxxxxxxxxxxxxxxxxx > https://lists.xenproject.org/mailman/listinfo/win-pv-devel _______________________________________________ win-pv-devel mailing list win-pv-devel@xxxxxxxxxxxxxxxxxxxx https://lists.xenproject.org/mailman/listinfo/win-pv-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |