|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] Re: [RFC PATCH 1/2] xenbus: Batch foreign page map and unmap hypercalls
On 10/11/2026 17:58, Rafał Wojdyła wrote:
> Submit up to 256 grant operations per hypercall with per-page status values. > > Signed-off-by: Rafał Wojdyła <omeg@xxxxxxxxxxxxxxxxxxxxxx> > --- > include/xen.h | 30 ++++-- > src/xen/grant_table.c | 210 +++++++++++++++++++++++++++++++++++------- > src/xenbus/gnttab.c | 200 ++++++++++++++++++++++++---------------- > 3 files changed, 323 insertions(+), 117 deletions(-) > > diff --git a/include/xen.h b/include/xen.h > index 0c3e8a4..ad98ba1 100644 > --- a/include/xen.h > +++ b/include/xen.h > @@ -262,15 +262,23 @@ GrantTableCopy( > _In_ ULONG Count > ); > > +// Maximum number of pages accepted by the map/unmap functions. > +#define XEN_GNTTAB_MAP_BATCH_MAX 256 > + > +// Zero is a valid Xen handle. > +#define XEN_GNTTAB_INVALID_HANDLE ((ULONG)-1) > + > _Check_return_ > XEN_API > NTSTATUS > -GrantTableMapForeignPage( > - _In_ USHORT Domain, > - _In_ ULONG GrantRef, > - _In_ PHYSICAL_ADDRESS Address, > - _In_ BOOLEAN ReadOnly, > - _Out_ ULONG *Handle > +GrantTableMapForeignPages( > + _In_ USHORT Domain, > + _In_ ULONG NumberPages, > + _In_reads_(NumberPages) PULONG GrantRefs, > + _In_ PHYSICAL_ADDRESS StartAddress, > + _In_ BOOLEAN ReadOnly, > + _Out_writes_(NumberPages) PULONG Handles, > + _Out_writes_(NumberPages) PNTSTATUS Results > ); > > _Check_return_ > @@ -281,6 +289,16 @@ GrantTableUnmapForeignPage( > _In_ PHYSICAL_ADDRESS Address > ); > > +_Check_return_ > +XEN_API > +NTSTATUS > +GrantTableUnmapForeignPages( > + _In_ ULONG NumberPages, > + _In_reads_(NumberPages) PULONG Handles, > + _In_ PHYSICAL_ADDRESS StartAddress, > + _Out_writes_(NumberPages) PNTSTATUS Results > + ); > + > _Check_return_ > XEN_API > NTSTATUS > diff --git a/src/xen/grant_table.c b/src/xen/grant_table.c > index 96a5a26..ec240df 100644 > --- a/src/xen/grant_table.c > +++ b/src/xen/grant_table.c > @@ -38,10 +38,13 @@ > #include "hypercall.h" > #include "dbg_print.h" > #include "assert.h" > +#include "util.h" > > #pragma warning(push) > #pragma warning(disable:4127) // conditional _expression_ is constant > > +#define XEN_GNTTAB_TAG 'TNGX' > + > // Most of the GNTST_* values don't have meaningful NTSTATUS counterparts, > // this macro translates those that do. > #define GNTST_TO_STATUS(_gntst, _status) \ > @@ -168,47 +171,90 @@ fail1: > _Check_return_ > XEN_API > NTSTATUS > -GrantTableMapForeignPage( > - _In_ USHORT Domain, > - _In_ ULONG GrantRef, > - _In_ PHYSICAL_ADDRESS Address, > - _In_ BOOLEAN ReadOnly, > - _Out_ ULONG *Handle > +GrantTableMapForeignPages( > + _In_ USHORT Domain, > + _In_ ULONG NumberPages, > + _In_reads_(NumberPages) PULONG GrantRefs, > + _In_ PHYSICAL_ADDRESS StartAddress, > + _In_ BOOLEAN ReadOnly, > + _Out_writes_(NumberPages) PULONG Handles, > + _Out_writes_(NumberPages) PNTSTATUS Results > ) > { > - struct gnttab_map_grant_ref op; > - LONG_PTR rc; > - NTSTATUS status; > + struct gnttab_map_grant_ref *ops; > + LONG_PTR rc; > + NTSTATUS status; > + ULONG i; > + ULONG fail_count; > + > + status = STATUS_INVALID_PARAMETER; > + if (NumberPages == 0 || NumberPages > XEN_GNTTAB_MAP_BATCH_MAX) > + goto fail1; > > - RtlZeroMemory(&op, sizeof(op)); > - op.dom = Domain; > - op.ref = GrantRef; > - op.flags = GNTMAP_host_map; > - if (ReadOnly) > - op.flags |= GNTMAP_readonly; > - op.host_addr = Address.QuadPart; > + for (i = 0; i < NumberPages; i++) > + Handles[i] = XEN_GNTTAB_INVALID_HANDLE; > > - rc = GrantTableOp(GNTTABOP_map_grant_ref, &op, 1); > + status = STATUS_NO_MEMORY; > + ops = __AllocatePoolWithTag(NonPagedPool, > + NumberPages * sizeof(*ops), > + XEN_GNTTAB_TAG); > + if (ops == NULL) > + goto fail2; > > - if (rc < 0) { > - ERRNO_TO_STATUS(-rc, status); > - goto fail1; > + for (i = 0; i < NumberPages; i++) { > + ops[i].dom = Domain; > + ops[i].ref = GrantRefs[i]; > + ops[i].flags = GNTMAP_host_map; > + if (ReadOnly) > + ops[i].flags |= GNTMAP_readonly; > + > + ops[i].host_addr = StartAddress.QuadPart + ((ULONGLONG)i << PAGE_SHIFT); > } > > - if (op.status != GNTST_okay) { > - Warning("%u:%u -> %u.%u failed (%d)\n", > - op.dom, > - op.ref, > - Address.HighPart, > - Address.LowPart, > - op.status); > + rc = GrantTableOp(GNTTABOP_map_grant_ref, ops, NumberPages); > + > + if (rc != 0) { > + Error("map_grant_ref: %u ops at 0x%08x%08x failed (%lld)\n", > + NumberPages, > + (ULONG)(StartAddress.QuadPart >> 32), > + (ULONG)StartAddress.QuadPart, > + rc); > + > + // Possible Xen errors during a hypercall: > + // EFAULT: Xen failed to access the output buffer. Some pages may > + // have been processed, but the state is unknown. > + // EINVAL: invalid arguments > + // ENOSYS: unsupported hypercall > + // EPERM: caller not running at ring 0 > + // All of these indicate either an unsafe state or corrupted > + // executing environment. > + BUG("map hypercall failed"); Can a hypercall failure simply come from userspace specifying an invalid domid, or some other sort of access denial? > + } > > - GNTST_TO_STATUS(op.status, status); > - goto fail2; > + fail_count = 0; > + for (i = 0; i < NumberPages; i++) { > + GNTST_TO_STATUS(ops[i].status, Results[i]); > + > + if (NT_SUCCESS(Results[i])) { > + Handles[i] = ops[i].handle; > + } else { > + if (fail_count == 0) { > + Warning("op[%u] %hu:%u -> 0x%08x%08x failed (%hd)\n", > + i, > + ops[i].dom, ops[i].ref, > + (ULONG)(ops[i].host_addr >> 32), > + (ULONG)ops[i].host_addr, > + ops[i].status); > + } > + > + fail_count++; > + } > } > > - *Handle = op.handle; > + if (fail_count > 0) > + Warning("%u of %u ops failed\n", fail_count, NumberPages); > > + __FreePoolWithTag(ops, XEN_GNTTAB_TAG); > return STATUS_SUCCESS; > > fail2: > @@ -244,7 +290,7 @@ GrantTableUnmapForeignPage( > } > > if (op.status != GNTST_okay) { > - Warning("%u.%u failed (%d)\n", > + Warning("0x%08x%08x failed (%hd)\n", > Address.HighPart, > Address.LowPart, > op.status); > @@ -264,6 +310,108 @@ fail1: > return status; > } > > +_Check_return_ > +XEN_API > +NTSTATUS > +GrantTableUnmapForeignPages( > + _In_ ULONG NumberPages, > + _In_reads_(NumberPages) PULONG Handles, > + _In_ PHYSICAL_ADDRESS StartAddress, > + _Out_writes_(NumberPages) PNTSTATUS Results > + ) > +{ > + struct gnttab_unmap_grant_ref *ops; > + LONG_PTR rc; > + NTSTATUS status; > + ULONG i; > + ULONG count; > + ULONG fail_count; > + > + status = STATUS_INVALID_PARAMETER; > + if (NumberPages == 0 || NumberPages > XEN_GNTTAB_MAP_BATCH_MAX) > + goto fail1; > + > + count = 0; > + for (i = 0; i < NumberPages; i++) { > + if (Handles[i] != XEN_GNTTAB_INVALID_HANDLE) > + count++; > + } > + > + if (count == 0) > + goto done; > + > + status = STATUS_NO_MEMORY; > + ops = __AllocatePoolWithTag(NonPagedPool, > + count * sizeof(*ops), > + XEN_GNTTAB_TAG); > + if (ops == NULL) > + goto fail2; > + > + count = 0; > + for (i = 0; i < NumberPages; i++) { > + if (Handles[i] == XEN_GNTTAB_INVALID_HANDLE) > + continue; > + > + ops[count].handle = Handles[i]; > + ops[count].host_addr = StartAddress.QuadPart + ((ULONGLONG)i << PAGE_SHIFT); > + count++; > + } > + > + rc = GrantTableOp(GNTTABOP_unmap_grant_ref, ops, count); > + > + if (rc != 0) { > + Error("unmap_grant_ref: %u ops at 0x%08x%08x failed (%lld)\n", > + count, > + (ULONG)(StartAddress.QuadPart >> 32), > + (ULONG)StartAddress.QuadPart, > + rc); > + > + // See GrantTableMapForeignPages() for explanation why this is fatal > + BUG("unmap hypercall failed"); > + } > + > + count = 0; > + fail_count = 0; > + for (i = 0; i < NumberPages; i++) { > + if (Handles[i] == XEN_GNTTAB_INVALID_HANDLE) { > + Results[i] = STATUS_SUCCESS; > + continue; > + } > + > + GNTST_TO_STATUS(ops[count].status, Results[i]); > + > + if (!NT_SUCCESS(Results[i])) { > + if (fail_count == 0) { > + Warning("op[%u] 0x%08x%08x failed (%hd)\n", > + i, > + (ULONG)(ops[count].host_addr >> 32), > + (ULONG)ops[count].host_addr, > + ops[count].status); > + } > + > + fail_count++; > + } > + > + count++; > + } > + > + if (fail_count > 0) > + Warning("%u of %u ops failed\n", fail_count, count); > + > + __FreePoolWithTag(ops, XEN_GNTTAB_TAG); > + > +done: > + return STATUS_SUCCESS; > + > +fail2: > + Error("fail2 (%08x)\n", status); > + > +fail1: > + Error("fail1 (%08x)\n", status); > + > + return status; > +} > + > _Check_return_ > XEN_API > NTSTATUS > diff --git a/src/xenbus/gnttab.c b/src/xenbus/gnttab.c > index bdf8b2b..6a4dd48 100644 > --- a/src/xenbus/gnttab.c > +++ b/src/xenbus/gnttab.c > @@ -656,56 +656,76 @@ fail1: > > static NTSTATUS > GnttabMapForeignPages( > - _In_ PINTERFACE Interface, > - _In_ USHORT Domain, > - _In_ ULONG NumberPages, > - _In_ PULONG References, > - _In_ BOOLEAN ReadOnly, > - _Out_ PHYSICAL_ADDRESS *Address > + _In_ PINTERFACE Interface, > + _In_ USHORT Domain, > + _In_ ULONG NumberPages, > + _In_reads_(NumberPages) PULONG References, > + _In_ BOOLEAN ReadOnly, > + _Out_ PPHYSICAL_ADDRESS StartAddress > ) > { > - PXENBUS_GNTTAB_CONTEXT Context = Interface->Context; > - PMDL Mdl; > - LONG PageIndex; > - PHYSICAL_ADDRESS PageAddress; > - PXENBUS_GNTTAB_MAP_ENTRY MapEntry; > - NTSTATUS status; > + PXENBUS_GNTTAB_CONTEXT Context = Interface->Context; > + PMDL Mdl; > + PXENBUS_GNTTAB_MAP_ENTRY MapEntry; > + NTSTATUS Results[XEN_GNTTAB_MAP_BATCH_MAX]; > + PHYSICAL_ADDRESS Address; > + ULONG Offset; > + ULONG Count; > + ULONG Index; > + NTSTATUS Status; > > - Mdl = FdoHoleAllocate(Context->Fdo, NumberPages); > + // Don't leave stale output address on failures. > + StartAddress->QuadPart = 0; > > - status = STATUS_NO_MEMORY; > - if (Mdl == NULL) > + Status = STATUS_INVALID_PARAMETER; > + // Sanity check for maximum MDL byte count. > + if (NumberPages == 0 || NumberPages >= 0x100000) > goto fail1; I keep seeing the 0x100000/0xfffff map limits in various places (here, as well as xeniface) so I hope it can be gathered together in 1 central check. > > - MapEntry = __GnttabAllocate(FIELD_OFFSET(XENBUS_GNTTAB_MAP_ENTRY, > - MapHandles) + > - (NumberPages * sizeof (ULONG))); > + Status = STATUS_NO_MEMORY; > + Mdl = FdoHoleAllocate(Context->Fdo, NumberPages); > + if (Mdl == NULL) > + goto fail2; > > - status = STATUS_NO_MEMORY; > + MapEntry = __GnttabAllocate(FIELD_OFFSET(XENBUS_GNTTAB_MAP_ENTRY, MapHandles) > + + NumberPages * sizeof (ULONG)); > if (MapEntry == NULL) > - goto fail2; > + goto fail3; > > MapEntry->Mdl = Mdl; > - > - Address->QuadPart = MmGetMdlPfnArray(Mdl)[0] << PAGE_SHIFT; > - PageAddress.QuadPart = Address->QuadPart; > - > - for (PageIndex = 0; PageIndex < (LONG)NumberPages; PageIndex++) { > - status = GrantTableMapForeignPage(Domain, > - References[PageIndex], > - PageAddress, > - ReadOnly, > - &MapEntry->MapHandles[PageIndex]); > - if (!NT_SUCCESS(status)) > - goto fail3; > - > - PageAddress.QuadPart += PAGE_SIZE; > + for (Index = 0; Index < NumberPages; Index++) > + MapEntry->MapHandles[Index] = XEN_GNTTAB_INVALID_HANDLE; > + > + Address.QuadPart = MmGetMdlPfnArray(Mdl)[0] << PAGE_SHIFT; > + > + *StartAddress = Address; > + > + for (Offset = 0; Offset < NumberPages; Offset += Count) { > + Count = __min(NumberPages - Offset, XEN_GNTTAB_MAP_BATCH_MAX); > + Address.QuadPart = StartAddress->QuadPart + ((ULONGLONG)Offset << PAGE_SHIFT); > + > + Status = GrantTableMapForeignPages(Domain, > + Count, > + &References[Offset], > + Address, > + ReadOnly, > + &MapEntry->MapHandles[Offset], > + Results); > + if (NT_SUCCESS(Status)) { > + for (Index = 0; Index < Count; Index++) { > + if (!NT_SUCCESS(Results[Index])) { > + Status = Results[Index]; > + break; > + } > + } > + } > + > + if (!NT_SUCCESS(Status)) > + goto fail4; > } > > - status = HashTableAdd(Context->MapTable, > - (ULONG_PTR)Address->QuadPart, > - (ULONG_PTR)MapEntry); > - if (!NT_SUCCESS(status)) > + Status = HashTableAdd(Context->MapTable, (ULONG_PTR)StartAddress->QuadPart, (ULONG_PTR)MapEntry); > + if (!NT_SUCCESS(Status)) > goto fail4; > > return STATUS_SUCCESS; > @@ -713,70 +733,90 @@ GnttabMapForeignPages( > fail4: > Error("fail4\n"); > > -fail3: > - Error("fail3\n"); > + // Legacy rollback behavior: ignore unmap failures, preserve the map error. > + for (Index = NumberPages; Index != 0; Index--) { > + if (MapEntry->MapHandles[Index - 1] == XEN_GNTTAB_INVALID_HANDLE) > + continue; > > - while (--PageIndex >= 0) { > - PageAddress.QuadPart -= PAGE_SIZE; > - (VOID) GrantTableUnmapForeignPage(MapEntry->MapHandles[PageIndex], > - PageAddress); > + Address.QuadPart = StartAddress->QuadPart + ((ULONGLONG)(Index - 1) << PAGE_SHIFT); > + (VOID) GrantTableUnmapForeignPage(MapEntry->MapHandles[Index - 1], Address); > } > > - Address->QuadPart = 0; > - > + StartAddress->QuadPart = 0; > __GnttabFree(MapEntry); > > -fail2: > - Error("fail2\n"); > +fail3: > + Error("fail3\n"); > > FdoHoleFree(Context->Fdo, Mdl); > > +fail2: > + Error("fail2\n"); > + > fail1: > - Error("fail1: (%08x)\n", status); > + Error("fail1 (%08x)\n", Status); > > - return status; > + return Status; > } > > static NTSTATUS > GnttabUnmapForeignPages( > - _In_ PINTERFACE Interface, > - _In_ PHYSICAL_ADDRESS Address > + _In_ PINTERFACE Interface, > + _In_ PHYSICAL_ADDRESS StartAddress > ) > { > - PXENBUS_GNTTAB_CONTEXT Context = Interface->Context; > - ULONG NumberPages; > - PHYSICAL_ADDRESS PageAddress; > - ULONG PageIndex; > - PXENBUS_GNTTAB_MAP_ENTRY MapEntry; > - PMDL Mdl; > - NTSTATUS status; > - > - status = HashTableLookup(Context->MapTable, > - (ULONG_PTR)Address.QuadPart, > - (PULONG_PTR)&MapEntry); > - if (!NT_SUCCESS(status)) > + PXENBUS_GNTTAB_CONTEXT Context = Interface->Context; > + PXENBUS_GNTTAB_MAP_ENTRY MapEntry; > + PMDL Mdl; > + NTSTATUS Results[XEN_GNTTAB_MAP_BATCH_MAX]; > + PHYSICAL_ADDRESS Address; > + ULONG NumberPages; > + ULONG Offset; > + ULONG Count; > + ULONG Index; > + NTSTATUS Status; > + NTSTATUS BatchStatus; > + > + Status = HashTableLookup(Context->MapTable, > + (ULONG_PTR)StartAddress.QuadPart, > + (PULONG_PTR)&MapEntry); > + if (!NT_SUCCESS(Status)) > goto fail1; > > - status = HashTableRemove(Context->MapTable, > - (ULONG_PTR)Address.QuadPart); > - if (!NT_SUCCESS(status)) > - goto fail2; > - > - PageAddress.QuadPart = Address.QuadPart; > - > Mdl = MapEntry->Mdl; > NumberPages = Mdl->ByteCount >> PAGE_SHIFT; > + Status = STATUS_SUCCESS; > + > + for (Offset = 0; Offset < NumberPages; Offset += Count) { > + Count = __min(NumberPages - Offset, XEN_GNTTAB_MAP_BATCH_MAX); > + Address.QuadPart = StartAddress.QuadPart + ((ULONGLONG)Offset << PAGE_SHIFT); > + > + BatchStatus = GrantTableUnmapForeignPages(Count, > + &MapEntry->MapHandles[Offset], > + Address, > + Results); > + > + if (NT_SUCCESS(BatchStatus)) { > + for (Index = 0; Index < Count; Index++) { > + if (!NT_SUCCESS(Results[Index])) { > + BatchStatus = Results[Index]; > + break; > + } > + } > + } > + > + if (!NT_SUCCESS(BatchStatus) && NT_SUCCESS(Status)) > + Status = BatchStatus; > + } > > - for (PageIndex = 0; PageIndex < NumberPages; PageIndex++) { > - status = GrantTableUnmapForeignPage(MapEntry->MapHandles[PageIndex], > - PageAddress); > - BUG_ON(!NT_SUCCESS(status)); > + // Legacy behavior: failures are fatal. > + BUG_ON(!NT_SUCCESS(Status)); > > - PageAddress.QuadPart += PAGE_SIZE; > - } > + Status = HashTableRemove(Context->MapTable, (ULONG_PTR)StartAddress.QuadPart); > + if (!NT_SUCCESS(Status)) > + goto fail2; > > __GnttabFree(MapEntry); > - > FdoHoleFree(Context->Fdo, Mdl); > > return STATUS_SUCCESS; > @@ -785,9 +825,9 @@ fail2: > Error("fail2\n"); > > fail1: > - Error("fail1: (%08x)\n", status); > + Error("fail1 (%08x)\n", Status); > > - return status; > + return Status; > } > > static VOID
--
|
![]() |
Lists.xenproject.org is hosted with RackSpace, monitoring our |