|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [RFC PATCH 1/2] xenbus: Batch foreign page map and unmap hypercalls
Submit up to 256 grant operations per hypercall with per-page status values.
Signed-off-by: Rafał Wojdyła <omeg@xxxxxxxxxxxxxxxxxxxxxx>
---
include/xen.h | 30 ++++--
src/xen/grant_table.c | 210 +++++++++++++++++++++++++++++++++++-------
src/xenbus/gnttab.c | 200 ++++++++++++++++++++++++----------------
3 files changed, 323 insertions(+), 117 deletions(-)
diff --git a/include/xen.h b/include/xen.h
index 0c3e8a4..ad98ba1 100644
--- a/include/xen.h
+++ b/include/xen.h
@@ -262,15 +262,23 @@ GrantTableCopy(
_In_ ULONG Count
);
+// Maximum number of pages accepted by the map/unmap functions.
+#define XEN_GNTTAB_MAP_BATCH_MAX 256
+
+// Zero is a valid Xen handle.
+#define XEN_GNTTAB_INVALID_HANDLE ((ULONG)-1)
+
_Check_return_
XEN_API
NTSTATUS
-GrantTableMapForeignPage(
- _In_ USHORT Domain,
- _In_ ULONG GrantRef,
- _In_ PHYSICAL_ADDRESS Address,
- _In_ BOOLEAN ReadOnly,
- _Out_ ULONG *Handle
+GrantTableMapForeignPages(
+ _In_ USHORT Domain,
+ _In_ ULONG NumberPages,
+ _In_reads_(NumberPages) PULONG GrantRefs,
+ _In_ PHYSICAL_ADDRESS StartAddress,
+ _In_ BOOLEAN ReadOnly,
+ _Out_writes_(NumberPages) PULONG Handles,
+ _Out_writes_(NumberPages) PNTSTATUS Results
);
_Check_return_
@@ -281,6 +289,16 @@ GrantTableUnmapForeignPage(
_In_ PHYSICAL_ADDRESS Address
);
+_Check_return_
+XEN_API
+NTSTATUS
+GrantTableUnmapForeignPages(
+ _In_ ULONG NumberPages,
+ _In_reads_(NumberPages) PULONG Handles,
+ _In_ PHYSICAL_ADDRESS StartAddress,
+ _Out_writes_(NumberPages) PNTSTATUS Results
+ );
+
_Check_return_
XEN_API
NTSTATUS
diff --git a/src/xen/grant_table.c b/src/xen/grant_table.c
index 96a5a26..ec240df 100644
--- a/src/xen/grant_table.c
+++ b/src/xen/grant_table.c
@@ -38,10 +38,13 @@
#include "hypercall.h"
#include "dbg_print.h"
#include "assert.h"
+#include "util.h"
#pragma warning(push)
#pragma warning(disable:4127) // conditional expression is constant
+#define XEN_GNTTAB_TAG 'TNGX'
+
// Most of the GNTST_* values don't have meaningful NTSTATUS counterparts,
// this macro translates those that do.
#define GNTST_TO_STATUS(_gntst, _status) \
@@ -168,47 +171,90 @@ fail1:
_Check_return_
XEN_API
NTSTATUS
-GrantTableMapForeignPage(
- _In_ USHORT Domain,
- _In_ ULONG GrantRef,
- _In_ PHYSICAL_ADDRESS Address,
- _In_ BOOLEAN ReadOnly,
- _Out_ ULONG *Handle
+GrantTableMapForeignPages(
+ _In_ USHORT Domain,
+ _In_ ULONG NumberPages,
+ _In_reads_(NumberPages) PULONG GrantRefs,
+ _In_ PHYSICAL_ADDRESS StartAddress,
+ _In_ BOOLEAN ReadOnly,
+ _Out_writes_(NumberPages) PULONG Handles,
+ _Out_writes_(NumberPages) PNTSTATUS Results
)
{
- struct gnttab_map_grant_ref op;
- LONG_PTR rc;
- NTSTATUS status;
+ struct gnttab_map_grant_ref *ops;
+ LONG_PTR rc;
+ NTSTATUS status;
+ ULONG i;
+ ULONG fail_count;
+
+ status = STATUS_INVALID_PARAMETER;
+ if (NumberPages == 0 || NumberPages > XEN_GNTTAB_MAP_BATCH_MAX)
+ goto fail1;
- RtlZeroMemory(&op, sizeof(op));
- op.dom = Domain;
- op.ref = GrantRef;
- op.flags = GNTMAP_host_map;
- if (ReadOnly)
- op.flags |= GNTMAP_readonly;
- op.host_addr = Address.QuadPart;
+ for (i = 0; i < NumberPages; i++)
+ Handles[i] = XEN_GNTTAB_INVALID_HANDLE;
- rc = GrantTableOp(GNTTABOP_map_grant_ref, &op, 1);
+ status = STATUS_NO_MEMORY;
+ ops = __AllocatePoolWithTag(NonPagedPool,
+ NumberPages * sizeof(*ops),
+ XEN_GNTTAB_TAG);
+ if (ops == NULL)
+ goto fail2;
- if (rc < 0) {
- ERRNO_TO_STATUS(-rc, status);
- goto fail1;
+ for (i = 0; i < NumberPages; i++) {
+ ops[i].dom = Domain;
+ ops[i].ref = GrantRefs[i];
+ ops[i].flags = GNTMAP_host_map;
+ if (ReadOnly)
+ ops[i].flags |= GNTMAP_readonly;
+
+ ops[i].host_addr = StartAddress.QuadPart + ((ULONGLONG)i <<
PAGE_SHIFT);
}
- if (op.status != GNTST_okay) {
- Warning("%u:%u -> %u.%u failed (%d)\n",
- op.dom,
- op.ref,
- Address.HighPart,
- Address.LowPart,
- op.status);
+ rc = GrantTableOp(GNTTABOP_map_grant_ref, ops, NumberPages);
+
+ if (rc != 0) {
+ Error("map_grant_ref: %u ops at 0x%08x%08x failed (%lld)\n",
+ NumberPages,
+ (ULONG)(StartAddress.QuadPart >> 32),
+ (ULONG)StartAddress.QuadPart,
+ rc);
+
+ // Possible Xen errors during a hypercall:
+ // EFAULT: Xen failed to access the output buffer. Some pages may
+ // have been processed, but the state is unknown.
+ // EINVAL: invalid arguments
+ // ENOSYS: unsupported hypercall
+ // EPERM: caller not running at ring 0
+ // All of these indicate either an unsafe state or corrupted
+ // executing environment.
+ BUG("map hypercall failed");
+ }
- GNTST_TO_STATUS(op.status, status);
- goto fail2;
+ fail_count = 0;
+ for (i = 0; i < NumberPages; i++) {
+ GNTST_TO_STATUS(ops[i].status, Results[i]);
+
+ if (NT_SUCCESS(Results[i])) {
+ Handles[i] = ops[i].handle;
+ } else {
+ if (fail_count == 0) {
+ Warning("op[%u] %hu:%u -> 0x%08x%08x failed (%hd)\n",
+ i,
+ ops[i].dom, ops[i].ref,
+ (ULONG)(ops[i].host_addr >> 32),
+ (ULONG)ops[i].host_addr,
+ ops[i].status);
+ }
+
+ fail_count++;
+ }
}
- *Handle = op.handle;
+ if (fail_count > 0)
+ Warning("%u of %u ops failed\n", fail_count, NumberPages);
+ __FreePoolWithTag(ops, XEN_GNTTAB_TAG);
return STATUS_SUCCESS;
fail2:
@@ -244,7 +290,7 @@ GrantTableUnmapForeignPage(
}
if (op.status != GNTST_okay) {
- Warning("%u.%u failed (%d)\n",
+ Warning("0x%08x%08x failed (%hd)\n",
Address.HighPart,
Address.LowPart,
op.status);
@@ -264,6 +310,108 @@ fail1:
return status;
}
+_Check_return_
+XEN_API
+NTSTATUS
+GrantTableUnmapForeignPages(
+ _In_ ULONG NumberPages,
+ _In_reads_(NumberPages) PULONG Handles,
+ _In_ PHYSICAL_ADDRESS StartAddress,
+ _Out_writes_(NumberPages) PNTSTATUS Results
+ )
+{
+ struct gnttab_unmap_grant_ref *ops;
+ LONG_PTR rc;
+ NTSTATUS status;
+ ULONG i;
+ ULONG count;
+ ULONG fail_count;
+
+ status = STATUS_INVALID_PARAMETER;
+ if (NumberPages == 0 || NumberPages > XEN_GNTTAB_MAP_BATCH_MAX)
+ goto fail1;
+
+ count = 0;
+ for (i = 0; i < NumberPages; i++) {
+ if (Handles[i] != XEN_GNTTAB_INVALID_HANDLE)
+ count++;
+ }
+
+ if (count == 0)
+ goto done;
+
+ status = STATUS_NO_MEMORY;
+ ops = __AllocatePoolWithTag(NonPagedPool,
+ count * sizeof(*ops),
+ XEN_GNTTAB_TAG);
+ if (ops == NULL)
+ goto fail2;
+
+ count = 0;
+ for (i = 0; i < NumberPages; i++) {
+ if (Handles[i] == XEN_GNTTAB_INVALID_HANDLE)
+ continue;
+
+ ops[count].handle = Handles[i];
+ ops[count].host_addr = StartAddress.QuadPart + ((ULONGLONG)i <<
PAGE_SHIFT);
+ count++;
+ }
+
+ rc = GrantTableOp(GNTTABOP_unmap_grant_ref, ops, count);
+
+ if (rc != 0) {
+ Error("unmap_grant_ref: %u ops at 0x%08x%08x failed (%lld)\n",
+ count,
+ (ULONG)(StartAddress.QuadPart >> 32),
+ (ULONG)StartAddress.QuadPart,
+ rc);
+
+ // See GrantTableMapForeignPages() for explanation why this is fatal
+ BUG("unmap hypercall failed");
+ }
+
+ count = 0;
+ fail_count = 0;
+ for (i = 0; i < NumberPages; i++) {
+ if (Handles[i] == XEN_GNTTAB_INVALID_HANDLE) {
+ Results[i] = STATUS_SUCCESS;
+ continue;
+ }
+
+ GNTST_TO_STATUS(ops[count].status, Results[i]);
+
+ if (!NT_SUCCESS(Results[i])) {
+ if (fail_count == 0) {
+ Warning("op[%u] 0x%08x%08x failed (%hd)\n",
+ i,
+ (ULONG)(ops[count].host_addr >> 32),
+ (ULONG)ops[count].host_addr,
+ ops[count].status);
+ }
+
+ fail_count++;
+ }
+
+ count++;
+ }
+
+ if (fail_count > 0)
+ Warning("%u of %u ops failed\n", fail_count, count);
+
+ __FreePoolWithTag(ops, XEN_GNTTAB_TAG);
+
+done:
+ return STATUS_SUCCESS;
+
+fail2:
+ Error("fail2 (%08x)\n", status);
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
+}
+
_Check_return_
XEN_API
NTSTATUS
diff --git a/src/xenbus/gnttab.c b/src/xenbus/gnttab.c
index bdf8b2b..6a4dd48 100644
--- a/src/xenbus/gnttab.c
+++ b/src/xenbus/gnttab.c
@@ -656,56 +656,76 @@ fail1:
static NTSTATUS
GnttabMapForeignPages(
- _In_ PINTERFACE Interface,
- _In_ USHORT Domain,
- _In_ ULONG NumberPages,
- _In_ PULONG References,
- _In_ BOOLEAN ReadOnly,
- _Out_ PHYSICAL_ADDRESS *Address
+ _In_ PINTERFACE Interface,
+ _In_ USHORT Domain,
+ _In_ ULONG NumberPages,
+ _In_reads_(NumberPages) PULONG References,
+ _In_ BOOLEAN ReadOnly,
+ _Out_ PPHYSICAL_ADDRESS StartAddress
)
{
- PXENBUS_GNTTAB_CONTEXT Context = Interface->Context;
- PMDL Mdl;
- LONG PageIndex;
- PHYSICAL_ADDRESS PageAddress;
- PXENBUS_GNTTAB_MAP_ENTRY MapEntry;
- NTSTATUS status;
+ PXENBUS_GNTTAB_CONTEXT Context = Interface->Context;
+ PMDL Mdl;
+ PXENBUS_GNTTAB_MAP_ENTRY MapEntry;
+ NTSTATUS Results[XEN_GNTTAB_MAP_BATCH_MAX];
+ PHYSICAL_ADDRESS Address;
+ ULONG Offset;
+ ULONG Count;
+ ULONG Index;
+ NTSTATUS Status;
- Mdl = FdoHoleAllocate(Context->Fdo, NumberPages);
+ // Don't leave stale output address on failures.
+ StartAddress->QuadPart = 0;
- status = STATUS_NO_MEMORY;
- if (Mdl == NULL)
+ Status = STATUS_INVALID_PARAMETER;
+ // Sanity check for maximum MDL byte count.
+ if (NumberPages == 0 || NumberPages >= 0x100000)
goto fail1;
- MapEntry = __GnttabAllocate(FIELD_OFFSET(XENBUS_GNTTAB_MAP_ENTRY,
- MapHandles) +
- (NumberPages * sizeof (ULONG)));
+ Status = STATUS_NO_MEMORY;
+ Mdl = FdoHoleAllocate(Context->Fdo, NumberPages);
+ if (Mdl == NULL)
+ goto fail2;
- status = STATUS_NO_MEMORY;
+ MapEntry = __GnttabAllocate(FIELD_OFFSET(XENBUS_GNTTAB_MAP_ENTRY,
MapHandles)
+ + NumberPages * sizeof (ULONG));
if (MapEntry == NULL)
- goto fail2;
+ goto fail3;
MapEntry->Mdl = Mdl;
-
- Address->QuadPart = MmGetMdlPfnArray(Mdl)[0] << PAGE_SHIFT;
- PageAddress.QuadPart = Address->QuadPart;
-
- for (PageIndex = 0; PageIndex < (LONG)NumberPages; PageIndex++) {
- status = GrantTableMapForeignPage(Domain,
- References[PageIndex],
- PageAddress,
- ReadOnly,
- &MapEntry->MapHandles[PageIndex]);
- if (!NT_SUCCESS(status))
- goto fail3;
-
- PageAddress.QuadPart += PAGE_SIZE;
+ for (Index = 0; Index < NumberPages; Index++)
+ MapEntry->MapHandles[Index] = XEN_GNTTAB_INVALID_HANDLE;
+
+ Address.QuadPart = MmGetMdlPfnArray(Mdl)[0] << PAGE_SHIFT;
+
+ *StartAddress = Address;
+
+ for (Offset = 0; Offset < NumberPages; Offset += Count) {
+ Count = __min(NumberPages - Offset, XEN_GNTTAB_MAP_BATCH_MAX);
+ Address.QuadPart = StartAddress->QuadPart + ((ULONGLONG)Offset <<
PAGE_SHIFT);
+
+ Status = GrantTableMapForeignPages(Domain,
+ Count,
+ &References[Offset],
+ Address,
+ ReadOnly,
+ &MapEntry->MapHandles[Offset],
+ Results);
+ if (NT_SUCCESS(Status)) {
+ for (Index = 0; Index < Count; Index++) {
+ if (!NT_SUCCESS(Results[Index])) {
+ Status = Results[Index];
+ break;
+ }
+ }
+ }
+
+ if (!NT_SUCCESS(Status))
+ goto fail4;
}
- status = HashTableAdd(Context->MapTable,
- (ULONG_PTR)Address->QuadPart,
- (ULONG_PTR)MapEntry);
- if (!NT_SUCCESS(status))
+ Status = HashTableAdd(Context->MapTable,
(ULONG_PTR)StartAddress->QuadPart, (ULONG_PTR)MapEntry);
+ if (!NT_SUCCESS(Status))
goto fail4;
return STATUS_SUCCESS;
@@ -713,70 +733,90 @@ GnttabMapForeignPages(
fail4:
Error("fail4\n");
-fail3:
- Error("fail3\n");
+ // Legacy rollback behavior: ignore unmap failures, preserve the map error.
+ for (Index = NumberPages; Index != 0; Index--) {
+ if (MapEntry->MapHandles[Index - 1] == XEN_GNTTAB_INVALID_HANDLE)
+ continue;
- while (--PageIndex >= 0) {
- PageAddress.QuadPart -= PAGE_SIZE;
- (VOID) GrantTableUnmapForeignPage(MapEntry->MapHandles[PageIndex],
- PageAddress);
+ Address.QuadPart = StartAddress->QuadPart + ((ULONGLONG)(Index - 1) <<
PAGE_SHIFT);
+ (VOID) GrantTableUnmapForeignPage(MapEntry->MapHandles[Index - 1],
Address);
}
- Address->QuadPart = 0;
-
+ StartAddress->QuadPart = 0;
__GnttabFree(MapEntry);
-fail2:
- Error("fail2\n");
+fail3:
+ Error("fail3\n");
FdoHoleFree(Context->Fdo, Mdl);
+fail2:
+ Error("fail2\n");
+
fail1:
- Error("fail1: (%08x)\n", status);
+ Error("fail1 (%08x)\n", Status);
- return status;
+ return Status;
}
static NTSTATUS
GnttabUnmapForeignPages(
- _In_ PINTERFACE Interface,
- _In_ PHYSICAL_ADDRESS Address
+ _In_ PINTERFACE Interface,
+ _In_ PHYSICAL_ADDRESS StartAddress
)
{
- PXENBUS_GNTTAB_CONTEXT Context = Interface->Context;
- ULONG NumberPages;
- PHYSICAL_ADDRESS PageAddress;
- ULONG PageIndex;
- PXENBUS_GNTTAB_MAP_ENTRY MapEntry;
- PMDL Mdl;
- NTSTATUS status;
-
- status = HashTableLookup(Context->MapTable,
- (ULONG_PTR)Address.QuadPart,
- (PULONG_PTR)&MapEntry);
- if (!NT_SUCCESS(status))
+ PXENBUS_GNTTAB_CONTEXT Context = Interface->Context;
+ PXENBUS_GNTTAB_MAP_ENTRY MapEntry;
+ PMDL Mdl;
+ NTSTATUS Results[XEN_GNTTAB_MAP_BATCH_MAX];
+ PHYSICAL_ADDRESS Address;
+ ULONG NumberPages;
+ ULONG Offset;
+ ULONG Count;
+ ULONG Index;
+ NTSTATUS Status;
+ NTSTATUS BatchStatus;
+
+ Status = HashTableLookup(Context->MapTable,
+ (ULONG_PTR)StartAddress.QuadPart,
+ (PULONG_PTR)&MapEntry);
+ if (!NT_SUCCESS(Status))
goto fail1;
- status = HashTableRemove(Context->MapTable,
- (ULONG_PTR)Address.QuadPart);
- if (!NT_SUCCESS(status))
- goto fail2;
-
- PageAddress.QuadPart = Address.QuadPart;
-
Mdl = MapEntry->Mdl;
NumberPages = Mdl->ByteCount >> PAGE_SHIFT;
+ Status = STATUS_SUCCESS;
+
+ for (Offset = 0; Offset < NumberPages; Offset += Count) {
+ Count = __min(NumberPages - Offset, XEN_GNTTAB_MAP_BATCH_MAX);
+ Address.QuadPart = StartAddress.QuadPart + ((ULONGLONG)Offset <<
PAGE_SHIFT);
+
+ BatchStatus = GrantTableUnmapForeignPages(Count,
+
&MapEntry->MapHandles[Offset],
+ Address,
+ Results);
+
+ if (NT_SUCCESS(BatchStatus)) {
+ for (Index = 0; Index < Count; Index++) {
+ if (!NT_SUCCESS(Results[Index])) {
+ BatchStatus = Results[Index];
+ break;
+ }
+ }
+ }
+
+ if (!NT_SUCCESS(BatchStatus) && NT_SUCCESS(Status))
+ Status = BatchStatus;
+ }
- for (PageIndex = 0; PageIndex < NumberPages; PageIndex++) {
- status = GrantTableUnmapForeignPage(MapEntry->MapHandles[PageIndex],
- PageAddress);
- BUG_ON(!NT_SUCCESS(status));
+ // Legacy behavior: failures are fatal.
+ BUG_ON(!NT_SUCCESS(Status));
- PageAddress.QuadPart += PAGE_SIZE;
- }
+ Status = HashTableRemove(Context->MapTable,
(ULONG_PTR)StartAddress.QuadPart);
+ if (!NT_SUCCESS(Status))
+ goto fail2;
__GnttabFree(MapEntry);
-
FdoHoleFree(Context->Fdo, Mdl);
return STATUS_SUCCESS;
@@ -785,9 +825,9 @@ fail2:
Error("fail2\n");
fail1:
- Error("fail1: (%08x)\n", status);
+ Error("fail1 (%08x)\n", Status);
- return status;
+ return Status;
}
static VOID
--
2.56.0
|
![]() |
Lists.xenproject.org is hosted with RackSpace, monitoring our |