|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-devel] [PATCH v6 09/10] libxl: vnuma nodes placement bits
Automatic numa placement cancels manual vnode placement
mechanism. If numa placement explicitly specified, try
to fit vnodes to the physical nodes.
Signed-off-by: Elena Ufimtseva <ufimtseva@xxxxxxxxx>
---
tools/libxl/libxl_create.c | 1 +
tools/libxl/libxl_dom.c | 148 ++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 149 insertions(+)
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index 0686f96..038f96e 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -188,6 +188,7 @@ int libxl__domain_build_info_setdefault(libxl__gc *gc,
return ERROR_FAIL;
libxl_defbool_setdefault(&b_info->numa_placement, true);
+ libxl_defbool_setdefault(&b_info->vnuma_autoplacement, true);
if (b_info->max_memkb == LIBXL_MEMKB_DEFAULT)
b_info->max_memkb = 32 * 1024;
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index 83eb29a..98e42b3 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -23,6 +23,7 @@
#include <xc_dom.h>
#include <xen/hvm/hvm_info_table.h>
#include <xen/hvm/hvm_xs_strings.h>
+#include <libxl_vnuma.h>
libxl_domain_type libxl__domain_type(libxl__gc *gc, uint32_t domid)
{
@@ -227,6 +228,61 @@ static void hvm_set_conf_params(xc_interface *handle,
uint32_t domid,
libxl_defbool_val(info->u.hvm.nested_hvm));
}
+/* sets vnode_to_pnode map. */
+static int libxl__init_vnode_to_pnode(libxl__gc *gc, uint32_t domid,
+ libxl_domain_build_info *info)
+{
+ unsigned int i, n;
+ int nr_nodes = 0;
+ uint64_t *vnodes_mem;
+ unsigned long long *nodes_claim = NULL;
+ libxl_numainfo *ninfo = NULL;
+
+ if (info->vnuma_vnodemap == NULL) {
+ info->vnuma_vnodemap = libxl__calloc(gc, info->nr_nodes,
+ sizeof(*info->vnuma_vnodemap));
+ }
+
+ /* default setting. */
+ for (i = 0; i < info->nr_nodes; i++)
+ info->vnuma_vnodemap[i] = VNUMA_NO_NODE;
+
+ /* Get NUMA info. */
+ ninfo = libxl_get_numainfo(CTX, &nr_nodes);
+ if (ninfo == NULL)
+ return ERROR_FAIL;
+ /* Nothing to see if only one NUMA node. */
+ if (nr_nodes <= 1)
+ return 0;
+
+ vnodes_mem = info->vnuma_mem;
+ /*
+ * TODO: change algorithm. The current just fits the nodes
+ * by its memory sizes. If no p-node found, will be used default
+ * value of NUMA_NO_NODE.
+ */
+ nodes_claim = libxl__calloc(gc, info->nr_nodes, sizeof(*nodes_claim));
+ if ( !nodes_claim )
+ return ERROR_FAIL;
+
+ libxl_for_each_set_bit(n, info->nodemap)
+ {
+ for (i = 0; i < info->nr_nodes; i++)
+ {
+ unsigned long mem_sz = vnodes_mem[i] << 20;
+ if ((nodes_claim[n] + mem_sz <= ninfo[n].free) &&
+ /* vnode was not set yet. */
+ (info->vnuma_vnodemap[i] == VNUMA_NO_NODE ) )
+ {
+ info->vnuma_vnodemap[i] = n;
+ nodes_claim[n] += mem_sz;
+ }
+ }
+ }
+
+ return 0;
+}
+
int libxl__build_pre(libxl__gc *gc, uint32_t domid,
libxl_domain_config *d_config, libxl__domain_build_state *state)
{
@@ -240,6 +296,21 @@ int libxl__build_pre(libxl__gc *gc, uint32_t domid,
return ERROR_FAIL;
}
+ struct vmemrange *memrange = libxl__calloc(gc, info->nr_nodes,
+ sizeof(*memrange));
+ if (libxl__vnuma_align_mem(gc, domid, info, memrange) < 0) {
+ LOG(DETAIL, "Failed to align memory map.\n");
+ return ERROR_FAIL;
+ }
+
+ /*
+ * NUMA placement and vNUMA autoplacement handling:
+ * If numa_placement is set to default, do not use vnode to pnode
+ * mapping as automatic placement algorithm will find best numa nodes.
+ * If numa_placement is not used, we can try and use domain vnode
+ * to pnode mask.
+ */
+
/*
* Check if the domain has any CPU or node affinity already. If not, try
* to build up the latter via automatic NUMA placement. In fact, in case
@@ -268,7 +339,33 @@ int libxl__build_pre(libxl__gc *gc, uint32_t domid,
rc = numa_place_domain(gc, domid, info);
if (rc)
return rc;
+
+ /*
+ * If vnode_to_pnode mask was defined, dont use it if we automatically
+ * place domain on NUMA nodes, just give warning.
+ */
+ if (!libxl_defbool_val(info->vnuma_autoplacement)) {
+ LOG(INFO, "Automatic NUMA placement for domain is turned on. \
+ vnode to physical nodes mapping will not be used.");
+ }
+ if (libxl__init_vnode_to_pnode(gc, domid, info) < 0) {
+ LOG(ERROR, "Failed to build vnode to pnode map\n");
+ return ERROR_FAIL;
+ }
+ } else {
+ if (!libxl_defbool_val(info->vnuma_autoplacement)) {
+ if (!libxl__vnodemap_is_usable(gc, info)) {
+ LOG(ERROR, "Defined vnode to pnode domain map cannot be
used.\n");
+ return ERROR_FAIL;
+ }
+ } else {
+ if (libxl__init_vnode_to_pnode(gc, domid, info) < 0) {
+ LOG(ERROR, "Failed to build vnode to pnode map.\n");
+ return ERROR_FAIL;
+ }
+ }
}
+
if (info->nodemap.size)
libxl_domain_set_nodeaffinity(ctx, domid, &info->nodemap);
/* As mentioned in libxl.h, vcpu_hard_array takes precedence */
@@ -294,6 +391,19 @@ int libxl__build_pre(libxl__gc *gc, uint32_t domid,
return ERROR_FAIL;
}
+ /*
+ * XEN_DOMCTL_setvnuma subop hypercall needs to know max mem
+ * for domain set by xc_domain_setmaxmem. So set vNUMA after
+ * maxmem is being set.
+ */
+ if (xc_domain_setvnuma(ctx->xch, domid, info->nr_nodes,
+ info->max_vcpus, memrange,
+ info->vdistance, info->vnuma_vcpumap,
+ info->vnuma_vnodemap) < 0) {
+ LIBXL__LOG_ERRNO(ctx, LIBXL__LOG_ERROR, "Couldn't set vnuma topology");
+ return ERROR_FAIL;
+ }
+
xs_domid = xs_read(ctx->xsh, XBT_NULL, "/tool/xenstored/domid", NULL);
state->store_domid = xs_domid ? atoi(xs_domid) : 0;
free(xs_domid);
@@ -389,6 +499,39 @@ retry_transaction:
return 0;
}
+static int libxl__dom_vnuma_init(struct libxl_domain_build_info *info,
+ struct xc_dom_image *dom)
+{
+ errno = ERROR_INVAL;
+
+ if (info->nr_nodes == 0)
+ return -1;
+
+ dom->vnode_to_pnode = (unsigned int *)malloc(
+ info->nr_nodes * sizeof(*info->vnuma_vnodemap));
+ dom->numa_memszs = (uint64_t *)malloc(
+ info->nr_nodes * sizeof(*info->vnuma_mem));
+
+ errno = ERROR_FAIL;
+ if ( dom->numa_memszs == NULL || dom->vnode_to_pnode == NULL ) {
+ info->nr_nodes = 0;
+ if (dom->vnode_to_pnode)
+ free(dom->vnode_to_pnode);
+ if (dom->numa_memszs)
+ free(dom->numa_memszs);
+ return -1;
+ }
+
+ memcpy(dom->numa_memszs, info->vnuma_mem,
+ sizeof(*info->vnuma_mem) * info->nr_nodes);
+ memcpy(dom->vnode_to_pnode, info->vnuma_vnodemap,
+ sizeof(*info->vnuma_vnodemap) * info->nr_nodes);
+
+ dom->nr_nodes = info->nr_nodes;
+
+ return 0;
+}
+
int libxl__build_pv(libxl__gc *gc, uint32_t domid,
libxl_domain_build_info *info, libxl__domain_build_state *state)
{
@@ -446,6 +589,11 @@ int libxl__build_pv(libxl__gc *gc, uint32_t domid,
dom->xenstore_domid = state->store_domid;
dom->claim_enabled = libxl_defbool_val(info->claim_mode);
+ if ( (ret = libxl__dom_vnuma_init(info, dom)) != 0 ) {
+ LOGE(ERROR, "Failed to set doman vnuma");
+ goto out;
+ }
+
if ( (ret = xc_dom_boot_xen_init(dom, ctx->xch, domid)) != 0 ) {
LOGE(ERROR, "xc_dom_boot_xen_init failed");
goto out;
--
1.7.10.4
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
http://lists.xen.org/xen-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |