[PATCH 10/21] tools/libxl: Add 'numa-node-id' property to DomU memory nodes

[Hirokazu Takahashi <taka@xxxxxxxxxxxxx>]

Add the 'numa-node-id' property to the memory nodes in the Device
Tree passed to DomU. The NUMA node to which each memory node
belongs is determined based on the virtual NUMA configuration
specified in the xl domain configuration file.
---
 tools/libs/light/libxl_arm.c | 81 ++++++++++++++++++++++++++++++++----
 1 file changed, 72 insertions(+), 9 deletions(-)

diff --git a/tools/libs/light/libxl_arm.c b/tools/libs/light/libxl_arm.c
index 7e9f8a1bc3..58a357858d 100644
--- a/tools/libs/light/libxl_arm.c
+++ b/tools/libs/light/libxl_arm.c
@@ -670,25 +670,84 @@ static int make_optee_node(libxl__gc *gc, void *fdt)
 }
 
 static int make_memory_nodes(libxl__gc *gc, void *fdt,
+                             const libxl_domain_build_info *b_info,
                              const struct xc_dom_image *dom)
 {
     int res, i;
     const char *name;
     const uint64_t bankbase[] = GUEST_RAM_BANK_BASES;
+    const uint64_t banksize[] = GUEST_RAM_BANK_SIZES;
+
+    int cur_bank = 0;
+    uint64_t cur_addr = bankbase[cur_bank];
+    uint64_t bank_remaining = banksize[cur_bank];
+
+    if (b_info->num_vnuma_nodes == 0) {
+        for (i = 0; i < GUEST_RAM_BANKS; i++) {
+            name = GCSPRINTF("memory@%"PRIx64, bankbase[i]);
+
+            LOG(DEBUG, "Creating placeholder node /%s", name);
+
+            res = fdt_begin_node(fdt, name);
+            if (res) return res;
+
+            res = fdt_property_string(fdt, "device_type", "memory");
+            if (res) return res;
+
+            res = fdt_property_regs(gc, fdt, GUEST_ROOT_ADDRESS_CELLS, 
GUEST_ROOT_SIZE_CELLS,
+                                1, 0, 0);
+            if (res) return res;
+
+            res = fdt_end_node(fdt);
+            if (res) return res;
+        }
+
+        return 0;
+    }
 
-    for (i = 0; i < GUEST_RAM_BANKS; i++) {
-        name = GCSPRINTF("memory@%"PRIx64, bankbase[i]);
+    for (i = 0; i < b_info->num_vnuma_nodes; i++) {
+        uint64_t vnode_rem = (uint64_t)b_info->vnuma_nodes[i].memkb << 10;
+        uint64_t regs[GUEST_RAM_BANKS * 2];
+        int reg_count = 0;
+        uint64_t vnode_start_addr = ~0ULL;
+        uint64_t chunk;
+
+        while (vnode_rem > 0) {
+            if (bank_remaining == 0) {
+                if (cur_bank < GUEST_RAM_BANKS - 1) {
+                    cur_bank++;
+                    cur_addr = bankbase[cur_bank];
+                    bank_remaining = banksize[cur_bank];
+                } else {
+                    LOG(ERROR, "Virtual node %d: Not enough RAM banks", i);
+                    return ERROR_INVAL;
+                }
+            }
+
+            if (vnode_start_addr == ~0ULL) vnode_start_addr = cur_addr;
 
-        LOG(DEBUG, "Creating placeholder node /%s", name);
+            chunk = (vnode_rem < bank_remaining) ? vnode_rem : bank_remaining;
+
+            regs[reg_count * 2] = cpu_to_fdt64(cur_addr);
+            regs[reg_count * 2 + 1] = cpu_to_fdt64(chunk);
+            reg_count++;
+
+            vnode_rem -= chunk;
+            cur_addr += chunk;
+            bank_remaining -= chunk;
+        }
 
+        name = GCSPRINTF("memory@%"PRIx64, vnode_start_addr);
         res = fdt_begin_node(fdt, name);
         if (res) return res;
 
         res = fdt_property_string(fdt, "device_type", "memory");
         if (res) return res;
 
-        res = fdt_property_regs(gc, fdt, GUEST_ROOT_ADDRESS_CELLS, 
GUEST_ROOT_SIZE_CELLS,
-                                1, 0, 0);
+        res = fdt_property(fdt, "reg", regs, sizeof(uint64_t) * 2 * reg_count);
+        if (res) return res;
+
+        res = fdt_property_u32(fdt, "numa-node-id", i);
         if (res) return res;
 
         res = fdt_end_node(fdt);
@@ -1365,7 +1424,7 @@ next_resize:
         FDT( make_cpus_node(gc, fdt, info->max_vcpus, ainfo) );
         FDT( make_psci_node(gc, fdt) );
 
-        FDT( make_memory_nodes(gc, fdt, dom) );
+        FDT( make_memory_nodes(gc, fdt, info, dom) );
 
         switch (info->arch_arm.gic_version) {
         case LIBXL_GIC_VERSION_V2:
@@ -1716,10 +1775,14 @@ int 
libxl__arch_domain_finalise_hw_description(libxl__gc *gc,
     if (res)
         return res;
 
-    for (i = 0; i < GUEST_RAM_BANKS; i++) {
-        const uint64_t size = (uint64_t)dom->rambank_size[i] << XC_PAGE_SHIFT;
+    if (d_config->b_info.num_vnuma_nodes == 0) {
+        for (i = 0; i < GUEST_RAM_BANKS; i++) {
+            const uint64_t size = (uint64_t)dom->rambank_size[i] << 
XC_PAGE_SHIFT;
 
-        finalise_one_node(gc, fdt, "/memory", bankbase[i], size);
+            finalise_one_node(gc, fdt, "/memory", bankbase[i], size);
+        }
+    } else {
+        LOG(DEBUG, "vNUMA enabled: skipping memory node finalisation as nodes 
are already populated");
     }
 
     if (dom->acpi_modules[0].data) {
-- 
2.43.0