[PATCH 15/21] xen/arm: Reflect physical NUMA node IDs in Dom0 memory DT nodes

[Hirokazu Takahashi <taka@xxxxxxxxxxxxx>]

Extract the physical NUMA node ID for each memory bank allocated for Dom0
and apply it to the corresponding memory node in the Dom0 Device Tree.
This implementation maps the physical NUMA node IDs directly as Dom0's
virtual NUMA node IDs.
---
 xen/arch/arm/domain_build.c           | 22 ++++++-
 xen/common/device-tree/domain-build.c | 88 +++++++++++++--------------
 2 files changed, 63 insertions(+), 47 deletions(-)

diff --git a/xen/arch/arm/domain_build.c b/xen/arch/arm/domain_build.c
index 2518909ed0..1deb4c1139 100644
--- a/xen/arch/arm/domain_build.c
+++ b/xen/arch/arm/domain_build.c
@@ -187,11 +187,26 @@ static bool __init insert_11_bank(struct domain *d,
     int res;
     mfn_t smfn;
     paddr_t start, size;
+    nodeid_t node = 0U;
 
     smfn = page_to_mfn(pg);
     start = mfn_to_maddr(smfn);
     size = pfn_to_paddr(1UL << order);
 
+    /* This code is temporal */
+    {
+        struct membanks *mem = bootinfo_get_mem();
+        for ( i = 0; i < mem->nr_banks; i++ )
+        {
+            if ( start >= mem->bank[i].start &&
+                 start < (mem->bank[i].start + mem->bank[i].size) )
+            {
+                node = get_numa_nodeid(&mem->bank[i]);
+                break;
+            }
+        }
+    }
+
     D11PRINT("Allocated %#"PRIpaddr"-%#"PRIpaddr" (%ldMB/%ldMB, order %d)\n",
              start, start + size,
              1UL << (order + PAGE_SHIFT - 20),
@@ -217,6 +232,7 @@ static bool __init insert_11_bank(struct domain *d,
     {
         mem->bank[0].start = start;
         mem->bank[0].size = size;
+        set_numa_nodeid(&mem->bank[0], node);
         mem->nr_banks = 1;
         return true;
     }
@@ -226,7 +242,7 @@ static bool __init insert_11_bank(struct domain *d,
         struct membank *bank = &mem->bank[i];
 
         /* If possible merge new memory into the start of the bank */
-        if ( bank->start == start+size )
+        if ( bank->start == start+size && get_numa_nodeid(bank) == node )
         {
             bank->start = start;
             bank->size += size;
@@ -234,7 +250,7 @@ static bool __init insert_11_bank(struct domain *d,
         }
 
         /* If possible merge new memory onto the end of the bank */
-        if ( start == bank->start + bank->size )
+        if ( start == bank->start + bank->size && get_numa_nodeid(bank) == 
node )
         {
             bank->size += size;
             return true;
@@ -253,6 +269,7 @@ static bool __init insert_11_bank(struct domain *d,
             mem->nr_banks++;
             bank->start = start;
             bank->size = size;
+            set_numa_nodeid(bank, node);
             return true;
         }
     }
@@ -263,6 +280,7 @@ static bool __init insert_11_bank(struct domain *d,
 
         bank->start = start;
         bank->size = size;
+        set_numa_nodeid(bank, node);
         mem->nr_banks++;
         return true;
     }
diff --git a/xen/common/device-tree/domain-build.c 
b/xen/common/device-tree/domain-build.c
index 2a760b007b..61e2e50062 100644
--- a/xen/common/device-tree/domain-build.c
+++ b/xen/common/device-tree/domain-build.c
@@ -493,15 +493,50 @@ int __init make_chosen_node(const struct kernel_info 
*kinfo)
     return res;
 }
 
+static int __init make_memory_sibling_node(const struct kernel_info *kinfo, 
int addrcells,
+                            int sizecells, const struct membank *bank)
+{
+    void *fdt = kinfo->fdt;
+    int res = 0;
+    u64 start = bank->start;
+    u64 size = bank->size;
+    __be32 reg[DT_MEM_NODE_REG_RANGE_SIZE];
+    __be32 *cells = &reg[0];
+    nodeid_t node = get_numa_nodeid(bank);
+    char name[32];
+
+    dt_dprintk("  Bank: %#"PRIx64"->%#"PRIx64" Node:%u\n", start, start + 
size, node);
+
+    snprintf(name, sizeof(name), "memory@%"PRIx64, start);
+    res = fdt_begin_node(fdt, name);
+    if ( res )
+        return res;
+
+    dt_child_set_range(&cells, addrcells, sizecells, start, size);
+    res = fdt_property_string(fdt, "device_type", "memory");
+    if ( res )
+        return res;
+
+    res = fdt_property(fdt, "reg", reg, (addrcells + sizecells) * 
sizeof(*reg));
+    if ( res )
+        return res;
+
+#ifdef CONFIG_NUMA
+    res = fdt_property_u32(fdt, "numa-node-id", node);
+    if ( res )
+        return res;
+#endif /* CONFIG_NUMA */
+
+    res = fdt_end_node(fdt);
+
+    return res;
+}
+
 int __init make_memory_node(const struct kernel_info *kinfo, int addrcells,
                             int sizecells, const struct membanks *mem)
 {
-    void *fdt = kinfo->fdt;
     unsigned int i;
-    int res, reg_size = addrcells + sizecells;
-    int nr_cells = 0;
-    __be32 reg[DT_MEM_NODE_REG_RANGE_SIZE];
-    __be32 *cells;
+    int res = 0;
 
     if ( mem->nr_banks == 0 )
         return -ENOENT;
@@ -516,54 +551,17 @@ int __init make_memory_node(const struct kernel_info 
*kinfo, int addrcells,
 
     dt_dprintk("Create memory node\n");
 
-    res = domain_fdt_begin_node(fdt, "memory", mem->bank[i].start);
-    if ( res )
-        return res;
-
-    res = fdt_property_string(fdt, "device_type", "memory");
-    if ( res )
-        return res;
-
-    cells = &reg[0];
     for ( ; i < mem->nr_banks; i++ )
     {
-        u64 start = mem->bank[i].start;
-        u64 size = mem->bank[i].size;
-
         if ( (mem->bank[i].type == MEMBANK_STATIC_DOMAIN) ||
              (mem->bank[i].type == MEMBANK_FDT_RESVMEM) )
             continue;
 
-        nr_cells += reg_size;
-        BUG_ON(nr_cells > ARRAY_SIZE(reg));
-        dt_child_set_range(&cells, addrcells, sizecells, start, size);
-    }
-
-    /*
-     * static shared memory banks need to be listed as /memory node, so when
-     * this function is handling the normal memory, add the banks.
-     */
-    if ( mem == kernel_info_get_mem_const(kinfo) )
-        shm_mem_node_fill_reg_range(kinfo, reg, &nr_cells, addrcells,
-                                    sizecells);
-
-    for ( cells = reg, i = 0; cells < reg + nr_cells; i++, cells += reg_size )
-    {
-        uint64_t start = dt_read_number(cells, addrcells);
-        uint64_t size = dt_read_number(cells + addrcells, sizecells);
-
-        dt_dprintk("  Bank %u: %#"PRIx64"->%#"PRIx64"\n",
-                   i, start, start + size);
+        res = make_memory_sibling_node(kinfo, addrcells, sizecells, 
&mem->bank[i]);
+        if ( res )
+            return res;
     }
 
-    dt_dprintk("(reg size %d, nr cells %d)\n", reg_size, nr_cells);
-
-    res = fdt_property(fdt, "reg", reg, nr_cells * sizeof(*reg));
-    if ( res )
-        return res;
-
-    res = fdt_end_node(fdt);
-
     return res;
 }
 
-- 
2.43.0