[PATCH v2 2/9] xen/x86: Use enumerations to indicate NUMA status

[Wei Chen <wei.chen@xxxxxxx>]

In current code, x86 is using two variables, numa_off and acpi_numa,
to indicate the NUMA status. This is because NUMA is not coupled with
ACPI, and ACPI still can work without NUMA on x86. With these two
variables' combinations, x86 can have several NUMA status:
NUMA swith on,
NUMA swith off,
NUMA swith on with NUMA emulation,
NUMA swith on with no-ACPI,
NUMA swith on with ACPI.

In this case, we introduce an enumeration numa_mode in this patch
to indicate above NUMA status, except NUMA on with emulation. Because
NUMA emulation has another variable, numa_fake, to indicate the number
of nodes for emulation. We can't use the enumeration to replace it at
the same time. But it still can be indicated by numa_on and numa_fake
as what it has been indicated.

Based on the enumeration we introduce numa_enabled_with_firmware for
callers to check NUMA status is enabled + ACPI. Using this helper is
because some NUMA implementation will use other firmware, this helper
will be easy to them to check enabled + others.

As we have touched srat_disabled, we have corrected its return value
from int to bool.

Signed-off-by: Wei Chen <wei.chen@xxxxxxx>
---
v1 -> v2:
1. Remove fw_numa.
2. Use enumeration to replace numa_off and acpi_numa.
3. Correct return value of srat_disabled.
4. Introduce numa_enabled_with_firmware.
---
 xen/arch/x86/include/asm/acpi.h |  1 -
 xen/arch/x86/include/asm/numa.h | 16 +++++++++++++---
 xen/arch/x86/numa.c             | 28 +++++++++++++++-------------
 xen/arch/x86/setup.c            |  3 ++-
 xen/arch/x86/srat.c             | 13 +++++++------
 5 files changed, 37 insertions(+), 24 deletions(-)

diff --git a/xen/arch/x86/include/asm/acpi.h b/xen/arch/x86/include/asm/acpi.h
index 9a9cc4c240..ab0d56dd70 100644
--- a/xen/arch/x86/include/asm/acpi.h
+++ b/xen/arch/x86/include/asm/acpi.h
@@ -101,7 +101,6 @@ extern unsigned long acpi_wakeup_address;
 
 #define ARCH_HAS_POWER_INIT    1
 
-extern s8 acpi_numa;
 extern int acpi_scan_nodes(u64 start, u64 end);
 #define NR_NODE_MEMBLKS (MAX_NUMNODES*2)
 
diff --git a/xen/arch/x86/include/asm/numa.h b/xen/arch/x86/include/asm/numa.h
index c32ccffde3..ee8262d969 100644
--- a/xen/arch/x86/include/asm/numa.h
+++ b/xen/arch/x86/include/asm/numa.h
@@ -28,12 +28,22 @@ extern nodeid_t pxm_to_node(unsigned int pxm);
 #define ZONE_ALIGN (1UL << (MAX_ORDER+PAGE_SHIFT))
 #define VIRTUAL_BUG_ON(x) 
 
+/* Enumerations for NUMA status. */
+enum numa_mode {
+       numa_on = 0,
+       numa_off,
+       /* NUMA turns on, but ACPI table is bad or disabled. */
+       numa_no_acpi,
+       /* NUMA turns on, and ACPI table works well. */
+       numa_acpi,
+};
+
 extern void numa_add_cpu(int cpu);
 extern void numa_init_array(void);
-extern bool numa_off;
-
+extern bool numa_enabled_with_firmware(void);
+extern enum numa_mode numa_status;
 
-extern int srat_disabled(void);
+extern bool srat_disabled(void);
 extern void numa_set_node(int cpu, nodeid_t node);
 extern nodeid_t setup_node(unsigned int pxm);
 extern void srat_detect_node(int cpu);
diff --git a/xen/arch/x86/numa.c b/xen/arch/x86/numa.c
index 627ae8aa95..0777a7518d 100644
--- a/xen/arch/x86/numa.c
+++ b/xen/arch/x86/numa.c
@@ -47,12 +47,16 @@ cpumask_t node_to_cpumask[MAX_NUMNODES] __read_mostly;
 
 nodemask_t __read_mostly node_online_map = { { [0] = 1UL } };
 
-bool numa_off;
-s8 acpi_numa = 0;
+enum numa_mode numa_status;
 
-int srat_disabled(void)
+bool srat_disabled(void)
 {
-    return numa_off || acpi_numa < 0;
+    return numa_status == numa_off || numa_status == numa_no_acpi;
+}
+
+bool __init numa_enabled_with_firmware(void)
+{
+    return numa_status == numa_acpi;
 }
 
 /*
@@ -254,12 +258,13 @@ void __init numa_initmem_init(unsigned long start_pfn, 
unsigned long end_pfn)
 #endif
 
 #ifdef CONFIG_ACPI_NUMA
-    if ( !numa_off && !acpi_scan_nodes(start, end) )
+    if ( numa_status != numa_off && !acpi_scan_nodes(start, end) )
         return;
 #endif
 
     printk(KERN_INFO "%s\n",
-           numa_off ? "NUMA turned off" : "No NUMA configuration found");
+           numa_status == numa_off ? "NUMA turned off"
+                                   : "No NUMA configuration found");
 
     printk(KERN_INFO "Faking a node at %"PRIpaddr"-%"PRIpaddr"\n",
            start, end);
@@ -292,13 +297,13 @@ void numa_set_node(int cpu, nodeid_t node)
 static int __init cf_check numa_setup(const char *opt)
 {
     if ( !strncmp(opt,"off",3) )
-        numa_off = true;
+        numa_status = numa_off;
     else if ( !strncmp(opt,"on",2) )
-        numa_off = false;
+        numa_status = numa_on;
 #ifdef CONFIG_NUMA_EMU
     else if ( !strncmp(opt, "fake=", 5) )
     {
-        numa_off = false;
+        numa_status = numa_on;
         numa_fake = simple_strtoul(opt+5,NULL,0);
         if ( numa_fake >= MAX_NUMNODES )
             numa_fake = MAX_NUMNODES;
@@ -306,10 +311,7 @@ static int __init cf_check numa_setup(const char *opt)
 #endif
 #ifdef CONFIG_ACPI_NUMA
     else if ( !strncmp(opt,"noacpi",6) )
-    {
-        numa_off = false;
-        acpi_numa = -1;
-    }
+        numa_status = numa_no_acpi;
 #endif
     else
         return -EINVAL;
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index f08b07b8de..4841af5926 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -329,7 +329,8 @@ void srat_detect_node(int cpu)
     node_set_online(node);
     numa_set_node(cpu, node);
 
-    if ( opt_cpu_info && acpi_numa > 0 )
+    /* Print CPU info when NUMA is enabled with ACPI. */
+    if ( opt_cpu_info && numa_enabled_with_firmware() )
         printk("CPU %d APIC %d -> Node %d\n", cpu, apicid, node);
 }
 
diff --git a/xen/arch/x86/srat.c b/xen/arch/x86/srat.c
index f53431f5e8..422e4c73e3 100644
--- a/xen/arch/x86/srat.c
+++ b/xen/arch/x86/srat.c
@@ -185,7 +185,7 @@ static __init void bad_srat(void)
 {
        int i;
        printk(KERN_ERR "SRAT: SRAT not used.\n");
-       acpi_numa = -1;
+       numa_status = numa_no_acpi;
        for (i = 0; i < MAX_LOCAL_APIC; i++)
                apicid_to_node[i] = NUMA_NO_NODE;
        for (i = 0; i < ARRAY_SIZE(pxm2node); i++)
@@ -260,7 +260,7 @@ acpi_numa_x2apic_affinity_init(const struct 
acpi_srat_x2apic_cpu_affinity *pa)
 
        apicid_to_node[pa->apic_id] = node;
        node_set(node, processor_nodes_parsed);
-       acpi_numa = 1;
+       numa_status = numa_acpi;
 
        if (opt_acpi_verbose)
                printk(KERN_INFO "SRAT: PXM %u -> APIC %08x -> Node %u\n",
@@ -295,7 +295,7 @@ acpi_numa_processor_affinity_init(const struct 
acpi_srat_cpu_affinity *pa)
        }
        apicid_to_node[pa->apic_id] = node;
        node_set(node, processor_nodes_parsed);
-       acpi_numa = 1;
+       numa_status = numa_acpi;
 
        if (opt_acpi_verbose)
                printk(KERN_INFO "SRAT: PXM %u -> APIC %02x -> Node %u\n",
@@ -484,7 +484,7 @@ static int __init cf_check srat_parse_region(
            (ma->flags & ACPI_SRAT_MEM_NON_VOLATILE))
                return 0;
 
-       if (numa_off)
+       if (numa_status == numa_off)
                printk(KERN_INFO "SRAT: %013"PRIx64"-%013"PRIx64"\n",
                       ma->base_address, ma->base_address + ma->length - 1);
 
@@ -499,7 +499,7 @@ void __init srat_parse_regions(paddr_t addr)
        u64 mask;
        unsigned int i;
 
-       if (acpi_disabled || acpi_numa < 0 ||
+       if (acpi_disabled || numa_status == numa_no_acpi ||
            acpi_table_parse(ACPI_SIG_SRAT, acpi_parse_srat))
                return;
 
@@ -528,7 +528,8 @@ int __init acpi_scan_nodes(paddr_t start, paddr_t end)
        for (i = 0; i < MAX_NUMNODES; i++)
                cutoff_node(i, start, end);
 
-       if (acpi_numa <= 0)
+       /* Only when numa_on with good firmware, we can do numa scan nodes. */
+       if (!numa_enabled_with_firmware())
                return -1;
 
        if (!nodes_cover_memory()) {
-- 
2.25.1