[PATCH v3 8/9] xen/arm: add Xen cache colors command line parameter

[Carlo Nonato <carlo.nonato@xxxxxxxxxxxxxxx>]

From: Luca Miccio <lucmiccio@xxxxxxxxx>

This commit adds a new command line parameter to configure Xen cache
colors. These colors can be dumped with the cache coloring info debug-key.

By default, Xen uses the first color.
Benchmarking the VM interrupt response time provides an estimation of
LLC usage by Xen's most latency-critical runtime task. Results on Arm
Cortex-A53 on Xilinx Zynq UltraScale+ XCZU9EG show that one color, which
reserves 64 KiB of L2, is enough to attain best responsiveness.

More colors are instead very likely to be needed on processors whose L1
cache is physically-indexed and physically-tagged, such as Cortex-A57.
In such cases, coloring applies to L1 also, and there typically are two
distinct L1-colors. Therefore, reserving only one color for Xen would
senselessly partitions a cache memory that is already private, i.e.
underutilize it. The default amount of Xen colors is thus set to one.

Signed-off-by: Luca Miccio <lucmiccio@xxxxxxxxx>
Signed-off-by: Marco Solieri <marco.solieri@xxxxxxxxxxxxxxx>
Signed-off-by: Carlo Nonato <carlo.nonato@xxxxxxxxxxxxxxx>
---
 docs/misc/arm/cache-coloring.rst  |  8 ++++----
 docs/misc/xen-command-line.pandoc |  9 +++++++++
 xen/arch/arm/coloring.c           | 30 ++++++++++++++++++++++++++++++
 3 files changed, 43 insertions(+), 4 deletions(-)

diff --git a/docs/misc/arm/cache-coloring.rst b/docs/misc/arm/cache-coloring.rst
index 0c89278aee..80eb259dfa 100644
--- a/docs/misc/arm/cache-coloring.rst
+++ b/docs/misc/arm/cache-coloring.rst
@@ -19,8 +19,8 @@ In order to enable and use it, few steps are needed.
 - If needed, change the amount of memory reserved for the buddy allocator 
either
   from the Xen configuration file, via the CONFIG_BUDDY_ALLOCATOR_SIZE value,
   or with the command line option. See `Colored allocator and buddy allocator`.
-- Assign colors to domains using the `Color selection format`_ (see
-  `Coloring parameters`_ for more documentation pointers).
+- Assign colors to each memory pool (Xen, Dom0/DomUs) using the
+  `Color selection format`_ for `Coloring parameters`_ configuration.
 
 Background
 **********
@@ -113,8 +113,8 @@ Examples:
 Coloring parameters
 *******************
 
-LLC way size (as previously discussed) and Dom0 colors can be set using the
-appropriate command line parameters. See the relevant documentation in
+LLC way size (as previously discussed), Xen colors and Dom0 colors can be set
+using the appropriate command line parameters. See the relevant documentation 
in
 "docs/misc/xen-command-line.pandoc".
 
 DomUs colors can be set either in the xl configuration file (relative
diff --git a/docs/misc/xen-command-line.pandoc 
b/docs/misc/xen-command-line.pandoc
index 25a59dd6a9..d831cf1196 100644
--- a/docs/misc/xen-command-line.pandoc
+++ b/docs/misc/xen-command-line.pandoc
@@ -2754,6 +2754,15 @@ In the case that x2apic is in use, this option switches 
between physical and
 clustered mode.  The default, given no hint from the **FADT**, is cluster
 mode.
 
+### xen-colors (arm64)
+> `= List of [ <integer> | <integer>-<integer> ]`
+
+> Default: `0: the lowermost color`
+
+Specify Xen color configuration.
+Two colors are most likely needed on platforms where private caches are
+physically indexed, e.g. the L1 instruction cache of the Arm Cortex-A57.
+
 ### xenheap_megabytes (arm32)
 > `= <size>`
 
diff --git a/xen/arch/arm/coloring.c b/xen/arch/arm/coloring.c
index 2cae215cd2..80c76c057f 100644
--- a/xen/arch/arm/coloring.c
+++ b/xen/arch/arm/coloring.c
@@ -32,6 +32,10 @@
 #include <asm/processor.h>
 #include <asm/sysregs.h>
 
+/* By default Xen uses the lowest color */
+#define XEN_DEFAULT_COLOR       0
+#define XEN_DEFAULT_NUM_COLORS  1
+
 /* Size of an LLC way */
 static unsigned int __ro_after_init llc_way_size;
 /* Number of colors available in the LLC */
@@ -43,6 +47,9 @@ static uint64_t __ro_after_init addr_col_mask;
 #define addr_set_color(addr, color) (((addr) & ~addr_col_mask) \
                                      | ((color) << PAGE_SHIFT))
 
+static unsigned int xen_colors[CONFIG_MAX_CACHE_COLORS];
+static unsigned int xen_num_colors;
+
 static unsigned int dom0_colors[CONFIG_MAX_CACHE_COLORS];
 static unsigned int dom0_num_colors;
 
@@ -94,6 +101,12 @@ static int parse_color_config(const char *buf, unsigned int 
*colors,
 
 size_param("llc-way-size", llc_way_size);
 
+static int __init parse_xen_colors(const char *s)
+{
+    return parse_color_config(s, xen_colors, &xen_num_colors);
+}
+custom_param("xen-colors", parse_xen_colors);
+
 static int __init parse_dom0_colors(const char *s)
 {
     return parse_color_config(s, dom0_colors, &dom0_num_colors);
@@ -188,6 +201,8 @@ static void dump_coloring_info(unsigned char key)
     printk("LLC way size: %u KiB\n", llc_way_size >> 10);
     printk("Number of LLC colors supported: %u\n", max_colors);
     printk("Address color mask: 0x%lx\n", addr_col_mask);
+    printk("Xen colors: ");
+    print_colors(xen_colors, xen_num_colors);
 }
 
 bool __init coloring_init(void)
@@ -225,6 +240,21 @@ bool __init coloring_init(void)
 
     addr_col_mask = (max_colors - 1) << PAGE_SHIFT;
 
+    if ( !xen_num_colors )
+    {
+        printk(XENLOG_WARNING
+               "Xen color config not found. Using default color: %u\n",
+               XEN_DEFAULT_COLOR);
+        xen_colors[0] = XEN_DEFAULT_COLOR;
+        xen_num_colors = XEN_DEFAULT_NUM_COLORS;
+    }
+
+    if ( !check_colors(xen_colors, xen_num_colors) )
+    {
+        printk(XENLOG_ERR "Bad color config for Xen\n");
+        return false;
+    }
+
     if ( !dom0_num_colors )
     {
         printk(XENLOG_WARNING
-- 
2.34.1