[v6,12/15] xen/arm: add Xen cache colors command line parameter

Commit Message

Carlo Nonato Jan. 29, 2024, 5:18 p.m. UTC

From: Luca Miccio <[e-mail]>

Add a new command line parameter to configure Xen cache colors.
These colors can be dumped with the cache coloring info debug-key.

By default, Xen uses the first color.
Benchmarking the VM interrupt response time provides an estimation of
LLC usage by Xen's most latency-critical runtime task. Results on Arm
Cortex-A53 on Xilinx Zynq UltraScale+ XCZU9EG show that one color, which
reserves 64 KiB of L2, is enough to attain best responsiveness:
- Xen 1 color latency:  3.1 us
- Xen 2 color latency:  3.1 us

More colors are instead very likely to be needed on processors whose L1
cache is physically-indexed and physically-tagged, such as Cortex-A57.
In such cases, coloring applies to L1 also, and there typically are two
distinct L1-colors. Therefore, reserving only one color for Xen would
senselessly partitions a cache memory that is already private, i.e.
underutilize it. The default amount of Xen colors is thus set to one.

Signed-off-by: Luca Miccio <[e-mail]>
Signed-off-by: Marco Solieri <[e-mail]>
Signed-off-by: Carlo Nonato <[e-mail]>
---
 docs/misc/xen-command-line.pandoc | 10 ++++++++++
 xen/common/llc-coloring.c         | 26 ++++++++++++++++++++++++++
 2 files changed, 36 insertions(+)

Comments

Jan Beulich Feb. 1, 2024, 3:10 p.m. UTC | #1

On 29.01.2024 18:18, Carlo Nonato wrote:
> --- a/xen/common/llc-coloring.c
> +++ b/xen/common/llc-coloring.c
> @@ -9,6 +9,9 @@
>  #include <xen/llc-coloring.h>
>  #include <xen/param.h>
>  
> +#define XEN_DEFAULT_COLOR       0
> +#define XEN_DEFAULT_NUM_COLORS  1
> +
>  bool __ro_after_init llc_coloring_enabled;
>  boolean_param("llc-coloring", llc_coloring_enabled);
>  
> @@ -21,6 +24,9 @@ static unsigned int __ro_after_init max_nr_colors = CONFIG_NR_LLC_COLORS;
>  static unsigned int __initdata dom0_colors[CONFIG_NR_LLC_COLORS];
>  static unsigned int __initdata dom0_num_colors;
>  
> +static unsigned int __ro_after_init xen_colors[CONFIG_NR_LLC_COLORS];

So unlike for Dom0 here you use the static buffer at runtime.

> +static unsigned int __ro_after_init xen_num_colors;

Taken together, I don't see the value in having XEN_DEFAULT_COLOR:
One can't simply change it and XEN_DEFAULT_NUM_COLORS to have Xen have,
say, 4 colors by default. I think you want to have xen_colors[] have
an initializer, with XEN_DEFAULT_COLOR dropped and XEN_DEFAULT_NUM_COLORS
moved. Or you actually allocate the runtime buffer if a command line
option is found, and just use ARRAY_SIZE() as the default count.

Jan

Patch

diff --git a/docs/misc/xen-command-line.pandoc b/docs/misc/xen-command-line.pandoc
index d52d38b97a..8c2b799f1e 100644
--- a/docs/misc/xen-command-line.pandoc
+++ b/docs/misc/xen-command-line.pandoc
@@ -2876,6 +2876,16 @@  mode.
 **WARNING: `x2apic_phys` is deprecated and superseded by `x2apic-mode`.
 The latter takes precedence if both are set.**
 
+### xen-llc-colors (arm64)
+> `= List of [ <integer> | <integer>-<integer> ]`
+
+> Default: `0: the lowermost color`
+
+Specify Xen LLC color configuration. This options is available only when
+`CONFIG_LLC_COLORING` is enabled.
+Two colors are most likely needed on platforms where private caches are
+physically indexed, e.g. the L1 instruction cache of the Arm Cortex-A57.
+
 ### xenheap_megabytes (arm32)
 > `= <size>`
 
diff --git a/xen/common/llc-coloring.c b/xen/common/llc-coloring.c
index 25e0861733..dace881b55 100644
--- a/xen/common/llc-coloring.c
+++ b/xen/common/llc-coloring.c
@@ -9,6 +9,9 @@ 
 #include <xen/llc-coloring.h>
 #include <xen/param.h>
 
+#define XEN_DEFAULT_COLOR       0
+#define XEN_DEFAULT_NUM_COLORS  1
+
 bool __ro_after_init llc_coloring_enabled;
 boolean_param("llc-coloring", llc_coloring_enabled);
 
@@ -21,6 +24,9 @@  static unsigned int __ro_after_init max_nr_colors = CONFIG_NR_LLC_COLORS;
 static unsigned int __initdata dom0_colors[CONFIG_NR_LLC_COLORS];
 static unsigned int __initdata dom0_num_colors;
 
+static unsigned int __ro_after_init xen_colors[CONFIG_NR_LLC_COLORS];
+static unsigned int __ro_after_init xen_num_colors;
+
 #define mfn_color_mask              (max_nr_colors - 1)
 #define mfn_to_color(mfn)           (mfn_x(mfn) & mfn_color_mask)
 
@@ -78,6 +84,12 @@  static int __init parse_dom0_colors(const char *s)
 }
 custom_param("dom0-llc-colors", parse_dom0_colors);
 
+static int __init parse_xen_colors(const char *s)
+{
+    return parse_color_config(s, xen_colors, max_nr_colors, &xen_num_colors);
+}
+custom_param("xen-llc-colors", parse_xen_colors);
+
 static void print_colors(const unsigned int *colors, unsigned int num_colors)
 {
     unsigned int i;
@@ -106,6 +118,8 @@  static void dump_coloring_info(unsigned char key)
     printk("'%c' pressed -> dumping LLC coloring general info\n", key);
     printk("LLC way size: %u KiB\n", llc_way_size >> 10);
     printk("Number of LLC colors supported: %u\n", max_nr_colors);
+    printk("Xen has %u LLC colors: ", xen_num_colors);
+    print_colors(xen_colors, xen_num_colors);
 }
 
 static bool check_colors(const unsigned int *colors, unsigned int num_colors)
@@ -149,6 +163,18 @@  void __init llc_coloring_init(void)
         panic("Number of LLC colors (%u) not in range [2, %u]\n",
               max_nr_colors, CONFIG_NR_LLC_COLORS);
 
+    if ( !xen_num_colors )
+    {
+        printk(XENLOG_WARNING
+               "Xen LLC color config not found. Using default color: %u\n",
+               XEN_DEFAULT_COLOR);
+        xen_colors[0] = XEN_DEFAULT_COLOR;
+        xen_num_colors = XEN_DEFAULT_NUM_COLORS;
+    }
+
+    if ( !check_colors(xen_colors, xen_num_colors) )
+        panic("Bad LLC color config for Xen\n");
+
     register_keyhandler('K', dump_coloring_info, "dump LLC coloring info", 1);
 
     arch_llc_coloring_init();