| Message ID | 20240102095138.17933-13-carlo.nonato@minervasys.tech (mailing list archive) |
|---|---|
| State | Superseded |
| Headers | show |
| Series | Arm cache coloring | expand |
Hi, On 02/01/2024 09:51, Carlo Nonato wrote: > From: Luca Miccio <lucmiccio@gmail.com> > > This commit adds a new command line parameter to configure Xen cache colors. > These colors can be dumped with the cache coloring info debug-key. > > By default, Xen uses the first color. > Benchmarking the VM interrupt response time provides an estimation of > LLC usage by Xen's most latency-critical runtime task. Results on Arm > Cortex-A53 on Xilinx Zynq UltraScale+ XCZU9EG show that one color, which > reserves 64 KiB of L2, is enough to attain best responsiveness. Would you be able to share some numbers? This is quite helpful if in the future we need to revise the default. > > More colors are instead very likely to be needed on processors whose L1 > cache is physically-indexed and physically-tagged, such as Cortex-A57. > In such cases, coloring applies to L1 also, and there typically are two > distinct L1-colors. Therefore, reserving only one color for Xen would > senselessly partitions a cache memory that is already private, i.e. > underutilize it. The default amount of Xen colors is thus set to one. > > Signed-off-by: Luca Miccio <lucmiccio@gmail.com> > Signed-off-by: Marco Solieri <marco.solieri@minervasys.tech> > Signed-off-by: Carlo Nonato <carlo.nonato@minervasys.tech> > --- > docs/misc/xen-command-line.pandoc | 10 ++++++++++ > xen/arch/arm/llc-coloring.c | 29 +++++++++++++++++++++++++++++ > 2 files changed, 39 insertions(+) > > diff --git a/docs/misc/xen-command-line.pandoc b/docs/misc/xen-command-line.pandoc > index 163fe7bcb1..f983f22796 100644 > --- a/docs/misc/xen-command-line.pandoc > +++ b/docs/misc/xen-command-line.pandoc > @@ -2877,6 +2877,16 @@ mode. > **WARNING: `x2apic_phys` is deprecated and superseded by `x2apic-mode`. > The latter takes precedence if both are set.** > > +### xen-llc-colors (arm64) > +> `= List of [ <integer> | <integer>-<integer> ]` > + > +> Default: `0: the lowermost color` > + > +Specify Xen LLC color configuration. This options is available only when > +`CONFIG_LLC_COLORING` is enabled. > +Two colors are most likely needed on platforms where private caches are > +physically indexed, e.g. the L1 instruction cache of the Arm Cortex-A57. > + > ### xenheap_megabytes (arm32) > > `= <size>` > > diff --git a/xen/arch/arm/llc-coloring.c b/xen/arch/arm/llc-coloring.c > index 526129cc43..99ea69ad39 100644 > --- a/xen/arch/arm/llc-coloring.c > +++ b/xen/arch/arm/llc-coloring.c > @@ -18,6 +18,9 @@ > #include <asm/processor.h> > #include <asm/sysregs.h> > > +#define XEN_DEFAULT_COLOR 0 > +#define XEN_DEFAULT_NUM_COLORS 1 > + > bool __ro_after_init llc_coloring_enabled; > boolean_param("llc-coloring", llc_coloring_enabled); > > @@ -30,6 +33,9 @@ static unsigned int __ro_after_init nr_colors = CONFIG_NR_LLC_COLORS; > static unsigned int __ro_after_init dom0_colors[CONFIG_NR_LLC_COLORS]; > static unsigned int __ro_after_init dom0_num_colors; > > +static unsigned int __ro_after_init xen_colors[CONFIG_NR_LLC_COLORS]; > +static unsigned int __ro_after_init xen_num_colors; > + > #define mfn_color_mask (nr_colors - 1) > #define mfn_to_color(mfn) (mfn_x(mfn) & mfn_color_mask) > > @@ -87,6 +93,12 @@ static int __init parse_dom0_colors(const char *s) > } > custom_param("dom0-llc-colors", parse_dom0_colors); > > +static int __init parse_xen_colors(const char *s) > +{ > + return parse_color_config(s, xen_colors, &xen_num_colors); > +} > +custom_param("xen-llc-colors", parse_xen_colors); > + > /* Return the LLC way size by probing the hardware */ > static unsigned int __init get_llc_way_size(void) > { > @@ -161,6 +173,8 @@ static void dump_coloring_info(unsigned char key) > printk("'%c' pressed -> dumping LLC coloring general info\n", key); > printk("LLC way size: %u KiB\n", llc_way_size >> 10); > printk("Number of LLC colors supported: %u\n", nr_colors); > + printk("Xen has %u LLC colors: ", xen_num_colors); > + print_colors(xen_colors, xen_num_colors); > } > > static bool check_colors(unsigned int *colors, unsigned int num_colors) > @@ -217,6 +231,21 @@ bool __init llc_coloring_init(void) > return false; > } > > + if ( !xen_num_colors ) > + { > + printk(XENLOG_WARNING > + "Xen LLC color config not found. Using default color: %u\n", > + XEN_DEFAULT_COLOR); > + xen_colors[0] = XEN_DEFAULT_COLOR; > + xen_num_colors = XEN_DEFAULT_NUM_COLORS; > + } > + > + if ( !check_colors(xen_colors, xen_num_colors) ) > + { > + printk(XENLOG_ERR "Bad LLC color config for Xen\n"); > + return false; > + } > + > register_keyhandler('K', dump_coloring_info, "dump LLC coloring info", 1); > > return true; Cheers,
On Fri, 5 Jan 2024, Julien Grall wrote: > Hi, > > On 02/01/2024 09:51, Carlo Nonato wrote: > > From: Luca Miccio <lucmiccio@gmail.com> > > > > This commit adds a new command line parameter to configure Xen cache colors. > > These colors can be dumped with the cache coloring info debug-key. > > > > By default, Xen uses the first color. > > Benchmarking the VM interrupt response time provides an estimation of > > LLC usage by Xen's most latency-critical runtime task. Results on Arm > > Cortex-A53 on Xilinx Zynq UltraScale+ XCZU9EG show that one color, which > > reserves 64 KiB of L2, is enough to attain best responsiveness. > > Would you be able to share some numbers? This is quite helpful if in the > future we need to revise the default. Here are the numbers for Xen 1 color vs Xen 2 colors. We are measuring IRQ lantecy using a baremetal app (a unikernel) that has 0.5 us latency on native without interference. Running the same application on Xen with 3 interference agents (3 other VMs that keep thrashing the cache): - Xen 1 color latency: 3.1 us - Xen 2 color2 latency: 3.1 us
Hi Stefano, On 05/01/2024 23:09, Stefano Stabellini wrote: > On Fri, 5 Jan 2024, Julien Grall wrote: >> Hi, >> >> On 02/01/2024 09:51, Carlo Nonato wrote: >>> From: Luca Miccio <lucmiccio@gmail.com> >>> >>> This commit adds a new command line parameter to configure Xen cache colors. >>> These colors can be dumped with the cache coloring info debug-key. >>> >>> By default, Xen uses the first color. >>> Benchmarking the VM interrupt response time provides an estimation of >>> LLC usage by Xen's most latency-critical runtime task. Results on Arm >>> Cortex-A53 on Xilinx Zynq UltraScale+ XCZU9EG show that one color, which >>> reserves 64 KiB of L2, is enough to attain best responsiveness. >> >> Would you be able to share some numbers? This is quite helpful if in the >> future we need to revise the default. > > Here are the numbers for Xen 1 color vs Xen 2 colors. > > We are measuring IRQ lantecy using a baremetal app (a unikernel) that > has 0.5 us latency on native without interference. > > Running the same application on Xen with 3 interference agents (3 other > VMs that keep thrashing the cache): > > - Xen 1 color latency: 3.1 us > - Xen 2 color2 latency: 3.1 us Thanks for sharing the numbers. Would it be possible to include them in the commit message? Cheers,
diff --git a/docs/misc/xen-command-line.pandoc b/docs/misc/xen-command-line.pandoc index 163fe7bcb1..f983f22796 100644 --- a/docs/misc/xen-command-line.pandoc +++ b/docs/misc/xen-command-line.pandoc @@ -2877,6 +2877,16 @@ mode. **WARNING: `x2apic_phys` is deprecated and superseded by `x2apic-mode`. The latter takes precedence if both are set.** +### xen-llc-colors (arm64) +> `= List of [ <integer> | <integer>-<integer> ]` + +> Default: `0: the lowermost color` + +Specify Xen LLC color configuration. This options is available only when +`CONFIG_LLC_COLORING` is enabled. +Two colors are most likely needed on platforms where private caches are +physically indexed, e.g. the L1 instruction cache of the Arm Cortex-A57. + ### xenheap_megabytes (arm32) > `= <size>` diff --git a/xen/arch/arm/llc-coloring.c b/xen/arch/arm/llc-coloring.c index 526129cc43..99ea69ad39 100644 --- a/xen/arch/arm/llc-coloring.c +++ b/xen/arch/arm/llc-coloring.c @@ -18,6 +18,9 @@ #include <asm/processor.h> #include <asm/sysregs.h> +#define XEN_DEFAULT_COLOR 0 +#define XEN_DEFAULT_NUM_COLORS 1 + bool __ro_after_init llc_coloring_enabled; boolean_param("llc-coloring", llc_coloring_enabled); @@ -30,6 +33,9 @@ static unsigned int __ro_after_init nr_colors = CONFIG_NR_LLC_COLORS; static unsigned int __ro_after_init dom0_colors[CONFIG_NR_LLC_COLORS]; static unsigned int __ro_after_init dom0_num_colors; +static unsigned int __ro_after_init xen_colors[CONFIG_NR_LLC_COLORS]; +static unsigned int __ro_after_init xen_num_colors; + #define mfn_color_mask (nr_colors - 1) #define mfn_to_color(mfn) (mfn_x(mfn) & mfn_color_mask) @@ -87,6 +93,12 @@ static int __init parse_dom0_colors(const char *s) } custom_param("dom0-llc-colors", parse_dom0_colors); +static int __init parse_xen_colors(const char *s) +{ + return parse_color_config(s, xen_colors, &xen_num_colors); +} +custom_param("xen-llc-colors", parse_xen_colors); + /* Return the LLC way size by probing the hardware */ static unsigned int __init get_llc_way_size(void) { @@ -161,6 +173,8 @@ static void dump_coloring_info(unsigned char key) printk("'%c' pressed -> dumping LLC coloring general info\n", key); printk("LLC way size: %u KiB\n", llc_way_size >> 10); printk("Number of LLC colors supported: %u\n", nr_colors); + printk("Xen has %u LLC colors: ", xen_num_colors); + print_colors(xen_colors, xen_num_colors); } static bool check_colors(unsigned int *colors, unsigned int num_colors) @@ -217,6 +231,21 @@ bool __init llc_coloring_init(void) return false; } + if ( !xen_num_colors ) + { + printk(XENLOG_WARNING + "Xen LLC color config not found. Using default color: %u\n", + XEN_DEFAULT_COLOR); + xen_colors[0] = XEN_DEFAULT_COLOR; + xen_num_colors = XEN_DEFAULT_NUM_COLORS; + } + + if ( !check_colors(xen_colors, xen_num_colors) ) + { + printk(XENLOG_ERR "Bad LLC color config for Xen\n"); + return false; + } + register_keyhandler('K', dump_coloring_info, "dump LLC coloring info", 1); return true;