[v2,4/7] x86: Make the maximum number of altp2m views configurable

Commit Message

Petr Beneš April 28, 2024, 4:52 p.m. UTC

From: Petr Beneš <[e-mail]>

This commit introduces the ability to configure the maximum number of altp2m
tables during domain creation. Previously, the limits were hardcoded to a
maximum of 10. This change allows for greater flexibility in environments that
require more or fewer altp2m views.

The maximum configurable limit for max_altp2m on x86 is now set to MAX_EPTP
(512). This cap is linked to the architectural limit of the EPTP-switching
VMFUNC, which supports up to 512 entries. Despite there being no inherent need
for limiting max_altp2m in scenarios not utilizing VMFUNC, decoupling these
components would necessitate substantial code changes.

Signed-off-by: Petr Beneš <[e-mail]>
---
Changed since v1:
  * Added xen_domctl_createdomain::max_altp2m field to xen/include/public/domctl.h
  * Bumped the XEN_DOMCTL_INTERFACE_VERSION
  * More elaborate commit message

 xen/arch/x86/domain.c             |  6 ++++
 xen/arch/x86/hvm/hvm.c            |  8 ++++-
 xen/arch/x86/hvm/vmx/vmx.c        |  2 +-
 xen/arch/x86/include/asm/domain.h |  7 ++---
 xen/arch/x86/include/asm/p2m.h    |  4 +--
 xen/arch/x86/mm/altp2m.c          | 27 +++++++++++++++--
 xen/arch/x86/mm/hap/hap.c         |  6 ++--
 xen/arch/x86/mm/mem_access.c      | 14 ++++-----
 xen/arch/x86/mm/mem_sharing.c     |  2 +-
 xen/arch/x86/mm/p2m-ept.c         |  6 ++--
 xen/arch/x86/mm/p2m.c             | 50 +++++++++++++++----------------
 xen/common/domain.c               |  7 +++++
 xen/include/public/domctl.h       |  3 +-
 xen/include/xen/sched.h           |  2 ++
 14 files changed, 94 insertions(+), 50 deletions(-)

--
2.34.1

Comments

Jan Beulich April 30, 2024, 2:27 p.m. UTC | #1

On 28.04.2024 18:52, Petr Beneš wrote:
> From: Petr Beneš <[e-mail]>
> 
> This commit introduces the ability to configure the maximum number of altp2m
> tables during domain creation. Previously, the limits were hardcoded to a
> maximum of 10. This change allows for greater flexibility in environments that
> require more or fewer altp2m views.
> 
> The maximum configurable limit for max_altp2m on x86 is now set to MAX_EPTP
> (512). This cap is linked to the architectural limit of the EPTP-switching
> VMFUNC, which supports up to 512 entries. Despite there being no inherent need
> for limiting max_altp2m in scenarios not utilizing VMFUNC, decoupling these
> components would necessitate substantial code changes.

While I don't mind this connection, ...

> --- a/xen/arch/x86/domain.c
> +++ b/xen/arch/x86/domain.c
> @@ -685,6 +685,12 @@ int arch_sanitise_domain_config(struct xen_domctl_createdomain *config)
>          return -EINVAL;
>      }
> 
> +    if ( config->max_altp2m > MAX_EPTP )
> +    {
> +        dprintk(XENLOG_INFO, "max_altp2m must be <= %u\n", MAX_EPTP);
> +        return -EINVAL;
> +    }

... using MAX_EPTP here feels like a layering violation to me. Imo there want
to be separate constants, tied together with a suitably placed BUILD_BUG_ON().

Furthermore comparisons like this (there are further ones elsewhere) suggest
there is a (continued) naming issue: A max_ or MAX_ prefix ought to name a
"maximum valid value", not "number of permitted values". This is not a
request to alter MAX_EPTP, but one to make sure the new struct fields really
have matching names and purposes.

> --- a/xen/arch/x86/include/asm/domain.h
> +++ b/xen/arch/x86/include/asm/domain.h
> @@ -258,11 +258,10 @@ struct paging_vcpu {
>      struct shadow_vcpu shadow;
>  };
> 
> +#define INVALID_ALTP2M  0xffff
> +#define MAX_EPTP        ((unsigned int)(PAGE_SIZE / sizeof(uint64_t)))

Aiui you add this cast because of the various min() uses. However, besides
wanting to avoid such casts (or in fact any, whenever possible), I don't
see why you need to retain all those min(): You bound d->max_altp2m against
MAX_EPTP during domain creation anyway.

> --- a/xen/arch/x86/mm/altp2m.c
> +++ b/xen/arch/x86/mm/altp2m.c
> @@ -13,6 +13,12 @@
>  void
>  altp2m_vcpu_initialise(struct vcpu *v)
>  {
> +    struct domain *d = v->domain;
> +
> +    /* Skip initialisation if no altp2m will be used. */
> +    if ( d->max_altp2m == 0 )
> +        return;

While I'm fine with this comment, ...

> @@ -28,8 +34,13 @@ altp2m_vcpu_initialise(struct vcpu *v)
>  void
>  altp2m_vcpu_destroy(struct vcpu *v)
>  {
> +    struct domain *d = v->domain;
>      struct p2m_domain *p2m;
> 
> +    /* Skip destruction if no altp2m was used. */
> +    if ( d->max_altp2m == 0 )
> +        return;

... this one doesn't look quite right: Even if altp2m-s were initialized,
none may have been used (yet).

> @@ -120,7 +131,13 @@ int p2m_init_altp2m(struct domain *d)
>      struct p2m_domain *hostp2m = p2m_get_hostp2m(d);
> 
>      mm_lock_init(&d->arch.altp2m_list_lock);
> -    for ( i = 0; i < MAX_ALTP2M; i++ )
> +
> +    if ( (d->arch.altp2m_p2m = xzalloc_array(struct p2m_domain *, d->max_altp2m)) == NULL )
> +    {
> +        return -ENOMEM;
> +    }

Nit: Overlong line and no need for the braces.

> +    for ( i = 0; i < d->max_altp2m; i++ )
>      {
>          d->arch.altp2m_p2m[i] = p2m = p2m_init_one(d);

This loop, btw, also demonstrates that "maximum" isn't true here. The
domain gets all of them allocated in one go.

> @@ -141,7 +158,10 @@ void p2m_teardown_altp2m(struct domain *d)
>      unsigned int i;
>      struct p2m_domain *p2m;
> 
> -    for ( i = 0; i < MAX_ALTP2M; i++ )
> +    if ( !d->arch.altp2m_p2m )
> +        return;
> +
> +    for ( i = 0; i < d->max_altp2m; i++ )
>      {
>          if ( !d->arch.altp2m_p2m[i] )
>              continue;
> @@ -149,6 +169,9 @@ void p2m_teardown_altp2m(struct domain *d)
>          d->arch.altp2m_p2m[i] = NULL;
>          p2m_free_one(p2m);
>      }
> +
> +    xfree(d->arch.altp2m_p2m);
> +    d->arch.altp2m_p2m = NULL;
>  }

Please don't (wrongly) open-code XFREE().

> @@ -2090,13 +2090,13 @@ int p2m_change_altp2m_gfn(struct domain *d, unsigned int idx,
>      mfn_t mfn;
>      int rc = -EINVAL;
> 
> -    if ( idx >=  min(ARRAY_SIZE(d->arch.altp2m_p2m), MAX_EPTP) ||
> +    if ( idx >=  min(d->max_altp2m, MAX_EPTP) ||

Nit: Please take the opportunity and remove the excess blank.

>           d->arch.altp2m_eptp[array_index_nospec(idx, MAX_EPTP)] ==

This use of MAX_EPTP also needs replacing, to avoid speculatively overrunning
the allocated array. At least one more instance elsewhere.

> @@ -2575,12 +2575,12 @@ int p2m_set_suppress_ve_multi(struct domain *d,
> 
>      if ( sve->view > 0 )
>      {
> -        if ( sve->view >= min(ARRAY_SIZE(d->arch.altp2m_p2m), MAX_EPTP) ||
> +        if ( sve->view >= min(d->max_altp2m, MAX_EPTP) ||
>               d->arch.altp2m_eptp[array_index_nospec(sve->view, MAX_EPTP)] ==
>               mfn_x(INVALID_MFN) )
>              return -EINVAL;
> 
> -        p2m = ap2m = array_access_nospec(d->arch.altp2m_p2m, sve->view);
> +        p2m = ap2m = d->arch.altp2m_p2m[array_index_nospec(sve->view, d->max_altp2m)];

Nit: Overlong line (more elsewhere).

> --- a/xen/common/domain.c
> +++ b/xen/common/domain.c
> @@ -568,6 +568,12 @@ static int sanitise_domain_config(struct xen_domctl_createdomain *config)
>          }
>      }
> 
> +    if ( config->max_altp2m && !hvm_altp2m_supported() )

This looks like it'll break the build on non-x86.

> --- a/xen/include/public/domctl.h
> +++ b/xen/include/public/domctl.h
> @@ -21,7 +21,7 @@
>  #include "hvm/save.h"
>  #include "memory.h"
> 
> -#define XEN_DOMCTL_INTERFACE_VERSION 0x00000016
> +#define XEN_DOMCTL_INTERFACE_VERSION 0x00000017
> 
>  /*
>   * NB. xen_domctl.domain is an IN/OUT parameter for this operation.
> @@ -77,6 +77,7 @@ struct xen_domctl_createdomain {
>       */
>      uint32_t max_vcpus;
>      uint32_t max_evtchn_port;
> +    uint32_t max_altp2m;
>      int32_t max_grant_frames;
>      int32_t max_maptrack_frames;

Both this and ...

> --- a/xen/include/xen/sched.h
> +++ b/xen/include/xen/sched.h
> @@ -602,6 +602,8 @@ struct domain
>          unsigned int guest_request_sync          : 1;
>      } monitor;
> 
> +    unsigned int max_altp2m; /* Maximum number of altp2m tables */
> +
>      unsigned int vmtrace_size; /* Buffer size in bytes, or 0 to disable. */

... this suggest you're confident other architectures will also want
to support altp2m. Yet nothing like that is said in the description.
In the absence of that common code should not require touching (much).

Jan

Petr Beneš April 30, 2024, 4 p.m. UTC | #2

On Tue, Apr 30, 2024 at 4:27 PM Jan Beulich <[e-mail]> wrote:
>
> > --- a/xen/arch/x86/domain.c
> > +++ b/xen/arch/x86/domain.c
> > @@ -685,6 +685,12 @@ int arch_sanitise_domain_config(struct xen_domctl_createdomain *config)
> >          return -EINVAL;
> >      }
> >
> > +    if ( config->max_altp2m > MAX_EPTP )
> > +    {
> > +        dprintk(XENLOG_INFO, "max_altp2m must be <= %u\n", MAX_EPTP);
> > +        return -EINVAL;
> > +    }
>
> ... using MAX_EPTP here feels like a layering violation to me. Imo there want
> to be separate constants, tied together with a suitably placed BUILD_BUG_ON().
>
> Furthermore comparisons like this (there are further ones elsewhere) suggest
> there is a (continued) naming issue: A max_ or MAX_ prefix ought to name a
> "maximum valid value", not "number of permitted values". This is not a
> request to alter MAX_EPTP, but one to make sure the new struct fields really
> have matching names and purposes.

Do you have any proposals? I was considering nr_altp2m. Another
question is what it should be named in xl.cfg - also nr_altp2m? I was
too hesitant to name it like that, since there aren't any nr_* fields
currently.

> > --- a/xen/arch/x86/include/asm/domain.h
> > +++ b/xen/arch/x86/include/asm/domain.h
> > @@ -258,11 +258,10 @@ struct paging_vcpu {
> >      struct shadow_vcpu shadow;
> >  };
> >
> > +#define INVALID_ALTP2M  0xffff
> > +#define MAX_EPTP        ((unsigned int)(PAGE_SIZE / sizeof(uint64_t)))
>
> Aiui you add this cast because of the various min() uses. However, besides
> wanting to avoid such casts (or in fact any, whenever possible), I don't
> see why you need to retain all those min(): You bound d->max_altp2m against
> MAX_EPTP during domain creation anyway.

Fair. I considered removing the min()s altogether. I left them in
place because I was too paranoid.

>
> > @@ -28,8 +34,13 @@ altp2m_vcpu_initialise(struct vcpu *v)
> >  void
> >  altp2m_vcpu_destroy(struct vcpu *v)
> >  {
> > +    struct domain *d = v->domain;
> >      struct p2m_domain *p2m;
> >
> > +    /* Skip destruction if no altp2m was used. */
> > +    if ( d->max_altp2m == 0 )
> > +        return;
>
> ... this one doesn't look quite right: Even if altp2m-s were initialized,
> none may have been used (yet).

Fair. Bad choice of words.

>
> > --- a/xen/include/xen/sched.h
> > +++ b/xen/include/xen/sched.h
> > @@ -602,6 +602,8 @@ struct domain
> >          unsigned int guest_request_sync          : 1;
> >      } monitor;
> >
> > +    unsigned int max_altp2m; /* Maximum number of altp2m tables */
> > +
> >      unsigned int vmtrace_size; /* Buffer size in bytes, or 0 to disable. */
>
> ... this suggest you're confident other architectures will also want
> to support altp2m. Yet nothing like that is said in the description.
> In the absence of that common code should not require touching (much).

Depends on the definition of "want". altp2m patches for arm/aarch64
were sent to Xen some 6 years ago but were unfortunately rejected. I
would very much welcome them.

I have added the field to domain instead of arch_domain simply because
it is not an arch-bound feature - similarly to vmtrace below, which
also doesn't have an equivalent implementation on other archs than x86
(yet).

As far as other comments/nits go - understood.

P.

Jan Beulich May 2, 2024, 6:19 a.m. UTC | #3

On 30.04.2024 18:00, Petr Beneš wrote:
> On Tue, Apr 30, 2024 at 4:27 PM Jan Beulich <[e-mail]> wrote:
>>> --- a/xen/arch/x86/domain.c
>>> +++ b/xen/arch/x86/domain.c
>>> @@ -685,6 +685,12 @@ int arch_sanitise_domain_config(struct xen_domctl_createdomain *config)
>>>          return -EINVAL;
>>>      }
>>>
>>> +    if ( config->max_altp2m > MAX_EPTP )
>>> +    {
>>> +        dprintk(XENLOG_INFO, "max_altp2m must be <= %u\n", MAX_EPTP);
>>> +        return -EINVAL;
>>> +    }
>>
>> ... using MAX_EPTP here feels like a layering violation to me. Imo there want
>> to be separate constants, tied together with a suitably placed BUILD_BUG_ON().
>>
>> Furthermore comparisons like this (there are further ones elsewhere) suggest
>> there is a (continued) naming issue: A max_ or MAX_ prefix ought to name a
>> "maximum valid value", not "number of permitted values". This is not a
>> request to alter MAX_EPTP, but one to make sure the new struct fields really
>> have matching names and purposes.
> 
> Do you have any proposals? I was considering nr_altp2m. Another
> question is what it should be named in xl.cfg - also nr_altp2m? I was
> too hesitant to name it like that, since there aren't any nr_* fields
> currently.

Internally nr_ or num_ are going to be fine. For xl whether either of
those would be, or maybe altp2ms= (along the lines of e.g. vcpus=), or
altp2m_count, or yet something else I simply don't know. That'll be
the maintainers there to help with.

Jan

Patch

diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c
index 20e83cf38b..6b458f330c 100644
--- a/xen/arch/x86/domain.c
+++ b/xen/arch/x86/domain.c
@@ -685,6 +685,12 @@  int arch_sanitise_domain_config(struct xen_domctl_createdomain *config)
         return -EINVAL;
     }

+    if ( config->max_altp2m > MAX_EPTP )
+    {
+        dprintk(XENLOG_INFO, "max_altp2m must be <= %u\n", MAX_EPTP);
+        return -EINVAL;
+    }
+
     if ( config->vmtrace_size )
     {
         unsigned int size = config->vmtrace_size;
diff --git a/xen/arch/x86/hvm/hvm.c b/xen/arch/x86/hvm/hvm.c
index 0ce45b177c..9b70fe7cfc 100644
--- a/xen/arch/x86/hvm/hvm.c
+++ b/xen/arch/x86/hvm/hvm.c
@@ -4633,6 +4633,12 @@  static int do_altp2m_op(
         goto out;
     }

+    if ( d->max_altp2m == 0 )
+    {
+        rc = -EINVAL;
+        goto out;
+    }
+
     if ( (rc = xsm_hvm_altp2mhvm_op(XSM_OTHER, d, mode, a.cmd)) )
         goto out;

@@ -5222,7 +5228,7 @@  void hvm_fast_singlestep(struct vcpu *v, uint16_t p2midx)
     if ( !hvm_is_singlestep_supported() )
         return;

-    if ( p2midx >= MAX_ALTP2M )
+    if ( p2midx >= v->domain->max_altp2m )
         return;

     v->arch.hvm.single_step = true;
diff --git a/xen/arch/x86/hvm/vmx/vmx.c b/xen/arch/x86/hvm/vmx/vmx.c
index 5f67a48592..8f57f3a7f5 100644
--- a/xen/arch/x86/hvm/vmx/vmx.c
+++ b/xen/arch/x86/hvm/vmx/vmx.c
@@ -4888,7 +4888,7 @@  bool asmlinkage vmx_vmenter_helper(const struct cpu_user_regs *regs)
         {
             unsigned int i;

-            for ( i = 0; i < MAX_ALTP2M; ++i )
+            for ( i = 0; i < currd->max_altp2m; ++i )
             {
                 if ( currd->arch.altp2m_eptp[i] == mfn_x(INVALID_MFN) )
                     continue;
diff --git a/xen/arch/x86/include/asm/domain.h b/xen/arch/x86/include/asm/domain.h
index f5daeb182b..5bb0bcae81 100644
--- a/xen/arch/x86/include/asm/domain.h
+++ b/xen/arch/x86/include/asm/domain.h
@@ -258,11 +258,10 @@  struct paging_vcpu {
     struct shadow_vcpu shadow;
 };

+#define INVALID_ALTP2M  0xffff
+#define MAX_EPTP        ((unsigned int)(PAGE_SIZE / sizeof(uint64_t)))
 #define MAX_NESTEDP2M 10

-#define MAX_ALTP2M      10 /* arbitrary */
-#define INVALID_ALTP2M  0xffff
-#define MAX_EPTP        (PAGE_SIZE / sizeof(uint64_t))
 struct p2m_domain;
 struct time_scale {
     int shift;
@@ -353,7 +352,7 @@  struct arch_domain

     /* altp2m: allow multiple copies of host p2m */
     bool altp2m_active;
-    struct p2m_domain *altp2m_p2m[MAX_ALTP2M];
+    struct p2m_domain **altp2m_p2m;
     mm_lock_t altp2m_list_lock;
     uint64_t *altp2m_eptp;
     uint64_t *altp2m_visible_eptp;
diff --git a/xen/arch/x86/include/asm/p2m.h b/xen/arch/x86/include/asm/p2m.h
index 111badf89a..2086bcb633 100644
--- a/xen/arch/x86/include/asm/p2m.h
+++ b/xen/arch/x86/include/asm/p2m.h
@@ -881,7 +881,7 @@  static inline struct p2m_domain *p2m_get_altp2m(struct vcpu *v)
     if ( index == INVALID_ALTP2M )
         return NULL;

-    BUG_ON(index >= MAX_ALTP2M);
+    BUG_ON(index >= v->domain->max_altp2m);

     return v->domain->arch.altp2m_p2m[index];
 }
@@ -891,7 +891,7 @@  static inline bool p2m_set_altp2m(struct vcpu *v, unsigned int idx)
 {
     struct p2m_domain *orig;

-    BUG_ON(idx >= MAX_ALTP2M);
+    BUG_ON(idx >= v->domain->max_altp2m);

     if ( idx == vcpu_altp2m(v).p2midx )
         return false;
diff --git a/xen/arch/x86/mm/altp2m.c b/xen/arch/x86/mm/altp2m.c
index a04297b646..c91e0fbfd1 100644
--- a/xen/arch/x86/mm/altp2m.c
+++ b/xen/arch/x86/mm/altp2m.c
@@ -13,6 +13,12 @@ 
 void
 altp2m_vcpu_initialise(struct vcpu *v)
 {
+    struct domain *d = v->domain;
+
+    /* Skip initialisation if no altp2m will be used. */
+    if ( d->max_altp2m == 0 )
+        return;
+
     if ( v != current )
         vcpu_pause(v);

@@ -28,8 +34,13 @@  altp2m_vcpu_initialise(struct vcpu *v)
 void
 altp2m_vcpu_destroy(struct vcpu *v)
 {
+    struct domain *d = v->domain;
     struct p2m_domain *p2m;

+    /* Skip destruction if no altp2m was used. */
+    if ( d->max_altp2m == 0 )
+        return;
+
     if ( v != current )
         vcpu_pause(v);

@@ -120,7 +131,13 @@  int p2m_init_altp2m(struct domain *d)
     struct p2m_domain *hostp2m = p2m_get_hostp2m(d);

     mm_lock_init(&d->arch.altp2m_list_lock);
-    for ( i = 0; i < MAX_ALTP2M; i++ )
+
+    if ( (d->arch.altp2m_p2m = xzalloc_array(struct p2m_domain *, d->max_altp2m)) == NULL )
+    {
+        return -ENOMEM;
+    }
+
+    for ( i = 0; i < d->max_altp2m; i++ )
     {
         d->arch.altp2m_p2m[i] = p2m = p2m_init_one(d);
         if ( p2m == NULL )
@@ -141,7 +158,10 @@  void p2m_teardown_altp2m(struct domain *d)
     unsigned int i;
     struct p2m_domain *p2m;

-    for ( i = 0; i < MAX_ALTP2M; i++ )
+    if ( !d->arch.altp2m_p2m )
+        return;
+
+    for ( i = 0; i < d->max_altp2m; i++ )
     {
         if ( !d->arch.altp2m_p2m[i] )
             continue;
@@ -149,6 +169,9 @@  void p2m_teardown_altp2m(struct domain *d)
         d->arch.altp2m_p2m[i] = NULL;
         p2m_free_one(p2m);
     }
+
+    xfree(d->arch.altp2m_p2m);
+    d->arch.altp2m_p2m = NULL;
 }

 /*
diff --git a/xen/arch/x86/mm/hap/hap.c b/xen/arch/x86/mm/hap/hap.c
index d2011fde24..7aff5fa664 100644
--- a/xen/arch/x86/mm/hap/hap.c
+++ b/xen/arch/x86/mm/hap/hap.c
@@ -515,7 +515,7 @@  int hap_enable(struct domain *d, u32 mode)
             d->arch.altp2m_visible_eptp[i] = mfn_x(INVALID_MFN);
         }

-        for ( i = 0; i < MAX_ALTP2M; i++ )
+        for ( i = 0; i < d->max_altp2m; i++ )
         {
             rv = p2m_alloc_table(d->arch.altp2m_p2m[i]);
             if ( rv != 0 )
@@ -538,7 +538,7 @@  void hap_final_teardown(struct domain *d)
     unsigned int i;

     if ( hvm_altp2m_supported() )
-        for ( i = 0; i < MAX_ALTP2M; i++ )
+        for ( i = 0; i < d->max_altp2m; i++ )
             p2m_teardown(d->arch.altp2m_p2m[i], true, NULL);

     /* Destroy nestedp2m's first */
@@ -590,7 +590,7 @@  void hap_teardown(struct domain *d, bool *preempted)
         FREE_XENHEAP_PAGE(d->arch.altp2m_eptp);
         FREE_XENHEAP_PAGE(d->arch.altp2m_visible_eptp);

-        for ( i = 0; i < MAX_ALTP2M; i++ )
+        for ( i = 0; i < d->max_altp2m; i++ )
         {
             p2m_teardown(d->arch.altp2m_p2m[i], false, preempted);
             if ( preempted && *preempted )
diff --git a/xen/arch/x86/mm/mem_access.c b/xen/arch/x86/mm/mem_access.c
index 60a0cce68a..1bf40cb746 100644
--- a/xen/arch/x86/mm/mem_access.c
+++ b/xen/arch/x86/mm/mem_access.c
@@ -347,12 +347,12 @@  long p2m_set_mem_access(struct domain *d, gfn_t gfn, uint32_t nr,
     /* altp2m view 0 is treated as the hostp2m */
     if ( altp2m_idx )
     {
-        if ( altp2m_idx >= min(ARRAY_SIZE(d->arch.altp2m_p2m), MAX_EPTP) ||
+        if ( altp2m_idx >= min(d->max_altp2m, MAX_EPTP) ||
              d->arch.altp2m_eptp[array_index_nospec(altp2m_idx, MAX_EPTP)] ==
              mfn_x(INVALID_MFN) )
             return -EINVAL;

-        ap2m = array_access_nospec(d->arch.altp2m_p2m, altp2m_idx);
+        ap2m = d->arch.altp2m_p2m[array_index_nospec(altp2m_idx, d->max_altp2m)];
     }

     if ( !xenmem_access_to_p2m_access(p2m, access, &a) )
@@ -403,12 +403,12 @@  long p2m_set_mem_access_multi(struct domain *d,
     /* altp2m view 0 is treated as the hostp2m */
     if ( altp2m_idx )
     {
-        if ( altp2m_idx >= min(ARRAY_SIZE(d->arch.altp2m_p2m), MAX_EPTP) ||
+        if ( altp2m_idx >= min(d->max_altp2m, MAX_EPTP) ||
              d->arch.altp2m_eptp[array_index_nospec(altp2m_idx, MAX_EPTP)] ==
              mfn_x(INVALID_MFN) )
             return -EINVAL;

-        ap2m = array_access_nospec(d->arch.altp2m_p2m, altp2m_idx);
+        ap2m = d->arch.altp2m_p2m[array_index_nospec(altp2m_idx, d->max_altp2m)];
     }

     p2m_lock(p2m);
@@ -466,12 +466,12 @@  int p2m_get_mem_access(struct domain *d, gfn_t gfn, xenmem_access_t *access,
     }
     else if ( altp2m_idx ) /* altp2m view 0 is treated as the hostp2m */
     {
-        if ( altp2m_idx >= min(ARRAY_SIZE(d->arch.altp2m_p2m), MAX_EPTP) ||
+        if ( altp2m_idx >= min(d->max_altp2m, MAX_EPTP) ||
              d->arch.altp2m_eptp[array_index_nospec(altp2m_idx, MAX_EPTP)] ==
              mfn_x(INVALID_MFN) )
             return -EINVAL;

-        p2m = array_access_nospec(d->arch.altp2m_p2m, altp2m_idx);
+        p2m = d->arch.altp2m_p2m[array_index_nospec(altp2m_idx, d->max_altp2m)];
     }

     return _p2m_get_mem_access(p2m, gfn, access);
@@ -486,7 +486,7 @@  void arch_p2m_set_access_required(struct domain *d, bool access_required)
     if ( altp2m_active(d) )
     {
         unsigned int i;
-        for ( i = 0; i < MAX_ALTP2M; i++ )
+        for ( i = 0; i < d->max_altp2m; i++ )
         {
             struct p2m_domain *p2m = d->arch.altp2m_p2m[i];

diff --git a/xen/arch/x86/mm/mem_sharing.c b/xen/arch/x86/mm/mem_sharing.c
index da28266ef0..3aaf1a3b8d 100644
--- a/xen/arch/x86/mm/mem_sharing.c
+++ b/xen/arch/x86/mm/mem_sharing.c
@@ -912,7 +912,7 @@  static int nominate_page(struct domain *d, gfn_t gfn,

         altp2m_list_lock(d);

-        for ( i = 0; i < MAX_ALTP2M; i++ )
+        for ( i = 0; i < d->max_altp2m; i++ )
         {
             ap2m = d->arch.altp2m_p2m[i];
             if ( !ap2m )
diff --git a/xen/arch/x86/mm/p2m-ept.c b/xen/arch/x86/mm/p2m-ept.c
index f83610cb8c..6b75fafd49 100644
--- a/xen/arch/x86/mm/p2m-ept.c
+++ b/xen/arch/x86/mm/p2m-ept.c
@@ -1293,7 +1293,7 @@  static void ept_set_ad_sync(struct domain *d, bool value)
     {
         unsigned int i;

-        for ( i = 0; i < MAX_ALTP2M; i++ )
+        for ( i = 0; i < d->max_altp2m; i++ )
         {
             struct p2m_domain *p2m;

@@ -1500,7 +1500,7 @@  void setup_ept_dump(void)

 void p2m_init_altp2m_ept(struct domain *d, unsigned int i)
 {
-    struct p2m_domain *p2m = array_access_nospec(d->arch.altp2m_p2m, i);
+    struct p2m_domain *p2m = d->arch.altp2m_p2m[array_index_nospec(i, d->max_altp2m)];
     struct p2m_domain *hostp2m = p2m_get_hostp2m(d);
     struct ept_data *ept;

@@ -1519,7 +1519,7 @@  unsigned int p2m_find_altp2m_by_eptp(struct domain *d, uint64_t eptp)

     altp2m_list_lock(d);

-    for ( i = 0; i < MAX_ALTP2M; i++ )
+    for ( i = 0; i < d->max_altp2m; i++ )
     {
         if ( d->arch.altp2m_eptp[i] == mfn_x(INVALID_MFN) )
             continue;
diff --git a/xen/arch/x86/mm/p2m.c b/xen/arch/x86/mm/p2m.c
index eb7996170d..a7144fc8e1 100644
--- a/xen/arch/x86/mm/p2m.c
+++ b/xen/arch/x86/mm/p2m.c
@@ -105,7 +105,7 @@  void p2m_change_entry_type_global(struct domain *d,
     {
         unsigned int i;

-        for ( i = 0; i < MAX_ALTP2M; i++ )
+        for ( i = 0; i < d->max_altp2m; i++ )
         {
             if ( d->arch.altp2m_eptp[i] != mfn_x(INVALID_MFN) )
             {
@@ -140,7 +140,7 @@  void p2m_memory_type_changed(struct domain *d)
     {
         unsigned int i;

-        for ( i = 0; i < MAX_ALTP2M; i++ )
+        for ( i = 0; i < d->max_altp2m; i++ )
         {
             if ( d->arch.altp2m_eptp[i] != mfn_x(INVALID_MFN) )
             {
@@ -913,7 +913,7 @@  void p2m_change_type_range(struct domain *d,
     {
         unsigned int i;

-        for ( i = 0; i < MAX_ALTP2M; i++ )
+        for ( i = 0; i < d->max_altp2m; i++ )
         {
             if ( d->arch.altp2m_eptp[i] != mfn_x(INVALID_MFN) )
             {
@@ -986,7 +986,7 @@  int p2m_finish_type_change(struct domain *d,
     {
         unsigned int i;

-        for ( i = 0; i < MAX_ALTP2M; i++ )
+        for ( i = 0; i < d->max_altp2m; i++ )
         {
             if ( d->arch.altp2m_eptp[i] != mfn_x(INVALID_MFN) )
             {
@@ -1780,7 +1780,7 @@  bool p2m_switch_vcpu_altp2m_by_id(struct vcpu *v, unsigned int idx)
     struct domain *d = v->domain;
     bool rc = false;

-    if ( idx >= MAX_ALTP2M )
+    if ( idx >= d->max_altp2m )
         return rc;

     altp2m_list_lock(d);
@@ -1886,8 +1886,8 @@  static void p2m_reset_altp2m(struct domain *d, unsigned int idx,
 {
     struct p2m_domain *p2m;

-    ASSERT(idx < MAX_ALTP2M);
-    p2m = array_access_nospec(d->arch.altp2m_p2m, idx);
+    ASSERT(idx < d->max_altp2m);
+    p2m = d->arch.altp2m_p2m[array_index_nospec(idx, d->max_altp2m)];

     p2m_lock(p2m);

@@ -1912,7 +1912,7 @@  void p2m_flush_altp2m(struct domain *d)

     altp2m_list_lock(d);

-    for ( i = 0; i < MAX_ALTP2M; i++ )
+    for ( i = 0; i < d->max_altp2m; i++ )
     {
         p2m_reset_altp2m(d, i, ALTP2M_DEACTIVATE);
         d->arch.altp2m_eptp[i] = mfn_x(INVALID_MFN);
@@ -1928,9 +1928,9 @@  static int p2m_activate_altp2m(struct domain *d, unsigned int idx,
     struct p2m_domain *hostp2m, *p2m;
     int rc;

-    ASSERT(idx < MAX_ALTP2M);
+    ASSERT(idx < d->max_altp2m);

-    p2m = array_access_nospec(d->arch.altp2m_p2m, idx);
+    p2m = d->arch.altp2m_p2m[array_index_nospec(idx, d->max_altp2m)];
     hostp2m = p2m_get_hostp2m(d);

     p2m_lock(p2m);
@@ -1968,7 +1968,7 @@  int p2m_init_altp2m_by_id(struct domain *d, unsigned int idx)
     int rc = -EINVAL;
     struct p2m_domain *hostp2m = p2m_get_hostp2m(d);

-    if ( idx >= min(ARRAY_SIZE(d->arch.altp2m_p2m), MAX_EPTP) )
+    if ( idx >= min(d->max_altp2m, MAX_EPTP) )
         return rc;

     altp2m_list_lock(d);
@@ -1995,7 +1995,7 @@  int p2m_init_next_altp2m(struct domain *d, uint16_t *idx,

     altp2m_list_lock(d);

-    for ( i = 0; i < MAX_ALTP2M; i++ )
+    for ( i = 0; i < d->max_altp2m; i++ )
     {
         if ( d->arch.altp2m_eptp[i] != mfn_x(INVALID_MFN) )
             continue;
@@ -2017,7 +2017,7 @@  int p2m_destroy_altp2m_by_id(struct domain *d, unsigned int idx)
     struct p2m_domain *p2m;
     int rc = -EBUSY;

-    if ( !idx || idx >= min(ARRAY_SIZE(d->arch.altp2m_p2m), MAX_EPTP) )
+    if ( !idx || idx >= min(d->max_altp2m, MAX_EPTP) )
         return rc;

     rc = domain_pause_except_self(d);
@@ -2030,7 +2030,7 @@  int p2m_destroy_altp2m_by_id(struct domain *d, unsigned int idx)
     if ( d->arch.altp2m_eptp[array_index_nospec(idx, MAX_EPTP)] !=
          mfn_x(INVALID_MFN) )
     {
-        p2m = array_access_nospec(d->arch.altp2m_p2m, idx);
+        p2m = d->arch.altp2m_p2m[array_index_nospec(idx, d->max_altp2m)];

         if ( !_atomic_read(p2m->active_vcpus) )
         {
@@ -2055,7 +2055,7 @@  int p2m_switch_domain_altp2m_by_id(struct domain *d, unsigned int idx)
     struct vcpu *v;
     int rc = -EINVAL;

-    if ( idx >= MAX_ALTP2M )
+    if ( idx >= d->max_altp2m )
         return rc;

     rc = domain_pause_except_self(d);
@@ -2090,13 +2090,13 @@  int p2m_change_altp2m_gfn(struct domain *d, unsigned int idx,
     mfn_t mfn;
     int rc = -EINVAL;

-    if ( idx >=  min(ARRAY_SIZE(d->arch.altp2m_p2m), MAX_EPTP) ||
+    if ( idx >=  min(d->max_altp2m, MAX_EPTP) ||
          d->arch.altp2m_eptp[array_index_nospec(idx, MAX_EPTP)] ==
          mfn_x(INVALID_MFN) )
         return rc;

     hp2m = p2m_get_hostp2m(d);
-    ap2m = array_access_nospec(d->arch.altp2m_p2m, idx);
+    ap2m = d->arch.altp2m_p2m[array_index_nospec(idx, d->max_altp2m)];

     p2m_lock(hp2m);
     p2m_lock(ap2m);
@@ -2152,7 +2152,7 @@  int p2m_altp2m_propagate_change(struct domain *d, gfn_t gfn,

     altp2m_list_lock(d);

-    for ( i = 0; i < MAX_ALTP2M; i++ )
+    for ( i = 0; i < d->max_altp2m; i++ )
     {
         p2m_type_t t;
         p2m_access_t a;
@@ -2175,7 +2175,7 @@  int p2m_altp2m_propagate_change(struct domain *d, gfn_t gfn,
             else
             {
                 /* At least 2 altp2m's impacted, so reset everything */
-                for ( i = 0; i < MAX_ALTP2M; i++ )
+                for ( i = 0; i < d->max_altp2m; i++ )
                 {
                     if ( i == last_reset_idx ||
                          d->arch.altp2m_eptp[i] == mfn_x(INVALID_MFN) )
@@ -2575,12 +2575,12 @@  int p2m_set_suppress_ve_multi(struct domain *d,

     if ( sve->view > 0 )
     {
-        if ( sve->view >= min(ARRAY_SIZE(d->arch.altp2m_p2m), MAX_EPTP) ||
+        if ( sve->view >= min(d->max_altp2m, MAX_EPTP) ||
              d->arch.altp2m_eptp[array_index_nospec(sve->view, MAX_EPTP)] ==
              mfn_x(INVALID_MFN) )
             return -EINVAL;

-        p2m = ap2m = array_access_nospec(d->arch.altp2m_p2m, sve->view);
+        p2m = ap2m = d->arch.altp2m_p2m[array_index_nospec(sve->view, d->max_altp2m)];
     }

     p2m_lock(host_p2m);
@@ -2643,12 +2643,12 @@  int p2m_get_suppress_ve(struct domain *d, gfn_t gfn, bool *suppress_ve,

     if ( altp2m_idx > 0 )
     {
-        if ( altp2m_idx >= min(ARRAY_SIZE(d->arch.altp2m_p2m), MAX_EPTP) ||
+        if ( altp2m_idx >= min(d->max_altp2m, MAX_EPTP) ||
              d->arch.altp2m_eptp[array_index_nospec(altp2m_idx, MAX_EPTP)] ==
              mfn_x(INVALID_MFN) )
             return -EINVAL;

-        p2m = ap2m = array_access_nospec(d->arch.altp2m_p2m, altp2m_idx);
+        p2m = ap2m = d->arch.altp2m_p2m[array_index_nospec(altp2m_idx, d->max_altp2m)];
     }
     else
         p2m = host_p2m;
@@ -2679,9 +2679,9 @@  int p2m_set_altp2m_view_visibility(struct domain *d, unsigned int altp2m_idx,

     /*
      * Eptp index is correlated with altp2m index and should not exceed
-     * min(MAX_ALTP2M, MAX_EPTP).
+     * min(d->max_altp2m, MAX_EPTP).
      */
-    if ( altp2m_idx >= min(ARRAY_SIZE(d->arch.altp2m_p2m), MAX_EPTP) ||
+    if ( altp2m_idx >= min(d->max_altp2m, MAX_EPTP) ||
          d->arch.altp2m_eptp[array_index_nospec(altp2m_idx, MAX_EPTP)] ==
          mfn_x(INVALID_MFN) )
         rc = -EINVAL;
diff --git a/xen/common/domain.c b/xen/common/domain.c
index 6773f7fb90..18785cc22a 100644
--- a/xen/common/domain.c
+++ b/xen/common/domain.c
@@ -568,6 +568,12 @@  static int sanitise_domain_config(struct xen_domctl_createdomain *config)
         }
     }

+    if ( config->max_altp2m && !hvm_altp2m_supported() )
+    {
+        dprintk(XENLOG_INFO, "altp2m requested but not available\n");
+        return -EINVAL;
+    }
+
     if ( config->vmtrace_size && !vmtrace_available )
     {
         dprintk(XENLOG_INFO, "vmtrace requested but not available\n");
@@ -610,6 +616,7 @@  struct domain *domain_create(domid_t domid,
     if ( config )
     {
         d->options = config->flags;
+        d->max_altp2m = config->max_altp2m;
         d->vmtrace_size = config->vmtrace_size;
     }

diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h
index a33f9ec32b..f9bd0ffe1a 100644
--- a/xen/include/public/domctl.h
+++ b/xen/include/public/domctl.h
@@ -21,7 +21,7 @@ 
 #include "hvm/save.h"
 #include "memory.h"

-#define XEN_DOMCTL_INTERFACE_VERSION 0x00000016
+#define XEN_DOMCTL_INTERFACE_VERSION 0x00000017

 /*
  * NB. xen_domctl.domain is an IN/OUT parameter for this operation.
@@ -77,6 +77,7 @@  struct xen_domctl_createdomain {
      */
     uint32_t max_vcpus;
     uint32_t max_evtchn_port;
+    uint32_t max_altp2m;
     int32_t max_grant_frames;
     int32_t max_maptrack_frames;

diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h
index 132b841995..46436fcb0b 100644
--- a/xen/include/xen/sched.h
+++ b/xen/include/xen/sched.h
@@ -602,6 +602,8 @@  struct domain
         unsigned int guest_request_sync          : 1;
     } monitor;

+    unsigned int max_altp2m; /* Maximum number of altp2m tables */
+
     unsigned int vmtrace_size; /* Buffer size in bytes, or 0 to disable. */

 #ifdef CONFIG_ARGO