diff mbox series

[v6,08/12] migration/multifd: Add new migration option for multifd DSA offloading.

Message ID 20241009234610.27039-9-yichen.wang@bytedance.com (mailing list archive)
State New, archived
Headers show
Series Use Intel DSA accelerator to offload zero page checking in multifd live migration. | expand

Commit Message

Yichen Wang Oct. 9, 2024, 11:46 p.m. UTC
From: Hao Xiang <hao.xiang@linux.dev>

Intel DSA offloading is an optional feature that turns on if
proper hardware and software stack is available. To turn on
DSA offloading in multifd live migration by setting:

zero-page-detection=dsa-accel
dsa-accel-path=[dsa_dev_path1] [dsa_dev_path2] ... [dsa_dev_pathX]

This feature is turned off by default.

Signed-off-by: Hao Xiang <hao.xiang@linux.dev>
Signed-off-by: Yichen Wang <yichen.wang@bytedance.com>
---
 hmp-commands.hx                |  2 +-
 include/qemu/dsa.h             | 13 +++++++++++++
 migration/migration-hmp-cmds.c | 19 ++++++++++++++++++-
 migration/options.c            | 30 ++++++++++++++++++++++++++++++
 migration/options.h            |  1 +
 qapi/migration.json            | 32 ++++++++++++++++++++++++++++----
 util/dsa.c                     | 31 +++++++++++++++++++++++++++++++
 7 files changed, 122 insertions(+), 6 deletions(-)

Comments

Dr. David Alan Gilbert Oct. 11, 2024, 5:14 p.m. UTC | #1
* Yichen Wang (yichen.wang@bytedance.com) wrote:
> From: Hao Xiang <hao.xiang@linux.dev>

Please split the cpuid stuff out into a separate patch; it feels like
it should be in some x86 specific place.

> Intel DSA offloading is an optional feature that turns on if
> proper hardware and software stack is available. To turn on
> DSA offloading in multifd live migration by setting:
> 
> zero-page-detection=dsa-accel
> dsa-accel-path=[dsa_dev_path1] [dsa_dev_path2] ... [dsa_dev_pathX]

I'd like to suggest changing that to:

accel-path=dsa:dev_path1 dsa:dev_path2 somethingelse:dev_path2 etc

that means we don't need a new option when someone adds a different
accelerator.

Dave
> This feature is turned off by default.
> 
> Signed-off-by: Hao Xiang <hao.xiang@linux.dev>
> Signed-off-by: Yichen Wang <yichen.wang@bytedance.com>
> ---
>  hmp-commands.hx                |  2 +-
>  include/qemu/dsa.h             | 13 +++++++++++++
>  migration/migration-hmp-cmds.c | 19 ++++++++++++++++++-
>  migration/options.c            | 30 ++++++++++++++++++++++++++++++
>  migration/options.h            |  1 +
>  qapi/migration.json            | 32 ++++++++++++++++++++++++++++----
>  util/dsa.c                     | 31 +++++++++++++++++++++++++++++++
>  7 files changed, 122 insertions(+), 6 deletions(-)
> 
> diff --git a/hmp-commands.hx b/hmp-commands.hx
> index 06746f0afc..0e04eac7c7 100644
> --- a/hmp-commands.hx
> +++ b/hmp-commands.hx
> @@ -1009,7 +1009,7 @@ ERST
>  
>      {
>          .name       = "migrate_set_parameter",
> -        .args_type  = "parameter:s,value:s",
> +        .args_type  = "parameter:s,value:S",

Can you show the case you need this for?

Dave

>          .params     = "parameter value",
>          .help       = "Set the parameter for migration",
>          .cmd        = hmp_migrate_set_parameter,
> diff --git a/include/qemu/dsa.h b/include/qemu/dsa.h
> index a3b502ee41..b1bb6daad2 100644
> --- a/include/qemu/dsa.h
> +++ b/include/qemu/dsa.h
> @@ -100,6 +100,13 @@ void qemu_dsa_stop(void);
>   */
>  void qemu_dsa_cleanup(void);
>  
> +/**
> + * @brief Check if DSA is supported.
> + *
> + * @return True if DSA is supported, otherwise false.
> + */
> +bool qemu_dsa_is_supported(void);
> +
>  /**
>   * @brief Check if DSA is running.
>   *
> @@ -141,6 +148,12 @@ buffer_is_zero_dsa_batch_sync(QemuDsaBatchTask *batch_task,
>  
>  typedef struct QemuDsaBatchTask {} QemuDsaBatchTask;
>  
> +static inline bool qemu_dsa_is_supported(void)
> +{
> +    return false;
> +}
> +
> +
>  static inline bool qemu_dsa_is_running(void)
>  {
>      return false;
> diff --git a/migration/migration-hmp-cmds.c b/migration/migration-hmp-cmds.c
> index 20d1a6e219..983f13b73c 100644
> --- a/migration/migration-hmp-cmds.c
> +++ b/migration/migration-hmp-cmds.c
> @@ -312,7 +312,16 @@ void hmp_info_migrate_parameters(Monitor *mon, const QDict *qdict)
>          monitor_printf(mon, "%s: '%s'\n",
>              MigrationParameter_str(MIGRATION_PARAMETER_TLS_AUTHZ),
>              params->tls_authz);
> -
> +        if (params->has_dsa_accel_path) {
> +            strList *dsa_accel_path = params->dsa_accel_path;
> +            monitor_printf(mon, "%s:",
> +                MigrationParameter_str(MIGRATION_PARAMETER_DSA_ACCEL_PATH));
> +            while (dsa_accel_path) {
> +                monitor_printf(mon, " '%s'", dsa_accel_path->value);
> +                dsa_accel_path = dsa_accel_path->next;
> +            }
> +            monitor_printf(mon, "\n");
> +        }
>          if (params->has_block_bitmap_mapping) {
>              const BitmapMigrationNodeAliasList *bmnal;
>  
> @@ -563,6 +572,14 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict)
>          p->has_x_checkpoint_delay = true;
>          visit_type_uint32(v, param, &p->x_checkpoint_delay, &err);
>          break;
> +    case MIGRATION_PARAMETER_DSA_ACCEL_PATH:
> +        p->has_dsa_accel_path = true;
> +        g_autofree char **strv = g_strsplit(valuestr ? : "", " ", -1);
> +        strList **tail = &p->dsa_accel_path;
> +        for (int i = 0; strv[i]; i++) {
> +            QAPI_LIST_APPEND(tail, strv[i]);
> +        }
> +        break;
>      case MIGRATION_PARAMETER_MULTIFD_CHANNELS:
>          p->has_multifd_channels = true;
>          visit_type_uint8(v, param, &p->multifd_channels, &err);
> diff --git a/migration/options.c b/migration/options.c
> index 147cd2b8fd..a0b3a7d291 100644
> --- a/migration/options.c
> +++ b/migration/options.c
> @@ -13,6 +13,7 @@
>  
>  #include "qemu/osdep.h"
>  #include "qemu/error-report.h"
> +#include "qemu/dsa.h"
>  #include "exec/target_page.h"
>  #include "qapi/clone-visitor.h"
>  #include "qapi/error.h"
> @@ -832,6 +833,13 @@ const char *migrate_tls_creds(void)
>      return s->parameters.tls_creds;
>  }
>  
> +const strList *migrate_dsa_accel_path(void)
> +{
> +    MigrationState *s = migrate_get_current();
> +
> +    return s->parameters.dsa_accel_path;
> +}
> +
>  const char *migrate_tls_hostname(void)
>  {
>      MigrationState *s = migrate_get_current();
> @@ -945,6 +953,8 @@ MigrationParameters *qmp_query_migrate_parameters(Error **errp)
>      params->zero_page_detection = s->parameters.zero_page_detection;
>      params->has_direct_io = true;
>      params->direct_io = s->parameters.direct_io;
> +    params->has_dsa_accel_path = true;
> +    params->dsa_accel_path = QAPI_CLONE(strList, s->parameters.dsa_accel_path);
>  
>      return params;
>  }
> @@ -953,6 +963,7 @@ void migrate_params_init(MigrationParameters *params)
>  {
>      params->tls_hostname = g_strdup("");
>      params->tls_creds = g_strdup("");
> +    params->dsa_accel_path = NULL;
>  
>      /* Set has_* up only for parameter checks */
>      params->has_throttle_trigger_threshold = true;
> @@ -1165,6 +1176,14 @@ bool migrate_params_check(MigrationParameters *params, Error **errp)
>          return false;
>      }
>  
> +    if (params->has_zero_page_detection &&
> +        params->zero_page_detection == ZERO_PAGE_DETECTION_DSA_ACCEL) {
> +        if (!qemu_dsa_is_supported()) {
> +            error_setg(errp, "DSA acceleration is not supported.");
> +            return false;
> +        }
> +    }
> +
>      return true;
>  }
>  
> @@ -1278,6 +1297,11 @@ static void migrate_params_test_apply(MigrateSetParameters *params,
>      if (params->has_direct_io) {
>          dest->direct_io = params->direct_io;
>      }
> +
> +    if (params->has_dsa_accel_path) {
> +        dest->has_dsa_accel_path = true;
> +        dest->dsa_accel_path = params->dsa_accel_path;
> +    }
>  }
>  
>  static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
> @@ -1410,6 +1434,12 @@ static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
>      if (params->has_direct_io) {
>          s->parameters.direct_io = params->direct_io;
>      }
> +    if (params->has_dsa_accel_path) {
> +        qapi_free_strList(s->parameters.dsa_accel_path);
> +        s->parameters.has_dsa_accel_path = true;
> +        s->parameters.dsa_accel_path =
> +            QAPI_CLONE(strList, params->dsa_accel_path);
> +    }
>  }
>  
>  void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp)
> diff --git a/migration/options.h b/migration/options.h
> index a0bd6edc06..8198b220bd 100644
> --- a/migration/options.h
> +++ b/migration/options.h
> @@ -86,6 +86,7 @@ const char *migrate_tls_creds(void);
>  const char *migrate_tls_hostname(void);
>  uint64_t migrate_xbzrle_cache_size(void);
>  ZeroPageDetection migrate_zero_page_detection(void);
> +const strList *migrate_dsa_accel_path(void);
>  
>  /* parameters helpers */
>  
> diff --git a/qapi/migration.json b/qapi/migration.json
> index b66cccf107..d8b42ceae6 100644
> --- a/qapi/migration.json
> +++ b/qapi/migration.json
> @@ -626,10 +626,14 @@
>  #     multifd migration is enabled, else in the main migration thread
>  #     as for @legacy.
>  #
> +# @dsa-accel: Perform zero page checking with the DSA accelerator
> +#     offloading in multifd sender thread if multifd migration is
> +#     enabled, else in the main migration thread as for @legacy.
> +#
>  # Since: 9.0
>  ##
>  { 'enum': 'ZeroPageDetection',
> -  'data': [ 'none', 'legacy', 'multifd' ] }
> +  'data': [ 'none', 'legacy', 'multifd', 'dsa-accel' ] }
>  
>  ##
>  # @BitmapMigrationBitmapAliasTransform:
> @@ -837,6 +841,12 @@
>  #     See description in @ZeroPageDetection.  Default is 'multifd'.
>  #     (since 9.0)
>  #
> +# @dsa-accel-path: If enabled, use DSA accelerator offloading for
> +#     certain memory operations. Enable DSA accelerator for zero
> +#     page detection offloading by setting the @zero-page-detection
> +#     to dsa-accel. This parameter defines the dsa device path, and
> +#     defaults to an empty list.  (Since 9.2)
> +#
>  # @direct-io: Open migration files with O_DIRECT when possible.  This
>  #     only has effect if the @mapped-ram capability is enabled.
>  #     (Since 9.1)
> @@ -855,7 +865,7 @@
>             'cpu-throttle-initial', 'cpu-throttle-increment',
>             'cpu-throttle-tailslow',
>             'tls-creds', 'tls-hostname', 'tls-authz', 'max-bandwidth',
> -           'avail-switchover-bandwidth', 'downtime-limit',
> +           'avail-switchover-bandwidth', 'downtime-limit', 'dsa-accel-path',
>             { 'name': 'x-checkpoint-delay', 'features': [ 'unstable' ] },
>             'multifd-channels',
>             'xbzrle-cache-size', 'max-postcopy-bandwidth',
> @@ -1018,6 +1028,12 @@
>  #     See description in @ZeroPageDetection.  Default is 'multifd'.
>  #     (since 9.0)
>  #
> +# @dsa-accel-path: If enabled, use DSA accelerator offloading for
> +#     certain memory operations. Enable DSA accelerator for zero
> +#     page detection offloading by setting the @zero-page-detection
> +#     to dsa-accel. This parameter defines the dsa device path, and
> +#     defaults to an empty list.  (Since 9.2)
> +#
>  # @direct-io: Open migration files with O_DIRECT when possible.  This
>  #     only has effect if the @mapped-ram capability is enabled.
>  #     (Since 9.1)
> @@ -1063,7 +1079,8 @@
>              '*vcpu-dirty-limit': 'uint64',
>              '*mode': 'MigMode',
>              '*zero-page-detection': 'ZeroPageDetection',
> -            '*direct-io': 'bool' } }
> +            '*direct-io': 'bool',
> +            '*dsa-accel-path': [ 'str' ] } }
>  
>  ##
>  # @migrate-set-parameters:
> @@ -1228,6 +1245,12 @@
>  #     See description in @ZeroPageDetection.  Default is 'multifd'.
>  #     (since 9.0)
>  #
> +# @dsa-accel-path: If enabled, use DSA accelerator offloading for
> +#     certain memory operations. Enable DSA accelerator for zero
> +#     page detection offloading by setting the @zero-page-detection
> +#     to dsa-accel. This parameter defines the dsa device path, and
> +#     defaults to an empty list.  (Since 9.2)
> +#
>  # @direct-io: Open migration files with O_DIRECT when possible.  This
>  #     only has effect if the @mapped-ram capability is enabled.
>  #     (Since 9.1)
> @@ -1270,7 +1293,8 @@
>              '*vcpu-dirty-limit': 'uint64',
>              '*mode': 'MigMode',
>              '*zero-page-detection': 'ZeroPageDetection',
> -            '*direct-io': 'bool' } }
> +            '*direct-io': 'bool',
> +            '*dsa-accel-path': [ 'str' ] } }
>  
>  ##
>  # @query-migrate-parameters:
> diff --git a/util/dsa.c b/util/dsa.c
> index cbaa47c360..eeede3c0c7 100644
> --- a/util/dsa.c
> +++ b/util/dsa.c
> @@ -23,6 +23,7 @@
>  #include "qemu/bswap.h"
>  #include "qemu/error-report.h"
>  #include "qemu/rcu.h"
> +#include <cpuid.h>
>  
>  #pragma GCC push_options
>  #pragma GCC target("enqcmd")
> @@ -691,6 +692,36 @@ static void dsa_completion_thread_stop(void *opaque)
>      qemu_sem_destroy(&thread_context->sem_init_done);
>  }
>  
> +/**
> + * @brief Check if DSA is supported.
> + *
> + * @return True if DSA is supported, otherwise false.
> + */
> +bool qemu_dsa_is_supported(void)
> +{
> +    /*
> +     * movdir64b is indicated by bit 28 of ecx in CPUID leaf 7, subleaf 0.
> +     * enqcmd is indicated by bit 29 of ecx in CPUID leaf 7, subleaf 0.
> +     * Doc: https://cdrdv2-public.intel.com/819680/architecture-instruction-\
> +     *      set-extensions-programming-reference.pdf
> +     */
> +    uint32_t eax, ebx, ecx, edx;
> +    bool movedirb_enabled;
> +    bool enqcmd_enabled;
> +
> +    __get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx);
> +    movedirb_enabled = (ecx >> 28) & 0x1;
> +    if (!movedirb_enabled) {
> +        return false;
> +    }
> +    enqcmd_enabled = (ecx >> 29) & 0x1;
> +    if (!enqcmd_enabled) {
> +        return false;
> +    }
> +
> +    return true;
> +}
> +
>  /**
>   * @brief Check if DSA is running.
>   *
> -- 
> Yichen Wang
>
Yichen Wang Oct. 15, 2024, 10:09 p.m. UTC | #2
On Fri, Oct 11, 2024 at 10:14 AM Dr. David Alan Gilbert
<dave@treblig.org> wrote:
>
> * Yichen Wang (yichen.wang@bytedance.com) wrote:
> > From: Hao Xiang <hao.xiang@linux.dev>
>
> Please split the cpuid stuff out into a separate patch; it feels like
> it should be in some x86 specific place.

DSA is an Intel feature/device, and it only makes sense on x86. I
mean, dsa.c already implies that it is a x86 specific feature. So I
guess keeping it in the dsa.c is a better option?

>
> > Intel DSA offloading is an optional feature that turns on if
> > proper hardware and software stack is available. To turn on
> > DSA offloading in multifd live migration by setting:
> >
> > zero-page-detection=dsa-accel
> > dsa-accel-path=[dsa_dev_path1] [dsa_dev_path2] ... [dsa_dev_pathX]
>
> I'd like to suggest changing that to:
>
> accel-path=dsa:dev_path1 dsa:dev_path2 somethingelse:dev_path2 etc
>
> that means we don't need a new option when someone adds a different
> accelerator.

It all depends on the implementation of "other accelerators". I am OK
with this change if you believe that is better.

>
> Dave
> > This feature is turned off by default.
> >
> > Signed-off-by: Hao Xiang <hao.xiang@linux.dev>
> > Signed-off-by: Yichen Wang <yichen.wang@bytedance.com>
> > ---
> >  hmp-commands.hx                |  2 +-
> >  include/qemu/dsa.h             | 13 +++++++++++++
> >  migration/migration-hmp-cmds.c | 19 ++++++++++++++++++-
> >  migration/options.c            | 30 ++++++++++++++++++++++++++++++
> >  migration/options.h            |  1 +
> >  qapi/migration.json            | 32 ++++++++++++++++++++++++++++----
> >  util/dsa.c                     | 31 +++++++++++++++++++++++++++++++
> >  7 files changed, 122 insertions(+), 6 deletions(-)
> >
> > diff --git a/hmp-commands.hx b/hmp-commands.hx
> > index 06746f0afc..0e04eac7c7 100644
> > --- a/hmp-commands.hx
> > +++ b/hmp-commands.hx
> > @@ -1009,7 +1009,7 @@ ERST
> >
> >      {
> >          .name       = "migrate_set_parameter",
> > -        .args_type  = "parameter:s,value:s",
> > +        .args_type  = "parameter:s,value:S",
>
> Can you show the case you need this for?
>

That is used for pass a strList in the QEMU CLI. Without this change,
I cannot do:
migrate_set_parameter dsa-accel-path /dev/dsa/wq0.1 /dev/dsa/wq0.1
and it will complain.

> Dave
>
> >          .params     = "parameter value",
> >          .help       = "Set the parameter for migration",
> >          .cmd        = hmp_migrate_set_parameter,
> > diff --git a/include/qemu/dsa.h b/include/qemu/dsa.h
> > index a3b502ee41..b1bb6daad2 100644
> > --- a/include/qemu/dsa.h
> > +++ b/include/qemu/dsa.h
> > @@ -100,6 +100,13 @@ void qemu_dsa_stop(void);
> >   */
> >  void qemu_dsa_cleanup(void);
> >
> > +/**
> > + * @brief Check if DSA is supported.
> > + *
> > + * @return True if DSA is supported, otherwise false.
> > + */
> > +bool qemu_dsa_is_supported(void);
> > +
> >  /**
> >   * @brief Check if DSA is running.
> >   *
> > @@ -141,6 +148,12 @@ buffer_is_zero_dsa_batch_sync(QemuDsaBatchTask *batch_task,
> >
> >  typedef struct QemuDsaBatchTask {} QemuDsaBatchTask;
> >
> > +static inline bool qemu_dsa_is_supported(void)
> > +{
> > +    return false;
> > +}
> > +
> > +
> >  static inline bool qemu_dsa_is_running(void)
> >  {
> >      return false;
> > diff --git a/migration/migration-hmp-cmds.c b/migration/migration-hmp-cmds.c
> > index 20d1a6e219..983f13b73c 100644
> > --- a/migration/migration-hmp-cmds.c
> > +++ b/migration/migration-hmp-cmds.c
> > @@ -312,7 +312,16 @@ void hmp_info_migrate_parameters(Monitor *mon, const QDict *qdict)
> >          monitor_printf(mon, "%s: '%s'\n",
> >              MigrationParameter_str(MIGRATION_PARAMETER_TLS_AUTHZ),
> >              params->tls_authz);
> > -
> > +        if (params->has_dsa_accel_path) {
> > +            strList *dsa_accel_path = params->dsa_accel_path;
> > +            monitor_printf(mon, "%s:",
> > +                MigrationParameter_str(MIGRATION_PARAMETER_DSA_ACCEL_PATH));
> > +            while (dsa_accel_path) {
> > +                monitor_printf(mon, " '%s'", dsa_accel_path->value);
> > +                dsa_accel_path = dsa_accel_path->next;
> > +            }
> > +            monitor_printf(mon, "\n");
> > +        }
> >          if (params->has_block_bitmap_mapping) {
> >              const BitmapMigrationNodeAliasList *bmnal;
> >
> > @@ -563,6 +572,14 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict)
> >          p->has_x_checkpoint_delay = true;
> >          visit_type_uint32(v, param, &p->x_checkpoint_delay, &err);
> >          break;
> > +    case MIGRATION_PARAMETER_DSA_ACCEL_PATH:
> > +        p->has_dsa_accel_path = true;
> > +        g_autofree char **strv = g_strsplit(valuestr ? : "", " ", -1);
> > +        strList **tail = &p->dsa_accel_path;
> > +        for (int i = 0; strv[i]; i++) {
> > +            QAPI_LIST_APPEND(tail, strv[i]);
> > +        }
> > +        break;
> >      case MIGRATION_PARAMETER_MULTIFD_CHANNELS:
> >          p->has_multifd_channels = true;
> >          visit_type_uint8(v, param, &p->multifd_channels, &err);
> > diff --git a/migration/options.c b/migration/options.c
> > index 147cd2b8fd..a0b3a7d291 100644
> > --- a/migration/options.c
> > +++ b/migration/options.c
> > @@ -13,6 +13,7 @@
> >
> >  #include "qemu/osdep.h"
> >  #include "qemu/error-report.h"
> > +#include "qemu/dsa.h"
> >  #include "exec/target_page.h"
> >  #include "qapi/clone-visitor.h"
> >  #include "qapi/error.h"
> > @@ -832,6 +833,13 @@ const char *migrate_tls_creds(void)
> >      return s->parameters.tls_creds;
> >  }
> >
> > +const strList *migrate_dsa_accel_path(void)
> > +{
> > +    MigrationState *s = migrate_get_current();
> > +
> > +    return s->parameters.dsa_accel_path;
> > +}
> > +
> >  const char *migrate_tls_hostname(void)
> >  {
> >      MigrationState *s = migrate_get_current();
> > @@ -945,6 +953,8 @@ MigrationParameters *qmp_query_migrate_parameters(Error **errp)
> >      params->zero_page_detection = s->parameters.zero_page_detection;
> >      params->has_direct_io = true;
> >      params->direct_io = s->parameters.direct_io;
> > +    params->has_dsa_accel_path = true;
> > +    params->dsa_accel_path = QAPI_CLONE(strList, s->parameters.dsa_accel_path);
> >
> >      return params;
> >  }
> > @@ -953,6 +963,7 @@ void migrate_params_init(MigrationParameters *params)
> >  {
> >      params->tls_hostname = g_strdup("");
> >      params->tls_creds = g_strdup("");
> > +    params->dsa_accel_path = NULL;
> >
> >      /* Set has_* up only for parameter checks */
> >      params->has_throttle_trigger_threshold = true;
> > @@ -1165,6 +1176,14 @@ bool migrate_params_check(MigrationParameters *params, Error **errp)
> >          return false;
> >      }
> >
> > +    if (params->has_zero_page_detection &&
> > +        params->zero_page_detection == ZERO_PAGE_DETECTION_DSA_ACCEL) {
> > +        if (!qemu_dsa_is_supported()) {
> > +            error_setg(errp, "DSA acceleration is not supported.");
> > +            return false;
> > +        }
> > +    }
> > +
> >      return true;
> >  }
> >
> > @@ -1278,6 +1297,11 @@ static void migrate_params_test_apply(MigrateSetParameters *params,
> >      if (params->has_direct_io) {
> >          dest->direct_io = params->direct_io;
> >      }
> > +
> > +    if (params->has_dsa_accel_path) {
> > +        dest->has_dsa_accel_path = true;
> > +        dest->dsa_accel_path = params->dsa_accel_path;
> > +    }
> >  }
> >
> >  static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
> > @@ -1410,6 +1434,12 @@ static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
> >      if (params->has_direct_io) {
> >          s->parameters.direct_io = params->direct_io;
> >      }
> > +    if (params->has_dsa_accel_path) {
> > +        qapi_free_strList(s->parameters.dsa_accel_path);
> > +        s->parameters.has_dsa_accel_path = true;
> > +        s->parameters.dsa_accel_path =
> > +            QAPI_CLONE(strList, params->dsa_accel_path);
> > +    }
> >  }
> >
> >  void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp)
> > diff --git a/migration/options.h b/migration/options.h
> > index a0bd6edc06..8198b220bd 100644
> > --- a/migration/options.h
> > +++ b/migration/options.h
> > @@ -86,6 +86,7 @@ const char *migrate_tls_creds(void);
> >  const char *migrate_tls_hostname(void);
> >  uint64_t migrate_xbzrle_cache_size(void);
> >  ZeroPageDetection migrate_zero_page_detection(void);
> > +const strList *migrate_dsa_accel_path(void);
> >
> >  /* parameters helpers */
> >
> > diff --git a/qapi/migration.json b/qapi/migration.json
> > index b66cccf107..d8b42ceae6 100644
> > --- a/qapi/migration.json
> > +++ b/qapi/migration.json
> > @@ -626,10 +626,14 @@
> >  #     multifd migration is enabled, else in the main migration thread
> >  #     as for @legacy.
> >  #
> > +# @dsa-accel: Perform zero page checking with the DSA accelerator
> > +#     offloading in multifd sender thread if multifd migration is
> > +#     enabled, else in the main migration thread as for @legacy.
> > +#
> >  # Since: 9.0
> >  ##
> >  { 'enum': 'ZeroPageDetection',
> > -  'data': [ 'none', 'legacy', 'multifd' ] }
> > +  'data': [ 'none', 'legacy', 'multifd', 'dsa-accel' ] }
> >
> >  ##
> >  # @BitmapMigrationBitmapAliasTransform:
> > @@ -837,6 +841,12 @@
> >  #     See description in @ZeroPageDetection.  Default is 'multifd'.
> >  #     (since 9.0)
> >  #
> > +# @dsa-accel-path: If enabled, use DSA accelerator offloading for
> > +#     certain memory operations. Enable DSA accelerator for zero
> > +#     page detection offloading by setting the @zero-page-detection
> > +#     to dsa-accel. This parameter defines the dsa device path, and
> > +#     defaults to an empty list.  (Since 9.2)
> > +#
> >  # @direct-io: Open migration files with O_DIRECT when possible.  This
> >  #     only has effect if the @mapped-ram capability is enabled.
> >  #     (Since 9.1)
> > @@ -855,7 +865,7 @@
> >             'cpu-throttle-initial', 'cpu-throttle-increment',
> >             'cpu-throttle-tailslow',
> >             'tls-creds', 'tls-hostname', 'tls-authz', 'max-bandwidth',
> > -           'avail-switchover-bandwidth', 'downtime-limit',
> > +           'avail-switchover-bandwidth', 'downtime-limit', 'dsa-accel-path',
> >             { 'name': 'x-checkpoint-delay', 'features': [ 'unstable' ] },
> >             'multifd-channels',
> >             'xbzrle-cache-size', 'max-postcopy-bandwidth',
> > @@ -1018,6 +1028,12 @@
> >  #     See description in @ZeroPageDetection.  Default is 'multifd'.
> >  #     (since 9.0)
> >  #
> > +# @dsa-accel-path: If enabled, use DSA accelerator offloading for
> > +#     certain memory operations. Enable DSA accelerator for zero
> > +#     page detection offloading by setting the @zero-page-detection
> > +#     to dsa-accel. This parameter defines the dsa device path, and
> > +#     defaults to an empty list.  (Since 9.2)
> > +#
> >  # @direct-io: Open migration files with O_DIRECT when possible.  This
> >  #     only has effect if the @mapped-ram capability is enabled.
> >  #     (Since 9.1)
> > @@ -1063,7 +1079,8 @@
> >              '*vcpu-dirty-limit': 'uint64',
> >              '*mode': 'MigMode',
> >              '*zero-page-detection': 'ZeroPageDetection',
> > -            '*direct-io': 'bool' } }
> > +            '*direct-io': 'bool',
> > +            '*dsa-accel-path': [ 'str' ] } }
> >
> >  ##
> >  # @migrate-set-parameters:
> > @@ -1228,6 +1245,12 @@
> >  #     See description in @ZeroPageDetection.  Default is 'multifd'.
> >  #     (since 9.0)
> >  #
> > +# @dsa-accel-path: If enabled, use DSA accelerator offloading for
> > +#     certain memory operations. Enable DSA accelerator for zero
> > +#     page detection offloading by setting the @zero-page-detection
> > +#     to dsa-accel. This parameter defines the dsa device path, and
> > +#     defaults to an empty list.  (Since 9.2)
> > +#
> >  # @direct-io: Open migration files with O_DIRECT when possible.  This
> >  #     only has effect if the @mapped-ram capability is enabled.
> >  #     (Since 9.1)
> > @@ -1270,7 +1293,8 @@
> >              '*vcpu-dirty-limit': 'uint64',
> >              '*mode': 'MigMode',
> >              '*zero-page-detection': 'ZeroPageDetection',
> > -            '*direct-io': 'bool' } }
> > +            '*direct-io': 'bool',
> > +            '*dsa-accel-path': [ 'str' ] } }
> >
> >  ##
> >  # @query-migrate-parameters:
> > diff --git a/util/dsa.c b/util/dsa.c
> > index cbaa47c360..eeede3c0c7 100644
> > --- a/util/dsa.c
> > +++ b/util/dsa.c
> > @@ -23,6 +23,7 @@
> >  #include "qemu/bswap.h"
> >  #include "qemu/error-report.h"
> >  #include "qemu/rcu.h"
> > +#include <cpuid.h>
> >
> >  #pragma GCC push_options
> >  #pragma GCC target("enqcmd")
> > @@ -691,6 +692,36 @@ static void dsa_completion_thread_stop(void *opaque)
> >      qemu_sem_destroy(&thread_context->sem_init_done);
> >  }
> >
> > +/**
> > + * @brief Check if DSA is supported.
> > + *
> > + * @return True if DSA is supported, otherwise false.
> > + */
> > +bool qemu_dsa_is_supported(void)
> > +{
> > +    /*
> > +     * movdir64b is indicated by bit 28 of ecx in CPUID leaf 7, subleaf 0.
> > +     * enqcmd is indicated by bit 29 of ecx in CPUID leaf 7, subleaf 0.
> > +     * Doc: https://cdrdv2-public.intel.com/819680/architecture-instruction-\
> > +     *      set-extensions-programming-reference.pdf
> > +     */
> > +    uint32_t eax, ebx, ecx, edx;
> > +    bool movedirb_enabled;
> > +    bool enqcmd_enabled;
> > +
> > +    __get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx);
> > +    movedirb_enabled = (ecx >> 28) & 0x1;
> > +    if (!movedirb_enabled) {
> > +        return false;
> > +    }
> > +    enqcmd_enabled = (ecx >> 29) & 0x1;
> > +    if (!enqcmd_enabled) {
> > +        return false;
> > +    }
> > +
> > +    return true;
> > +}
> > +
> >  /**
> >   * @brief Check if DSA is running.
> >   *
> > --
> > Yichen Wang
> >
> --
>  -----Open up your eyes, open up your mind, open up your code -------
> / Dr. David Alan Gilbert    |       Running GNU/Linux       | Happy  \
> \        dave @ treblig.org |                               | In Hex /
>  \ _________________________|_____ http://www.treblig.org   |_______/
Dr. David Alan Gilbert Oct. 15, 2024, 10:51 p.m. UTC | #3
* Yichen Wang (yichen.wang@bytedance.com) wrote:
> On Fri, Oct 11, 2024 at 10:14 AM Dr. David Alan Gilbert
> <dave@treblig.org> wrote:
> >
> > * Yichen Wang (yichen.wang@bytedance.com) wrote:
> > > From: Hao Xiang <hao.xiang@linux.dev>
> >
> > Please split the cpuid stuff out into a separate patch; it feels like
> > it should be in some x86 specific place.
> 
> DSA is an Intel feature/device, and it only makes sense on x86. I
> mean, dsa.c already implies that it is a x86 specific feature. So I
> guess keeping it in the dsa.c is a better option?

Yes, the dsa.c might be OK; we just shouldn't have very x86 specific
code in the other places.

> >
> > > Intel DSA offloading is an optional feature that turns on if
> > > proper hardware and software stack is available. To turn on
> > > DSA offloading in multifd live migration by setting:
> > >
> > > zero-page-detection=dsa-accel
> > > dsa-accel-path=[dsa_dev_path1] [dsa_dev_path2] ... [dsa_dev_pathX]
> >
> > I'd like to suggest changing that to:
> >
> > accel-path=dsa:dev_path1 dsa:dev_path2 somethingelse:dev_path2 etc
> >
> > that means we don't need a new option when someone adds a different
> > accelerator.
> 
> It all depends on the implementation of "other accelerators". I am OK
> with this change if you believe that is better.

I think so; it makes no assumption about what any other accelerator might
need, but means we don't need to add yet another command for them.

> >
> > Dave
> > > This feature is turned off by default.
> > >
> > > Signed-off-by: Hao Xiang <hao.xiang@linux.dev>
> > > Signed-off-by: Yichen Wang <yichen.wang@bytedance.com>
> > > ---
> > >  hmp-commands.hx                |  2 +-
> > >  include/qemu/dsa.h             | 13 +++++++++++++
> > >  migration/migration-hmp-cmds.c | 19 ++++++++++++++++++-
> > >  migration/options.c            | 30 ++++++++++++++++++++++++++++++
> > >  migration/options.h            |  1 +
> > >  qapi/migration.json            | 32 ++++++++++++++++++++++++++++----
> > >  util/dsa.c                     | 31 +++++++++++++++++++++++++++++++
> > >  7 files changed, 122 insertions(+), 6 deletions(-)
> > >
> > > diff --git a/hmp-commands.hx b/hmp-commands.hx
> > > index 06746f0afc..0e04eac7c7 100644
> > > --- a/hmp-commands.hx
> > > +++ b/hmp-commands.hx
> > > @@ -1009,7 +1009,7 @@ ERST
> > >
> > >      {
> > >          .name       = "migrate_set_parameter",
> > > -        .args_type  = "parameter:s,value:s",
> > > +        .args_type  = "parameter:s,value:S",
> >
> > Can you show the case you need this for?
> >
> 
> That is used for pass a strList in the QEMU CLI. Without this change,
> I cannot do:
> migrate_set_parameter dsa-accel-path /dev/dsa/wq0.1 /dev/dsa/wq0.1
> and it will complain.

OK, please make sure you get a sensible error if you try and pass
a string list to one of the existing parameters.

Dave

> > Dave
> >
> > >          .params     = "parameter value",
> > >          .help       = "Set the parameter for migration",
> > >          .cmd        = hmp_migrate_set_parameter,
> > > diff --git a/include/qemu/dsa.h b/include/qemu/dsa.h
> > > index a3b502ee41..b1bb6daad2 100644
> > > --- a/include/qemu/dsa.h
> > > +++ b/include/qemu/dsa.h
> > > @@ -100,6 +100,13 @@ void qemu_dsa_stop(void);
> > >   */
> > >  void qemu_dsa_cleanup(void);
> > >
> > > +/**
> > > + * @brief Check if DSA is supported.
> > > + *
> > > + * @return True if DSA is supported, otherwise false.
> > > + */
> > > +bool qemu_dsa_is_supported(void);
> > > +
> > >  /**
> > >   * @brief Check if DSA is running.
> > >   *
> > > @@ -141,6 +148,12 @@ buffer_is_zero_dsa_batch_sync(QemuDsaBatchTask *batch_task,
> > >
> > >  typedef struct QemuDsaBatchTask {} QemuDsaBatchTask;
> > >
> > > +static inline bool qemu_dsa_is_supported(void)
> > > +{
> > > +    return false;
> > > +}
> > > +
> > > +
> > >  static inline bool qemu_dsa_is_running(void)
> > >  {
> > >      return false;
> > > diff --git a/migration/migration-hmp-cmds.c b/migration/migration-hmp-cmds.c
> > > index 20d1a6e219..983f13b73c 100644
> > > --- a/migration/migration-hmp-cmds.c
> > > +++ b/migration/migration-hmp-cmds.c
> > > @@ -312,7 +312,16 @@ void hmp_info_migrate_parameters(Monitor *mon, const QDict *qdict)
> > >          monitor_printf(mon, "%s: '%s'\n",
> > >              MigrationParameter_str(MIGRATION_PARAMETER_TLS_AUTHZ),
> > >              params->tls_authz);
> > > -
> > > +        if (params->has_dsa_accel_path) {
> > > +            strList *dsa_accel_path = params->dsa_accel_path;
> > > +            monitor_printf(mon, "%s:",
> > > +                MigrationParameter_str(MIGRATION_PARAMETER_DSA_ACCEL_PATH));
> > > +            while (dsa_accel_path) {
> > > +                monitor_printf(mon, " '%s'", dsa_accel_path->value);
> > > +                dsa_accel_path = dsa_accel_path->next;
> > > +            }
> > > +            monitor_printf(mon, "\n");
> > > +        }
> > >          if (params->has_block_bitmap_mapping) {
> > >              const BitmapMigrationNodeAliasList *bmnal;
> > >
> > > @@ -563,6 +572,14 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict)
> > >          p->has_x_checkpoint_delay = true;
> > >          visit_type_uint32(v, param, &p->x_checkpoint_delay, &err);
> > >          break;
> > > +    case MIGRATION_PARAMETER_DSA_ACCEL_PATH:
> > > +        p->has_dsa_accel_path = true;
> > > +        g_autofree char **strv = g_strsplit(valuestr ? : "", " ", -1);
> > > +        strList **tail = &p->dsa_accel_path;
> > > +        for (int i = 0; strv[i]; i++) {
> > > +            QAPI_LIST_APPEND(tail, strv[i]);
> > > +        }
> > > +        break;
> > >      case MIGRATION_PARAMETER_MULTIFD_CHANNELS:
> > >          p->has_multifd_channels = true;
> > >          visit_type_uint8(v, param, &p->multifd_channels, &err);
> > > diff --git a/migration/options.c b/migration/options.c
> > > index 147cd2b8fd..a0b3a7d291 100644
> > > --- a/migration/options.c
> > > +++ b/migration/options.c
> > > @@ -13,6 +13,7 @@
> > >
> > >  #include "qemu/osdep.h"
> > >  #include "qemu/error-report.h"
> > > +#include "qemu/dsa.h"
> > >  #include "exec/target_page.h"
> > >  #include "qapi/clone-visitor.h"
> > >  #include "qapi/error.h"
> > > @@ -832,6 +833,13 @@ const char *migrate_tls_creds(void)
> > >      return s->parameters.tls_creds;
> > >  }
> > >
> > > +const strList *migrate_dsa_accel_path(void)
> > > +{
> > > +    MigrationState *s = migrate_get_current();
> > > +
> > > +    return s->parameters.dsa_accel_path;
> > > +}
> > > +
> > >  const char *migrate_tls_hostname(void)
> > >  {
> > >      MigrationState *s = migrate_get_current();
> > > @@ -945,6 +953,8 @@ MigrationParameters *qmp_query_migrate_parameters(Error **errp)
> > >      params->zero_page_detection = s->parameters.zero_page_detection;
> > >      params->has_direct_io = true;
> > >      params->direct_io = s->parameters.direct_io;
> > > +    params->has_dsa_accel_path = true;
> > > +    params->dsa_accel_path = QAPI_CLONE(strList, s->parameters.dsa_accel_path);
> > >
> > >      return params;
> > >  }
> > > @@ -953,6 +963,7 @@ void migrate_params_init(MigrationParameters *params)
> > >  {
> > >      params->tls_hostname = g_strdup("");
> > >      params->tls_creds = g_strdup("");
> > > +    params->dsa_accel_path = NULL;
> > >
> > >      /* Set has_* up only for parameter checks */
> > >      params->has_throttle_trigger_threshold = true;
> > > @@ -1165,6 +1176,14 @@ bool migrate_params_check(MigrationParameters *params, Error **errp)
> > >          return false;
> > >      }
> > >
> > > +    if (params->has_zero_page_detection &&
> > > +        params->zero_page_detection == ZERO_PAGE_DETECTION_DSA_ACCEL) {
> > > +        if (!qemu_dsa_is_supported()) {
> > > +            error_setg(errp, "DSA acceleration is not supported.");
> > > +            return false;
> > > +        }
> > > +    }
> > > +
> > >      return true;
> > >  }
> > >
> > > @@ -1278,6 +1297,11 @@ static void migrate_params_test_apply(MigrateSetParameters *params,
> > >      if (params->has_direct_io) {
> > >          dest->direct_io = params->direct_io;
> > >      }
> > > +
> > > +    if (params->has_dsa_accel_path) {
> > > +        dest->has_dsa_accel_path = true;
> > > +        dest->dsa_accel_path = params->dsa_accel_path;
> > > +    }
> > >  }
> > >
> > >  static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
> > > @@ -1410,6 +1434,12 @@ static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
> > >      if (params->has_direct_io) {
> > >          s->parameters.direct_io = params->direct_io;
> > >      }
> > > +    if (params->has_dsa_accel_path) {
> > > +        qapi_free_strList(s->parameters.dsa_accel_path);
> > > +        s->parameters.has_dsa_accel_path = true;
> > > +        s->parameters.dsa_accel_path =
> > > +            QAPI_CLONE(strList, params->dsa_accel_path);
> > > +    }
> > >  }
> > >
> > >  void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp)
> > > diff --git a/migration/options.h b/migration/options.h
> > > index a0bd6edc06..8198b220bd 100644
> > > --- a/migration/options.h
> > > +++ b/migration/options.h
> > > @@ -86,6 +86,7 @@ const char *migrate_tls_creds(void);
> > >  const char *migrate_tls_hostname(void);
> > >  uint64_t migrate_xbzrle_cache_size(void);
> > >  ZeroPageDetection migrate_zero_page_detection(void);
> > > +const strList *migrate_dsa_accel_path(void);
> > >
> > >  /* parameters helpers */
> > >
> > > diff --git a/qapi/migration.json b/qapi/migration.json
> > > index b66cccf107..d8b42ceae6 100644
> > > --- a/qapi/migration.json
> > > +++ b/qapi/migration.json
> > > @@ -626,10 +626,14 @@
> > >  #     multifd migration is enabled, else in the main migration thread
> > >  #     as for @legacy.
> > >  #
> > > +# @dsa-accel: Perform zero page checking with the DSA accelerator
> > > +#     offloading in multifd sender thread if multifd migration is
> > > +#     enabled, else in the main migration thread as for @legacy.
> > > +#
> > >  # Since: 9.0
> > >  ##
> > >  { 'enum': 'ZeroPageDetection',
> > > -  'data': [ 'none', 'legacy', 'multifd' ] }
> > > +  'data': [ 'none', 'legacy', 'multifd', 'dsa-accel' ] }
> > >
> > >  ##
> > >  # @BitmapMigrationBitmapAliasTransform:
> > > @@ -837,6 +841,12 @@
> > >  #     See description in @ZeroPageDetection.  Default is 'multifd'.
> > >  #     (since 9.0)
> > >  #
> > > +# @dsa-accel-path: If enabled, use DSA accelerator offloading for
> > > +#     certain memory operations. Enable DSA accelerator for zero
> > > +#     page detection offloading by setting the @zero-page-detection
> > > +#     to dsa-accel. This parameter defines the dsa device path, and
> > > +#     defaults to an empty list.  (Since 9.2)
> > > +#
> > >  # @direct-io: Open migration files with O_DIRECT when possible.  This
> > >  #     only has effect if the @mapped-ram capability is enabled.
> > >  #     (Since 9.1)
> > > @@ -855,7 +865,7 @@
> > >             'cpu-throttle-initial', 'cpu-throttle-increment',
> > >             'cpu-throttle-tailslow',
> > >             'tls-creds', 'tls-hostname', 'tls-authz', 'max-bandwidth',
> > > -           'avail-switchover-bandwidth', 'downtime-limit',
> > > +           'avail-switchover-bandwidth', 'downtime-limit', 'dsa-accel-path',
> > >             { 'name': 'x-checkpoint-delay', 'features': [ 'unstable' ] },
> > >             'multifd-channels',
> > >             'xbzrle-cache-size', 'max-postcopy-bandwidth',
> > > @@ -1018,6 +1028,12 @@
> > >  #     See description in @ZeroPageDetection.  Default is 'multifd'.
> > >  #     (since 9.0)
> > >  #
> > > +# @dsa-accel-path: If enabled, use DSA accelerator offloading for
> > > +#     certain memory operations. Enable DSA accelerator for zero
> > > +#     page detection offloading by setting the @zero-page-detection
> > > +#     to dsa-accel. This parameter defines the dsa device path, and
> > > +#     defaults to an empty list.  (Since 9.2)
> > > +#
> > >  # @direct-io: Open migration files with O_DIRECT when possible.  This
> > >  #     only has effect if the @mapped-ram capability is enabled.
> > >  #     (Since 9.1)
> > > @@ -1063,7 +1079,8 @@
> > >              '*vcpu-dirty-limit': 'uint64',
> > >              '*mode': 'MigMode',
> > >              '*zero-page-detection': 'ZeroPageDetection',
> > > -            '*direct-io': 'bool' } }
> > > +            '*direct-io': 'bool',
> > > +            '*dsa-accel-path': [ 'str' ] } }
> > >
> > >  ##
> > >  # @migrate-set-parameters:
> > > @@ -1228,6 +1245,12 @@
> > >  #     See description in @ZeroPageDetection.  Default is 'multifd'.
> > >  #     (since 9.0)
> > >  #
> > > +# @dsa-accel-path: If enabled, use DSA accelerator offloading for
> > > +#     certain memory operations. Enable DSA accelerator for zero
> > > +#     page detection offloading by setting the @zero-page-detection
> > > +#     to dsa-accel. This parameter defines the dsa device path, and
> > > +#     defaults to an empty list.  (Since 9.2)
> > > +#
> > >  # @direct-io: Open migration files with O_DIRECT when possible.  This
> > >  #     only has effect if the @mapped-ram capability is enabled.
> > >  #     (Since 9.1)
> > > @@ -1270,7 +1293,8 @@
> > >              '*vcpu-dirty-limit': 'uint64',
> > >              '*mode': 'MigMode',
> > >              '*zero-page-detection': 'ZeroPageDetection',
> > > -            '*direct-io': 'bool' } }
> > > +            '*direct-io': 'bool',
> > > +            '*dsa-accel-path': [ 'str' ] } }
> > >
> > >  ##
> > >  # @query-migrate-parameters:
> > > diff --git a/util/dsa.c b/util/dsa.c
> > > index cbaa47c360..eeede3c0c7 100644
> > > --- a/util/dsa.c
> > > +++ b/util/dsa.c
> > > @@ -23,6 +23,7 @@
> > >  #include "qemu/bswap.h"
> > >  #include "qemu/error-report.h"
> > >  #include "qemu/rcu.h"
> > > +#include <cpuid.h>
> > >
> > >  #pragma GCC push_options
> > >  #pragma GCC target("enqcmd")
> > > @@ -691,6 +692,36 @@ static void dsa_completion_thread_stop(void *opaque)
> > >      qemu_sem_destroy(&thread_context->sem_init_done);
> > >  }
> > >
> > > +/**
> > > + * @brief Check if DSA is supported.
> > > + *
> > > + * @return True if DSA is supported, otherwise false.
> > > + */
> > > +bool qemu_dsa_is_supported(void)
> > > +{
> > > +    /*
> > > +     * movdir64b is indicated by bit 28 of ecx in CPUID leaf 7, subleaf 0.
> > > +     * enqcmd is indicated by bit 29 of ecx in CPUID leaf 7, subleaf 0.
> > > +     * Doc: https://cdrdv2-public.intel.com/819680/architecture-instruction-\
> > > +     *      set-extensions-programming-reference.pdf
> > > +     */
> > > +    uint32_t eax, ebx, ecx, edx;
> > > +    bool movedirb_enabled;
> > > +    bool enqcmd_enabled;
> > > +
> > > +    __get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx);
> > > +    movedirb_enabled = (ecx >> 28) & 0x1;
> > > +    if (!movedirb_enabled) {
> > > +        return false;
> > > +    }
> > > +    enqcmd_enabled = (ecx >> 29) & 0x1;
> > > +    if (!enqcmd_enabled) {
> > > +        return false;
> > > +    }
> > > +
> > > +    return true;
> > > +}
> > > +
> > >  /**
> > >   * @brief Check if DSA is running.
> > >   *
> > > --
> > > Yichen Wang
> > >
> > --
> >  -----Open up your eyes, open up your mind, open up your code -------
> > / Dr. David Alan Gilbert    |       Running GNU/Linux       | Happy  \
> > \        dave @ treblig.org |                               | In Hex /
> >  \ _________________________|_____ http://www.treblig.org   |_______/
diff mbox series

Patch

diff --git a/hmp-commands.hx b/hmp-commands.hx
index 06746f0afc..0e04eac7c7 100644
--- a/hmp-commands.hx
+++ b/hmp-commands.hx
@@ -1009,7 +1009,7 @@  ERST
 
     {
         .name       = "migrate_set_parameter",
-        .args_type  = "parameter:s,value:s",
+        .args_type  = "parameter:s,value:S",
         .params     = "parameter value",
         .help       = "Set the parameter for migration",
         .cmd        = hmp_migrate_set_parameter,
diff --git a/include/qemu/dsa.h b/include/qemu/dsa.h
index a3b502ee41..b1bb6daad2 100644
--- a/include/qemu/dsa.h
+++ b/include/qemu/dsa.h
@@ -100,6 +100,13 @@  void qemu_dsa_stop(void);
  */
 void qemu_dsa_cleanup(void);
 
+/**
+ * @brief Check if DSA is supported.
+ *
+ * @return True if DSA is supported, otherwise false.
+ */
+bool qemu_dsa_is_supported(void);
+
 /**
  * @brief Check if DSA is running.
  *
@@ -141,6 +148,12 @@  buffer_is_zero_dsa_batch_sync(QemuDsaBatchTask *batch_task,
 
 typedef struct QemuDsaBatchTask {} QemuDsaBatchTask;
 
+static inline bool qemu_dsa_is_supported(void)
+{
+    return false;
+}
+
+
 static inline bool qemu_dsa_is_running(void)
 {
     return false;
diff --git a/migration/migration-hmp-cmds.c b/migration/migration-hmp-cmds.c
index 20d1a6e219..983f13b73c 100644
--- a/migration/migration-hmp-cmds.c
+++ b/migration/migration-hmp-cmds.c
@@ -312,7 +312,16 @@  void hmp_info_migrate_parameters(Monitor *mon, const QDict *qdict)
         monitor_printf(mon, "%s: '%s'\n",
             MigrationParameter_str(MIGRATION_PARAMETER_TLS_AUTHZ),
             params->tls_authz);
-
+        if (params->has_dsa_accel_path) {
+            strList *dsa_accel_path = params->dsa_accel_path;
+            monitor_printf(mon, "%s:",
+                MigrationParameter_str(MIGRATION_PARAMETER_DSA_ACCEL_PATH));
+            while (dsa_accel_path) {
+                monitor_printf(mon, " '%s'", dsa_accel_path->value);
+                dsa_accel_path = dsa_accel_path->next;
+            }
+            monitor_printf(mon, "\n");
+        }
         if (params->has_block_bitmap_mapping) {
             const BitmapMigrationNodeAliasList *bmnal;
 
@@ -563,6 +572,14 @@  void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict)
         p->has_x_checkpoint_delay = true;
         visit_type_uint32(v, param, &p->x_checkpoint_delay, &err);
         break;
+    case MIGRATION_PARAMETER_DSA_ACCEL_PATH:
+        p->has_dsa_accel_path = true;
+        g_autofree char **strv = g_strsplit(valuestr ? : "", " ", -1);
+        strList **tail = &p->dsa_accel_path;
+        for (int i = 0; strv[i]; i++) {
+            QAPI_LIST_APPEND(tail, strv[i]);
+        }
+        break;
     case MIGRATION_PARAMETER_MULTIFD_CHANNELS:
         p->has_multifd_channels = true;
         visit_type_uint8(v, param, &p->multifd_channels, &err);
diff --git a/migration/options.c b/migration/options.c
index 147cd2b8fd..a0b3a7d291 100644
--- a/migration/options.c
+++ b/migration/options.c
@@ -13,6 +13,7 @@ 
 
 #include "qemu/osdep.h"
 #include "qemu/error-report.h"
+#include "qemu/dsa.h"
 #include "exec/target_page.h"
 #include "qapi/clone-visitor.h"
 #include "qapi/error.h"
@@ -832,6 +833,13 @@  const char *migrate_tls_creds(void)
     return s->parameters.tls_creds;
 }
 
+const strList *migrate_dsa_accel_path(void)
+{
+    MigrationState *s = migrate_get_current();
+
+    return s->parameters.dsa_accel_path;
+}
+
 const char *migrate_tls_hostname(void)
 {
     MigrationState *s = migrate_get_current();
@@ -945,6 +953,8 @@  MigrationParameters *qmp_query_migrate_parameters(Error **errp)
     params->zero_page_detection = s->parameters.zero_page_detection;
     params->has_direct_io = true;
     params->direct_io = s->parameters.direct_io;
+    params->has_dsa_accel_path = true;
+    params->dsa_accel_path = QAPI_CLONE(strList, s->parameters.dsa_accel_path);
 
     return params;
 }
@@ -953,6 +963,7 @@  void migrate_params_init(MigrationParameters *params)
 {
     params->tls_hostname = g_strdup("");
     params->tls_creds = g_strdup("");
+    params->dsa_accel_path = NULL;
 
     /* Set has_* up only for parameter checks */
     params->has_throttle_trigger_threshold = true;
@@ -1165,6 +1176,14 @@  bool migrate_params_check(MigrationParameters *params, Error **errp)
         return false;
     }
 
+    if (params->has_zero_page_detection &&
+        params->zero_page_detection == ZERO_PAGE_DETECTION_DSA_ACCEL) {
+        if (!qemu_dsa_is_supported()) {
+            error_setg(errp, "DSA acceleration is not supported.");
+            return false;
+        }
+    }
+
     return true;
 }
 
@@ -1278,6 +1297,11 @@  static void migrate_params_test_apply(MigrateSetParameters *params,
     if (params->has_direct_io) {
         dest->direct_io = params->direct_io;
     }
+
+    if (params->has_dsa_accel_path) {
+        dest->has_dsa_accel_path = true;
+        dest->dsa_accel_path = params->dsa_accel_path;
+    }
 }
 
 static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
@@ -1410,6 +1434,12 @@  static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
     if (params->has_direct_io) {
         s->parameters.direct_io = params->direct_io;
     }
+    if (params->has_dsa_accel_path) {
+        qapi_free_strList(s->parameters.dsa_accel_path);
+        s->parameters.has_dsa_accel_path = true;
+        s->parameters.dsa_accel_path =
+            QAPI_CLONE(strList, params->dsa_accel_path);
+    }
 }
 
 void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp)
diff --git a/migration/options.h b/migration/options.h
index a0bd6edc06..8198b220bd 100644
--- a/migration/options.h
+++ b/migration/options.h
@@ -86,6 +86,7 @@  const char *migrate_tls_creds(void);
 const char *migrate_tls_hostname(void);
 uint64_t migrate_xbzrle_cache_size(void);
 ZeroPageDetection migrate_zero_page_detection(void);
+const strList *migrate_dsa_accel_path(void);
 
 /* parameters helpers */
 
diff --git a/qapi/migration.json b/qapi/migration.json
index b66cccf107..d8b42ceae6 100644
--- a/qapi/migration.json
+++ b/qapi/migration.json
@@ -626,10 +626,14 @@ 
 #     multifd migration is enabled, else in the main migration thread
 #     as for @legacy.
 #
+# @dsa-accel: Perform zero page checking with the DSA accelerator
+#     offloading in multifd sender thread if multifd migration is
+#     enabled, else in the main migration thread as for @legacy.
+#
 # Since: 9.0
 ##
 { 'enum': 'ZeroPageDetection',
-  'data': [ 'none', 'legacy', 'multifd' ] }
+  'data': [ 'none', 'legacy', 'multifd', 'dsa-accel' ] }
 
 ##
 # @BitmapMigrationBitmapAliasTransform:
@@ -837,6 +841,12 @@ 
 #     See description in @ZeroPageDetection.  Default is 'multifd'.
 #     (since 9.0)
 #
+# @dsa-accel-path: If enabled, use DSA accelerator offloading for
+#     certain memory operations. Enable DSA accelerator for zero
+#     page detection offloading by setting the @zero-page-detection
+#     to dsa-accel. This parameter defines the dsa device path, and
+#     defaults to an empty list.  (Since 9.2)
+#
 # @direct-io: Open migration files with O_DIRECT when possible.  This
 #     only has effect if the @mapped-ram capability is enabled.
 #     (Since 9.1)
@@ -855,7 +865,7 @@ 
            'cpu-throttle-initial', 'cpu-throttle-increment',
            'cpu-throttle-tailslow',
            'tls-creds', 'tls-hostname', 'tls-authz', 'max-bandwidth',
-           'avail-switchover-bandwidth', 'downtime-limit',
+           'avail-switchover-bandwidth', 'downtime-limit', 'dsa-accel-path',
            { 'name': 'x-checkpoint-delay', 'features': [ 'unstable' ] },
            'multifd-channels',
            'xbzrle-cache-size', 'max-postcopy-bandwidth',
@@ -1018,6 +1028,12 @@ 
 #     See description in @ZeroPageDetection.  Default is 'multifd'.
 #     (since 9.0)
 #
+# @dsa-accel-path: If enabled, use DSA accelerator offloading for
+#     certain memory operations. Enable DSA accelerator for zero
+#     page detection offloading by setting the @zero-page-detection
+#     to dsa-accel. This parameter defines the dsa device path, and
+#     defaults to an empty list.  (Since 9.2)
+#
 # @direct-io: Open migration files with O_DIRECT when possible.  This
 #     only has effect if the @mapped-ram capability is enabled.
 #     (Since 9.1)
@@ -1063,7 +1079,8 @@ 
             '*vcpu-dirty-limit': 'uint64',
             '*mode': 'MigMode',
             '*zero-page-detection': 'ZeroPageDetection',
-            '*direct-io': 'bool' } }
+            '*direct-io': 'bool',
+            '*dsa-accel-path': [ 'str' ] } }
 
 ##
 # @migrate-set-parameters:
@@ -1228,6 +1245,12 @@ 
 #     See description in @ZeroPageDetection.  Default is 'multifd'.
 #     (since 9.0)
 #
+# @dsa-accel-path: If enabled, use DSA accelerator offloading for
+#     certain memory operations. Enable DSA accelerator for zero
+#     page detection offloading by setting the @zero-page-detection
+#     to dsa-accel. This parameter defines the dsa device path, and
+#     defaults to an empty list.  (Since 9.2)
+#
 # @direct-io: Open migration files with O_DIRECT when possible.  This
 #     only has effect if the @mapped-ram capability is enabled.
 #     (Since 9.1)
@@ -1270,7 +1293,8 @@ 
             '*vcpu-dirty-limit': 'uint64',
             '*mode': 'MigMode',
             '*zero-page-detection': 'ZeroPageDetection',
-            '*direct-io': 'bool' } }
+            '*direct-io': 'bool',
+            '*dsa-accel-path': [ 'str' ] } }
 
 ##
 # @query-migrate-parameters:
diff --git a/util/dsa.c b/util/dsa.c
index cbaa47c360..eeede3c0c7 100644
--- a/util/dsa.c
+++ b/util/dsa.c
@@ -23,6 +23,7 @@ 
 #include "qemu/bswap.h"
 #include "qemu/error-report.h"
 #include "qemu/rcu.h"
+#include <cpuid.h>
 
 #pragma GCC push_options
 #pragma GCC target("enqcmd")
@@ -691,6 +692,36 @@  static void dsa_completion_thread_stop(void *opaque)
     qemu_sem_destroy(&thread_context->sem_init_done);
 }
 
+/**
+ * @brief Check if DSA is supported.
+ *
+ * @return True if DSA is supported, otherwise false.
+ */
+bool qemu_dsa_is_supported(void)
+{
+    /*
+     * movdir64b is indicated by bit 28 of ecx in CPUID leaf 7, subleaf 0.
+     * enqcmd is indicated by bit 29 of ecx in CPUID leaf 7, subleaf 0.
+     * Doc: https://cdrdv2-public.intel.com/819680/architecture-instruction-\
+     *      set-extensions-programming-reference.pdf
+     */
+    uint32_t eax, ebx, ecx, edx;
+    bool movedirb_enabled;
+    bool enqcmd_enabled;
+
+    __get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx);
+    movedirb_enabled = (ecx >> 28) & 0x1;
+    if (!movedirb_enabled) {
+        return false;
+    }
+    enqcmd_enabled = (ecx >> 29) & 0x1;
+    if (!enqcmd_enabled) {
+        return false;
+    }
+
+    return true;
+}
+
 /**
  * @brief Check if DSA is running.
  *