[v2,1/1] Introduce "xen-load-devices-state"

Commit Message

Changlong Xie March 14, 2016, 8:03 a.m. UTC

From: Wen Congyang <[e-mail]>

Introduce a "xen-load-devices-state" QAPI command that can be used to
load the state of all devices, but not the RAM or the block devices of
the VM.

We only have hmp commands savevm/loadvm, and qmp commands
xen-save-devices-state.

We use this new command for COLO:
1. suspend both primary vm and secondary vm
2. sync the state
3. resume both primary vm and secondary vm

In such case, we need to update all devices' state in any time.

Signed-off-by: Wen Congyang <[e-mail]>
Signed-off-by: Changlong Xie <[e-mail]>
---
 migration/savevm.c | 36 ++++++++++++++++++++++++++++++++++++
 qapi-schema.json   | 14 ++++++++++++++
 qmp-commands.hx    | 27 +++++++++++++++++++++++++++
 3 files changed, 77 insertions(+)

Comments

Changlong Xie March 22, 2016, 4:21 a.m. UTC | #1

ping......

On 03/14/2016 04:03 PM, Changlong Xie wrote:
> From: Wen Congyang <[e-mail]>
>
> Introduce a "xen-load-devices-state" QAPI command that can be used to
> load the state of all devices, but not the RAM or the block devices of
> the VM.
>
> We only have hmp commands savevm/loadvm, and qmp commands
> xen-save-devices-state.
>
> We use this new command for COLO:
> 1. suspend both primary vm and secondary vm
> 2. sync the state
> 3. resume both primary vm and secondary vm
>
> In such case, we need to update all devices' state in any time.
>
> Signed-off-by: Wen Congyang <[e-mail]>
> Signed-off-by: Changlong Xie <[e-mail]>
> ---
>   migration/savevm.c | 36 ++++++++++++++++++++++++++++++++++++
>   qapi-schema.json   | 14 ++++++++++++++
>   qmp-commands.hx    | 27 +++++++++++++++++++++++++++
>   3 files changed, 77 insertions(+)
>
> diff --git a/migration/savevm.c b/migration/savevm.c
> index 96e7db5..aaead12 100644
> --- a/migration/savevm.c
> +++ b/migration/savevm.c
> @@ -50,6 +50,7 @@
>   #include "qemu/iov.h"
>   #include "block/snapshot.h"
>   #include "block/qapi.h"
> +#include "hw/xen/xen.h"
>
>
>   #ifndef ETH_P_RARP
> @@ -1768,6 +1769,12 @@ qemu_loadvm_section_start_full(QEMUFile *f, MigrationIncomingState *mis)
>           return -EINVAL;
>       }
>
> +    /* Validate if it is a device's state */
> +    if (xen_enabled() && se->is_ram) {
> +        error_report("loadvm: %s RAM loading not allowed on Xen", idstr);
> +        return -EINVAL;
> +    }
> +
>       /* Add entry */
>       le = g_malloc0(sizeof(*le));
>
> @@ -2077,6 +2084,35 @@ void qmp_xen_save_devices_state(const char *filename, Error **errp)
>       }
>   }
>
> +void qmp_xen_load_devices_state(const char *filename, Error **errp)
> +{
> +    QEMUFile *f;
> +    int saved_vm_running;
> +    int ret;
> +
> +    saved_vm_running = runstate_is_running();
> +    vm_stop(RUN_STATE_RESTORE_VM);
> +
> +    f = qemu_fopen(filename, "rb");
> +    if (!f) {
> +        error_setg_file_open(errp, errno, filename);
> +        goto out;
> +    }
> +
> +    migration_incoming_state_new(f);
> +    ret = qemu_loadvm_state(f);
> +    qemu_fclose(f);
> +    migration_incoming_state_destroy();
> +    if (ret < 0) {
> +        error_setg(errp, QERR_IO_ERROR);
> +    }
> +
> +out:
> +    if (saved_vm_running) {
> +        vm_start();
> +    }
> +}
> +
>   int load_vmstate(const char *name)
>   {
>       BlockDriverState *bs, *bs_vm_state;
> diff --git a/qapi-schema.json b/qapi-schema.json
> index 362c9d8..8cca59d 100644
> --- a/qapi-schema.json
> +++ b/qapi-schema.json
> @@ -4122,3 +4122,17 @@
>   ##
>   { 'enum': 'ReplayMode',
>     'data': [ 'none', 'record', 'play' ] }
> +
> +##
> +# @xen-load-devices-state:
> +#
> +# Load the state of all devices from file. The RAM and the block devices
> +# of the VM are not loaded by this command.
> +#
> +# @filename: the file to load the state of the devices from as binary
> +# data. See xen-save-devices-state.txt for a description of the binary
> +# format.
> +#
> +# Since: 2.7
> +##
> +{ 'command': 'xen-load-devices-state', 'data': {'filename': 'str'} }
> diff --git a/qmp-commands.hx b/qmp-commands.hx
> index b629673..4925702 100644
> --- a/qmp-commands.hx
> +++ b/qmp-commands.hx
> @@ -587,6 +587,33 @@ Example:
>   EQMP
>
>       {
> +        .name       = "xen-load-devices-state",
> +        .args_type  = "filename:F",
> +        .mhandler.cmd_new = qmp_marshal_xen_load_devices_state,
> +    },
> +
> +SQMP
> +xen-load-devices-state
> +----------------------
> +
> +Load the state of all devices from file. The RAM and the block devices
> +of the VM are not loaded by this command.
> +
> +Arguments:
> +
> +- "filename": the file to load the state of the devices from as binary
> +data. See xen-save-devices-state.txt for a description of the binary
> +format.
> +
> +Example:
> +
> +-> { "execute": "xen-load-devices-state",
> +     "arguments": { "filename": "/tmp/resume" } }
> +<- { "return": {} }
> +
> +EQMP
> +
> +    {
>           .name       = "xen-set-global-dirty-log",
>           .args_type  = "enable:b",
>           .mhandler.cmd_new = qmp_marshal_xen_set_global_dirty_log,
>

Stefano Stabellini March 22, 2016, 11:27 a.m. UTC | #2

On Mon, 14 Mar 2016, Changlong Xie wrote:
> From: Wen Congyang <[e-mail]>
> 
> Introduce a "xen-load-devices-state" QAPI command that can be used to
> load the state of all devices, but not the RAM or the block devices of
> the VM.
> 
> We only have hmp commands savevm/loadvm, and qmp commands
> xen-save-devices-state.
> 
> We use this new command for COLO:
> 1. suspend both primary vm and secondary vm
> 2. sync the state
> 3. resume both primary vm and secondary vm
> 
> In such case, we need to update all devices' state in any time.
> 
> Signed-off-by: Wen Congyang <[e-mail]>
> Signed-off-by: Changlong Xie <[e-mail]>

It looks OK to me.


>  migration/savevm.c | 36 ++++++++++++++++++++++++++++++++++++
>  qapi-schema.json   | 14 ++++++++++++++
>  qmp-commands.hx    | 27 +++++++++++++++++++++++++++
>  3 files changed, 77 insertions(+)
> 
> diff --git a/migration/savevm.c b/migration/savevm.c
> index 96e7db5..aaead12 100644
> --- a/migration/savevm.c
> +++ b/migration/savevm.c
> @@ -50,6 +50,7 @@
>  #include "qemu/iov.h"
>  #include "block/snapshot.h"
>  #include "block/qapi.h"
> +#include "hw/xen/xen.h"
>  
>  
>  #ifndef ETH_P_RARP
> @@ -1768,6 +1769,12 @@ qemu_loadvm_section_start_full(QEMUFile *f, MigrationIncomingState *mis)
>          return -EINVAL;
>      }
>  
> +    /* Validate if it is a device's state */
> +    if (xen_enabled() && se->is_ram) {
> +        error_report("loadvm: %s RAM loading not allowed on Xen", idstr);
> +        return -EINVAL;
> +    }
> +
>      /* Add entry */
>      le = g_malloc0(sizeof(*le));
>  
> @@ -2077,6 +2084,35 @@ void qmp_xen_save_devices_state(const char *filename, Error **errp)
>      }
>  }
>  
> +void qmp_xen_load_devices_state(const char *filename, Error **errp)
> +{
> +    QEMUFile *f;
> +    int saved_vm_running;
> +    int ret;
> +
> +    saved_vm_running = runstate_is_running();
> +    vm_stop(RUN_STATE_RESTORE_VM);
> +
> +    f = qemu_fopen(filename, "rb");
> +    if (!f) {
> +        error_setg_file_open(errp, errno, filename);
> +        goto out;
> +    }
> +
> +    migration_incoming_state_new(f);
> +    ret = qemu_loadvm_state(f);
> +    qemu_fclose(f);
> +    migration_incoming_state_destroy();
> +    if (ret < 0) {
> +        error_setg(errp, QERR_IO_ERROR);
> +    }
> +
> +out:
> +    if (saved_vm_running) {
> +        vm_start();
> +    }
> +}
> +
>  int load_vmstate(const char *name)
>  {
>      BlockDriverState *bs, *bs_vm_state;
> diff --git a/qapi-schema.json b/qapi-schema.json
> index 362c9d8..8cca59d 100644
> --- a/qapi-schema.json
> +++ b/qapi-schema.json
> @@ -4122,3 +4122,17 @@
>  ##
>  { 'enum': 'ReplayMode',
>    'data': [ 'none', 'record', 'play' ] }
> +
> +##
> +# @xen-load-devices-state:
> +#
> +# Load the state of all devices from file. The RAM and the block devices
> +# of the VM are not loaded by this command.
> +#
> +# @filename: the file to load the state of the devices from as binary
> +# data. See xen-save-devices-state.txt for a description of the binary
> +# format.
> +#
> +# Since: 2.7
> +##
> +{ 'command': 'xen-load-devices-state', 'data': {'filename': 'str'} }
> diff --git a/qmp-commands.hx b/qmp-commands.hx
> index b629673..4925702 100644
> --- a/qmp-commands.hx
> +++ b/qmp-commands.hx
> @@ -587,6 +587,33 @@ Example:
>  EQMP
>  
>      {
> +        .name       = "xen-load-devices-state",
> +        .args_type  = "filename:F",
> +        .mhandler.cmd_new = qmp_marshal_xen_load_devices_state,
> +    },
> +
> +SQMP
> +xen-load-devices-state
> +----------------------
> +
> +Load the state of all devices from file. The RAM and the block devices
> +of the VM are not loaded by this command.
> +
> +Arguments:
> +
> +- "filename": the file to load the state of the devices from as binary
> +data. See xen-save-devices-state.txt for a description of the binary
> +format.
> +
> +Example:
> +
> +-> { "execute": "xen-load-devices-state",
> +     "arguments": { "filename": "/tmp/resume" } }
> +<- { "return": {} }
> +
> +EQMP
> +
> +    {
>          .name       = "xen-set-global-dirty-log",
>          .args_type  = "enable:b",
>          .mhandler.cmd_new = qmp_marshal_xen_set_global_dirty_log,
> -- 
> 1.9.3
> 
> 
>

Dr. David Alan Gilbert March 22, 2016, 12:22 p.m. UTC | #3

* Changlong Xie ([e-mail]) wrote:
> From: Wen Congyang <[e-mail]>
> 
> Introduce a "xen-load-devices-state" QAPI command that can be used to
> load the state of all devices, but not the RAM or the block devices of
> the VM.
> 
> We only have hmp commands savevm/loadvm, and qmp commands
> xen-save-devices-state.

Can you explain on Xen how the RAM actually gets loaded?

> 
> We use this new command for COLO:
> 1. suspend both primary vm and secondary vm
> 2. sync the state
> 3. resume both primary vm and secondary vm
> 
> In such case, we need to update all devices' state in any time.
> 
> Signed-off-by: Wen Congyang <[e-mail]>
> Signed-off-by: Changlong Xie <[e-mail]>
> ---
>  migration/savevm.c | 36 ++++++++++++++++++++++++++++++++++++
>  qapi-schema.json   | 14 ++++++++++++++
>  qmp-commands.hx    | 27 +++++++++++++++++++++++++++
>  3 files changed, 77 insertions(+)
> 
> diff --git a/migration/savevm.c b/migration/savevm.c
> index 96e7db5..aaead12 100644
> --- a/migration/savevm.c
> +++ b/migration/savevm.c
> @@ -50,6 +50,7 @@
>  #include "qemu/iov.h"
>  #include "block/snapshot.h"
>  #include "block/qapi.h"
> +#include "hw/xen/xen.h"
>  
>  
>  #ifndef ETH_P_RARP
> @@ -1768,6 +1769,12 @@ qemu_loadvm_section_start_full(QEMUFile *f, MigrationIncomingState *mis)
>          return -EINVAL;
>      }
>  
> +    /* Validate if it is a device's state */
> +    if (xen_enabled() && se->is_ram) {
> +        error_report("loadvm: %s RAM loading not allowed on Xen", idstr);
> +        return -EINVAL;
> +    }
> +
>      /* Add entry */
>      le = g_malloc0(sizeof(*le));
>  
> @@ -2077,6 +2084,35 @@ void qmp_xen_save_devices_state(const char *filename, Error **errp)
>      }
>  }
>  
> +void qmp_xen_load_devices_state(const char *filename, Error **errp)
> +{
> +    QEMUFile *f;
> +    int saved_vm_running;
> +    int ret;
> +
> +    saved_vm_running = runstate_is_running();
> +    vm_stop(RUN_STATE_RESTORE_VM);
> +
> +    f = qemu_fopen(filename, "rb");
> +    if (!f) {
> +        error_setg_file_open(errp, errno, filename);
> +        goto out;
> +    }
> +
> +    migration_incoming_state_new(f);
> +    ret = qemu_loadvm_state(f);
> +    qemu_fclose(f);
> +    migration_incoming_state_destroy();
> +    if (ret < 0) {
> +        error_setg(errp, QERR_IO_ERROR);
> +    }
> +
> +out:
> +    if (saved_vm_running) {
> +        vm_start();
> +    }

Does it ever happen that you had it running immediately
before you did this command? Somehow you'd have to have loaded the RAM
at just the right point, and I don't see how that would happen if the guest
was running.

Dave

> +}
> +
>  int load_vmstate(const char *name)
>  {
>      BlockDriverState *bs, *bs_vm_state;
> diff --git a/qapi-schema.json b/qapi-schema.json
> index 362c9d8..8cca59d 100644
> --- a/qapi-schema.json
> +++ b/qapi-schema.json
> @@ -4122,3 +4122,17 @@
>  ##
>  { 'enum': 'ReplayMode',
>    'data': [ 'none', 'record', 'play' ] }
> +
> +##
> +# @xen-load-devices-state:
> +#
> +# Load the state of all devices from file. The RAM and the block devices
> +# of the VM are not loaded by this command.
> +#
> +# @filename: the file to load the state of the devices from as binary
> +# data. See xen-save-devices-state.txt for a description of the binary
> +# format.
> +#
> +# Since: 2.7
> +##
> +{ 'command': 'xen-load-devices-state', 'data': {'filename': 'str'} }
> diff --git a/qmp-commands.hx b/qmp-commands.hx
> index b629673..4925702 100644
> --- a/qmp-commands.hx
> +++ b/qmp-commands.hx
> @@ -587,6 +587,33 @@ Example:
>  EQMP
>  
>      {
> +        .name       = "xen-load-devices-state",
> +        .args_type  = "filename:F",
> +        .mhandler.cmd_new = qmp_marshal_xen_load_devices_state,
> +    },
> +
> +SQMP
> +xen-load-devices-state
> +----------------------
> +
> +Load the state of all devices from file. The RAM and the block devices
> +of the VM are not loaded by this command.
> +
> +Arguments:
> +
> +- "filename": the file to load the state of the devices from as binary
> +data. See xen-save-devices-state.txt for a description of the binary
> +format.
> +
> +Example:
> +
> +-> { "execute": "xen-load-devices-state",
> +     "arguments": { "filename": "/tmp/resume" } }
> +<- { "return": {} }
> +
> +EQMP
> +
> +    {
>          .name       = "xen-set-global-dirty-log",
>          .args_type  = "enable:b",
>          .mhandler.cmd_new = qmp_marshal_xen_set_global_dirty_log,
> -- 
> 1.9.3
> 
> 
> 
--
Dr. David Alan Gilbert / [e-mail] / Manchester, UK

Changlong Xie March 23, 2016, 7:25 a.m. UTC | #4

On 03/22/2016 08:22 PM, Dr. David Alan Gilbert wrote:
> * Changlong Xie ([e-mail]) wrote:
>> From: Wen Congyang <[e-mail]>
>>
>> Introduce a "xen-load-devices-state" QAPI command that can be used to
>> load the state of all devices, but not the RAM or the block devices of
>> the VM.
>>
>> We only have hmp commands savevm/loadvm, and qmp commands
>> xen-save-devices-state.
>
> Can you explain on Xen how the RAM actually gets loaded?

Xen use xc(xen toolstack) to do RAM restore/save

>
>>
>> We use this new command for COLO:
>> 1. suspend both primary vm and secondary vm
>> 2. sync the state
>> 3. resume both primary vm and secondary vm
>>
>> In such case, we need to update all devices' state in any time.
>>
>> Signed-off-by: Wen Congyang <[e-mail]>
>> Signed-off-by: Changlong Xie <[e-mail]>
>> ---
>>   migration/savevm.c | 36 ++++++++++++++++++++++++++++++++++++
>>   qapi-schema.json   | 14 ++++++++++++++
>>   qmp-commands.hx    | 27 +++++++++++++++++++++++++++
>>   3 files changed, 77 insertions(+)
>>
>> diff --git a/migration/savevm.c b/migration/savevm.c
>> index 96e7db5..aaead12 100644
>> --- a/migration/savevm.c
>> +++ b/migration/savevm.c
>> @@ -50,6 +50,7 @@
>>   #include "qemu/iov.h"
>>   #include "block/snapshot.h"
>>   #include "block/qapi.h"
>> +#include "hw/xen/xen.h"
>>
>>
>>   #ifndef ETH_P_RARP
>> @@ -1768,6 +1769,12 @@ qemu_loadvm_section_start_full(QEMUFile *f, MigrationIncomingState *mis)
>>           return -EINVAL;
>>       }
>>
>> +    /* Validate if it is a device's state */
>> +    if (xen_enabled() && se->is_ram) {
>> +        error_report("loadvm: %s RAM loading not allowed on Xen", idstr);
>> +        return -EINVAL;
>> +    }
>> +
>>       /* Add entry */
>>       le = g_malloc0(sizeof(*le));
>>
>> @@ -2077,6 +2084,35 @@ void qmp_xen_save_devices_state(const char *filename, Error **errp)
>>       }
>>   }
>>
>> +void qmp_xen_load_devices_state(const char *filename, Error **errp)
>> +{
>> +    QEMUFile *f;
>> +    int saved_vm_running;
>> +    int ret;
>> +
>> +    saved_vm_running = runstate_is_running();
>> +    vm_stop(RUN_STATE_RESTORE_VM);
>> +
>> +    f = qemu_fopen(filename, "rb");
>> +    if (!f) {
>> +        error_setg_file_open(errp, errno, filename);
>> +        goto out;
>> +    }
>> +
>> +    migration_incoming_state_new(f);
>> +    ret = qemu_loadvm_state(f);
>> +    qemu_fclose(f);
>> +    migration_incoming_state_destroy();
>> +    if (ret < 0) {
>> +        error_setg(errp, QERR_IO_ERROR);
>> +    }
>> +
>> +out:
>> +    if (saved_vm_running) {
>> +        vm_start();
>> +    }
>
> Does it ever happen that you had it running immediately
> before you did this command? Somehow you'd have to have loaded the RAM

No, we suspend vm before running this command in xen to make sure the 
condition that you discribed never happen.

Thanks
	-Xie

> at just the right point, and I don't see how that would happen if the guest
> was running.
>
> Dave
>
>> +}
>> +
>>   int load_vmstate(const char *name)
>>   {
>>       BlockDriverState *bs, *bs_vm_state;
>> diff --git a/qapi-schema.json b/qapi-schema.json
>> index 362c9d8..8cca59d 100644
>> --- a/qapi-schema.json
>> +++ b/qapi-schema.json
>> @@ -4122,3 +4122,17 @@
>>   ##
>>   { 'enum': 'ReplayMode',
>>     'data': [ 'none', 'record', 'play' ] }
>> +
>> +##
>> +# @xen-load-devices-state:
>> +#
>> +# Load the state of all devices from file. The RAM and the block devices
>> +# of the VM are not loaded by this command.
>> +#
>> +# @filename: the file to load the state of the devices from as binary
>> +# data. See xen-save-devices-state.txt for a description of the binary
>> +# format.
>> +#
>> +# Since: 2.7
>> +##
>> +{ 'command': 'xen-load-devices-state', 'data': {'filename': 'str'} }
>> diff --git a/qmp-commands.hx b/qmp-commands.hx
>> index b629673..4925702 100644
>> --- a/qmp-commands.hx
>> +++ b/qmp-commands.hx
>> @@ -587,6 +587,33 @@ Example:
>>   EQMP
>>
>>       {
>> +        .name       = "xen-load-devices-state",
>> +        .args_type  = "filename:F",
>> +        .mhandler.cmd_new = qmp_marshal_xen_load_devices_state,
>> +    },
>> +
>> +SQMP
>> +xen-load-devices-state
>> +----------------------
>> +
>> +Load the state of all devices from file. The RAM and the block devices
>> +of the VM are not loaded by this command.
>> +
>> +Arguments:
>> +
>> +- "filename": the file to load the state of the devices from as binary
>> +data. See xen-save-devices-state.txt for a description of the binary
>> +format.
>> +
>> +Example:
>> +
>> +-> { "execute": "xen-load-devices-state",
>> +     "arguments": { "filename": "/tmp/resume" } }
>> +<- { "return": {} }
>> +
>> +EQMP
>> +
>> +    {
>>           .name       = "xen-set-global-dirty-log",
>>           .args_type  = "enable:b",
>>           .mhandler.cmd_new = qmp_marshal_xen_set_global_dirty_log,
>> --
>> 1.9.3
>>
>>
>>
> --
> Dr. David Alan Gilbert / [e-mail] / Manchester, UK
>
>
> .
>

Dr. David Alan Gilbert March 23, 2016, 8:56 a.m. UTC | #5

* Changlong Xie ([e-mail]) wrote:
> On 03/22/2016 08:22 PM, Dr. David Alan Gilbert wrote:
> >* Changlong Xie ([e-mail]) wrote:
> >>From: Wen Congyang <[e-mail]>
> >>
> >>Introduce a "xen-load-devices-state" QAPI command that can be used to
> >>load the state of all devices, but not the RAM or the block devices of
> >>the VM.
> >>
> >>We only have hmp commands savevm/loadvm, and qmp commands
> >>xen-save-devices-state.
> >
> >Can you explain on Xen how the RAM actually gets loaded?
> 
> Xen use xc(xen toolstack) to do RAM restore/save
> 
> >
> >>
> >>We use this new command for COLO:
> >>1. suspend both primary vm and secondary vm
> >>2. sync the state
> >>3. resume both primary vm and secondary vm
> >>
> >>In such case, we need to update all devices' state in any time.
> >>
> >>Signed-off-by: Wen Congyang <[e-mail]>
> >>Signed-off-by: Changlong Xie <[e-mail]>
> >>---
> >>  migration/savevm.c | 36 ++++++++++++++++++++++++++++++++++++
> >>  qapi-schema.json   | 14 ++++++++++++++
> >>  qmp-commands.hx    | 27 +++++++++++++++++++++++++++
> >>  3 files changed, 77 insertions(+)
> >>
> >>diff --git a/migration/savevm.c b/migration/savevm.c
> >>index 96e7db5..aaead12 100644
> >>--- a/migration/savevm.c
> >>+++ b/migration/savevm.c
> >>@@ -50,6 +50,7 @@
> >>  #include "qemu/iov.h"
> >>  #include "block/snapshot.h"
> >>  #include "block/qapi.h"
> >>+#include "hw/xen/xen.h"
> >>
> >>
> >>  #ifndef ETH_P_RARP
> >>@@ -1768,6 +1769,12 @@ qemu_loadvm_section_start_full(QEMUFile *f, MigrationIncomingState *mis)
> >>          return -EINVAL;
> >>      }
> >>
> >>+    /* Validate if it is a device's state */
> >>+    if (xen_enabled() && se->is_ram) {
> >>+        error_report("loadvm: %s RAM loading not allowed on Xen", idstr);
> >>+        return -EINVAL;
> >>+    }
> >>+
> >>      /* Add entry */
> >>      le = g_malloc0(sizeof(*le));
> >>
> >>@@ -2077,6 +2084,35 @@ void qmp_xen_save_devices_state(const char *filename, Error **errp)
> >>      }
> >>  }
> >>
> >>+void qmp_xen_load_devices_state(const char *filename, Error **errp)
> >>+{
> >>+    QEMUFile *f;
> >>+    int saved_vm_running;
> >>+    int ret;
> >>+
> >>+    saved_vm_running = runstate_is_running();
> >>+    vm_stop(RUN_STATE_RESTORE_VM);
> >>+
> >>+    f = qemu_fopen(filename, "rb");
> >>+    if (!f) {
> >>+        error_setg_file_open(errp, errno, filename);
> >>+        goto out;
> >>+    }
> >>+
> >>+    migration_incoming_state_new(f);
> >>+    ret = qemu_loadvm_state(f);
> >>+    qemu_fclose(f);
> >>+    migration_incoming_state_destroy();
> >>+    if (ret < 0) {
> >>+        error_setg(errp, QERR_IO_ERROR);
> >>+    }
> >>+
> >>+out:
> >>+    if (saved_vm_running) {
> >>+        vm_start();
> >>+    }
> >
> >Does it ever happen that you had it running immediately
> >before you did this command? Somehow you'd have to have loaded the RAM
> 
> No, we suspend vm before running this command in xen to make sure the
> condition that you discribed never happen.

OK, so then if the VM is suspended, what does the:

    if (saved_vm_running) {
        vm_start();
    }

at the end of your routine do?

Dave


> 
> Thanks
> 	-Xie
> 
> >at just the right point, and I don't see how that would happen if the guest
> >was running.
> >
> >Dave
> >
> >>+}
> >>+
> >>  int load_vmstate(const char *name)
> >>  {
> >>      BlockDriverState *bs, *bs_vm_state;
> >>diff --git a/qapi-schema.json b/qapi-schema.json
> >>index 362c9d8..8cca59d 100644
> >>--- a/qapi-schema.json
> >>+++ b/qapi-schema.json
> >>@@ -4122,3 +4122,17 @@
> >>  ##
> >>  { 'enum': 'ReplayMode',
> >>    'data': [ 'none', 'record', 'play' ] }
> >>+
> >>+##
> >>+# @xen-load-devices-state:
> >>+#
> >>+# Load the state of all devices from file. The RAM and the block devices
> >>+# of the VM are not loaded by this command.
> >>+#
> >>+# @filename: the file to load the state of the devices from as binary
> >>+# data. See xen-save-devices-state.txt for a description of the binary
> >>+# format.
> >>+#
> >>+# Since: 2.7
> >>+##
> >>+{ 'command': 'xen-load-devices-state', 'data': {'filename': 'str'} }
> >>diff --git a/qmp-commands.hx b/qmp-commands.hx
> >>index b629673..4925702 100644
> >>--- a/qmp-commands.hx
> >>+++ b/qmp-commands.hx
> >>@@ -587,6 +587,33 @@ Example:
> >>  EQMP
> >>
> >>      {
> >>+        .name       = "xen-load-devices-state",
> >>+        .args_type  = "filename:F",
> >>+        .mhandler.cmd_new = qmp_marshal_xen_load_devices_state,
> >>+    },
> >>+
> >>+SQMP
> >>+xen-load-devices-state
> >>+----------------------
> >>+
> >>+Load the state of all devices from file. The RAM and the block devices
> >>+of the VM are not loaded by this command.
> >>+
> >>+Arguments:
> >>+
> >>+- "filename": the file to load the state of the devices from as binary
> >>+data. See xen-save-devices-state.txt for a description of the binary
> >>+format.
> >>+
> >>+Example:
> >>+
> >>+-> { "execute": "xen-load-devices-state",
> >>+     "arguments": { "filename": "/tmp/resume" } }
> >>+<- { "return": {} }
> >>+
> >>+EQMP
> >>+
> >>+    {
> >>          .name       = "xen-set-global-dirty-log",
> >>          .args_type  = "enable:b",
> >>          .mhandler.cmd_new = qmp_marshal_xen_set_global_dirty_log,
> >>--
> >>1.9.3
> >>
> >>
> >>
> >--
> >Dr. David Alan Gilbert / [e-mail] / Manchester, UK
> >
> >
> >.
> >
> 
> 
--
Dr. David Alan Gilbert / [e-mail] / Manchester, UK

Wen Congyang March 23, 2016, 9:02 a.m. UTC | #6

On 03/23/2016 04:56 PM, Dr. David Alan Gilbert wrote:
> * Changlong Xie ([e-mail]) wrote:
>> On 03/22/2016 08:22 PM, Dr. David Alan Gilbert wrote:
>>> * Changlong Xie ([e-mail]) wrote:
>>>> From: Wen Congyang <[e-mail]>
>>>>
>>>> Introduce a "xen-load-devices-state" QAPI command that can be used to
>>>> load the state of all devices, but not the RAM or the block devices of
>>>> the VM.
>>>>
>>>> We only have hmp commands savevm/loadvm, and qmp commands
>>>> xen-save-devices-state.
>>>
>>> Can you explain on Xen how the RAM actually gets loaded?
>>
>> Xen use xc(xen toolstack) to do RAM restore/save
>>
>>>
>>>>
>>>> We use this new command for COLO:
>>>> 1. suspend both primary vm and secondary vm
>>>> 2. sync the state
>>>> 3. resume both primary vm and secondary vm
>>>>
>>>> In such case, we need to update all devices' state in any time.
>>>>
>>>> Signed-off-by: Wen Congyang <[e-mail]>
>>>> Signed-off-by: Changlong Xie <[e-mail]>
>>>> ---
>>>>  migration/savevm.c | 36 ++++++++++++++++++++++++++++++++++++
>>>>  qapi-schema.json   | 14 ++++++++++++++
>>>>  qmp-commands.hx    | 27 +++++++++++++++++++++++++++
>>>>  3 files changed, 77 insertions(+)
>>>>
>>>> diff --git a/migration/savevm.c b/migration/savevm.c
>>>> index 96e7db5..aaead12 100644
>>>> --- a/migration/savevm.c
>>>> +++ b/migration/savevm.c
>>>> @@ -50,6 +50,7 @@
>>>>  #include "qemu/iov.h"
>>>>  #include "block/snapshot.h"
>>>>  #include "block/qapi.h"
>>>> +#include "hw/xen/xen.h"
>>>>
>>>>
>>>>  #ifndef ETH_P_RARP
>>>> @@ -1768,6 +1769,12 @@ qemu_loadvm_section_start_full(QEMUFile *f, MigrationIncomingState *mis)
>>>>          return -EINVAL;
>>>>      }
>>>>
>>>> +    /* Validate if it is a device's state */
>>>> +    if (xen_enabled() && se->is_ram) {
>>>> +        error_report("loadvm: %s RAM loading not allowed on Xen", idstr);
>>>> +        return -EINVAL;
>>>> +    }
>>>> +
>>>>      /* Add entry */
>>>>      le = g_malloc0(sizeof(*le));
>>>>
>>>> @@ -2077,6 +2084,35 @@ void qmp_xen_save_devices_state(const char *filename, Error **errp)
>>>>      }
>>>>  }
>>>>
>>>> +void qmp_xen_load_devices_state(const char *filename, Error **errp)
>>>> +{
>>>> +    QEMUFile *f;
>>>> +    int saved_vm_running;
>>>> +    int ret;
>>>> +
>>>> +    saved_vm_running = runstate_is_running();
>>>> +    vm_stop(RUN_STATE_RESTORE_VM);
>>>> +
>>>> +    f = qemu_fopen(filename, "rb");
>>>> +    if (!f) {
>>>> +        error_setg_file_open(errp, errno, filename);
>>>> +        goto out;
>>>> +    }
>>>> +
>>>> +    migration_incoming_state_new(f);
>>>> +    ret = qemu_loadvm_state(f);
>>>> +    qemu_fclose(f);
>>>> +    migration_incoming_state_destroy();
>>>> +    if (ret < 0) {
>>>> +        error_setg(errp, QERR_IO_ERROR);
>>>> +    }
>>>> +
>>>> +out:
>>>> +    if (saved_vm_running) {
>>>> +        vm_start();
>>>> +    }
>>>
>>> Does it ever happen that you had it running immediately
>>> before you did this command? Somehow you'd have to have loaded the RAM
>>
>> No, we suspend vm before running this command in xen to make sure the
>> condition that you discribed never happen.
> 
> OK, so then if the VM is suspended, what does the:
> 
>     if (saved_vm_running) {
>         vm_start();
>     }
> 
> at the end of your routine do?

+    saved_vm_running = runstate_is_running();
+    vm_stop(RUN_STATE_RESTORE_VM);

It is copied from the other codes in qemu.
I think we should return failure if the vm is running:
if (runstate_is_running()) {
    error_setg(xxx);
    return;
}

Thanks
Wen Congyang

> 
> Dave
> 
> 
>>
>> Thanks
>> 	-Xie
>>
>>> at just the right point, and I don't see how that would happen if the guest
>>> was running.
>>>
>>> Dave
>>>
>>>> +}
>>>> +
>>>>  int load_vmstate(const char *name)
>>>>  {
>>>>      BlockDriverState *bs, *bs_vm_state;
>>>> diff --git a/qapi-schema.json b/qapi-schema.json
>>>> index 362c9d8..8cca59d 100644
>>>> --- a/qapi-schema.json
>>>> +++ b/qapi-schema.json
>>>> @@ -4122,3 +4122,17 @@
>>>>  ##
>>>>  { 'enum': 'ReplayMode',
>>>>    'data': [ 'none', 'record', 'play' ] }
>>>> +
>>>> +##
>>>> +# @xen-load-devices-state:
>>>> +#
>>>> +# Load the state of all devices from file. The RAM and the block devices
>>>> +# of the VM are not loaded by this command.
>>>> +#
>>>> +# @filename: the file to load the state of the devices from as binary
>>>> +# data. See xen-save-devices-state.txt for a description of the binary
>>>> +# format.
>>>> +#
>>>> +# Since: 2.7
>>>> +##
>>>> +{ 'command': 'xen-load-devices-state', 'data': {'filename': 'str'} }
>>>> diff --git a/qmp-commands.hx b/qmp-commands.hx
>>>> index b629673..4925702 100644
>>>> --- a/qmp-commands.hx
>>>> +++ b/qmp-commands.hx
>>>> @@ -587,6 +587,33 @@ Example:
>>>>  EQMP
>>>>
>>>>      {
>>>> +        .name       = "xen-load-devices-state",
>>>> +        .args_type  = "filename:F",
>>>> +        .mhandler.cmd_new = qmp_marshal_xen_load_devices_state,
>>>> +    },
>>>> +
>>>> +SQMP
>>>> +xen-load-devices-state
>>>> +----------------------
>>>> +
>>>> +Load the state of all devices from file. The RAM and the block devices
>>>> +of the VM are not loaded by this command.
>>>> +
>>>> +Arguments:
>>>> +
>>>> +- "filename": the file to load the state of the devices from as binary
>>>> +data. See xen-save-devices-state.txt for a description of the binary
>>>> +format.
>>>> +
>>>> +Example:
>>>> +
>>>> +-> { "execute": "xen-load-devices-state",
>>>> +     "arguments": { "filename": "/tmp/resume" } }
>>>> +<- { "return": {} }
>>>> +
>>>> +EQMP
>>>> +
>>>> +    {
>>>>          .name       = "xen-set-global-dirty-log",
>>>>          .args_type  = "enable:b",
>>>>          .mhandler.cmd_new = qmp_marshal_xen_set_global_dirty_log,
>>>> --
>>>> 1.9.3
>>>>
>>>>
>>>>
>>> --
>>> Dr. David Alan Gilbert / [e-mail] / Manchester, UK
>>>
>>>
>>> .
>>>
>>
>>
> --
> Dr. David Alan Gilbert / [e-mail] / Manchester, UK
> 
> 
> .
>

Dr. David Alan Gilbert March 23, 2016, 9:41 a.m. UTC | #7

* Wen Congyang ([e-mail]) wrote:
> On 03/23/2016 04:56 PM, Dr. David Alan Gilbert wrote:
> > * Changlong Xie ([e-mail]) wrote:
> >> On 03/22/2016 08:22 PM, Dr. David Alan Gilbert wrote:
> >>> * Changlong Xie ([e-mail]) wrote:
> >>>> From: Wen Congyang <[e-mail]>
> >>>>
> >>>> Introduce a "xen-load-devices-state" QAPI command that can be used to
> >>>> load the state of all devices, but not the RAM or the block devices of
> >>>> the VM.
> >>>>
> >>>> We only have hmp commands savevm/loadvm, and qmp commands
> >>>> xen-save-devices-state.
> >>>
> >>> Can you explain on Xen how the RAM actually gets loaded?
> >>
> >> Xen use xc(xen toolstack) to do RAM restore/save
> >>
> >>>
> >>>>
> >>>> We use this new command for COLO:
> >>>> 1. suspend both primary vm and secondary vm
> >>>> 2. sync the state
> >>>> 3. resume both primary vm and secondary vm
> >>>>
> >>>> In such case, we need to update all devices' state in any time.
> >>>>
> >>>> Signed-off-by: Wen Congyang <[e-mail]>
> >>>> Signed-off-by: Changlong Xie <[e-mail]>
> >>>> ---
> >>>>  migration/savevm.c | 36 ++++++++++++++++++++++++++++++++++++
> >>>>  qapi-schema.json   | 14 ++++++++++++++
> >>>>  qmp-commands.hx    | 27 +++++++++++++++++++++++++++
> >>>>  3 files changed, 77 insertions(+)
> >>>>
> >>>> diff --git a/migration/savevm.c b/migration/savevm.c
> >>>> index 96e7db5..aaead12 100644
> >>>> --- a/migration/savevm.c
> >>>> +++ b/migration/savevm.c
> >>>> @@ -50,6 +50,7 @@
> >>>>  #include "qemu/iov.h"
> >>>>  #include "block/snapshot.h"
> >>>>  #include "block/qapi.h"
> >>>> +#include "hw/xen/xen.h"
> >>>>
> >>>>
> >>>>  #ifndef ETH_P_RARP
> >>>> @@ -1768,6 +1769,12 @@ qemu_loadvm_section_start_full(QEMUFile *f, MigrationIncomingState *mis)
> >>>>          return -EINVAL;
> >>>>      }
> >>>>
> >>>> +    /* Validate if it is a device's state */
> >>>> +    if (xen_enabled() && se->is_ram) {
> >>>> +        error_report("loadvm: %s RAM loading not allowed on Xen", idstr);
> >>>> +        return -EINVAL;
> >>>> +    }
> >>>> +
> >>>>      /* Add entry */
> >>>>      le = g_malloc0(sizeof(*le));
> >>>>
> >>>> @@ -2077,6 +2084,35 @@ void qmp_xen_save_devices_state(const char *filename, Error **errp)
> >>>>      }
> >>>>  }
> >>>>
> >>>> +void qmp_xen_load_devices_state(const char *filename, Error **errp)
> >>>> +{
> >>>> +    QEMUFile *f;
> >>>> +    int saved_vm_running;
> >>>> +    int ret;
> >>>> +
> >>>> +    saved_vm_running = runstate_is_running();
> >>>> +    vm_stop(RUN_STATE_RESTORE_VM);
> >>>> +
> >>>> +    f = qemu_fopen(filename, "rb");
> >>>> +    if (!f) {
> >>>> +        error_setg_file_open(errp, errno, filename);
> >>>> +        goto out;
> >>>> +    }
> >>>> +
> >>>> +    migration_incoming_state_new(f);
> >>>> +    ret = qemu_loadvm_state(f);
> >>>> +    qemu_fclose(f);
> >>>> +    migration_incoming_state_destroy();
> >>>> +    if (ret < 0) {
> >>>> +        error_setg(errp, QERR_IO_ERROR);
> >>>> +    }
> >>>> +
> >>>> +out:
> >>>> +    if (saved_vm_running) {
> >>>> +        vm_start();
> >>>> +    }
> >>>
> >>> Does it ever happen that you had it running immediately
> >>> before you did this command? Somehow you'd have to have loaded the RAM
> >>
> >> No, we suspend vm before running this command in xen to make sure the
> >> condition that you discribed never happen.
> > 
> > OK, so then if the VM is suspended, what does the:
> > 
> >     if (saved_vm_running) {
> >         vm_start();
> >     }
> > 
> > at the end of your routine do?
> 
> +    saved_vm_running = runstate_is_running();
> +    vm_stop(RUN_STATE_RESTORE_VM);
> 
> It is copied from the other codes in qemu.
> I think we should return failure if the vm is running:
> if (runstate_is_running()) {
>     error_setg(xxx);
>     return;
> }

OK, yes; and add a comment to explain.

Dave

> 
> Thanks
> Wen Congyang
> 
> > 
> > Dave
> > 
> > 
> >>
> >> Thanks
> >> 	-Xie
> >>
> >>> at just the right point, and I don't see how that would happen if the guest
> >>> was running.
> >>>
> >>> Dave
> >>>
> >>>> +}
> >>>> +
> >>>>  int load_vmstate(const char *name)
> >>>>  {
> >>>>      BlockDriverState *bs, *bs_vm_state;
> >>>> diff --git a/qapi-schema.json b/qapi-schema.json
> >>>> index 362c9d8..8cca59d 100644
> >>>> --- a/qapi-schema.json
> >>>> +++ b/qapi-schema.json
> >>>> @@ -4122,3 +4122,17 @@
> >>>>  ##
> >>>>  { 'enum': 'ReplayMode',
> >>>>    'data': [ 'none', 'record', 'play' ] }
> >>>> +
> >>>> +##
> >>>> +# @xen-load-devices-state:
> >>>> +#
> >>>> +# Load the state of all devices from file. The RAM and the block devices
> >>>> +# of the VM are not loaded by this command.
> >>>> +#
> >>>> +# @filename: the file to load the state of the devices from as binary
> >>>> +# data. See xen-save-devices-state.txt for a description of the binary
> >>>> +# format.
> >>>> +#
> >>>> +# Since: 2.7
> >>>> +##
> >>>> +{ 'command': 'xen-load-devices-state', 'data': {'filename': 'str'} }
> >>>> diff --git a/qmp-commands.hx b/qmp-commands.hx
> >>>> index b629673..4925702 100644
> >>>> --- a/qmp-commands.hx
> >>>> +++ b/qmp-commands.hx
> >>>> @@ -587,6 +587,33 @@ Example:
> >>>>  EQMP
> >>>>
> >>>>      {
> >>>> +        .name       = "xen-load-devices-state",
> >>>> +        .args_type  = "filename:F",
> >>>> +        .mhandler.cmd_new = qmp_marshal_xen_load_devices_state,
> >>>> +    },
> >>>> +
> >>>> +SQMP
> >>>> +xen-load-devices-state
> >>>> +----------------------
> >>>> +
> >>>> +Load the state of all devices from file. The RAM and the block devices
> >>>> +of the VM are not loaded by this command.
> >>>> +
> >>>> +Arguments:
> >>>> +
> >>>> +- "filename": the file to load the state of the devices from as binary
> >>>> +data. See xen-save-devices-state.txt for a description of the binary
> >>>> +format.
> >>>> +
> >>>> +Example:
> >>>> +
> >>>> +-> { "execute": "xen-load-devices-state",
> >>>> +     "arguments": { "filename": "/tmp/resume" } }
> >>>> +<- { "return": {} }
> >>>> +
> >>>> +EQMP
> >>>> +
> >>>> +    {
> >>>>          .name       = "xen-set-global-dirty-log",
> >>>>          .args_type  = "enable:b",
> >>>>          .mhandler.cmd_new = qmp_marshal_xen_set_global_dirty_log,
> >>>> --
> >>>> 1.9.3
> >>>>
> >>>>
> >>>>
> >>> --
> >>> Dr. David Alan Gilbert / [e-mail] / Manchester, UK
> >>>
> >>>
> >>> .
> >>>
> >>
> >>
> > --
> > Dr. David Alan Gilbert / [e-mail] / Manchester, UK
> > 
> > 
> > .
> > 
> 
> 
> 
--
Dr. David Alan Gilbert / [e-mail] / Manchester, UK

Changlong Xie March 23, 2016, 10:12 a.m. UTC | #8

On 03/23/2016 05:41 PM, Dr. David Alan Gilbert wrote:
> * Wen Congyang ([e-mail]) wrote:
>> On 03/23/2016 04:56 PM, Dr. David Alan Gilbert wrote:
>>> * Changlong Xie ([e-mail]) wrote:
>>>> On 03/22/2016 08:22 PM, Dr. David Alan Gilbert wrote:
>>>>> * Changlong Xie ([e-mail]) wrote:
>>>>>> From: Wen Congyang <[e-mail]>
>>>>>>
>>>>>> Introduce a "xen-load-devices-state" QAPI command that can be used to
>>>>>> load the state of all devices, but not the RAM or the block devices of
>>>>>> the VM.
>>>>>>
>>>>>> We only have hmp commands savevm/loadvm, and qmp commands
>>>>>> xen-save-devices-state.
>>>>>
>>>>> Can you explain on Xen how the RAM actually gets loaded?
>>>>
>>>> Xen use xc(xen toolstack) to do RAM restore/save
>>>>
>>>>>
>>>>>>
>>>>>> We use this new command for COLO:
>>>>>> 1. suspend both primary vm and secondary vm
>>>>>> 2. sync the state
>>>>>> 3. resume both primary vm and secondary vm
>>>>>>
>>>>>> In such case, we need to update all devices' state in any time.
>>>>>>
>>>>>> Signed-off-by: Wen Congyang <[e-mail]>
>>>>>> Signed-off-by: Changlong Xie <[e-mail]>
>>>>>> ---
>>>>>>   migration/savevm.c | 36 ++++++++++++++++++++++++++++++++++++
>>>>>>   qapi-schema.json   | 14 ++++++++++++++
>>>>>>   qmp-commands.hx    | 27 +++++++++++++++++++++++++++
>>>>>>   3 files changed, 77 insertions(+)
>>>>>>
>>>>>> diff --git a/migration/savevm.c b/migration/savevm.c
>>>>>> index 96e7db5..aaead12 100644
>>>>>> --- a/migration/savevm.c
>>>>>> +++ b/migration/savevm.c
>>>>>> @@ -50,6 +50,7 @@
>>>>>>   #include "qemu/iov.h"
>>>>>>   #include "block/snapshot.h"
>>>>>>   #include "block/qapi.h"
>>>>>> +#include "hw/xen/xen.h"
>>>>>>
>>>>>>
>>>>>>   #ifndef ETH_P_RARP
>>>>>> @@ -1768,6 +1769,12 @@ qemu_loadvm_section_start_full(QEMUFile *f, MigrationIncomingState *mis)
>>>>>>           return -EINVAL;
>>>>>>       }
>>>>>>
>>>>>> +    /* Validate if it is a device's state */
>>>>>> +    if (xen_enabled() && se->is_ram) {
>>>>>> +        error_report("loadvm: %s RAM loading not allowed on Xen", idstr);
>>>>>> +        return -EINVAL;
>>>>>> +    }
>>>>>> +
>>>>>>       /* Add entry */
>>>>>>       le = g_malloc0(sizeof(*le));
>>>>>>
>>>>>> @@ -2077,6 +2084,35 @@ void qmp_xen_save_devices_state(const char *filename, Error **errp)
>>>>>>       }
>>>>>>   }
>>>>>>
>>>>>> +void qmp_xen_load_devices_state(const char *filename, Error **errp)
>>>>>> +{
>>>>>> +    QEMUFile *f;
>>>>>> +    int saved_vm_running;
>>>>>> +    int ret;
>>>>>> +
>>>>>> +    saved_vm_running = runstate_is_running();
>>>>>> +    vm_stop(RUN_STATE_RESTORE_VM);
>>>>>> +
>>>>>> +    f = qemu_fopen(filename, "rb");
>>>>>> +    if (!f) {
>>>>>> +        error_setg_file_open(errp, errno, filename);
>>>>>> +        goto out;
>>>>>> +    }
>>>>>> +
>>>>>> +    migration_incoming_state_new(f);
>>>>>> +    ret = qemu_loadvm_state(f);
>>>>>> +    qemu_fclose(f);
>>>>>> +    migration_incoming_state_destroy();
>>>>>> +    if (ret < 0) {
>>>>>> +        error_setg(errp, QERR_IO_ERROR);
>>>>>> +    }
>>>>>> +
>>>>>> +out:
>>>>>> +    if (saved_vm_running) {
>>>>>> +        vm_start();
>>>>>> +    }
>>>>>
>>>>> Does it ever happen that you had it running immediately
>>>>> before you did this command? Somehow you'd have to have loaded the RAM
>>>>
>>>> No, we suspend vm before running this command in xen to make sure the
>>>> condition that you discribed never happen.
>>>
>>> OK, so then if the VM is suspended, what does the:
>>>
>>>      if (saved_vm_running) {
>>>          vm_start();
>>>      }
>>>
>>> at the end of your routine do?
>>
>> +    saved_vm_running = runstate_is_running();
>> +    vm_stop(RUN_STATE_RESTORE_VM);
>>
>> It is copied from the other codes in qemu.
>> I think we should return failure if the vm is running:
>> if (runstate_is_running()) {
>>      error_setg(xxx);
>>      return;
>> }
>
> OK, yes; and add a comment to explain.

+    if (runstate_is_running()) {
+        error_setg(errp, "Cannot update device state while vm is running");
+        return;
+    }

Will fix in next version.

Thanks
	-Xie
>
> Dave
>
>>
>> Thanks
>> Wen Congyang
>>
>>>
>>> Dave
>>>
>>>
>>>>
>>>> Thanks
>>>> 	-Xie
>>>>
>>>>> at just the right point, and I don't see how that would happen if the guest
>>>>> was running.
>>>>>
>>>>> Dave
>>>>>
>>>>>> +}
>>>>>> +
>>>>>>   int load_vmstate(const char *name)
>>>>>>   {
>>>>>>       BlockDriverState *bs, *bs_vm_state;
>>>>>> diff --git a/qapi-schema.json b/qapi-schema.json
>>>>>> index 362c9d8..8cca59d 100644
>>>>>> --- a/qapi-schema.json
>>>>>> +++ b/qapi-schema.json
>>>>>> @@ -4122,3 +4122,17 @@
>>>>>>   ##
>>>>>>   { 'enum': 'ReplayMode',
>>>>>>     'data': [ 'none', 'record', 'play' ] }
>>>>>> +
>>>>>> +##
>>>>>> +# @xen-load-devices-state:
>>>>>> +#
>>>>>> +# Load the state of all devices from file. The RAM and the block devices
>>>>>> +# of the VM are not loaded by this command.
>>>>>> +#
>>>>>> +# @filename: the file to load the state of the devices from as binary
>>>>>> +# data. See xen-save-devices-state.txt for a description of the binary
>>>>>> +# format.
>>>>>> +#
>>>>>> +# Since: 2.7
>>>>>> +##
>>>>>> +{ 'command': 'xen-load-devices-state', 'data': {'filename': 'str'} }
>>>>>> diff --git a/qmp-commands.hx b/qmp-commands.hx
>>>>>> index b629673..4925702 100644
>>>>>> --- a/qmp-commands.hx
>>>>>> +++ b/qmp-commands.hx
>>>>>> @@ -587,6 +587,33 @@ Example:
>>>>>>   EQMP
>>>>>>
>>>>>>       {
>>>>>> +        .name       = "xen-load-devices-state",
>>>>>> +        .args_type  = "filename:F",
>>>>>> +        .mhandler.cmd_new = qmp_marshal_xen_load_devices_state,
>>>>>> +    },
>>>>>> +
>>>>>> +SQMP
>>>>>> +xen-load-devices-state
>>>>>> +----------------------
>>>>>> +
>>>>>> +Load the state of all devices from file. The RAM and the block devices
>>>>>> +of the VM are not loaded by this command.
>>>>>> +
>>>>>> +Arguments:
>>>>>> +
>>>>>> +- "filename": the file to load the state of the devices from as binary
>>>>>> +data. See xen-save-devices-state.txt for a description of the binary
>>>>>> +format.
>>>>>> +
>>>>>> +Example:
>>>>>> +
>>>>>> +-> { "execute": "xen-load-devices-state",
>>>>>> +     "arguments": { "filename": "/tmp/resume" } }
>>>>>> +<- { "return": {} }
>>>>>> +
>>>>>> +EQMP
>>>>>> +
>>>>>> +    {
>>>>>>           .name       = "xen-set-global-dirty-log",
>>>>>>           .args_type  = "enable:b",
>>>>>>           .mhandler.cmd_new = qmp_marshal_xen_set_global_dirty_log,
>>>>>> --
>>>>>> 1.9.3
>>>>>>
>>>>>>
>>>>>>
>>>>> --
>>>>> Dr. David Alan Gilbert / [e-mail] / Manchester, UK
>>>>>
>>>>>
>>>>> .
>>>>>
>>>>
>>>>
>>> --
>>> Dr. David Alan Gilbert / [e-mail] / Manchester, UK
>>>
>>>
>>> .
>>>
>>
>>
>>
> --
> Dr. David Alan Gilbert / [e-mail] / Manchester, UK
>
>
> .
>

Dr. David Alan Gilbert March 23, 2016, 10:15 a.m. UTC | #9

* Changlong Xie ([e-mail]) wrote:
> On 03/23/2016 05:41 PM, Dr. David Alan Gilbert wrote:
> >* Wen Congyang ([e-mail]) wrote:
> >>On 03/23/2016 04:56 PM, Dr. David Alan Gilbert wrote:
> >>>* Changlong Xie ([e-mail]) wrote:
> >>>>On 03/22/2016 08:22 PM, Dr. David Alan Gilbert wrote:
> >>>>>* Changlong Xie ([e-mail]) wrote:
> >>>>>>From: Wen Congyang <[e-mail]>
> >>>>>>
> >>>>>>Introduce a "xen-load-devices-state" QAPI command that can be used to
> >>>>>>load the state of all devices, but not the RAM or the block devices of
> >>>>>>the VM.
> >>>>>>
> >>>>>>We only have hmp commands savevm/loadvm, and qmp commands
> >>>>>>xen-save-devices-state.
> >>>>>
> >>>>>Can you explain on Xen how the RAM actually gets loaded?
> >>>>
> >>>>Xen use xc(xen toolstack) to do RAM restore/save
> >>>>
> >>>>>
> >>>>>>
> >>>>>>We use this new command for COLO:
> >>>>>>1. suspend both primary vm and secondary vm
> >>>>>>2. sync the state
> >>>>>>3. resume both primary vm and secondary vm
> >>>>>>
> >>>>>>In such case, we need to update all devices' state in any time.
> >>>>>>
> >>>>>>Signed-off-by: Wen Congyang <[e-mail]>
> >>>>>>Signed-off-by: Changlong Xie <[e-mail]>
> >>>>>>---
> >>>>>>  migration/savevm.c | 36 ++++++++++++++++++++++++++++++++++++
> >>>>>>  qapi-schema.json   | 14 ++++++++++++++
> >>>>>>  qmp-commands.hx    | 27 +++++++++++++++++++++++++++
> >>>>>>  3 files changed, 77 insertions(+)
> >>>>>>
> >>>>>>diff --git a/migration/savevm.c b/migration/savevm.c
> >>>>>>index 96e7db5..aaead12 100644
> >>>>>>--- a/migration/savevm.c
> >>>>>>+++ b/migration/savevm.c
> >>>>>>@@ -50,6 +50,7 @@
> >>>>>>  #include "qemu/iov.h"
> >>>>>>  #include "block/snapshot.h"
> >>>>>>  #include "block/qapi.h"
> >>>>>>+#include "hw/xen/xen.h"
> >>>>>>
> >>>>>>
> >>>>>>  #ifndef ETH_P_RARP
> >>>>>>@@ -1768,6 +1769,12 @@ qemu_loadvm_section_start_full(QEMUFile *f, MigrationIncomingState *mis)
> >>>>>>          return -EINVAL;
> >>>>>>      }
> >>>>>>
> >>>>>>+    /* Validate if it is a device's state */
> >>>>>>+    if (xen_enabled() && se->is_ram) {
> >>>>>>+        error_report("loadvm: %s RAM loading not allowed on Xen", idstr);
> >>>>>>+        return -EINVAL;
> >>>>>>+    }
> >>>>>>+
> >>>>>>      /* Add entry */
> >>>>>>      le = g_malloc0(sizeof(*le));
> >>>>>>
> >>>>>>@@ -2077,6 +2084,35 @@ void qmp_xen_save_devices_state(const char *filename, Error **errp)
> >>>>>>      }
> >>>>>>  }
> >>>>>>
> >>>>>>+void qmp_xen_load_devices_state(const char *filename, Error **errp)
> >>>>>>+{
> >>>>>>+    QEMUFile *f;
> >>>>>>+    int saved_vm_running;
> >>>>>>+    int ret;
> >>>>>>+
> >>>>>>+    saved_vm_running = runstate_is_running();
> >>>>>>+    vm_stop(RUN_STATE_RESTORE_VM);
> >>>>>>+
> >>>>>>+    f = qemu_fopen(filename, "rb");
> >>>>>>+    if (!f) {
> >>>>>>+        error_setg_file_open(errp, errno, filename);
> >>>>>>+        goto out;
> >>>>>>+    }
> >>>>>>+
> >>>>>>+    migration_incoming_state_new(f);
> >>>>>>+    ret = qemu_loadvm_state(f);
> >>>>>>+    qemu_fclose(f);
> >>>>>>+    migration_incoming_state_destroy();
> >>>>>>+    if (ret < 0) {
> >>>>>>+        error_setg(errp, QERR_IO_ERROR);
> >>>>>>+    }
> >>>>>>+
> >>>>>>+out:
> >>>>>>+    if (saved_vm_running) {
> >>>>>>+        vm_start();
> >>>>>>+    }
> >>>>>
> >>>>>Does it ever happen that you had it running immediately
> >>>>>before you did this command? Somehow you'd have to have loaded the RAM
> >>>>
> >>>>No, we suspend vm before running this command in xen to make sure the
> >>>>condition that you discribed never happen.
> >>>
> >>>OK, so then if the VM is suspended, what does the:
> >>>
> >>>     if (saved_vm_running) {
> >>>         vm_start();
> >>>     }
> >>>
> >>>at the end of your routine do?
> >>
> >>+    saved_vm_running = runstate_is_running();
> >>+    vm_stop(RUN_STATE_RESTORE_VM);
> >>
> >>It is copied from the other codes in qemu.
> >>I think we should return failure if the vm is running:
> >>if (runstate_is_running()) {
> >>     error_setg(xxx);
> >>     return;
> >>}
> >
> >OK, yes; and add a comment to explain.
> 
> +    if (runstate_is_running()) {
> +        error_setg(errp, "Cannot update device state while vm is running");
> +        return;
> +    }

No, a *comment*; e.g.


    /* Guest must be paused before loading the device state; the RAM state will
     * already have been loaded using xc.
     */
    if (runstate_is_running()) {
        error_setg(errp, "Cannot update device state while vm is running");
        return;
    }

Dave

> Will fix in next version.
> 
> Thanks
> 	-Xie
> >
> >Dave
> >
> >>
> >>Thanks
> >>Wen Congyang
> >>
> >>>
> >>>Dave
> >>>
> >>>
> >>>>
> >>>>Thanks
> >>>>	-Xie
> >>>>
> >>>>>at just the right point, and I don't see how that would happen if the guest
> >>>>>was running.
> >>>>>
> >>>>>Dave
> >>>>>
> >>>>>>+}
> >>>>>>+
> >>>>>>  int load_vmstate(const char *name)
> >>>>>>  {
> >>>>>>      BlockDriverState *bs, *bs_vm_state;
> >>>>>>diff --git a/qapi-schema.json b/qapi-schema.json
> >>>>>>index 362c9d8..8cca59d 100644
> >>>>>>--- a/qapi-schema.json
> >>>>>>+++ b/qapi-schema.json
> >>>>>>@@ -4122,3 +4122,17 @@
> >>>>>>  ##
> >>>>>>  { 'enum': 'ReplayMode',
> >>>>>>    'data': [ 'none', 'record', 'play' ] }
> >>>>>>+
> >>>>>>+##
> >>>>>>+# @xen-load-devices-state:
> >>>>>>+#
> >>>>>>+# Load the state of all devices from file. The RAM and the block devices
> >>>>>>+# of the VM are not loaded by this command.
> >>>>>>+#
> >>>>>>+# @filename: the file to load the state of the devices from as binary
> >>>>>>+# data. See xen-save-devices-state.txt for a description of the binary
> >>>>>>+# format.
> >>>>>>+#
> >>>>>>+# Since: 2.7
> >>>>>>+##
> >>>>>>+{ 'command': 'xen-load-devices-state', 'data': {'filename': 'str'} }
> >>>>>>diff --git a/qmp-commands.hx b/qmp-commands.hx
> >>>>>>index b629673..4925702 100644
> >>>>>>--- a/qmp-commands.hx
> >>>>>>+++ b/qmp-commands.hx
> >>>>>>@@ -587,6 +587,33 @@ Example:
> >>>>>>  EQMP
> >>>>>>
> >>>>>>      {
> >>>>>>+        .name       = "xen-load-devices-state",
> >>>>>>+        .args_type  = "filename:F",
> >>>>>>+        .mhandler.cmd_new = qmp_marshal_xen_load_devices_state,
> >>>>>>+    },
> >>>>>>+
> >>>>>>+SQMP
> >>>>>>+xen-load-devices-state
> >>>>>>+----------------------
> >>>>>>+
> >>>>>>+Load the state of all devices from file. The RAM and the block devices
> >>>>>>+of the VM are not loaded by this command.
> >>>>>>+
> >>>>>>+Arguments:
> >>>>>>+
> >>>>>>+- "filename": the file to load the state of the devices from as binary
> >>>>>>+data. See xen-save-devices-state.txt for a description of the binary
> >>>>>>+format.
> >>>>>>+
> >>>>>>+Example:
> >>>>>>+
> >>>>>>+-> { "execute": "xen-load-devices-state",
> >>>>>>+     "arguments": { "filename": "/tmp/resume" } }
> >>>>>>+<- { "return": {} }
> >>>>>>+
> >>>>>>+EQMP
> >>>>>>+
> >>>>>>+    {
> >>>>>>          .name       = "xen-set-global-dirty-log",
> >>>>>>          .args_type  = "enable:b",
> >>>>>>          .mhandler.cmd_new = qmp_marshal_xen_set_global_dirty_log,
> >>>>>>--
> >>>>>>1.9.3
> >>>>>>
> >>>>>>
> >>>>>>
> >>>>>--
> >>>>>Dr. David Alan Gilbert / [e-mail] / Manchester, UK
> >>>>>
> >>>>>
> >>>>>.
> >>>>>
> >>>>
> >>>>
> >>>--
> >>>Dr. David Alan Gilbert / [e-mail] / Manchester, UK
> >>>
> >>>
> >>>.
> >>>
> >>
> >>
> >>
> >--
> >Dr. David Alan Gilbert / [e-mail] / Manchester, UK
> >
> >
> >.
> >
> 
> 
--
Dr. David Alan Gilbert / [e-mail] / Manchester, UK

Patch

diff --git a/migration/savevm.c b/migration/savevm.c
index 96e7db5..aaead12 100644
--- a/migration/savevm.c
+++ b/migration/savevm.c
@@ -50,6 +50,7 @@ 
 #include "qemu/iov.h"
 #include "block/snapshot.h"
 #include "block/qapi.h"
+#include "hw/xen/xen.h"
 
 
 #ifndef ETH_P_RARP
@@ -1768,6 +1769,12 @@  qemu_loadvm_section_start_full(QEMUFile *f, MigrationIncomingState *mis)
         return -EINVAL;
     }
 
+    /* Validate if it is a device's state */
+    if (xen_enabled() && se->is_ram) {
+        error_report("loadvm: %s RAM loading not allowed on Xen", idstr);
+        return -EINVAL;
+    }
+
     /* Add entry */
     le = g_malloc0(sizeof(*le));
 
@@ -2077,6 +2084,35 @@  void qmp_xen_save_devices_state(const char *filename, Error **errp)
     }
 }
 
+void qmp_xen_load_devices_state(const char *filename, Error **errp)
+{
+    QEMUFile *f;
+    int saved_vm_running;
+    int ret;
+
+    saved_vm_running = runstate_is_running();
+    vm_stop(RUN_STATE_RESTORE_VM);
+
+    f = qemu_fopen(filename, "rb");
+    if (!f) {
+        error_setg_file_open(errp, errno, filename);
+        goto out;
+    }
+
+    migration_incoming_state_new(f);
+    ret = qemu_loadvm_state(f);
+    qemu_fclose(f);
+    migration_incoming_state_destroy();
+    if (ret < 0) {
+        error_setg(errp, QERR_IO_ERROR);
+    }
+
+out:
+    if (saved_vm_running) {
+        vm_start();
+    }
+}
+
 int load_vmstate(const char *name)
 {
     BlockDriverState *bs, *bs_vm_state;
diff --git a/qapi-schema.json b/qapi-schema.json
index 362c9d8..8cca59d 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -4122,3 +4122,17 @@ 
 ##
 { 'enum': 'ReplayMode',
   'data': [ 'none', 'record', 'play' ] }
+
+##
+# @xen-load-devices-state:
+#
+# Load the state of all devices from file. The RAM and the block devices
+# of the VM are not loaded by this command.
+#
+# @filename: the file to load the state of the devices from as binary
+# data. See xen-save-devices-state.txt for a description of the binary
+# format.
+#
+# Since: 2.7
+##
+{ 'command': 'xen-load-devices-state', 'data': {'filename': 'str'} }
diff --git a/qmp-commands.hx b/qmp-commands.hx
index b629673..4925702 100644
--- a/qmp-commands.hx
+++ b/qmp-commands.hx
@@ -587,6 +587,33 @@  Example:
 EQMP
 
     {
+        .name       = "xen-load-devices-state",
+        .args_type  = "filename:F",
+        .mhandler.cmd_new = qmp_marshal_xen_load_devices_state,
+    },
+
+SQMP
+xen-load-devices-state
+----------------------
+
+Load the state of all devices from file. The RAM and the block devices
+of the VM are not loaded by this command.
+
+Arguments:
+
+- "filename": the file to load the state of the devices from as binary
+data. See xen-save-devices-state.txt for a description of the binary
+format.
+
+Example:
+
+-> { "execute": "xen-load-devices-state",
+     "arguments": { "filename": "/tmp/resume" } }
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "xen-set-global-dirty-log",
         .args_type  = "enable:b",
         .mhandler.cmd_new = qmp_marshal_xen_set_global_dirty_log,