@@ -10,3 +10,4 @@ Migration has plenty of features to support different use cases.
dirty-limit
vfio
virtio
+ mapped-ram
new file mode 100644
@@ -0,0 +1,138 @@
+Mapped-ram
+==========
+
+Mapped-ram is a new stream format for the RAM section designed to
+supplement the existing ``file:`` migration and make it compatible
+with ``multifd``. This enables parallel migration of a guest's RAM to
+a file.
+
+The core of the feature is to ensure that RAM pages are mapped
+directly to offsets in the resulting migration file. This enables the
+``multifd`` threads to write exclusively to those offsets even if the
+guest is constantly dirtying pages (i.e. live migration). Another
+benefit is that the resulting file will have a bounded size, since
+pages which are dirtied multiple times will always go to a fixed
+location in the file, rather than constantly being added to a
+sequential stream. Having the pages at fixed offsets also allows the
+usage of O_DIRECT for save/restore of the migration stream as the
+pages are ensured to be written respecting O_DIRECT alignment
+restrictions (direct-io support not yet implemented).
+
+Usage
+-----
+
+On both source and destination, enable the ``multifd`` and
+``mapped-ram`` capabilities:
+
+ ``migrate_set_capability multifd on``
+
+ ``migrate_set_capability mapped-ram on``
+
+Use a ``file:`` URL for migration:
+
+ ``migrate file:/path/to/migration/file``
+
+Mapped-ram migration is best done non-live, i.e. by stopping the VM on
+the source side before migrating.
+
+Use-cases
+---------
+
+The mapped-ram feature was designed for use cases where the migration
+stream will be directed to a file in the filesystem and not
+immediately restored on the destination VM [#]_. These could be
+thought of as snapshots. We can further categorize them into live and
+non-live.
+
+- Non-live snapshot
+
+If the use case requires a VM to be stopped before taking a snapshot,
+that's the ideal scenario for mapped-ram migration. Not having to
+track dirty pages, the migration will write the RAM pages to the disk
+as fast as it can.
+
+Note: if a snapshot is taken of a running VM, but the VM will be
+stopped after the snapshot by the admin, then consider stopping it
+right before the snapshot to take benefit of the performance gains
+mentioned above.
+
+- Live snapshot
+
+If the use case requires that the VM keeps running during and after
+the snapshot operation, then mapped-ram migration can still be used,
+but will be less performant. Other strategies such as
+background-snapshot should be evaluated as well. One benefit of
+mapped-ram in this scenario is portability since background-snapshot
+depends on async dirty tracking (KVM_GET_DIRTY_LOG) which is not
+supported outside of Linux.
+
+.. [#] While this same effect could be obtained with the usage of
+ snapshots or the ``file:`` migration alone, mapped-ram provides
+ a performance increase for VMs with larger RAM sizes (10s to
+ 100s of GiBs), specially if the VM has been stopped beforehand.
+
+RAM section format
+------------------
+
+Instead of having a sequential stream of pages that follow the
+RAMBlock headers, the dirty pages for a RAMBlock follow its header
+instead. This ensures that each RAM page has a fixed offset in the
+resulting migration file.
+
+A bitmap is introduced to track which pages have been written in the
+migration file. Pages are written at a fixed location for every
+ramblock. Zero pages are ignored as they'd be zero in the destination
+migration as well.
+
+::
+
+ Without mapped-ram: With mapped-ram:
+
+ --------------------- --------------------------------
+ | ramblock 1 header | | ramblock 1 header |
+ --------------------- --------------------------------
+ | ramblock 2 header | | ramblock 1 mapped-ram header |
+ --------------------- --------------------------------
+ | ... | | padding to next 1MB boundary |
+ --------------------- | ... |
+ | ramblock n header | --------------------------------
+ --------------------- | ramblock 1 pages |
+ | RAM_SAVE_FLAG_EOS | | ... |
+ --------------------- --------------------------------
+ | stream of pages | | ramblock 2 header |
+ | (iter 1) | --------------------------------
+ | ... | | ramblock 2 mapped-ram header |
+ --------------------- --------------------------------
+ | RAM_SAVE_FLAG_EOS | | padding to next 1MB boundary |
+ --------------------- | ... |
+ | stream of pages | --------------------------------
+ | (iter 2) | | ramblock 2 pages |
+ | ... | | ... |
+ --------------------- --------------------------------
+ | ... | | ... |
+ --------------------- --------------------------------
+ | RAM_SAVE_FLAG_EOS |
+ --------------------------------
+ | ... |
+ --------------------------------
+
+where:
+ - ramblock header: the generic information for a ramblock, such as
+ idstr, used_len, etc.
+
+ - ramblock mapped-ram header: the information added by this feature:
+ bitmap of pages written, bitmap size and offset of pages in the
+ migration file.
+
+Restrictions
+------------
+
+Since pages are written to their relative offsets and out of order
+(due to the memory dirtying patterns), streaming channels such as
+sockets are not supported. A seekable channel such as a file is
+required. This can be verified in the QIOChannel by the presence of
+the QIO_CHANNEL_FEATURE_SEEKABLE.
+
+The improvements brought by this feature apply only to guest physical
+RAM. Other types of memory such as VRAM are migrated as part of device
+states.
@@ -1953,6 +1953,13 @@ static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc,
return false;
}
+ if (migrate_mapped_ram()) {
+ if (migrate_tls()) {
+ error_setg(errp, "Cannot use TLS with mapped-ram");
+ return false;
+ }
+ }
+
if (migrate_mode_is_cpr(s)) {
const char *conflict = NULL;
@@ -204,6 +204,7 @@ Property migration_properties[] = {
DEFINE_PROP_MIG_CAP("x-switchover-ack",
MIGRATION_CAPABILITY_SWITCHOVER_ACK),
DEFINE_PROP_MIG_CAP("x-dirty-limit", MIGRATION_CAPABILITY_DIRTY_LIMIT),
+ DEFINE_PROP_MIG_CAP("mapped-ram", MIGRATION_CAPABILITY_MAPPED_RAM),
DEFINE_PROP_END_OF_LIST(),
};
@@ -263,6 +264,13 @@ bool migrate_events(void)
return s->capabilities[MIGRATION_CAPABILITY_EVENTS];
}
+bool migrate_mapped_ram(void)
+{
+ MigrationState *s = migrate_get_current();
+
+ return s->capabilities[MIGRATION_CAPABILITY_MAPPED_RAM];
+}
+
bool migrate_ignore_shared(void)
{
MigrationState *s = migrate_get_current();
@@ -645,6 +653,32 @@ bool migrate_caps_check(bool *old_caps, bool *new_caps, Error **errp)
}
}
+ if (new_caps[MIGRATION_CAPABILITY_MAPPED_RAM]) {
+ if (new_caps[MIGRATION_CAPABILITY_MULTIFD]) {
+ error_setg(errp,
+ "Mapped-ram migration is incompatible with multifd");
+ return false;
+ }
+
+ if (new_caps[MIGRATION_CAPABILITY_XBZRLE]) {
+ error_setg(errp,
+ "Mapped-ram migration is incompatible with xbzrle");
+ return false;
+ }
+
+ if (new_caps[MIGRATION_CAPABILITY_COMPRESS]) {
+ error_setg(errp,
+ "Mapped-ram migration is incompatible with compression");
+ return false;
+ }
+
+ if (new_caps[MIGRATION_CAPABILITY_POSTCOPY_RAM]) {
+ error_setg(errp,
+ "Mapped-ram migration is incompatible with postcopy");
+ return false;
+ }
+ }
+
return true;
}
@@ -31,6 +31,7 @@ bool migrate_compress(void);
bool migrate_dirty_bitmaps(void);
bool migrate_dirty_limit(void);
bool migrate_events(void);
+bool migrate_mapped_ram(void);
bool migrate_ignore_shared(void);
bool migrate_late_block_activate(void);
bool migrate_multifd(void);
@@ -245,6 +245,7 @@ static bool should_validate_capability(int capability)
/* Validate only new capabilities to keep compatibility. */
switch (capability) {
case MIGRATION_CAPABILITY_X_IGNORE_SHARED:
+ case MIGRATION_CAPABILITY_MAPPED_RAM:
return true;
default:
return false;
@@ -531,6 +531,10 @@
# and can result in more stable read performance. Requires KVM
# with accelerator property "dirty-ring-size" set. (Since 8.1)
#
+# @mapped-ram: Migrate using fixed offsets in the migration file for
+# each RAM page. Requires a migration URI that supports seeking,
+# such as a file. (since 9.0)
+#
# Features:
#
# @deprecated: Member @block is deprecated. Use blockdev-mirror with
@@ -555,7 +559,7 @@
{ 'name': 'x-ignore-shared', 'features': [ 'unstable' ] },
'validate-uuid', 'background-snapshot',
'zero-copy-send', 'postcopy-preempt', 'switchover-ack',
- 'dirty-limit'] }
+ 'dirty-limit', 'mapped-ram'] }
##
# @MigrationCapabilityStatus: