[v10,21/31] libxc/save: support COLO save

Commit Message

Wen Congyang Feb. 22, 2016, 2:52 a.m. UTC

After suspend primary vm, get dirty bitmap on secondary vm,
and send pages both dirty on primary/secondary to secondary.

Signed-off-by: Yang Hongyang <[e-mail]>
Signed-off-by: Wen Congyang <[e-mail]>
CC: Andrew Cooper <[e-mail]>
---
 tools/libxc/xc_sr_common.h |   2 +
 tools/libxc/xc_sr_save.c   | 104 ++++++++++++++++++++++++++++++++++++++++++---
 2 files changed, 101 insertions(+), 5 deletions(-)

Comments

Wei Liu Feb. 25, 2016, 3:58 p.m. UTC | #1

On Mon, Feb 22, 2016 at 10:52:25AM +0800, Wen Congyang wrote:
[...]
>  /*
>   * Suspend the domain and send dirty memory.
>   * This is the last iteration of the live migration and the
> @@ -558,6 +610,16 @@ static int suspend_and_send_dirty(struct xc_sr_context *ctx)
>  
>      bitmap_or(dirty_bitmap, ctx->save.deferred_pages, ctx->save.p2m_size);
>  
> +    if ( !ctx->save.live && ctx->save.checkpointed == MIG_STREAM_COLO )
> +    {
> +        rc = merge_secondary_dirty_bitmap(ctx);
> +        if ( rc )
> +        {
> +            PERROR("Failed to get secondary vm's dirty pages");
> +            goto out;
> +        }
> +    }
> +
>      rc = send_dirty_pages(ctx, stats.dirty_count + ctx->save.nr_deferred_pages);
>      if ( rc )
>          goto out;
> @@ -791,13 +853,42 @@ static int save(struct xc_sr_context *ctx, uint16_t guest_type)
>              if ( rc )
>                  goto err;
>  
> -            rc = ctx->save.callbacks->postcopy(ctx->save.callbacks->data);
> -            if ( rc <= 0 )
> -                goto err;
> +            if ( ctx->save.checkpointed == MIG_STREAM_COLO )
> +            {
> +                rc = ctx->save.callbacks->checkpoint(ctx->save.callbacks->data);
> +                if ( !rc )
> +                {
> +                    rc = -1;
> +                    goto err;
> +                }
> +            }
>  
> -            rc = ctx->save.callbacks->checkpoint(ctx->save.callbacks->data);
> -            if ( rc <= 0 )
> +            rc = ctx->save.callbacks->postcopy(ctx->save.callbacks->data);
> +            if ( !rc )

This original code for checking postcopy return value is if ( rc <= 0 ).

Wei.

Wen Congyang Feb. 26, 2016, 2:35 a.m. UTC | #2

On 02/25/2016 11:58 PM, Wei Liu wrote:
> On Mon, Feb 22, 2016 at 10:52:25AM +0800, Wen Congyang wrote:
> [...]
>>  /*
>>   * Suspend the domain and send dirty memory.
>>   * This is the last iteration of the live migration and the
>> @@ -558,6 +610,16 @@ static int suspend_and_send_dirty(struct xc_sr_context *ctx)
>>  
>>      bitmap_or(dirty_bitmap, ctx->save.deferred_pages, ctx->save.p2m_size);
>>  
>> +    if ( !ctx->save.live && ctx->save.checkpointed == MIG_STREAM_COLO )
>> +    {
>> +        rc = merge_secondary_dirty_bitmap(ctx);
>> +        if ( rc )
>> +        {
>> +            PERROR("Failed to get secondary vm's dirty pages");
>> +            goto out;
>> +        }
>> +    }
>> +
>>      rc = send_dirty_pages(ctx, stats.dirty_count + ctx->save.nr_deferred_pages);
>>      if ( rc )
>>          goto out;
>> @@ -791,13 +853,42 @@ static int save(struct xc_sr_context *ctx, uint16_t guest_type)
>>              if ( rc )
>>                  goto err;
>>  
>> -            rc = ctx->save.callbacks->postcopy(ctx->save.callbacks->data);
>> -            if ( rc <= 0 )
>> -                goto err;
>> +            if ( ctx->save.checkpointed == MIG_STREAM_COLO )
>> +            {
>> +                rc = ctx->save.callbacks->checkpoint(ctx->save.callbacks->data);
>> +                if ( !rc )
>> +                {
>> +                    rc = -1;
>> +                    goto err;
>> +                }
>> +            }
>>  
>> -            rc = ctx->save.callbacks->checkpoint(ctx->save.callbacks->data);
>> -            if ( rc <= 0 )
>> +            rc = ctx->save.callbacks->postcopy(ctx->save.callbacks->data);
>> +            if ( !rc )
> 
> This original code for checking postcopy return value is if ( rc <= 0 ).

OK, I will check it.

Thanks
Wen Congyang

> 
> Wei.
> 
> 
> .
>

Patch

diff --git a/tools/libxc/xc_sr_common.h b/tools/libxc/xc_sr_common.h
index 29ab4eb..5301746 100644
--- a/tools/libxc/xc_sr_common.h
+++ b/tools/libxc/xc_sr_common.h
@@ -196,6 +196,8 @@  struct xc_sr_context
     {
         struct /* Save data. */
         {
+            int recv_fd;
+
             struct xc_sr_save_ops ops;
             struct save_callbacks *callbacks;
 
diff --git a/tools/libxc/xc_sr_save.c b/tools/libxc/xc_sr_save.c
index 7393355..89a5eaf 100644
--- a/tools/libxc/xc_sr_save.c
+++ b/tools/libxc/xc_sr_save.c
@@ -517,6 +517,58 @@  static int send_memory_live(struct xc_sr_context *ctx)
     return rc;
 }
 
+static int merge_secondary_dirty_bitmap(struct xc_sr_context *ctx)
+{
+    xc_interface *xch = ctx->xch;
+    struct xc_sr_record rec;
+    uint64_t *pfns = NULL;
+    uint64_t pfn;
+    unsigned count, i;
+    int rc;
+    DECLARE_HYPERCALL_BUFFER_SHADOW(unsigned long, dirty_bitmap,
+                                    &ctx->save.dirty_bitmap_hbuf);
+
+    rc = read_record(ctx, ctx->save.recv_fd, &rec);
+    if ( rc )
+        goto err;
+
+    if ( rec.type != REC_TYPE_CHECKPOINT_DIRTY_PFN_LIST )
+    {
+        PERROR("Expect dirty bitmap record, but received %u", rec.type );
+        rc = -1;
+        goto err;
+    }
+
+    if ( rec.length % sizeof(*pfns) )
+    {
+        PERROR("Invalid dirty pfn list record length %u", rec.length );
+        rc = -1;
+        goto err;
+    }
+
+    count = rec.length / sizeof(*pfns);
+    pfns = rec.data;
+
+    for ( i = 0; i < count; i++ )
+    {
+        pfn = pfns[i];
+        if (pfn > ctx->save.p2m_size)
+        {
+            PERROR("Invalid pfn %#lx", pfn );
+            rc = -1;
+            goto err;
+        }
+
+        set_bit(pfn, dirty_bitmap);
+    }
+
+    rc = 0;
+
+ err:
+    free(rec.data);
+    return rc;
+}
+
 /*
  * Suspend the domain and send dirty memory.
  * This is the last iteration of the live migration and the
@@ -558,6 +610,16 @@  static int suspend_and_send_dirty(struct xc_sr_context *ctx)
 
     bitmap_or(dirty_bitmap, ctx->save.deferred_pages, ctx->save.p2m_size);
 
+    if ( !ctx->save.live && ctx->save.checkpointed == MIG_STREAM_COLO )
+    {
+        rc = merge_secondary_dirty_bitmap(ctx);
+        if ( rc )
+        {
+            PERROR("Failed to get secondary vm's dirty pages");
+            goto out;
+        }
+    }
+
     rc = send_dirty_pages(ctx, stats.dirty_count + ctx->save.nr_deferred_pages);
     if ( rc )
         goto out;
@@ -791,13 +853,42 @@  static int save(struct xc_sr_context *ctx, uint16_t guest_type)
             if ( rc )
                 goto err;
 
-            rc = ctx->save.callbacks->postcopy(ctx->save.callbacks->data);
-            if ( rc <= 0 )
-                goto err;
+            if ( ctx->save.checkpointed == MIG_STREAM_COLO )
+            {
+                rc = ctx->save.callbacks->checkpoint(ctx->save.callbacks->data);
+                if ( !rc )
+                {
+                    rc = -1;
+                    goto err;
+                }
+            }
 
-            rc = ctx->save.callbacks->checkpoint(ctx->save.callbacks->data);
-            if ( rc <= 0 )
+            rc = ctx->save.callbacks->postcopy(ctx->save.callbacks->data);
+            if ( !rc )
+            {
+                rc = -1;
                 goto err;
+            }
+
+            if ( ctx->save.checkpointed == MIG_STREAM_COLO )
+            {
+                rc = ctx->save.callbacks->wait_checkpoint(
+                                                    ctx->save.callbacks->data);
+                if ( rc <= 0 )
+                    goto err;
+            }
+            else if ( ctx->save.checkpointed == MIG_STREAM_REMUS )
+            {
+                rc = ctx->save.callbacks->checkpoint(ctx->save.callbacks->data);
+                if ( rc <= 0 )
+                    goto err;
+            }
+            else
+            {
+                ERROR("Unknown checkpointed stream");
+                rc = -1;
+                goto err;
+            }
         }
     } while ( ctx->save.checkpointed != MIG_STREAM_NONE );
 
@@ -843,6 +934,7 @@  int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom,
     ctx.save.live  = !!(flags & XCFLAGS_LIVE);
     ctx.save.debug = !!(flags & XCFLAGS_DEBUG);
     ctx.save.checkpointed = checkpointed_stream;
+    ctx.save.recv_fd = recv_fd;
 
     /* If altering migration_stream update this assert too. */
     assert(checkpointed_stream == MIG_STREAM_NONE ||
@@ -863,6 +955,8 @@  int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom,
         assert(callbacks->switch_qemu_logdirty);
     if ( ctx.save.checkpointed )
         assert(callbacks->checkpoint && callbacks->postcopy);
+    if ( ctx.save.checkpointed == MIG_STREAM_COLO )
+        assert(callbacks->wait_checkpoint);
 
     DPRINTF("fd %d, dom %u, max_iters %u, max_factor %u, flags %u, hvm %d",
             io_fd, dom, max_iters, max_factor, flags, hvm);