From patchwork Wed Dec 30 02:37:42 2015 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Wen Congyang X-Patchwork-Id: 7931601 Return-Path: X-Original-To: patchwork-xen-devel@patchwork.kernel.org Delivered-To: patchwork-parsemail@patchwork2.web.kernel.org Received: from mail.kernel.org (mail.kernel.org [198.145.29.136]) by patchwork2.web.kernel.org (Postfix) with ESMTP id 00296BF0A9 for ; Wed, 30 Dec 2015 02:40:34 +0000 (UTC) Received: from mail.kernel.org (localhost [127.0.0.1]) by mail.kernel.org (Postfix) with ESMTP id EFC3D201FA for ; Wed, 30 Dec 2015 02:40:31 +0000 (UTC) Received: from lists.xen.org (lists.xenproject.org [50.57.142.19]) (using TLSv1 with cipher AES256-SHA (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id C5306201EF for ; Wed, 30 Dec 2015 02:40:29 +0000 (UTC) Received: from localhost ([127.0.0.1] helo=lists.xen.org) by lists.xen.org with esmtp (Exim 4.72) (envelope-from ) id 1aE6f8-000307-R8; Wed, 30 Dec 2015 02:38:54 +0000 Received: from mail6.bemta14.messagelabs.com ([193.109.254.103]) by lists.xen.org with esmtp (Exim 4.72) (envelope-from ) id 1aE6f5-0002v4-NF for xen-devel@lists.xen.org; Wed, 30 Dec 2015 02:38:51 +0000 Received: from [193.109.254.147] by server-7.bemta-14.messagelabs.com id 04/A3-28221-BB343865; Wed, 30 Dec 2015 02:38:51 +0000 X-Env-Sender: wency@cn.fujitsu.com X-Msg-Ref: server-8.tower-27.messagelabs.com!1451443108!10039549!4 X-Originating-IP: [59.151.112.132] X-SpamReason: No, hits=0.5 required=7.0 tests=BODY_RANDOM_LONG X-StarScan-Received: X-StarScan-Version: 7.35.1; banners=-,-,- X-VirusChecked: Checked Received: (qmail 11168 invoked from network); 30 Dec 2015 02:38:42 -0000 Received: from cn.fujitsu.com (HELO heian.cn.fujitsu.com) (59.151.112.132) by server-8.tower-27.messagelabs.com with SMTP; 30 Dec 2015 02:38:42 -0000 X-IronPort-AV: E=Sophos;i="5.20,346,1444665600"; d="scan'208";a="2080175" Received: from unknown (HELO cn.fujitsu.com) ([10.167.33.5]) by heian.cn.fujitsu.com with ESMTP; 30 Dec 2015 10:38:29 +0800 Received: from G08CNEXCHPEKD01.g08.fujitsu.local (unknown [10.167.33.80]) by cn.fujitsu.com (Postfix) with ESMTP id 35B5641890F8; Wed, 30 Dec 2015 10:38:11 +0800 (CST) Received: from G08FNSTD140052.g08.fujitsu.local (10.167.226.52) by G08CNEXCHPEKD01.g08.fujitsu.local (10.167.33.89) with Microsoft SMTP Server (TLS) id 14.3.181.6; Wed, 30 Dec 2015 10:38:10 +0800 From: Wen Congyang To: xen devel , Andrew Cooper , Ian Campbell , Ian Jackson , Wei Liu Date: Wed, 30 Dec 2015 10:37:42 +0800 Message-ID: <1451443075-27428-13-git-send-email-wency@cn.fujitsu.com> X-Mailer: git-send-email 2.5.0 In-Reply-To: <1451443075-27428-1-git-send-email-wency@cn.fujitsu.com> References: <1451443075-27428-1-git-send-email-wency@cn.fujitsu.com> MIME-Version: 1.0 X-Originating-IP: [10.167.226.52] X-yoursite-MailScanner-ID: 35B5641890F8.A7287 X-yoursite-MailScanner: Found to be clean X-yoursite-MailScanner-From: wency@cn.fujitsu.com X-Spam-Status: No, score=-4.2 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_MED, UNPARSEABLE_RELAY autolearn=unavailable version=3.3.1 Cc: Lars Kurth , Changlong Xie , Wen Congyang , Gui Jianfeng , Jiang Yunhong , Dong Eddie , Shriram Rajagopalan , Yang Hongyang Subject: [Xen-devel] [PATCH v9 12/25] primary vm suspend/resume/checkpoint code X-BeenThere: xen-devel@lists.xen.org X-Mailman-Version: 2.1.13 Precedence: list List-Id: Xen developer discussion List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Sender: xen-devel-bounces@lists.xen.org Errors-To: xen-devel-bounces@lists.xen.org X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on mail.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP We will do the following things again and again: 1. Suspend primary vm a. Suspend primary vm b. do postsuspend c. Read CHECKPOINT_SVM_SUSPENDED sent by secondary 2. Resume primary vm a. Read CHECKPOINT_SVM_READY from slave b. Do presume c. Resume primary vm d. Read CHECKPOINT_SVM_RESUMED from slave 3. Wait a new checkpoint a. Wait a new checkpoint(not implemented) b. Send CHECKPOINT_NEW to slave Signed-off-by: Wen Congyang Signed-off-by: Yang Hongyang --- tools/libxl/Makefile | 2 +- tools/libxl/libxl.c | 6 +- tools/libxl/libxl_colo.h | 10 + tools/libxl/libxl_colo_save.c | 552 ++++++++++++++++++++++++++++++++++++++++++ tools/libxl/libxl_dom_save.c | 13 +- tools/libxl/libxl_internal.h | 168 +++++++------ tools/libxl/libxl_types.idl | 1 + 7 files changed, 672 insertions(+), 80 deletions(-) create mode 100644 tools/libxl/libxl_colo_save.c diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile index 19a95a9..b11cf34 100644 --- a/tools/libxl/Makefile +++ b/tools/libxl/Makefile @@ -63,7 +63,7 @@ LIBXL_OBJS-y += libxl_no_convert_callout.o endif LIBXL_OBJS-y += libxl_remus.o libxl_checkpoint_device.o libxl_remus_disk_drbd.o -LIBXL_OBJS-y += libxl_colo_restore.o +LIBXL_OBJS-y += libxl_colo_restore.o libxl_colo_save.o LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o libxl_psr.o LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_arm.o libxl_libfdt_compat.o diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c index 481824d..7d227d7 100644 --- a/tools/libxl/libxl.c +++ b/tools/libxl/libxl.c @@ -17,6 +17,7 @@ #include "libxl_osdeps.h" #include "libxl_internal.h" +#include "libxl_colo.h" #define PAGE_TO_MEMKB(pages) ((pages) * 4) #define BACKEND_STRING_SIZE 5 @@ -882,7 +883,10 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info, assert(info); /* Point of no return */ - libxl__remus_setup(egc, &dss->rs); + if (libxl_defbool_val(info->colo)) + libxl__colo_save_setup(egc, &dss->css); + else + libxl__remus_setup(egc, &dss->rs); return AO_INPROGRESS; out: diff --git a/tools/libxl/libxl_colo.h b/tools/libxl/libxl_colo.h index 8bea1a2..39515c4 100644 --- a/tools/libxl/libxl_colo.h +++ b/tools/libxl/libxl_colo.h @@ -21,4 +21,14 @@ extern void libxl__colo_restore_setup(libxl__egc *egc, extern void libxl__colo_restore_teardown(libxl__egc *egc, void *dcs_void, int ret, int retval, int errnoval); +extern void libxl__colo_save_domain_suspend_callback(void *data); +extern void libxl__colo_save_domain_checkpoint_callback(void *data); +extern void libxl__colo_save_domain_resume_callback(void *data); +extern void libxl__colo_save_domain_should_checkpoint_callback(void *data); +extern void libxl__colo_save_setup(libxl__egc *egc, + libxl__colo_save_state *css); +extern void libxl__colo_save_teardown(libxl__egc *egc, + libxl__colo_save_state *css, + int rc); + #endif diff --git a/tools/libxl/libxl_colo_save.c b/tools/libxl/libxl_colo_save.c new file mode 100644 index 0000000..d6b4e7b --- /dev/null +++ b/tools/libxl/libxl_colo_save.c @@ -0,0 +1,552 @@ +/* + * Copyright (C) 2014 FUJITSU LIMITED + * Author: Wen Congyang + * Yang Hongyang + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; version 2.1 only. with the special + * exception on linking described in file LICENSE. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + */ + +#include "libxl_osdeps.h" /* must come before any other headers */ + +#include "libxl_internal.h" +#include "libxl_colo.h" + +static const libxl__checkpoint_device_instance_ops *colo_ops[] = { + NULL, +}; + +/* ================= helper functions ================= */ +static int init_device_subkind(libxl__checkpoint_devices_state *cds) +{ + /* init device subkind-specific state in the libxl ctx */ + int rc; + STATE_AO_GC(cds->ao); + + rc = 0; + return rc; +} + +static void cleanup_device_subkind(libxl__checkpoint_devices_state *cds) +{ + /* cleanup device subkind-specific state in the libxl ctx */ + STATE_AO_GC(cds->ao); +} + +/* ================= colo: setup save environment ================= */ +static void colo_save_setup_done(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc); +static void colo_save_setup_failed(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc); + +void libxl__colo_save_setup(libxl__egc *egc, libxl__colo_save_state *css) +{ + libxl__domain_save_state *dss = CONTAINER_OF(css, *dss, css); + + /* Convenience aliases */ + libxl__checkpoint_devices_state *const cds = &dss->cds; + + STATE_AO_GC(dss->ao); + + if (dss->type != LIBXL_DOMAIN_TYPE_HVM) { + LOG(ERROR, "COLO only supports hvm now"); + goto out; + } + + css->send_fd = dss->fd; + css->recv_fd = dss->recv_fd; + css->svm_running = false; + + /* TODO: disk/nic support */ + cds->device_kind_flags = 0; + cds->ops = colo_ops; + cds->callback = colo_save_setup_done; + cds->ao = ao; + cds->domid = dss->domid; + cds->concrete_data = css; + + css->srs.ao = ao; + css->srs.fd = css->recv_fd; + css->srs.back_channel = true; + libxl__stream_read_start(egc, &css->srs); + + if (init_device_subkind(cds)) + goto out; + + libxl__checkpoint_devices_setup(egc, &dss->cds); + + return; + +out: + libxl__ao_complete(egc, ao, ERROR_FAIL); +} + +static void colo_save_setup_done(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc) +{ + libxl__colo_save_state *css = cds->concrete_data; + libxl__domain_save_state *dss = CONTAINER_OF(css, *dss, css); + EGC_GC; + + if (!rc) { + libxl__domain_save(egc, dss); + return; + } + + LOG(ERROR, "COLO: failed to setup device for guest with domid %u", + dss->domid); + cds->callback = colo_save_setup_failed; + libxl__checkpoint_devices_teardown(egc, cds); +} + +static void colo_save_setup_failed(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc) +{ + STATE_AO_GC(cds->ao); + + if (rc) + LOG(ERROR, "COLO: failed to teardown device after setup failed" + " for guest with domid %u, rc %d", cds->domid, rc); + + cleanup_device_subkind(cds); + libxl__ao_complete(egc, ao, rc); +} + + +/* ================= colo: teardown save environment ================= */ +static void colo_teardown_done(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc); + +void libxl__colo_save_teardown(libxl__egc *egc, + libxl__colo_save_state *css, + int rc) +{ + libxl__domain_save_state *dss = CONTAINER_OF(css, *dss, css); + + EGC_GC; + + LOG(WARN, "COLO: Domain suspend terminated with rc %d," + " teardown COLO devices...", rc); + + libxl__stream_read_abort(egc, &css->srs, 1); + + dss->cds.callback = colo_teardown_done; + libxl__checkpoint_devices_teardown(egc, &dss->cds); + return; +} + +static void colo_teardown_done(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc) +{ + libxl__colo_save_state *css = cds->concrete_data; + libxl__domain_save_state *dss = CONTAINER_OF(css, *dss, css); + + cleanup_device_subkind(cds); + dss->callback(egc, dss, rc); +} + +/* + * checkpoint callbacks are called in the following order: + * 1. suspend + * 2. checkpoint + * 3. resume + * 4. should_checkpoint + */ +static void colo_common_write_stream_done(libxl__egc *egc, + libxl__stream_write_state *stream, + int rc); +static void colo_common_read_stream_done(libxl__egc *egc, + libxl__stream_read_state *stream, + int rc); +/* ===================== colo: suspend primary vm ===================== */ + +static void colo_read_svm_suspended_done(libxl__egc *egc, + libxl__colo_save_state *css, + int id); +/* + * Do the following things when suspending primary vm: + * 1. suspend primary vm + * 2. do postsuspend + * 3. read CHECKPOINT_SVM_SUSPENDED + * 4. read secondary vm's dirty pages + */ +static void colo_suspend_primary_vm_done(libxl__egc *egc, + libxl__domain_suspend_state *dsps, + int ok); +static void colo_postsuspend_cb(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc); + +void libxl__colo_save_domain_suspend_callback(void *data) +{ + libxl__save_helper_state *shs = data; + libxl__egc *egc = shs->egc; + libxl__stream_write_state *sws = CONTAINER_OF(shs, *sws, shs); + libxl__domain_save_state *dss = sws->dss; + + /* Convenience aliases */ + libxl__domain_suspend_state *dsps = &dss->dsps; + + dsps->callback_common_done = colo_suspend_primary_vm_done; + libxl__domain_suspend(egc, dsps); +} + +static void colo_suspend_primary_vm_done(libxl__egc *egc, + libxl__domain_suspend_state *dsps, + int rc) +{ + libxl__domain_save_state *dss = CONTAINER_OF(dsps, *dss, dsps); + + EGC_GC; + + if (rc) { + LOG(ERROR, "cannot suspend primary vm"); + goto out; + } + + /* Convenience aliases */ + libxl__checkpoint_devices_state *const cds = &dss->cds; + + cds->callback = colo_postsuspend_cb; + libxl__checkpoint_devices_postsuspend(egc, cds); + return; + +out: + dss->rc = rc; + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, !rc); +} + +static void colo_postsuspend_cb(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc) +{ + libxl__colo_save_state *css = cds->concrete_data; + libxl__domain_save_state *dss = CONTAINER_OF(css, *dss, css); + + EGC_GC; + + if (rc) { + LOG(ERROR, "postsuspend fails"); + goto out; + } + + if (!css->svm_running) { + rc = 0; + goto out; + } + + /* + * read CHECKPOINT_SVM_SUSPENDED + */ + css->callback = colo_read_svm_suspended_done; + css->srs.checkpoint_callback = colo_common_read_stream_done; + libxl__stream_read_checkpoint_state(egc, &css->srs); + + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, !rc); +} + +static void colo_read_svm_suspended_done(libxl__egc *egc, + libxl__colo_save_state *css, + int id) +{ + int ok = 0; + libxl__domain_save_state *dss = CONTAINER_OF(css, *dss, css); + + EGC_GC; + + if (id != CHECKPOINT_SVM_SUSPENDED) { + LOG(ERROR, "invalid section: %d, expected: %d", id, + CHECKPOINT_SVM_SUSPENDED); + goto out; + } + + ok = 1; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, ok); +} + + +/* ===================== colo: send tailbuf ========================== */ +void libxl__colo_save_domain_checkpoint_callback(void *data) +{ + libxl__save_helper_state *shs = data; + libxl__stream_write_state *sws = CONTAINER_OF(shs, *sws, shs); + libxl__domain_save_state *dss = sws->dss; + + /* Convenience aliases */ + libxl__colo_save_state *const css = &dss->css; + + /* write emulator xenstore data, emulator context, and checkpoint end */ + css->callback = NULL; + dss->sws.checkpoint_callback = colo_common_write_stream_done; + libxl__stream_write_start_checkpoint(shs->egc, &dss->sws); +} + +/* ===================== colo: resume primary vm ===================== */ +/* + * Do the following things when resuming primary vm: + * 1. read CHECKPOINT_SVM_READY + * 2. do preresume + * 3. resume primary vm + * 4. read CHECKPOINT_SVM_RESUMED + */ +static void colo_read_svm_ready_done(libxl__egc *egc, + libxl__colo_save_state *css, + int id); +static void colo_preresume_cb(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc); +static void colo_read_svm_resumed_done(libxl__egc *egc, + libxl__colo_save_state *css, + int id); + +void libxl__colo_save_domain_resume_callback(void *data) +{ + libxl__save_helper_state *shs = data; + libxl__egc *egc = shs->egc; + libxl__stream_write_state *sws = CONTAINER_OF(shs, *sws, shs); + libxl__domain_save_state *dss = sws->dss; + + /* Convenience aliases */ + libxl__colo_save_state *const css = &dss->css; + + EGC_GC; + + /* read CHECKPOINT_SVM_READY */ + css->callback = colo_read_svm_ready_done; + css->srs.checkpoint_callback = colo_common_read_stream_done; + libxl__stream_read_checkpoint_state(egc, &css->srs); +} + +static void colo_read_svm_ready_done(libxl__egc *egc, + libxl__colo_save_state *css, + int id) +{ + libxl__domain_save_state *dss = CONTAINER_OF(css, *dss, css); + + EGC_GC; + + if (id != CHECKPOINT_SVM_READY) { + LOG(ERROR, "invalid section: %d, expected: %d", id, + CHECKPOINT_SVM_READY); + goto out; + } + + css->svm_running = true; + dss->cds.callback = colo_preresume_cb; + libxl__checkpoint_devices_preresume(egc, &dss->cds); + + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, 0); +} + +static void colo_preresume_cb(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc) +{ + libxl__colo_save_state *css = cds->concrete_data; + libxl__domain_save_state *dss = CONTAINER_OF(css, *dss, css); + + EGC_GC; + + if (rc) { + LOG(ERROR, "preresume fails"); + goto out; + } + + /* Resumes the domain and the device model */ + if (libxl__domain_resume(gc, dss->domid, /* Fast Suspend */1)) { + LOG(ERROR, "cannot resume primary vm"); + goto out; + } + + /* read CHECKPOINT_SVM_RESUMED */ + css->callback = colo_read_svm_resumed_done; + css->srs.checkpoint_callback = colo_common_read_stream_done; + libxl__stream_read_checkpoint_state(egc, &css->srs); + + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, 0); +} + +static void colo_read_svm_resumed_done(libxl__egc *egc, + libxl__colo_save_state *css, + int id) +{ + int ok = 0; + libxl__domain_save_state *dss = CONTAINER_OF(css, *dss, css); + + EGC_GC; + + if (id != CHECKPOINT_SVM_RESUMED) { + LOG(ERROR, "invalid section: %d, expected: %d", id, + CHECKPOINT_SVM_RESUMED); + goto out; + } + + ok = 1; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, ok); +} + + +/* ===================== colo: wait new checkpoint ===================== */ +/* + * Do the following things: + * 1. do commit + * 2. wait for a new checkpoint + * 3. write CHECKPOINT_NEW + */ +static void colo_device_commit_cb(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc); +static void colo_start_new_checkpoint(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc); + +void libxl__colo_save_domain_should_checkpoint_callback(void *data) +{ + libxl__save_helper_state *shs = data; + libxl__stream_write_state *sws = CONTAINER_OF(shs, *sws, shs); + libxl__domain_save_state *dss = sws->dss; + libxl__egc *egc = dss->sws.shs.egc; + + /* Convenience aliases */ + libxl__checkpoint_devices_state *const cds = &dss->cds; + + cds->callback = colo_device_commit_cb; + libxl__checkpoint_devices_commit(egc, cds); +} + +static void colo_device_commit_cb(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc) +{ + libxl__colo_save_state *css = cds->concrete_data; + libxl__domain_save_state *dss = CONTAINER_OF(css, *dss, css); + + EGC_GC; + + if (rc) { + LOG(ERROR, "commit fails"); + goto out; + } + + /* TODO: wait a new checkpoint */ + colo_start_new_checkpoint(egc, cds, 0); + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, 0); +} + +static void colo_start_new_checkpoint(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc) +{ + libxl__colo_save_state *css = cds->concrete_data; + libxl__domain_save_state *dss = CONTAINER_OF(css, *dss, css); + libxl_sr_checkpoint_state srcs = { .id = CHECKPOINT_NEW }; + + if (rc) + goto out; + + /* write CHECKPOINT_NEW */ + css->callback = NULL; + dss->sws.checkpoint_callback = colo_common_write_stream_done; + libxl__stream_write_checkpoint_state(egc, &dss->sws, &srcs); + + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, 0); +} + + +/* ===================== colo: common callback ===================== */ +static void colo_common_write_stream_done(libxl__egc *egc, + libxl__stream_write_state *stream, + int rc) +{ + libxl__domain_save_state *dss = CONTAINER_OF(stream, *dss, sws); + int ok; + + /* Convenience aliases */ + libxl__colo_save_state *const css = &dss->css; + + EGC_GC; + + if (rc < 0) { + /* TODO: it may be a internal error, but we don't know */ + LOG(ERROR, "sending data fails"); + ok = 0; + goto out; + } + + if (!css->callback) { + /* Everythins is OK */ + ok = 1; + goto out; + } + + css->callback(egc, css, 0); + + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, ok); +} + +static void colo_common_read_stream_done(libxl__egc *egc, + libxl__stream_read_state *stream, + int rc) +{ + libxl__colo_save_state *css = CONTAINER_OF(stream, *css, srs); + libxl__domain_save_state *dss = CONTAINER_OF(css, *dss, css); + int ok; + + EGC_GC; + + if (rc < 0) { + /* TODO: it may be a internal error, but we don't know */ + LOG(ERROR, "reading data fails"); + ok = 0; + goto out; + } + + if (!css->callback) { + /* Everythins is OK */ + ok = 1; + goto out; + } + + /* rc contains the id */ + css->callback(egc, css, rc); + + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, ok); +} diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c index 5fd9db2..776306f 100644 --- a/tools/libxl/libxl_dom_save.c +++ b/tools/libxl/libxl_dom_save.c @@ -16,6 +16,7 @@ #include "libxl_osdeps.h" /* must come before any other headers */ #include "libxl_internal.h" +#include "libxl_colo.h" #include @@ -401,6 +402,11 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss) callbacks->suspend = libxl__remus_domain_suspend_callback; callbacks->postcopy = libxl__remus_domain_resume_callback; callbacks->checkpoint = libxl__remus_domain_save_checkpoint_callback; + } else if (dss->checkpointed_stream == LIBXL_CHECKPOINTED_STREAM_COLO) { + callbacks->suspend = libxl__colo_save_domain_suspend_callback; + callbacks->postcopy = libxl__colo_save_domain_resume_callback; + callbacks->checkpoint = libxl__colo_save_domain_checkpoint_callback; + callbacks->should_checkpoint = libxl__colo_save_domain_should_checkpoint_callback; } else callbacks->suspend = libxl__domain_suspend_callback; @@ -442,12 +448,15 @@ static void domain_save_done(libxl__egc *egc, if (dss->remus) { /* - * With Remus, if we reach this point, it means either + * With Remus/COLO, if we reach this point, it means either * backup died or some network error occurred preventing us * from sending checkpoints. Teardown the network buffers and * release netlink resources. This is an async op. */ - libxl__remus_teardown(egc, &dss->rs, rc); + if (libxl_defbool_val(dss->remus->colo)) + libxl__colo_save_teardown(egc, &dss->css, rc); + else + libxl__remus_teardown(egc, &dss->rs, rc); return; } diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h index 79b0c6d..54903af 100644 --- a/tools/libxl/libxl_internal.h +++ b/tools/libxl/libxl_internal.h @@ -2798,7 +2798,7 @@ typedef struct libxl__save_helper_state { /* * The abstract checkpoint device layer exposes a common * set of API to [external] libxl for manipulating devices attached to - * a guest protected by Remus. The device layer also exposes a set of + * a guest protected by Remus/COLO. The device layer also exposes a set of * [internal] interfaces that every device type must implement. * * The following API are exposed to libxl: @@ -2816,7 +2816,7 @@ typedef struct libxl__save_helper_state { * +libxl__checkpoint_devices_commit * * Each device type needs to implement the interfaces specified in - * the libxl__checkpoint_device_instance_ops if it wishes to support Remus. + * the libxl__checkpoint_device_instance_ops if it wishes to support Remus/COLO. * * The high-level control flow through the checkpoint device layer is shown * below: @@ -2836,7 +2836,7 @@ typedef struct libxl__checkpoint_device_instance_ops libxl__checkpoint_device_in /* * Interfaces to be implemented by every device subkind that wishes to - * support Remus. Functions must be implemented unless otherwise + * support Remus/COLO. Functions must be implemented unless otherwise * stated. Many of these functions are asynchronous. They call * dev->aodev.callback when done. The actual implementations may be * synchronous and call dev->aodev.callback directly (as the last @@ -3013,6 +3013,89 @@ static inline bool libxl__conversion_helper_inuse (const libxl__conversion_helper_state *chs) { return libxl__ev_child_inuse(&chs->child); } +/* State for manipulating a libxl migration v2 stream */ +typedef struct libxl__domain_create_state libxl__domain_create_state; + +typedef void libxl__domain_create_cb(libxl__egc *egc, + libxl__domain_create_state*, + int rc, uint32_t domid); + +typedef struct libxl__stream_read_state libxl__stream_read_state; + +typedef struct libxl__sr_record_buf { + /* private to stream read helper */ + LIBXL_STAILQ_ENTRY(struct libxl__sr_record_buf) entry; + libxl__sr_rec_hdr hdr; + void *body; /* iff hdr.length != 0 */ +} libxl__sr_record_buf; + +struct libxl__stream_read_state { + /* filled by the user */ + libxl__ao *ao; + libxl__domain_create_state *dcs; + int fd; + bool legacy; + bool back_channel; + void (*completion_callback)(libxl__egc *egc, + libxl__stream_read_state *srs, + int rc); + void (*checkpoint_callback)(libxl__egc *egc, + libxl__stream_read_state *srs, + int rc); + /* Private */ + int rc; + bool running; + bool in_checkpoint; + bool sync_teardown; /* Only used to coordinate shutdown on error path. */ + bool in_checkpoint_state; + libxl__save_helper_state shs; + libxl__conversion_helper_state chs; + + /* Main stream-reading data. */ + libxl__datacopier_state dc; /* Only used when reading a record */ + libxl__sr_hdr hdr; + LIBXL_STAILQ_HEAD(, libxl__sr_record_buf) record_queue; /* NOGC */ + enum { + SRS_PHASE_NORMAL, + SRS_PHASE_BUFFERING, + SRS_PHASE_UNBUFFERING, + } phase; + bool recursion_guard; + + /* Only used while actively reading a record from the stream. */ + libxl__sr_record_buf *incoming_record; /* NOGC */ + + /* Both only used when processing an EMULATOR record. */ + libxl__datacopier_state emu_dc; + libxl__carefd *emu_carefd; +}; + +_hidden void libxl__stream_read_init(libxl__stream_read_state *stream); +_hidden void libxl__stream_read_start(libxl__egc *egc, + libxl__stream_read_state *stream); +_hidden void libxl__stream_read_start_checkpoint(libxl__egc *egc, + libxl__stream_read_state *stream); +_hidden void libxl__stream_read_checkpoint_state(libxl__egc *egc, + libxl__stream_read_state *stream); +_hidden void libxl__stream_read_abort(libxl__egc *egc, + libxl__stream_read_state *stream, int rc); +static inline bool +libxl__stream_read_inuse(const libxl__stream_read_state *stream) +{ + return stream->running; +} + +/*----- colo related state structure -----*/ +typedef struct libxl__colo_save_state libxl__colo_save_state; +struct libxl__colo_save_state { + int send_fd; + int recv_fd; + + /* private */ + libxl__stream_read_state srs; + void (*callback)(libxl__egc *, libxl__colo_save_state *, int); + bool svm_running; +}; /*----- Domain suspend (save) state structure -----*/ /* @@ -3146,7 +3229,12 @@ struct libxl__domain_save_state { int hvm; int xcflags; libxl__domain_suspend_state dsps; - libxl__remus_state rs; + union { + /* for Remus */ + libxl__remus_state rs; + /* for COLO */ + libxl__colo_save_state css; + }; libxl__checkpoint_devices_state cds; libxl__stream_write_state sws; libxl__logdirty_switch logdirty; @@ -3396,78 +3484,6 @@ _hidden int libxl__destroy_qdisk_backend(libxl__gc *gc, uint32_t domid); /*----- Domain creation -----*/ -typedef struct libxl__domain_create_state libxl__domain_create_state; - -typedef void libxl__domain_create_cb(libxl__egc *egc, - libxl__domain_create_state*, - int rc, uint32_t domid); - -/* State for manipulating a libxl migration v2 stream */ -typedef struct libxl__stream_read_state libxl__stream_read_state; - -typedef struct libxl__sr_record_buf { - /* private to stream read helper */ - LIBXL_STAILQ_ENTRY(struct libxl__sr_record_buf) entry; - libxl__sr_rec_hdr hdr; - void *body; /* iff hdr.length != 0 */ -} libxl__sr_record_buf; - -struct libxl__stream_read_state { - /* filled by the user */ - libxl__ao *ao; - libxl__domain_create_state *dcs; - int fd; - bool legacy; - bool back_channel; - void (*completion_callback)(libxl__egc *egc, - libxl__stream_read_state *srs, - int rc); - void (*checkpoint_callback)(libxl__egc *egc, - libxl__stream_read_state *srs, - int rc); - /* Private */ - int rc; - bool running; - bool in_checkpoint; - bool sync_teardown; /* Only used to coordinate shutdown on error path. */ - bool in_checkpoint_state; - libxl__save_helper_state shs; - libxl__conversion_helper_state chs; - - /* Main stream-reading data. */ - libxl__datacopier_state dc; /* Only used when reading a record */ - libxl__sr_hdr hdr; - LIBXL_STAILQ_HEAD(, libxl__sr_record_buf) record_queue; /* NOGC */ - enum { - SRS_PHASE_NORMAL, - SRS_PHASE_BUFFERING, - SRS_PHASE_UNBUFFERING, - } phase; - bool recursion_guard; - - /* Only used while actively reading a record from the stream. */ - libxl__sr_record_buf *incoming_record; /* NOGC */ - - /* Both only used when processing an EMULATOR record. */ - libxl__datacopier_state emu_dc; - libxl__carefd *emu_carefd; -}; - -_hidden void libxl__stream_read_init(libxl__stream_read_state *stream); -_hidden void libxl__stream_read_start(libxl__egc *egc, - libxl__stream_read_state *stream); -_hidden void libxl__stream_read_start_checkpoint(libxl__egc *egc, - libxl__stream_read_state *stream); -_hidden void libxl__stream_read_checkpoint_state(libxl__egc *egc, - libxl__stream_read_state *stream); -_hidden void libxl__stream_read_abort(libxl__egc *egc, - libxl__stream_read_state *stream, int rc); -static inline bool -libxl__stream_read_inuse(const libxl__stream_read_state *stream) -{ - return stream->running; -} - /* colo related structure */ typedef struct libxl__colo_restore_state libxl__colo_restore_state; typedef void libxl__colo_callback(libxl__egc *, diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl index db001ad..7c46bc2 100644 --- a/tools/libxl/libxl_types.idl +++ b/tools/libxl/libxl_types.idl @@ -762,6 +762,7 @@ libxl_domain_remus_info = Struct("domain_remus_info",[ ("netbuf", libxl_defbool), ("netbufscript", string), ("diskbuf", libxl_defbool), + ("colo", libxl_defbool) ]) libxl_event_type = Enumeration("event_type", [