|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] Re: [Xen-devel] [PATCH 20/27] tools/libxl: Infrastructure for writing a v2 stream
On 06/15/2015 09:44 PM, Andrew Cooper wrote:
> From: Ross Lagerwall <ross.lagerwall@xxxxxxxxxx>
>
> Signed-off-by: Ross Lagerwall <ross.lagerwall@xxxxxxxxxx>
> Signed-off-by: Andrew Cooper <andrew.cooper3@xxxxxxxxxx>
> CC: Ian Campbell <Ian.Campbell@xxxxxxxxxx>
> CC: Ian Jackson <Ian.Jackson@xxxxxxxxxxxxx>
> CC: Wei Liu <wei.liu2@xxxxxxxxxx>
> ---
> tools/libxl/Makefile | 2 +-
> tools/libxl/libxl_internal.h | 33 +++
> tools/libxl/libxl_stream_write.c | 536
> ++++++++++++++++++++++++++++++++++++++
> 3 files changed, 570 insertions(+), 1 deletion(-)
> create mode 100644 tools/libxl/libxl_stream_write.c
>
> diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
> index ca0ae3e..63e32f7 100644
> --- a/tools/libxl/Makefile
> +++ b/tools/libxl/Makefile
> @@ -94,7 +94,7 @@ LIBXL_OBJS = flexarray.o libxl.o libxl_create.o libxl_dm.o
> libxl_pci.o \
> libxl_dom.o libxl_exec.o libxl_xshelp.o libxl_device.o \
> libxl_internal.o libxl_utils.o libxl_uuid.o \
> libxl_json.o libxl_aoutils.o libxl_numa.o libxl_vnuma.o
> \
> - libxl_stream_read.o \
> + libxl_stream_read.o libxl_stream_write.o \
> libxl_save_callout.o _libxl_save_msgs_callout.o \
> libxl_convert_callout.o \
> libxl_qmp.o libxl_event.o libxl_fork.o $(LIBXL_OBJS-y)
> diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
> index 5482950..82cd792 100644
> --- a/tools/libxl/libxl_internal.h
> +++ b/tools/libxl/libxl_internal.h
> @@ -2868,6 +2868,38 @@ typedef void libxl__domain_suspend_cb(libxl__egc*,
> typedef void libxl__save_device_model_cb(libxl__egc*,
> libxl__domain_suspend_state*, int
> rc);
>
> +/* State for writing a libxl migration v2 stream */
> +typedef struct libxl__stream_write_state libxl__stream_write_state;
> +
> +struct libxl__stream_write_state {
> + /* filled by the user */
> + libxl__ao *ao;
> + int fd;
> + uint32_t domid;
> + void (*completion_callback)(libxl__egc *egc,
> + libxl__domain_suspend_state *dss,
> + int rc);
> + /* Private */
> + int rc;
> + int joined_rc;
> + size_t padding;
> + bool running;
> + libxl__datacopier_state dc;
> +};
> +
> +_hidden void libxl__stream_write_start(libxl__egc *egc,
> + libxl__stream_write_state *stream);
> +
> +_hidden void libxl__stream_write_abort(libxl__egc *egc,
> + libxl__stream_write_state *stream,
> + int rc);
> +
> +static inline bool libxl__stream_write_inuse(
> + const libxl__stream_write_state *stream)
> +{
> + return stream->running;
> +}
> +
> typedef struct libxl__logdirty_switch {
> const char *cmd;
> const char *cmd_path;
> @@ -2907,6 +2939,7 @@ struct libxl__domain_suspend_state {
> /* private for libxl__domain_save_device_model */
> libxl__save_device_model_cb *save_dm_callback;
> libxl__datacopier_state save_dm_datacopier;
> + libxl__stream_write_state sws;
> };
>
>
> diff --git a/tools/libxl/libxl_stream_write.c
> b/tools/libxl/libxl_stream_write.c
> new file mode 100644
> index 0000000..856d72e
> --- /dev/null
> +++ b/tools/libxl/libxl_stream_write.c
> @@ -0,0 +1,536 @@
> +/*
> + * Copyright (C) 2015 Citrix Ltd.
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU Lesser General Public License as published
> + * by the Free Software Foundation; version 2.1 only. with the special
> + * exception on linking described in file LICENSE.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> + * GNU Lesser General Public License for more details.
> + */
> +
> +#include "libxl_osdeps.h" /* must come before any other headers */
> +
> +#include "libxl_internal.h"
> +
> +/*
> + * Infrastructure for writing a domain to a libxl migration v2 stream.
> + *
> + * Entry points from outside:
> + * - libxl__stream_write_start()
> + * - Start writing a stream from the start.
> + *
> + * In normal operation, there are two tasks running at once; this stream
> + * processing, and the the libxl-save-helper. check_stream_finished() is
> used
> + * to join all the tasks in both success and error cases.
> + *
> + * Nomenclature for event callbacks:
> + * - $FOO_done(): Completion callback for $FOO
> + * - write_$FOO(): Set up writing a $FOO
> + * - $BAR_header(): A $BAR record header only
> + * - $BAR_record(): A complete $BAR record with header and content
> + *
> + * The main loop for a plain VM writes:
> + * - Stream header
> + * - Libxc record
> + * - Toolstack record
> + * - if (hvm), Qemu record
> + * - End record
> + */
> +
> +static const uint8_t zero_padding[1U << REC_ALIGN_ORDER] = { 0 };
> +
> +static void stream_success(libxl__egc *egc,
> + libxl__stream_write_state *stream);
> +static void stream_failed(libxl__egc *egc,
> + libxl__stream_write_state *stream, int ret);
> +static void stream_done(libxl__egc *egc,
> + libxl__stream_write_state *stream);
> +
> +static void check_stream_finished(libxl__egc *egc,
> + libxl__domain_suspend_state *dcs,
> + int rc, const char *what);
> +
> +/* Event callbacks for plain VM. */
> +static void stream_header_done(libxl__egc *egc,
> + libxl__datacopier_state *dc,
> + int onwrite, int errnoval);
> +static void libxc_header_done(libxl__egc *egc,
> + libxl__datacopier_state *dc,
> + int onwrite, int errnoval);
> +/* libxl__xc_domain_save_done() lives here, event-order wise. */
> +static void write_toolstack_record(libxl__egc *egc,
> + libxl__stream_write_state *stream);
> +static void toolstack_record_done(libxl__egc *egc,
> + libxl__datacopier_state *dc,
> + int onwrite, int errnoval);
> +static void write_emulator_record(libxl__egc *egc,
> + libxl__stream_write_state *stream);
> +static void emulator_body_done(libxl__egc *egc,
> + libxl__datacopier_state *dc,
> + int onwrite, int errnoval);
> +static void emulator_padding_done(libxl__egc *egc,
> + libxl__datacopier_state *dc,
> + int onwrite, int errnoval);
> +static void write_end_record(libxl__egc *egc,
> + libxl__stream_write_state *stream);
> +static void end_record_done(libxl__egc *egc,
> + libxl__datacopier_state *dc,
> + int onwrite, int errnoval);
> +
> +void libxl__stream_write_start(libxl__egc *egc,
> + libxl__stream_write_state *stream)
> +{
> + libxl__datacopier_state *dc = &stream->dc;
> + STATE_AO_GC(stream->ao);
> + struct libxl_sr_hdr hdr = { 0 };
> + int ret = 0;
> +
> + assert(!stream->running);
> + stream->running = true;
> +
> + memset(dc, 0, sizeof(*dc));
> + dc->readwhat = "";
> + dc->copywhat = "suspend header";
> + dc->writewhat = "save/migration stream";
> + dc->ao = ao;
> + dc->readfd = -1;
> + dc->writefd = stream->fd;
> + dc->maxsz = INT_MAX;
> + dc->bytes_to_read = INT_MAX;
> + dc->callback = stream_header_done;
> +
> + ret = libxl__datacopier_start(dc);
> + if (ret)
> + goto err;
> +
> + hdr.ident = htobe64(RESTORE_STREAM_IDENT);
> + hdr.version = htobe32(RESTORE_STREAM_VERSION);
> + hdr.options = htobe32(0);
> +
> + libxl__datacopier_prefixdata(egc, dc, &hdr, sizeof(hdr));
> + return;
> +
> + err:
> + assert(ret);
> + stream_failed(egc, stream, ret);
> +}
> +
> +void libxl__stream_write_abort(libxl__egc *egc,
> + libxl__stream_write_state *stream, int rc)
> +{
> + stream_failed(egc, stream, rc);
> +}
> +
> +static void stream_success(libxl__egc *egc, libxl__stream_write_state
> *stream)
> +{
> + stream->rc = 0;
> + stream->running = false;
> +
> + stream_done(egc, stream);
> +}
> +
> +static void stream_failed(libxl__egc *egc,
> + libxl__stream_write_state *stream, int rc)
> +{
> + assert(rc);
> + stream->rc = rc;
> +
> + if (stream->running) {
> + stream->running = false;
> + stream_done(egc, stream);
> + }
> +}
> +
> +static void stream_done(libxl__egc *egc,
> + libxl__stream_write_state *stream)
> +{
> + libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws);
> +
> + assert(!stream->running);
> +
> + check_stream_finished(egc, dss, stream->rc, "stream");
> +}
> +
> +static void check_stream_finished(libxl__egc *egc,
> + libxl__domain_suspend_state *dss,
> + int rc, const char *what)
> +{
> + libxl__stream_write_state *stream = &dss->sws;
> + STATE_AO_GC(dss->ao);
> +
> + LOG(INFO, "Task '%s' joining (rc %d)", what, rc);
> +
> + if (rc && !stream->joined_rc) {
> + bool skip = false;
> + /* First reported failure from joining tasks. Tear everything down
> */
> + stream->joined_rc = rc;
> +
> + if (libxl__stream_write_inuse(&dss->sws)) {
> + skip = true;
> + libxl__stream_write_abort(egc, &dss->sws, rc);
> + }
> +
> + if (libxl__save_helper_inuse(&dss->shs)) {
> + skip = true;
> + libxl__save_helper_abort(egc, &dss->shs);
> + }
> +
> + /* There is at least one more active task to join - wait for its
> + callback */
> + if ( skip )
> + return;
> + }
> +
> + if (libxl__stream_write_inuse(&dss->sws))
> + LOG(DEBUG, "stream still in use");
> + else if (libxl__save_helper_inuse(&dss->shs))
> + LOG(DEBUG, "save/restore still in use");
> + else {
> + LOG(INFO, "Join complete: result %d", stream->joined_rc);
> + stream->completion_callback(egc, dss, stream->joined_rc);
> + }
> +}
> +
> +static void stream_header_done(libxl__egc *egc,
> + libxl__datacopier_state *dc,
> + int onwrite, int errnoval)
> +{
> + libxl__stream_write_state *stream = CONTAINER_OF(dc, *stream, dc);
> + STATE_AO_GC(stream->ao);
> + struct libxl_sr_rec_hdr rec = { REC_TYPE_LIBXC_CONTEXT, 0 };
> + int ret = 0;
> +
> + if (onwrite || errnoval) {
> + ret = ERROR_FAIL;
> + goto err;
> + }
> +
> + dc->copywhat = "suspend footer";
> + dc->writewhat = "save/migration stream";
> + dc->callback = libxc_header_done;
> +
> + ret = libxl__datacopier_start(dc);
> + if (ret)
> + goto err;
> +
> + libxl__datacopier_prefixdata(egc, dc, &rec, sizeof(rec));
> + return;
> +
> + err:
> + assert(ret);
> + stream_failed(egc, stream, ret);
> +}
> +
> +static void libxc_header_done(libxl__egc *egc,
> + libxl__datacopier_state *dc,
> + int onwrite, int errnoval)
> +{
> + libxl__stream_write_state *stream = CONTAINER_OF(dc, *stream, dc);
> + libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws);
> + STATE_AO_GC(stream->ao);
> + int ret = 0;
> +
> + if (onwrite || errnoval) {
> + ret = ERROR_FAIL;
> + goto err;
> + }
> +
> + libxl__xc_domain_save(egc, dss);
> + return;
> +
> + err:
> + assert(ret);
> + stream_failed(egc, stream, ret);
> +}
> +
> +static void __attribute__((used))
> +will_be_libxl__xc_domain_save_done(libxl__egc *egc, void *dss_void,
> + int rc, int retval, int errnoval)
> +{
> + libxl__domain_suspend_state *dss = dss_void;
> + libxl__stream_write_state *stream = &dss->sws;
> + STATE_AO_GC(dss->ao);
> +
> + if (rc)
> + goto err;
> +
> + if (retval) {
> + LOGEV(ERROR, errnoval, "saving domain: %s",
> + dss->guest_responded ?
> + "domain responded to suspend request" :
> + "domain did not respond to suspend request");
> + if ( !dss->guest_responded )
> + rc = ERROR_GUEST_TIMEDOUT;
> + else
> + rc = ERROR_FAIL;
> + goto err;
> + }
> +
> + write_toolstack_record(egc, stream);
> + return;
> +
> + err:
> + assert(rc);
> + check_stream_finished(egc, dss, rc, "save/restore helper");
> +}
> +
> +static void write_toolstack_record(libxl__egc *egc,
> + libxl__stream_write_state *stream)
> +{
> + libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws);
> + libxl__datacopier_state *dc = &stream->dc;
> + STATE_AO_GC(stream->ao);
> + struct libxl_sr_rec_hdr rec = { REC_TYPE_XENSTORE_DATA, 0 };
> + int ret = 0;
> + uint8_t *toolstack_buf = NULL; /* We must free this. */
> + uint32_t toolstack_len, padding_len;
> +
> + ret = libxl__toolstack_save(dss->domid, &toolstack_buf,
> + &toolstack_len, dss);
> + if (ret)
> + goto err;
> +
> + dc->copywhat = "toolstack record";
> + dc->writewhat = "save/migration stream";
> + dc->callback = toolstack_record_done;
> +
> + ret = libxl__datacopier_start(dc);
> + if (ret)
> + goto err;
> +
> + rec.length = toolstack_len;
> +
> + libxl__datacopier_prefixdata(egc, dc, &rec, sizeof(rec));
> + libxl__datacopier_prefixdata(egc, dc, toolstack_buf, toolstack_len);
> +
> + padding_len = ROUNDUP(rec.length, REC_ALIGN_ORDER) - rec.length;
> + if (padding_len)
> + libxl__datacopier_prefixdata(egc, dc, zero_padding, padding_len);
> +
> + free(toolstack_buf);
> + return;
> +
> + err:
> + assert(ret);
> + free(toolstack_buf);
> + stream_failed(egc, stream, ret);
> +}
> +
> +static void toolstack_record_done(libxl__egc *egc,
> + libxl__datacopier_state *dc,
> + int onwrite, int errnoval)
> +{
> + libxl__stream_write_state *stream = CONTAINER_OF(dc, *stream, dc);
> + libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws);
> + STATE_AO_GC(stream->ao);
> + int ret = 0;
> +
> + if (onwrite || errnoval) {
> + ret = ERROR_FAIL;
> + goto err;
> + }
> +
> + if (dss->type == LIBXL_DOMAIN_TYPE_HVM)
> + write_emulator_record(egc, stream);
> + else
> + write_end_record(egc, stream);
> +
> + return;
> +
> + err:
> + assert(ret);
> + stream_failed(egc, stream, ret);
> +}
> +
> +static void write_emulator_record(libxl__egc *egc,
> + libxl__stream_write_state *stream)
> +{
> + libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws);
> + libxl__datacopier_state *dc = &stream->dc;
> + STATE_AO_GC(stream->ao);
> + struct libxl_sr_rec_hdr rec = { REC_TYPE_EMULATOR_CONTEXT, 0 };
> + struct libxl_sr_emulator_hdr ehdr = { 0 };
> + struct stat st;
> + int ret = 0;
> + uint32_t qemu_state_len;
> +
> + assert(dss->type == LIBXL_DOMAIN_TYPE_HVM);
> +
> + /* Convenience aliases */
> + const char *const filename = dss->dm_savefile;
> + const uint32_t domid = dss->domid;
> +
> + switch(libxl__device_model_version_running(gc, domid)) {
> + case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL:
> + ehdr.id = EMULATOR_QEMU_TRADITIONAL;
> + break;
> +
> + case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
> + ehdr.id = EMULATOR_QEMU_UPSTREAM;
> + break;
> +
> + default:
> + ret = ERROR_FAIL;
> + goto err;
> + }
> +
> + ret = libxl__domain_suspend_device_model(gc, dss);
We have called it when suspending the guest. For migration, calling
this function twice is OK, but for remus, it will cause a problem:
the guest is running, and this device-model will be stopped by this
function.
Thanks
Wen Congyang
> + if (ret)
> + goto err;
> +
> + dc->readwhat = GCSPRINTF("qemu save file %s", filename);
> + dc->copywhat = "emulator record";
> + dc->writewhat = "save/migration stream";
> + dc->callback = emulator_body_done;
> +
> + dc->readfd = open(filename, O_RDONLY);
> + if (dc->readfd < 0) {
> + LOGE(ERROR, "unable to open %s", dc->readwhat);
> + goto err;
> + }
> +
> + if (fstat(dc->readfd, &st))
> + {
> + LOGE(ERROR, "unable to fstat %s", dc->readwhat);
> + goto err;
> + }
> +
> + if (!S_ISREG(st.st_mode)) {
> + LOG(ERROR, "%s is not a plain file!", dc->readwhat);
> + goto err;
> + }
> +
> + qemu_state_len = st.st_size;
> + rec.length = qemu_state_len + sizeof(ehdr);
> +
> + ret = libxl__datacopier_start(dc);
> + if (ret)
> + goto err;
> +
> + libxl__datacopier_prefixdata(egc, dc, &rec, sizeof(rec));
> + libxl__datacopier_prefixdata(egc, dc, &ehdr, sizeof(ehdr));
> +
> + stream->padding = ROUNDUP(qemu_state_len, REC_ALIGN_ORDER) -
> qemu_state_len;
> + return;
> +
> + err:
> + assert(ret);
> + stream_failed(egc, stream, ret);
> +}
> +
> +static void emulator_body_done(libxl__egc *egc,
> + libxl__datacopier_state *dc,
> + int onwrite, int errnoval)
> +{
> + libxl__stream_write_state *stream = CONTAINER_OF(dc, *stream, dc);
> + STATE_AO_GC(stream->ao);
> + int ret = 0;
> +
> + if (onwrite || errnoval) {
> + ret = ERROR_FAIL;
> + goto err;
> + }
> +
> + dc->readwhat = "";
> + dc->readfd = -1;
> +
> + if (stream->padding) {
> + assert(stream->padding < (1U << REC_ALIGN_ORDER));
> +
> + dc->copywhat = "emulator padding";
> + dc->writewhat = "save/migration stream";
> + dc->callback = emulator_padding_done;
> +
> + ret = libxl__datacopier_start(dc);
> + if (ret)
> + goto err;
> +
> + libxl__datacopier_prefixdata(egc, dc, zero_padding, stream->padding);
> + return;
> + }
> +
> + emulator_padding_done(egc, dc, 0, 0);
> + return;
> +
> + err:
> + assert(ret);
> + stream_failed(egc, stream, ret);
> +}
> +
> +static void emulator_padding_done(libxl__egc *egc,
> + libxl__datacopier_state *dc,
> + int onwrite, int errnoval)
> +{
> + libxl__stream_write_state *stream = CONTAINER_OF(dc, *stream, dc);
> + STATE_AO_GC(stream->ao);
> + int ret = 0;
> +
> + if (onwrite || errnoval) {
> + ret = ERROR_FAIL;
> + goto err;
> + }
> +
> + write_end_record(egc, stream);
> + return;
> +
> + err:
> + assert(ret);
> + stream_failed(egc, stream, ret);
> +}
> +
> +static void write_end_record(libxl__egc *egc,
> + libxl__stream_write_state *stream)
> +{
> + libxl__datacopier_state *dc = &stream->dc;
> + STATE_AO_GC(stream->ao);
> + struct libxl_sr_rec_hdr rec = { REC_TYPE_END, 0 };
> + int ret = 0;
> +
> + dc->copywhat = "suspend footer";
> + dc->writewhat = "save/migration stream";
> + dc->callback = end_record_done;
> +
> + ret = libxl__datacopier_start(dc);
> + if (ret)
> + goto err;
> +
> + libxl__datacopier_prefixdata(egc, dc, &rec, sizeof(rec));
> + return;
> +
> + err:
> + assert(ret);
> + stream_failed(egc, stream, ret);
> +}
> +
> +static void end_record_done(libxl__egc *egc,
> + libxl__datacopier_state *dc,
> + int onwrite, int errnoval)
> +{
> + libxl__stream_write_state *stream = CONTAINER_OF(dc, *stream, dc);
> + STATE_AO_GC(stream->ao);
> + int ret = 0;
> +
> + if (onwrite || errnoval) {
> + ret = ERROR_FAIL;
> + goto err;
> + }
> +
> + stream_success(egc, stream);
> + return;
> +
> + err:
> + assert(ret);
> + stream_failed(egc, stream, ret);
> +}
> +
> +/*
> + * Local variables:
> + * mode: C
> + * c-basic-offset: 4
> + * indent-tabs-mode: nil
> + * End:
> + */
>
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
http://lists.xen.org/xen-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |