|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] Re: [Xen-devel] [PATCH 05/21] libxl: domain save: API changes for asynchrony
> @@ -648,32 +648,51 @@ libxl_vminfo * libxl_list_vm(libxl_ctx *ctx, int *nb_vm)
> return ptr;
> }
>
> +static void remus_crashed_cb(libxl__egc *egc,
> + libxl__domain_suspend_state *dss, int rc);
I think you were going to rename this remus_failover_cb but
nevertheless:
Acked-by: Ian Campbell <ian.campbell@xxxxxxxxxx>
Were you going to ping Shriram to test remus with this series applied?
> +
> /* TODO: Explicit Checkpoint acknowledgements via recv_fd. */
> int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info,
> - uint32_t domid, int send_fd, int recv_fd)
> + uint32_t domid, int send_fd, int recv_fd,
> + const libxl_asyncop_how *ao_how)
> {
> - GC_INIT(ctx);
> - libxl_domain_type type = libxl__domain_type(gc, domid);
> - int rc = 0;
> + AO_CREATE(ctx, domid, ao_how);
> + libxl__domain_suspend_state *dss;
> + int rc;
>
> + libxl_domain_type type = libxl__domain_type(gc, domid);
> if (type == LIBXL_DOMAIN_TYPE_INVALID) {
> rc = ERROR_FAIL;
> - goto remus_fail;
> + goto out;
> }
>
> - if (info == NULL) {
> - LIBXL__LOG(ctx, LIBXL__LOG_ERROR,
> - "No remus_info structure supplied for domain %d", domid);
> - rc = ERROR_INVAL;
> - goto remus_fail;
> - }
> + GCNEW(dss);
> + dss->ao = ao;
> + dss->callback = remus_crashed_cb;
> + dss->domid = domid;
> + dss->fd = send_fd;
> + /* TODO do something with recv_fd */
> + dss->type = type;
> + dss->live = 1;
> + dss->debug = 0;
> + dss->remus = info;
> +
> + assert(info);
>
> /* TBD: Remus setup - i.e. attach qdisc, enable disk buffering, etc */
>
> /* Point of no return */
> - rc = libxl__domain_suspend_common(gc, domid, send_fd, type, /* live */ 1,
> - /* debug */ 0, info);
> + libxl__domain_suspend(egc, dss);
> + return AO_INPROGRESS;
> +
> + out:
> + return AO_ABORT(rc);
> +}
>
> +static void remus_crashed_cb(libxl__egc *egc,
> + libxl__domain_suspend_state *dss, int rc)
> +{
> + STATE_AO_GC(dss->ao);
> /*
> * With Remus, if we reach this point, it means either
> * backup died or some network error occurred preventing us
> @@ -683,27 +702,46 @@ int libxl_domain_remus_start(libxl_ctx *ctx,
> libxl_domain_remus_info *info,
> /* TBD: Remus cleanup - i.e. detach qdisc, release other
> * resources.
> */
> - remus_fail:
> - GC_FREE;
> - return rc;
> + libxl__ao_complete(egc, ao, rc);
> }
>
> -int libxl_domain_suspend(libxl_ctx *ctx, libxl_domain_suspend_info *info,
> - uint32_t domid, int fd)
> +static void domain_suspend_cb(libxl__egc *egc,
> + libxl__domain_suspend_state *dss, int rc)
> {
> - GC_INIT(ctx);
> + STATE_AO_GC(dss->ao);
> + libxl__ao_complete(egc,ao,rc);
> +
> +}
> +
> +int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
> + const libxl_asyncop_how *ao_how)
> +{
> + AO_CREATE(ctx, domid, ao_how);
> + int rc;
> +
> libxl_domain_type type = libxl__domain_type(gc, domid);
> - int live = info != NULL && info->flags & XL_SUSPEND_LIVE;
> - int debug = info != NULL && info->flags & XL_SUSPEND_DEBUG;
> - int rc = 0;
> + if (type == LIBXL_DOMAIN_TYPE_INVALID) {
> + rc = ERROR_FAIL;
> + goto out_err;
> + }
>
> - rc = libxl__domain_suspend_common(gc, domid, fd, type, live, debug,
> - /* No Remus */ NULL);
> + libxl__domain_suspend_state *dss;
> + GCNEW(dss);
>
> - if (!rc && type == LIBXL_DOMAIN_TYPE_HVM)
> - rc = libxl__domain_save_device_model(gc, domid, fd);
> - GC_FREE;
> - return rc;
> + dss->ao = ao;
> + dss->callback = domain_suspend_cb;
> +
> + dss->domid = domid;
> + dss->fd = fd;
> + dss->type = type;
> + dss->live = flags & LIBXL_SUSPEND_LIVE;
> + dss->debug = flags & LIBXL_SUSPEND_DEBUG;
> +
> + libxl__domain_suspend(egc, dss);
> + return AO_INPROGRESS;
> +
> + out_err:
> + return AO_ABORT(rc);
> }
>
> int libxl_domain_pause(libxl_ctx *ctx, uint32_t domid)
> diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
> index 05f0e01..10d7115 100644
> --- a/tools/libxl/libxl.h
> +++ b/tools/libxl/libxl.h
> @@ -347,13 +347,6 @@ typedef struct libxl__ctx libxl_ctx;
>
> const libxl_version_info* libxl_get_version_info(libxl_ctx *ctx);
>
> -typedef struct {
> -#define XL_SUSPEND_DEBUG 1
> -#define XL_SUSPEND_LIVE 2
> - int flags;
> - int (*suspend_callback)(void *, int);
> -} libxl_domain_suspend_info;
> -
> enum {
> ERROR_NONSPECIFIC = -1,
> ERROR_VERSION = -2,
> @@ -514,16 +507,23 @@ int libxl_domain_create_restore(libxl_ctx *ctx,
> libxl_domain_config *d_config,
>
> void libxl_domain_config_init(libxl_domain_config *d_config);
> void libxl_domain_config_dispose(libxl_domain_config *d_config);
> -int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info,
> - uint32_t domid, int send_fd, int recv_fd);
> -int libxl_domain_suspend(libxl_ctx *ctx, libxl_domain_suspend_info *info,
> - uint32_t domid, int fd);
> +
> +int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd,
> + int flags, /* LIBXL_SUSPEND_* */
> + const libxl_asyncop_how *ao_how);
> +#define LIBXL_SUSPEND_DEBUG 1
> +#define LIBXL_SUSPEND_LIVE 2
>
> /* @param suspend_cancel [from xenctrl.h:xc_domain_resume( @param fast )]
> * If this parameter is true, use co-operative resume. The guest
> * must support this.
> */
> int libxl_domain_resume(libxl_ctx *ctx, uint32_t domid, int suspend_cancel);
> +
> +int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info,
> + uint32_t domid, int send_fd, int recv_fd,
> + const libxl_asyncop_how *ao_how);
> +
> int libxl_domain_shutdown(libxl_ctx *ctx, uint32_t domid);
> int libxl_domain_reboot(libxl_ctx *ctx, uint32_t domid);
> int libxl_domain_destroy(libxl_ctx *ctx, uint32_t domid);
> diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
> index 1d4e809..b48452c 100644
> --- a/tools/libxl/libxl_dom.c
> +++ b/tools/libxl/libxl_dom.c
> @@ -17,13 +17,11 @@
>
> #include <glob.h>
>
> -#include <xenctrl.h>
> -#include <xc_dom.h>
> +#include "libxl_internal.h"
>
> +#include <xc_dom.h>
> #include <xen/hvm/hvm_info_table.h>
>
> -#include "libxl_internal.h"
> -
> libxl_domain_type libxl__domain_type(libxl__gc *gc, uint32_t domid)
> {
> libxl_ctx *ctx = libxl__gc_owner(gc);
> @@ -521,11 +519,18 @@ int libxl__toolstack_restore(uint32_t domid, const
> uint8_t *buf,
> return 0;
> }
>
> -static int libxl__domain_suspend_common_switch_qemu_logdirty
> +/*==================== Domain suspend (save) ====================*/
> +
> +static void domain_suspend_done(libxl__egc *egc,
> + libxl__domain_suspend_state *dss, int rc);
> +
> +/*----- callbacks, called by xc_domain_save -----*/
> +
> +int libxl__domain_suspend_common_switch_qemu_logdirty
> (int domid, unsigned int enable, void *data)
> {
> libxl__domain_suspend_state *dss = data;
> - libxl__gc *gc = dss->gc;
> + STATE_AO_GC(dss->ao);
> char *path;
> bool rc;
>
> @@ -590,10 +595,10 @@ int libxl__domain_resume_device_model(libxl__gc *gc,
> uint32_t domid)
> return 0;
> }
>
> -static int libxl__domain_suspend_common_callback(void *data)
> +int libxl__domain_suspend_common_callback(void *data)
> {
> libxl__domain_suspend_state *dss = data;
> - libxl__gc *gc = dss->gc;
> + STATE_AO_GC(dss->ao);
> unsigned long hvm_s_state = 0, hvm_pvdrv = 0;
> int ret;
> char *state = "suspend";
> @@ -714,7 +719,7 @@ static int libxl__domain_suspend_common_callback(void
> *data)
>
> guest_suspended:
> if (dss->hvm) {
> - ret = libxl__domain_suspend_device_model(dss->gc, dss->domid);
> + ret = libxl__domain_suspend_device_model(gc, dss->domid);
> if (ret) {
> LOG(ERROR, "libxl__domain_suspend_device_model failed ret=%d",
> ret);
> return 0;
> @@ -731,11 +736,11 @@ static inline char *save_helper(libxl__gc *gc, uint32_t
> domid,
> domid, phys_offset, node);
> }
>
> -static int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
> +int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
> uint32_t *len, void *data)
> {
> libxl__domain_suspend_state *dss = data;
> - libxl__gc *gc = dss->gc;
> + STATE_AO_GC(dss->ao);
> int i = 0;
> char *start_addr = NULL, *size = NULL, *phys_offset = NULL, *name = NULL;
> unsigned int num = 0;
> @@ -806,6 +811,8 @@ static int libxl__toolstack_save(uint32_t domid, uint8_t
> **buf,
> return 0;
> }
>
> +/*----- remus callbacks -----*/
> +
> static int libxl__remus_domain_suspend_callback(void *data)
> {
> /* TODO: Issue disk and network checkpoint reqs. */
> @@ -815,7 +822,7 @@ static int libxl__remus_domain_suspend_callback(void
> *data)
> static int libxl__remus_domain_resume_callback(void *data)
> {
> libxl__domain_suspend_state *dss = data;
> - libxl__gc *gc = dss->gc;
> + STATE_AO_GC(dss->ao);
>
> /* Resumes the domain and the device model */
> if (libxl_domain_resume(CTX, dss->domid, /* Fast Suspend */1))
> @@ -828,10 +835,11 @@ static int libxl__remus_domain_resume_callback(void
> *data)
> static int libxl__remus_domain_checkpoint_callback(void *data)
> {
> libxl__domain_suspend_state *dss = data;
> + STATE_AO_GC(dss->ao);
>
> /* This would go into tailbuf. */
> if (dss->hvm &&
> - libxl__domain_save_device_model(dss->gc, dss->domid, dss->save_fd))
> + libxl__domain_save_device_model(gc, dss->domid, dss->fd))
> return 0;
>
> /* TODO: Wait for disk and memory ack, release network buffer */
> @@ -839,17 +847,23 @@ static int libxl__remus_domain_checkpoint_callback(void
> *data)
> return 1;
> }
>
> -int libxl__domain_suspend_common(libxl__gc *gc, uint32_t domid, int fd,
> - libxl_domain_type type,
> - int live, int debug,
> - const libxl_domain_remus_info *r_info)
> +/*----- main code for suspending, in order of execution -----*/
> +
> +void libxl__domain_suspend(libxl__egc *egc, libxl__domain_suspend_state *dss)
> {
> + STATE_AO_GC(dss->ao);
> int port;
> - struct save_callbacks callbacks[1];
> - libxl__domain_suspend_state dss[1];
> int rc = ERROR_FAIL;
> unsigned long vm_generationid_addr;
>
> + /* Convenience aliases */
> + const uint32_t domid = dss->domid;
> + const libxl_domain_type type = dss->type;
> + const int live = dss->live;
> + const int debug = dss->debug;
> + const libxl_domain_remus_info *const r_info = dss->remus;
> + struct save_callbacks *const callbacks = &dss->callbacks;
> +
> switch (type) {
> case LIBXL_DOMAIN_TYPE_HVM: {
> char *path;
> @@ -868,15 +882,13 @@ int libxl__domain_suspend_common(libxl__gc *gc,
> uint32_t domid, int fd,
> dss->hvm = 0;
> break;
> default:
> - return ERROR_INVAL;
> + abort();
> }
>
> dss->xcflags = (live) ? XCFLAGS_LIVE : 0
> | (debug) ? XCFLAGS_DEBUG : 0
> | (dss->hvm) ? XCFLAGS_HVM : 0;
>
> - dss->domid = domid;
> - dss->gc = gc;
> dss->suspend_eventchn = -1;
> dss->guest_responded = 0;
>
> @@ -884,10 +896,7 @@ int libxl__domain_suspend_common(libxl__gc *gc, uint32_t
> domid, int fd,
> dss->interval = r_info->interval;
> if (r_info->compression)
> dss->xcflags |= XCFLAGS_CHECKPOINT_COMPRESS;
> - dss->save_fd = fd;
> }
> - else
> - dss->save_fd = -1;
>
> dss->xce = xc_evtchn_open(NULL, 0);
> if (dss->xce == NULL)
> @@ -917,10 +926,28 @@ int libxl__domain_suspend_common(libxl__gc *gc,
> uint32_t domid, int fd,
> callbacks->toolstack_save = libxl__toolstack_save;
> callbacks->data = dss;
>
> - rc = xc_domain_save(CTX->xch, fd, domid, 0, 0, dss->xcflags, callbacks,
> - dss->hvm, vm_generationid_addr);
> - if ( rc ) {
> - LOGE(ERROR, "saving domain: %s",
> + libxl__xc_domain_save(egc, dss, vm_generationid_addr);
> + return;
> +
> + out:
> + domain_suspend_done(egc, dss, rc);
> +}
> +
> +void libxl__xc_domain_save_done(libxl__egc *egc,
> + libxl__domain_suspend_state *dss,
> + int rc, int retval, int errnoval)
> +{
> + STATE_AO_GC(dss->ao);
> +
> + /* Convenience aliases */
> + const libxl_domain_type type = dss->type;
> + const uint32_t domid = dss->domid;
> +
> + if (rc)
> + goto out;
> +
> + if (retval) {
> + LOGEV(ERROR, errnoval, "saving domain: %s",
> dss->guest_responded ?
> "domain responded to suspend request" :
> "domain did not respond to suspend request");
> @@ -928,16 +955,21 @@ int libxl__domain_suspend_common(libxl__gc *gc,
> uint32_t domid, int fd,
> rc = ERROR_GUEST_TIMEDOUT;
> else
> rc = ERROR_FAIL;
> + goto out;
> }
>
> - if (dss->suspend_eventchn > 0)
> - xc_suspend_evtchn_release(CTX->xch, dss->xce, domid,
> - dss->suspend_eventchn);
> - if (dss->xce != NULL)
> - xc_evtchn_close(dss->xce);
> + if (type == LIBXL_DOMAIN_TYPE_HVM) {
> + rc = libxl__domain_suspend_device_model(gc, domid);
> + if (rc) goto out;
> +
> + rc = libxl__domain_save_device_model(gc, domid, dss->fd);
> + if (rc) goto out;
> + }
> +
> + rc = 0;
>
> out:
> - return rc;
> + domain_suspend_done(egc, dss, rc);
> }
>
> int libxl__domain_save_device_model(libxl__gc *gc, uint32_t domid, int fd)
> @@ -992,6 +1024,25 @@ out:
> return rc;
> }
>
> +static void domain_suspend_done(libxl__egc *egc,
> + libxl__domain_suspend_state *dss, int rc)
> +{
> + STATE_AO_GC(dss->ao);
> +
> + /* Convenience aliases */
> + const uint32_t domid = dss->domid;
> +
> + if (dss->suspend_eventchn > 0)
> + xc_suspend_evtchn_release(CTX->xch, dss->xce, domid,
> + dss->suspend_eventchn);
> + if (dss->xce != NULL)
> + xc_evtchn_close(dss->xce);
> +
> + dss->callback(egc, dss, rc);
> +}
> +
> +/*==================== Miscellaneous ====================*/
> +
> char *libxl__uuid2string(libxl__gc *gc, const libxl_uuid uuid)
> {
> char *s = libxl__sprintf(gc, LIBXL_UUID_FMT, LIBXL_UUID_BYTES(uuid));
> diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
> index 28478ea..7cf1b04 100644
> --- a/tools/libxl/libxl_internal.h
> +++ b/tools/libxl/libxl_internal.h
> @@ -1777,16 +1777,28 @@ _hidden int
> libxl__datacopier_start(libxl__datacopier_state *dc);
>
> typedef struct libxl__domain_suspend_state libxl__domain_suspend_state;
>
> +typedef void libxl__domain_suspend_cb(libxl__egc*,
> + libxl__domain_suspend_state*, int rc);
> +
> struct libxl__domain_suspend_state {
> - libxl__gc *gc;
> + /* set by caller of libxl__domain_suspend */
> + libxl__ao *ao;
> + libxl__domain_suspend_cb *callback;
> +
> + uint32_t domid;
> + int fd;
> + libxl_domain_type type;
> + int live;
> + int debug;
> + const libxl_domain_remus_info *remus;
> + /* private */
> xc_evtchn *xce; /* event channel handle */
> int suspend_eventchn;
> - int domid;
> int hvm;
> - unsigned int xcflags;
> + int xcflags;
> int guest_responded;
> - int save_fd; /* Migration stream fd (for Remus) */
> int interval; /* checkpoint interval (for Remus) */
> + struct save_callbacks callbacks;
> };
>
>
> @@ -1903,10 +1915,27 @@ struct libxl__domain_create_state {
>
> /*----- Domain suspend (save) functions -----*/
>
> -_hidden int libxl__domain_suspend_common(libxl__gc *gc, uint32_t domid, int
> fd,
> - libxl_domain_type type,
> - int live, int debug,
> - const libxl_domain_remus_info
> *r_info);
> +/* calls dss->callback when done */
> +_hidden void libxl__domain_suspend(libxl__egc *egc,
> + libxl__domain_suspend_state *dss);
> +
> +
> +/* calls libxl__xc_domain_suspend_done when done */
> +_hidden void libxl__xc_domain_save(libxl__egc*, libxl__domain_suspend_state*,
> + unsigned long vm_generationid_addr);
> +/* If rc==0 then retval is the return value from xc_domain_save
> + * and errnoval is the errno value it provided.
> + * If rc!=0, retval and errnoval are undefined. */
> +_hidden void libxl__xc_domain_save_done(libxl__egc*,
> + libxl__domain_suspend_state*,
> + int rc, int retval, int errnoval);
> +
> +_hidden int libxl__domain_suspend_common_callback(void *data);
> +_hidden int libxl__domain_suspend_common_switch_qemu_logdirty
> + (int domid, unsigned int enable, void *data);
> +_hidden int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
> + uint32_t *len, void *data);
> +
>
> /* calls libxl__xc_domain_restore_done when done */
> _hidden void libxl__xc_domain_restore(libxl__egc *egc,
> diff --git a/tools/libxl/libxl_save_callout.c
> b/tools/libxl/libxl_save_callout.c
> index 2f8db9f..1b481ab 100644
> --- a/tools/libxl/libxl_save_callout.c
> +++ b/tools/libxl/libxl_save_callout.c
> @@ -35,3 +35,14 @@ void libxl__xc_domain_restore(libxl__egc *egc,
> libxl__domain_create_state *dcs,
> &state->vm_generationid_addr, &dcs->callbacks);
> libxl__xc_domain_restore_done(egc, dcs, 0, r, errno);
> }
> +
> +void libxl__xc_domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss,
> + unsigned long vm_generationid_addr)
> +{
> + STATE_AO_GC(dss->ao);
> + int r;
> +
> + r = xc_domain_save(CTX->xch, dss->fd, dss->domid, 0, 0, dss->xcflags,
> + &dss->callbacks, dss->hvm, vm_generationid_addr);
> + libxl__xc_domain_save_done(egc, dss, 0, r, errno);
> +}
> diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
> index 3ea95ef..19daa1c 100644
> --- a/tools/libxl/xl_cmdimpl.c
> +++ b/tools/libxl/xl_cmdimpl.c
> @@ -2846,7 +2846,7 @@ static int save_domain(const char *p, const char
> *filename, int checkpoint,
>
> save_domain_core_writeconfig(fd, filename, config_data, config_len);
>
> - CHK_ERRNO(libxl_domain_suspend(ctx, NULL, domid, fd));
> + CHK_ERRNO(libxl_domain_suspend(ctx, domid, fd, 0, NULL));
> close(fd);
>
> if (checkpoint)
> @@ -3008,7 +3008,6 @@ static void migrate_domain(const char *domain_spec,
> const char *rune,
> pid_t child = -1;
> int rc;
> int send_fd = -1, recv_fd = -1;
> - libxl_domain_suspend_info suspinfo;
> char *away_domname;
> char rc_buf;
> uint8_t *config_data;
> @@ -3030,9 +3029,7 @@ static void migrate_domain(const char *domain_spec,
> const char *rune,
>
> xtl_stdiostream_adjust_flags(logger, XTL_STDIOSTREAM_HIDE_PROGRESS, 0);
>
> - memset(&suspinfo, 0, sizeof(suspinfo));
> - suspinfo.flags |= XL_SUSPEND_LIVE;
> - rc = libxl_domain_suspend(ctx, &suspinfo, domid, send_fd);
> + rc = libxl_domain_suspend(ctx, domid, send_fd, LIBXL_SUSPEND_LIVE, NULL);
> if (rc) {
> fprintf(stderr, "migration sender: libxl_domain_suspend failed"
> " (rc=%d)\n", rc);
> @@ -6604,7 +6601,7 @@ int main_remus(int argc, char **argv)
> }
>
> /* Point of no return */
> - rc = libxl_domain_remus_start(ctx, &r_info, domid, send_fd, recv_fd);
> + rc = libxl_domain_remus_start(ctx, &r_info, domid, send_fd, recv_fd, 0);
>
> /* If we are here, it means backup has failed/domain suspend failed.
> * Try to resume the domain and exit gracefully.
> --
> 1.7.2.5
>
>
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@xxxxxxxxxxxxx
> http://lists.xen.org/xen-devel
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
http://lists.xen.org/xen-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |