[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-devel] [PATCH v17 2/8] remus: introduce remus device
introduce remus device, an abstract layer of remus devices(nic, disk, etc).It provides the following APIs for libxl: >libxl__remus_devices_setup setup remus devices, like attach qdisc, enable disk buffering, etc >libxl__remus_devices_teardown teardown devices >libxl__remus_devices_postsuspend >libxl__remus_devices_preresume >libxl__remus_devices_commit above three are for checkpoint. through remus device layer, the remus execution flow will be like this: xl remus -> remus device setup |-> remus checkpoint(postsuspend, preresume, commit) ... |-> remus device teardown, failover or abort the remus device layer provides an interface libxl__remus_device_subkind_ops which a remus device must implement. the whole remus structure: |remus| | |remus device| | |nic| |drbd disks| |qemu disks| ... a device(nic, drbd disks, qemu disks, etc) must implement libxl__remus_device_subkind_ops to support remus. Signed-off-by: Yang Hongyang <yanghy@xxxxxxxxxxxxxx> Signed-off-by: Wen Congyang <wency@xxxxxxxxxxxxxx> Signed-off-by: Lai Jiangshan <laijs@xxxxxxxxxxxxxx> Signed-off-by: Ian Jackson <ian.jackson@xxxxxxxxxxxxx> --- tools/libxl/Makefile | 2 + tools/libxl/libxl.c | 52 +++++- tools/libxl/libxl_dom.c | 160 +++++++++++++++-- tools/libxl/libxl_internal.h | 191 ++++++++++++++++++++ tools/libxl/libxl_remus_device.c | 364 +++++++++++++++++++++++++++++++++++++++ tools/libxl/libxl_types.idl | 2 + 6 files changed, 752 insertions(+), 19 deletions(-) create mode 100644 tools/libxl/libxl_remus_device.c diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile index eb63510..202f1bb 100644 --- a/tools/libxl/Makefile +++ b/tools/libxl/Makefile @@ -56,6 +56,8 @@ else LIBXL_OBJS-y += libxl_nonetbuffer.o endif +LIBXL_OBJS-y += libxl_remus_device.o + LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_arm.o diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c index a9205d1..95d9953 100644 --- a/tools/libxl/libxl.c +++ b/tools/libxl/libxl.c @@ -781,9 +781,17 @@ out: return ptr; } +static void libxl__remus_setup_done(libxl__egc *egc, + libxl__remus_device_state *rds, int rc); +static void libxl__remus_setup_failed(libxl__egc *egc, + libxl__remus_device_state *rds, int rc); static void remus_failover_cb(libxl__egc *egc, libxl__domain_suspend_state *dss, int rc); +static const libxl__remus_device_subkind_ops *remus_ops[] = { + NULL, +}; + /* TODO: Explicit Checkpoint acknowledgements via recv_fd. */ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info, uint32_t domid, int send_fd, int recv_fd, @@ -812,16 +820,52 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info, assert(info); - /* TBD: Remus setup - i.e. attach qdisc, enable disk buffering, etc */ + /* Convenience aliases */ + libxl__remus_device_state *const rds = &dss->rds; + rds->ao = ao; + rds->egc = egc; + rds->domid = domid; + rds->callback = libxl__remus_setup_done; + rds->ops = remus_ops; /* Point of no return */ - libxl__domain_suspend(egc, dss); + libxl__remus_devices_setup(egc, rds); return AO_INPROGRESS; out: return AO_ABORT(rc); } +static void libxl__remus_setup_done(libxl__egc *egc, + libxl__remus_device_state *rds, int rc) +{ + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + STATE_AO_GC(dss->ao); + + if (!rc) { + libxl__domain_suspend(egc, dss); + return; + } + + LOG(ERROR, "Remus: failed to setup device for guest with domid %u, rc %d", + dss->domid, rc); + rds->callback = libxl__remus_setup_failed; + libxl__remus_devices_teardown(egc, rds); +} + +static void libxl__remus_setup_failed(libxl__egc *egc, + libxl__remus_device_state *rds, int rc) +{ + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + STATE_AO_GC(dss->ao); + + if (rc) + LOG(ERROR, "Remus: failed to teardown device after setup failed" + " for guest with domid %u, rc %d", dss->domid, rc); + + dss->callback(egc, dss, rc); +} + static void remus_failover_cb(libxl__egc *egc, libxl__domain_suspend_state *dss, int rc) { @@ -831,10 +875,6 @@ static void remus_failover_cb(libxl__egc *egc, * backup died or some network error occurred preventing us * from sending checkpoints. */ - - /* TBD: Remus cleanup - i.e. detach qdisc, release other - * resources. - */ libxl__ao_complete(egc, ao, rc); } diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c index 83eb29a..49bd1ad 100644 --- a/tools/libxl/libxl_dom.c +++ b/tools/libxl/libxl_dom.c @@ -798,8 +798,6 @@ static void domain_suspend_done(libxl__egc *egc, libxl__domain_suspend_state *dss, int rc); static void domain_suspend_callback_common_done(libxl__egc *egc, libxl__domain_suspend_state *dss, int ok); -static void remus_domain_suspend_callback_common_done(libxl__egc *egc, - libxl__domain_suspend_state *dss, int ok); /*----- complicated callback, called by xc_domain_save -----*/ @@ -1461,6 +1459,14 @@ static void domain_suspend_callback_common_done(libxl__egc *egc, } /*----- remus callbacks -----*/ +static void remus_domain_suspend_callback_common_done(libxl__egc *egc, + libxl__domain_suspend_state *dss, int ok); +static void remus_device_postsuspend_cb(libxl__egc *egc, + libxl__remus_device_state *rds, + int rc); +static void remus_device_preresume_cb(libxl__egc *egc, + libxl__remus_device_state *rds, + int rc); static void libxl__remus_domain_suspend_callback(void *data) { @@ -1475,32 +1481,74 @@ static void libxl__remus_domain_suspend_callback(void *data) static void remus_domain_suspend_callback_common_done(libxl__egc *egc, libxl__domain_suspend_state *dss, int ok) { - /* REMUS TODO: Issue disk and network checkpoint reqs. */ + if (!ok) + goto out; + + libxl__remus_device_state *const rds = &dss->rds; + rds->callback = remus_device_postsuspend_cb; + libxl__remus_devices_postsuspend(egc, rds); + return; + +out: libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok); } -static void libxl__remus_domain_resume_callback(void *data) +static void remus_device_postsuspend_cb(libxl__egc *egc, + libxl__remus_device_state *rds, + int rc) { int ok = 0; + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + + if (rc) + goto out; + + ok = 1; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok); +} + +static void libxl__remus_domain_resume_callback(void *data) +{ libxl__save_helper_state *shs = data; libxl__egc *egc = shs->egc; libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs); STATE_AO_GC(dss->ao); - /* Resumes the domain and the device model */ - if (libxl__domain_resume(gc, dss->domid, /* Fast Suspend */1)) + libxl__remus_device_state *const rds = &dss->rds; + rds->callback = remus_device_preresume_cb; + libxl__remus_devices_preresume(egc, rds); +} + +static void remus_device_preresume_cb(libxl__egc *egc, + libxl__remus_device_state *rds, + int rc) +{ + int ok = 0; + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + STATE_AO_GC(dss->ao); + + if (rc) goto out; - /* REMUS TODO: Deal with disk. Start a new network output buffer */ - ok = 1; + /* Resumes the domain and the device model */ + if (!libxl__domain_resume(gc, dss->domid, /* Fast Suspend */1)) + ok = 1; + out: - libxl__xc_domain_saverestore_async_callback_done(egc, shs, ok); + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok); } /*----- remus asynchronous checkpoint callback -----*/ static void remus_checkpoint_dm_saved(libxl__egc *egc, libxl__domain_suspend_state *dss, int rc); +static void remus_device_commit_cb(libxl__egc *egc, + libxl__remus_device_state *rds, + int rc); +static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev, + const struct timeval *requested_abs); static void libxl__remus_domain_checkpoint_callback(void *data) { @@ -1520,10 +1568,64 @@ static void libxl__remus_domain_checkpoint_callback(void *data) static void remus_checkpoint_dm_saved(libxl__egc *egc, libxl__domain_suspend_state *dss, int rc) { - /* REMUS TODO: Wait for disk and memory ack, release network buffer */ - /* REMUS TODO: make this asynchronous */ - assert(!rc); /* REMUS TODO handle this error properly */ - usleep(dss->interval * 1000); + /* Convenience aliases */ + libxl__remus_device_state *const rds = &dss->rds; + + STATE_AO_GC(dss->ao); + + if (rc) { + LOG(ERROR, "Failed to save device model. Terminating Remus.."); + goto out; + } + + rds->callback = remus_device_commit_cb; + libxl__remus_devices_commit(egc, rds); + + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0); +} + +static void remus_device_commit_cb(libxl__egc *egc, + libxl__remus_device_state *rds, + int rc) +{ + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + + STATE_AO_GC(dss->ao); + + if (rc) { + LOG(ERROR, "Failed to do device commit op." + " Terminating Remus.."); + goto out; + } + + /* Set checkpoint interval timeout */ + rc = libxl__ev_time_register_rel(gc, &dss->checkpoint_timeout, + remus_next_checkpoint, + dss->interval); + + if (rc) { + LOG(ERROR, "unable to register timeout for next epoch." + " Terminating Remus.."); + goto out; + } + + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0); +} + +static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev, + const struct timeval *requested_abs) +{ + libxl__domain_suspend_state *dss = + CONTAINER_OF(ev, *dss, checkpoint_timeout); + + STATE_AO_GC(dss->ao); + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 1); } @@ -1738,6 +1840,10 @@ static void save_device_model_datacopier_done(libxl__egc *egc, dss->save_dm_callback(egc, dss, our_rc); } +static void remus_teardown_done(libxl__egc *egc, + libxl__remus_device_state *rds, + int rc); + static void domain_suspend_done(libxl__egc *egc, libxl__domain_suspend_state *dss, int rc) { @@ -1752,6 +1858,34 @@ static void domain_suspend_done(libxl__egc *egc, xc_suspend_evtchn_release(CTX->xch, CTX->xce, domid, dss->guest_evtchn.port, &dss->guest_evtchn_lockfd); + if (dss->remus) { + /* + * With Remus, if we reach this point, it means either + * backup died or some network error occurred preventing us + * from sending checkpoints. Teardown the network buffers and + * release netlink resources. This is an async op. + */ + LOG(WARN, "Remus: Domain suspend terminated with rc %d," + " teardown Remus devices...", rc); + dss->rds.callback = remus_teardown_done; + libxl__remus_devices_teardown(egc, &dss->rds); + return; + } + + dss->callback(egc, dss, rc); +} + +static void remus_teardown_done(libxl__egc *egc, + libxl__remus_device_state *rds, + int rc) +{ + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + STATE_AO_GC(dss->ao); + + if (rc) + LOG(ERROR, "Remus: failed to teardown device for guest with domid %u," + " rc %d", dss->domid, rc); + dss->callback(egc, dss, rc); } diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h index db10efb..ac9d2b4 100644 --- a/tools/libxl/libxl_internal.h +++ b/tools/libxl/libxl_internal.h @@ -2470,6 +2470,195 @@ typedef struct libxl__save_helper_state { * marshalling and xc callback functions */ } libxl__save_helper_state; +/*----- remus device related state structure -----*/ +/* remus device is an abstract layer of remus devices(nic, disk, + * etc).It provides the following APIs for libxl: + * >libxl__remus_devices_setup + * setup remus devices, like attach qdisc, enable disk buffering, etc + * >libxl__remus_devices_teardown + * teardown devices + * >libxl__remus_devices_postsuspend + * >libxl__remus_devices_preresume + * >libxl__remus_devices_commit + * above three are for checkpoint. + * through remus device layer, the remus execution flow will be like + * this: + * xl remus -> remus device setup + * |-> remus checkpoint(postsuspend, preresume, commit) + * ... + * |-> remus device teardown, failover or abort + * the remus device layer provides an interface + * libxl__remus_device_subkind_ops + * which a remus device must implement. the whole remus structure: + * |remus| + * | + * |remus device| + * | + * |nic| |drbd disks| |qemu disks| ... + * a device(nic, drbd disks, qemu disks, etc) must implement + * libxl__remus_device_subkind_ops to support remus. + */ + +typedef enum libxl__remus_device_kind { + LIBXL__REMUS_DEVICE_NIC = (1 << 0), + LIBXL__REMUS_DEVICE_DISK = (1 << 1), +} libxl__remus_device_kind; + +typedef struct libxl__remus_device libxl__remus_device; +typedef struct libxl__remus_device_state libxl__remus_device_state; +typedef struct libxl__remus_device_subkind_ops libxl__remus_device_subkind_ops; + +/* + * These operations are provided by a device subkind and + * called by the main remus code. Functions must be + * implemented unless otherwise stated. + * Many of these functions are asynchronous. They call + * dev->callback when done. The actual implementations may be + * synchronously and call dev->callback directly (as the last + * thing they do). + */ +struct libxl__remus_device_subkind_ops { + /* the device kind this ops belongs to... */ + libxl__remus_device_kind kind; + + /* + * init() and cleanup() relate to the subkind-specific state in + * the libxl ctx, not to any specific device. + * Synchronous. cleanup() cannot fail. + */ + int (*init)(libxl__remus_device_state *rds); + void (*cleanup)(libxl__remus_device_state *rds); + + /* + * Checkpoint operations. May be NULL, meaning the op is not + * implemented and the caller should treat them as a no-op (and do + * nothing when checkpointing). + * Asynchronous. + */ + + void (*postsuspend)(libxl__remus_device *dev); + void (*preresume)(libxl__remus_device *dev); + void (*commit)(libxl__remus_device *dev); + + /* + * Determines whether the subkind matches the specific device. + * init() will have been called for the subkind. If match() + * succeeds, the device will then be managed with this set of + * subkind operations. + * Asynchronous. Yields 0 if the device matches, or + * REMUS_DEVOPS_NOT_MATCH if it does not; any other rc indicates + * failure. + * May be NULL, meaning this subkind matches any device. + */ + void (*match)(libxl__remus_device *dev); + + /* + * setup() and teardown() are refer to the actual remus device. + * Asynchronous. + * teardown is called even if setup fails. + */ + void (*setup)(libxl__remus_device *dev); + void (*teardown)(libxl__remus_device *dev); +}; + +typedef void libxl__remus_callback(libxl__egc *, + libxl__remus_device_state *, int rc); + +/* + * State associated with a remus invocation, including parameters + * passed to the remus abstract device layer by the remus + * save/restore machinery. + */ +struct libxl__remus_device_state { + /*---- must be set by caller of libxl__remus_device_(setup|teardown) ----*/ + + libxl__ao *ao; + libxl__egc *egc; + uint32_t domid; + libxl__remus_callback *callback; + /* the last ops must be NULL */ + const libxl__remus_device_subkind_ops **ops; + int device_kind_flags; + + /*----- private -----*/ + + int saved_rc; + /* number of devices which have been handled */ + int num_devices; + int num_set_up; + /* + * this array is allocated before setup the remus devices by the + * remus abstract layer. + * the allocated size of this array is the total number of libxl + * nic devices and disk devices(num_nics + num_disks). + * this array stores the remus devices which have been set up, + * so the actual size of this array is num_set_up. + * we add devices that have been set up to the array no matter + * the setup process succeed or failed because we need to ensure + * the device been teardown while setup failed. If any of the + * device setup failed, we will quit remus, but before we exit, + * we will teardown the devices that have been added to **dev. + */ + libxl__remus_device **dev; + + libxl_device_nic *nics; + int num_nics; + libxl_device_disk *disks; + int num_disks; +}; + +typedef void libxl__remus_device_callback(libxl__egc *, + libxl__remus_device *, + int rc); + +/* + * Information about a single device being handled by remus. + * Allocated by the remus abstract layer. + */ +struct libxl__remus_device { + /*----- shared between abstract and concrete layers -----*/ + /*----- all set by remus device abstruct layer -----*/ + + /* libxl__device_* which this remus device related to */ + const void *backend_dev; + libxl__remus_device_kind kind; + libxl__remus_device_state *rds; + libxl__remus_device_callback *callback; + + /*----- private for abstract layer only -----*/ + + /* + * Control and state variables for the asynchronous callback + * based loops which iterate over device subkinds, and over + * individual devices. + */ + int ops_index; + const libxl__remus_device_subkind_ops *ops; + + /*----- private for concrete (device-specific) layer -----*/ + + /* concrete device's private data */ + void *concrete_data; + libxl__async_exec_state aes; + /* + * for async func calls, in the implementation of device ops, we + * may use fork to do async ops. this is owned by device-specific + * ops methods + */ + libxl__ev_child child; +}; + +/* the following 5 APIs are async ops, call rds->callback when done */ +_hidden void libxl__remus_devices_setup(libxl__egc *egc, + libxl__remus_device_state *rds); +_hidden void libxl__remus_devices_teardown(libxl__egc *egc, + libxl__remus_device_state *rds); +_hidden void libxl__remus_devices_postsuspend(libxl__egc *egc, + libxl__remus_device_state *rds); +_hidden void libxl__remus_devices_preresume(libxl__egc *egc, + libxl__remus_device_state *rds); +_hidden void libxl__remus_devices_commit(libxl__egc *egc, + libxl__remus_device_state *rds); _hidden int libxl__netbuffer_enabled(libxl__gc *gc); /*----- Domain suspend (save) state structure -----*/ @@ -2510,6 +2699,8 @@ struct libxl__domain_suspend_state { libxl__ev_xswatch guest_watch; libxl__ev_time guest_timeout; const char *dm_savefile; + libxl__remus_device_state rds; + libxl__ev_time checkpoint_timeout; /* used for Remus checkpoint */ int interval; /* checkpoint interval (for Remus) */ libxl__save_helper_state shs; libxl__logdirty_switch logdirty; diff --git a/tools/libxl/libxl_remus_device.c b/tools/libxl/libxl_remus_device.c new file mode 100644 index 0000000..ffe3c0e --- /dev/null +++ b/tools/libxl/libxl_remus_device.c @@ -0,0 +1,364 @@ +/* + * Copyright (C) 2014 FUJITSU LIMITED + * Author: Yang Hongyang <yanghy@xxxxxxxxxxxxxx> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; version 2.1 only. with the special + * exception on linking described in file LICENSE. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + */ + +#include "libxl_osdeps.h" /* must come before any other headers */ + +#include "libxl_internal.h" + +typedef enum remus_operation_phase { + REMUS_DEVICE_SETUP, + REMUS_DEVICE_CHECKPOINT, + REMUS_DEVICE_TEARDOWN, +} remus_operation_phase; + +/*----- helper functions -----*/ + +static int init_device_subkind(libxl__remus_device_state *rds) +{ + int rc; + const libxl__remus_device_subkind_ops **ops; + + for (ops = rds->ops; *ops; ops++) { + rc = (*ops)->init(rds); + if (rc) + goto out; + } + + rc = 0; +out: + return rc; + +} + +static void cleanup_device_subkind(libxl__remus_device_state *rds) +{ + const libxl__remus_device_subkind_ops **ops; + + for (ops = rds->ops; *ops; ops++) + (*ops)->cleanup(rds); +} + +static void all_devices_teared_down(libxl__egc *egc, + libxl__remus_device_state *rds); + +static void handled_one_device(libxl__egc *egc, + libxl__remus_device_state *rds, + remus_operation_phase phase) +{ + rds->num_devices++; + + switch(phase) { + case REMUS_DEVICE_SETUP: + if (rds->num_devices == (rds->num_nics + rds->num_disks)) + rds->callback(egc, rds, rds->saved_rc); + break; + case REMUS_DEVICE_CHECKPOINT: + if (rds->num_devices == rds->num_set_up) + rds->callback(egc, rds, rds->saved_rc); + break; + case REMUS_DEVICE_TEARDOWN: + if (rds->num_devices == rds->num_set_up) + all_devices_teared_down(egc, rds); + break; + default: + break; + } +} + +/*----- setup() and teardown() -----*/ + +/* callbacks */ + +static void device_match_cb(libxl__egc *egc, + libxl__remus_device *dev, + int rc); +static void device_setup_cb(libxl__egc *egc, + libxl__remus_device *dev, + int rc); +static void device_teardown_cb(libxl__egc *egc, + libxl__remus_device *dev, + int rc); + +/* remus device setup and teardown */ + +static void remus_device_init(libxl__egc *egc, + libxl__remus_device_state *rds, + libxl__remus_device_kind kind, + void *libxl_dev); + +void libxl__remus_devices_setup(libxl__egc *egc, libxl__remus_device_state *rds) +{ + int i; + + STATE_AO_GC(rds->ao); + + rds->saved_rc = init_device_subkind(rds); + if (rds->saved_rc) + goto out; + + rds->num_devices = 0; + rds->num_nics = 0; + rds->num_disks = 0; + + if (rds->device_kind_flags & LIBXL__REMUS_DEVICE_NIC) + rds->nics = libxl_device_nic_list(CTX, rds->domid, &rds->num_nics); + + if (rds->device_kind_flags & LIBXL__REMUS_DEVICE_DISK) + rds->disks = libxl_device_disk_list(CTX, rds->domid, &rds->num_disks); + + if (rds->num_nics == 0 && rds->num_disks == 0) + goto out; + + GCNEW_ARRAY(rds->dev, rds->num_nics + rds->num_disks); + + for (i = 0; i < rds->num_nics; i++) { + remus_device_init(egc, rds, + LIBXL__REMUS_DEVICE_NIC, &rds->nics[i]); + } + + for (i = 0; i < rds->num_disks; i++) { + remus_device_init(egc, rds, + LIBXL__REMUS_DEVICE_DISK, &rds->disks[i]); + } + + return; + +out: + rds->callback(egc, rds, rds->saved_rc); + return; +} + +static void remus_device_init(libxl__egc *egc, + libxl__remus_device_state *rds, + libxl__remus_device_kind kind, + void *libxl_dev) +{ + libxl__remus_device *dev = NULL; + + STATE_AO_GC(rds->ao); + GCNEW(dev); + dev->backend_dev = libxl_dev; + dev->kind = kind; + dev->rds = rds; + + libxl__async_exec_init(&dev->aes); + libxl__ev_child_init(&dev->child); + + /* match the ops begin */ + dev->ops_index = -1; + dev->callback = device_match_cb; + device_match_cb(egc, dev, ERROR_REMUS_DEVOPS_NOT_MATCH); +} + +static void device_match_cb(libxl__egc *egc, + libxl__remus_device *dev, + int rc) +{ + libxl__remus_device_state *const rds = dev->rds; + + STATE_AO_GC(rds->ao); + + if (rds->saved_rc) { + /* there's already an error happened, we do not need to continue */ + handled_one_device(egc, rds, REMUS_DEVICE_SETUP); + return; + } + + if (rc) { + /* the ops does not match, try next ops */ + dev->ops = rds->ops[++dev->ops_index]; + if (!dev->ops || rc != ERROR_REMUS_DEVOPS_NOT_MATCH) { + /* the device can not be matched */ + rds->saved_rc = ERROR_REMUS_DEVICE_NOT_SUPPORTED; + handled_one_device(egc, rds, REMUS_DEVICE_SETUP); + return; + } + for ( ; dev->ops; dev->ops = rds->ops[++dev->ops_index]) { + if (dev->ops->kind == dev->kind) { + if (dev->ops->match) { + dev->ops->match(dev); + break; + } else { + /* + * This devops do not have match() implementation. + * That means this kind of device's ops is always + * matched with the kind of device. + */ + goto matched; + } + } + } + + return; + } + +matched: + /* the ops matched, setup the device */ + dev->callback = device_setup_cb; + dev->ops->setup(dev); +} + +static void device_setup_cb(libxl__egc *egc, + libxl__remus_device *dev, + int rc) +{ + /* Convenience aliases */ + libxl__remus_device_state *const rds = dev->rds; + + STATE_AO_GC(rds->ao); + + /* + * we add devices that have been set up to the array no matter + * the setup process succeed or failed because we need to ensure + * the device been teardown while setup failed. If any of the + * device setup failed, we will quit remus, but before we exit, + * we will teardown the devices that have been added to **dev. + */ + rds->dev[rds->num_set_up++] = dev; + /* we preserve the first error that happened */ + if (rc && !rds->saved_rc) + rds->saved_rc = rc; + + handled_one_device(egc, rds, REMUS_DEVICE_SETUP); +} + +void libxl__remus_devices_teardown(libxl__egc *egc, + libxl__remus_device_state *rds) +{ + int i; + libxl__remus_device *dev; + + STATE_AO_GC(rds->ao); + + rds->num_devices = 0; + rds->saved_rc = 0; + + if (rds->num_set_up == 0) + goto out; + + for (i = 0; i < rds->num_set_up; i++) { + dev = rds->dev[i]; + dev->callback = device_teardown_cb; + dev->ops->teardown(dev); + } + + return; + +out: + all_devices_teared_down(egc, rds); +} + +static void device_teardown_cb(libxl__egc *egc, + libxl__remus_device *dev, + int rc) +{ + libxl__remus_device_state *const rds = dev->rds; + + STATE_AO_GC(rds->ao); + + /* we preserve the first error that happened */ + if (rc && !rds->saved_rc) + rds->saved_rc = rc; + + /* ignore teardown errors to teardown as many devs as possible*/ + handled_one_device(egc, rds, REMUS_DEVICE_TEARDOWN); +} + +static void all_devices_teared_down(libxl__egc *egc, + libxl__remus_device_state *rds) +{ + int i; + + /* clean nic */ + for (i = 0; i < rds->num_nics; i++) + libxl_device_nic_dispose(&rds->nics[i]); + free(rds->nics); + rds->nics = NULL; + rds->num_nics = 0; + + /* clean disk */ + for (i = 0; i < rds->num_disks; i++) + libxl_device_disk_dispose(&rds->disks[i]); + free(rds->disks); + rds->disks = NULL; + rds->num_disks = 0; + + cleanup_device_subkind(rds); + + rds->callback(egc, rds, rds->saved_rc); +} + +/*----- checkpointing APIs -----*/ + +/* callbacks */ + +static void device_checkpoint_cb(libxl__egc *egc, + libxl__remus_device *dev, + int rc); + +/* API implementations */ + +#define define_remus_device_checkpoint_api(api) \ +void libxl__remus_devices_##api(libxl__egc *egc, \ + libxl__remus_device_state *rds) \ +{ \ + int i; \ + libxl__remus_device *dev; \ + \ + STATE_AO_GC(rds->ao); \ + \ + rds->num_devices = 0; \ + rds->saved_rc = 0; \ + \ + if (rds->num_set_up == 0) \ + goto out; \ + \ + for (i = 0; i < rds->num_set_up; i++) { \ + dev = rds->dev[i]; \ + dev->callback = device_checkpoint_cb; \ + if (dev->ops->api) { \ + dev->ops->api(dev); \ + } else { \ + handled_one_device(egc, rds, REMUS_DEVICE_CHECKPOINT); \ + } \ + } \ + \ + return; \ + \ +out: \ + rds->callback(egc, rds, rds->saved_rc); \ +} + +define_remus_device_checkpoint_api(postsuspend); + +define_remus_device_checkpoint_api(preresume); + +define_remus_device_checkpoint_api(commit); + +static void device_checkpoint_cb(libxl__egc *egc, + libxl__remus_device *dev, + int rc) +{ + /* Convenience aliases */ + libxl__remus_device_state *const rds = dev->rds; + + STATE_AO_GC(rds->ao); + + /* we preserve the first error that happened */ + if (rc && !rds->saved_rc) + rds->saved_rc = rc; + + handled_one_device(egc, rds, REMUS_DEVICE_CHECKPOINT); +} diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl index a412f9c..61d31c1 100644 --- a/tools/libxl/libxl_types.idl +++ b/tools/libxl/libxl_types.idl @@ -58,6 +58,8 @@ libxl_error = Enumeration("error", [ (-12, "OSEVENT_REG_FAIL"), (-13, "BUFFERFULL"), (-14, "UNKNOWN_CHILD"), + (-15, "REMUS_DEVOPS_NOT_MATCH"), + (-16, "REMUS_DEVICE_NOT_SUPPORTED"), ], value_namespace = "") libxl_domain_type = Enumeration("domain_type", [ -- 1.9.1 _______________________________________________ Xen-devel mailing list Xen-devel@xxxxxxxxxxxxx http://lists.xen.org/xen-devel
|
Lists.xenproject.org is hosted with RackSpace, monitoring our |