[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-devel] [PATCH 6/7] remus: implement remus replicated checkpointing disk
Signed-off-by: Lai Jiangshan <laijs@xxxxxxxxxxxxxx> Signed-off-by: Wen Congyang <wency@xxxxxxxxxxxxxx> --- tools/libxl/Makefile | 1 + tools/libxl/libxl_dom.c | 19 +++- tools/libxl/libxl_internal.h | 16 +++ tools/libxl/libxl_remus.c | 18 +++ tools/libxl/libxl_remus_disk.c | 285 ++++++++++++++++++++++++++++++++++++++++ tools/libxl/libxl_remus_disk.h | 74 +++++++++++ 6 files changed, 411 insertions(+), 2 deletions(-) create mode 100644 tools/libxl/libxl_remus_disk.c create mode 100644 tools/libxl/libxl_remus_disk.h diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile index 670a2bc..b040a79 100644 --- a/tools/libxl/Makefile +++ b/tools/libxl/Makefile @@ -53,6 +53,7 @@ LIBXL_OBJS-y += libxl_nonetbuffer.o endif LIBXL_OBJS-y += libxl_remus.o +LIBXL_OBJS-y += libxl_remus_disk.o LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_arm.o diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c index 1272bf6..ebaece2 100644 --- a/tools/libxl/libxl_dom.c +++ b/tools/libxl/libxl_dom.c @@ -1437,9 +1437,14 @@ static void remus_domain_suspend_callback_common_done(libxl__egc *egc, STATE_AO_GC(dss->ao); - /* REMUS TODO: Issue disk checkpoint reqs. */ if (!remus_state->netbuf_state || !ok) goto out; + /* Issue disk checkpoint reqs. */ + if (libxl__remus_disk_postsuspend(remus_state)) { + ok = 0; + goto out; + } + /* The domain was suspended successfully. Start a new network * buffer for the next epoch. If this operation fails, then act * as though domain suspend failed -- libxc exits its infinite @@ -1463,7 +1468,10 @@ static int libxl__remus_domain_resume_callback(void *data) if (libxl__domain_resume(gc, dss->domid, /* Fast Suspend */1)) return 0; - /* REMUS TODO: Deal with disk. */ + /* Deal with disk. */ + if (libxl__remus_disk_preresume(dss->remus_state)) + return 0; + return 1; } @@ -1506,6 +1514,13 @@ static void remus_checkpoint_dm_saved(libxl__egc *egc, goto out; } + rc = libxl__remus_disk_commit(remus_state); + if (rc) { + LOG(ERROR, "Failed to commit disk state" + " Terminating Remus.."); + goto out; + } + if (remus_state->netbuf_state) { rc = libxl__remus_netbuf_release_prev_epoch(gc, dss->domid, remus_state); diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h index 4e07969..7c30d9a 100644 --- a/tools/libxl/libxl_internal.h +++ b/tools/libxl/libxl_internal.h @@ -2446,6 +2446,9 @@ typedef struct libxl__remus_state { void *netbuf_state; /* used for checkpoint */ libxl__ev_time timeout; + + /* Opaque context containing disk related stuff */ + void *disk_state; } libxl__remus_state; _hidden int libxl__netbuffer_enabled(libxl__gc *gc); @@ -2474,6 +2477,19 @@ _hidden int libxl__remus_netbuf_release_prev_epoch(libxl__gc *gc, uint32_t domid, libxl__remus_state *remus_state); +_hidden int libxl__remus_disk_postsuspend(libxl__remus_state *state); +_hidden int libxl__remus_disk_preresume(libxl__remus_state *state); +_hidden int libxl__remus_disk_commit(libxl__remus_state *state); +_hidden void libxl__remus_disk_setup(libxl__egc *egc, + libxl__domain_suspend_state *dss); +_hidden void libxl__remus_disk_setup_done(libxl__egc *egc, + libxl__domain_suspend_state *dss, + int rc); +_hidden void libxl__remus_disk_teardown(libxl__egc *egc, + libxl__domain_suspend_state *dss); +_hidden void libxl__remus_disk_teardown_done(libxl__egc *egc, + libxl__domain_suspend_state *dss); + _hidden void libxl__remus_setup_initiate(libxl__egc *egc, libxl__domain_suspend_state *dss); diff --git a/tools/libxl/libxl_remus.c b/tools/libxl/libxl_remus.c index 78652d2..9eab98d 100644 --- a/tools/libxl/libxl_remus.c +++ b/tools/libxl/libxl_remus.c @@ -47,6 +47,18 @@ void libxl__remus_netbuf_setup_done(libxl__egc *egc, libxl__domain_suspend_state *dss, int rc) { + if (rc) { + libxl__remus_setup_done(egc, dss, rc); + return; + } + + libxl__remus_disk_setup(egc, dss); +} + +void libxl__remus_disk_setup_done(libxl__egc *egc, + libxl__domain_suspend_state *dss, + int rc) +{ libxl__remus_setup_done(egc, dss, rc); } @@ -72,5 +84,11 @@ static void libxl__remus_teardown_done(libxl__egc *egc, void libxl__remus_netbuf_teardown_done(libxl__egc *egc, libxl__domain_suspend_state *dss) { + libxl__remus_disk_teardown(egc, dss); +} + +void libxl__remus_disk_teardown_done(libxl__egc *egc, + libxl__domain_suspend_state *dss) +{ libxl__remus_teardown_done(egc, dss); } diff --git a/tools/libxl/libxl_remus_disk.c b/tools/libxl/libxl_remus_disk.c new file mode 100644 index 0000000..ca3e879 --- /dev/null +++ b/tools/libxl/libxl_remus_disk.c @@ -0,0 +1,285 @@ +/* + * Copyright (C) 2014 FUJITSU LIMITED + * Author Lai Jiangshan <laijs@xxxxxxxxxxxxxx> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; version 2.1 only. with the special + * exception on linking described in file LICENSE. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + */ + +#include "libxl_osdeps.h" /* must come before any other headers */ + +#include "libxl_internal.h" + +#include "libxl_remus_disk.h" + +typedef struct libxl__remus_disk_state { + libxl__remus_state *remus_state; + libxl_device_disk *disks; + struct libxl__remus_disk **remus_disks; + uint32_t num_disks; + uint32_t curr_disk_id; + uint32_t curr_disktype_id; + + libxl_async_exec async_exec; +} libxl__remus_disk_state; + +/*** checkpoint disks states and callbacks ***/ +static const libxl__remus_disk_type *remus_disk_types[] = +{ +}; + +int libxl__remus_disk_postsuspend(libxl__remus_state *remus_state) +{ + int i; + int rc = 0; + libxl__remus_disk *remus_disk; + + /* Convenience aliases */ + libxl__remus_disk_state *disk_state = remus_state->disk_state; + + for (i = 0; rc == 0 && i < disk_state->num_disks; i++) { + remus_disk = disk_state->remus_disks[i]; + rc = remus_disk->type->postsuspend(remus_disk); + } + + return rc; +} + +int libxl__remus_disk_preresume(libxl__remus_state *remus_state) +{ + int i; + int rc = 0; + libxl__remus_disk *remus_disk; + + /* Convenience aliases */ + libxl__remus_disk_state *disk_state = remus_state->disk_state; + + for (i = 0; rc == 0 && i < disk_state->num_disks; i++) { + remus_disk = disk_state->remus_disks[i]; + rc = remus_disk->type->preresume(remus_disk); + } + + return rc; +} + +int libxl__remus_disk_commit(libxl__remus_state *remus_state) +{ + int i; + int rc = 0; + libxl__remus_disk *remus_disk; + + /* Convenience aliases */ + libxl__remus_disk_state *disk_state = remus_state->disk_state; + + for (i = 0; rc == 0 && i < disk_state->num_disks; i++) { + remus_disk = disk_state->remus_disks[i]; + rc = remus_disk->type->commit(remus_disk); + } + + return rc; +} + +static void alloc_remus_disk(libxl__domain_suspend_state *dss, + libxl__remus_disk **remus_disk, + const libxl_device_disk *disk, + const libxl__remus_disk_type *disk_type) +{ + STATE_AO_GC(dss->ao); + libxl__remus_disk *new_disk; + + new_disk = libxl__zalloc(gc, sizeof(libxl__remus_disk) + disk_type->size); + new_disk->type = disk_type; + new_disk->disk = disk; + new_disk->disk_state = dss->remus_state->disk_state; + new_disk->opaque = &new_disk[1]; + + *remus_disk = new_disk; +} + +static void setup_all_disks(libxl__remus_disk_state *disk_state) +{ + int i, rc = 0; + libxl__remus_disk *remus_disk; + + /* Convenience aliases */ + libxl__egc *egc = disk_state->remus_state->egc; + libxl__domain_suspend_state *dss = disk_state->remus_state->dss; + + for (i = 0; i < disk_state->num_disks; i++) { + remus_disk = disk_state->remus_disks[i]; + rc = remus_disk->type->setup(remus_disk); + if (rc) { + libxl__remus_disk_setup_done(egc, dss, rc); + return; + } + } + + libxl__remus_disk_setup_done(egc, dss, 0); +} + +static int disk_match_once(libxl__remus_disk_state *disk_state) +{ + int i, j, rc = 0; + const libxl__remus_disk_type *disk_type; + const libxl_device_disk *disk; + + /* Convenience aliases */ + libxl__egc *egc = disk_state->remus_state->egc; + libxl__domain_suspend_state *dss = disk_state->remus_state->dss; + + i = disk_state->curr_disk_id; + j = disk_state->curr_disktype_id; + for (; i < disk_state->num_disks; i++) { + disk = &disk_state->disks[i]; + for (; j < ARRAY_SIZE(remus_disk_types); j++) { + disk_type = remus_disk_types[j]; + + rc = disk_type->match(dss, disk, &disk_state->async_exec, + disk_state); + if (rc) + goto out; + + if (rc == 0) { + alloc_remus_disk(dss, &disk_state->remus_disks[i], + disk, disk_type); + break; + } + } + j = 0; + } + +out: + if (rc < 0) + libxl__remus_disk_setup_done(egc, dss, rc); + else if (i < disk_state->num_disks) { + disk_state->curr_disk_id = i; + disk_state->curr_disktype_id = j; + } + + return rc; +} + +void disk_match_script_cb(void *opaque, int status) +{ + libxl__remus_disk_state *disk_state = opaque; + int rc; + + /* Convenience aliases */ + int curr_disktype_id = disk_state->curr_disktype_id; + int curr_disk_id = disk_state->curr_disk_id; + libxl_device_disk *disk = &disk_state->disks[disk_state->curr_disk_id]; + const libxl__remus_disk_type *disk_type = remus_disk_types[curr_disktype_id]; + libxl__egc *egc = disk_state->remus_state->egc; + libxl__domain_suspend_state *dss = disk_state->remus_state->dss; + + if (!status) { + alloc_remus_disk(dss, &disk_state->remus_disks[curr_disk_id], + disk, disk_type); + disk_state->curr_disk_id++; + disk_state->curr_disktype_id = 0; + } else { + disk_state->curr_disktype_id++; + if (++disk_state->curr_disk_id > ARRAY_SIZE(remus_disk_types)) { + /* no disktype matches with this disk */ + libxl__remus_disk_setup_done(egc, dss, ERROR_FAIL); + return; + } + } + + rc = disk_match_once(disk_state); + if (rc) + return; + + setup_all_disks(disk_state); +} + +void libxl__remus_disk_setup(libxl__egc *egc, libxl__domain_suspend_state *dss) +{ + int rc, num_disks; + libxl__remus_disk_state *disk_state = NULL; + + STATE_AO_GC(dss->ao); + + GCNEW(disk_state); + + dss->remus_state->disk_state = disk_state; + libxl__ev_child_init(&disk_state->async_exec.child); + disk_state->async_exec.ao = dss->ao; + disk_state->remus_state = dss->remus_state; + + disk_state->disks = libxl_device_disk_list(CTX, dss->domid, &num_disks); + disk_state->num_disks = num_disks; + GCNEW_ARRAY(disk_state->remus_disks, num_disks); + + disk_state->curr_disk_id = 0; + disk_state->curr_disktype_id = 0; + rc = disk_match_once(disk_state); + if (rc) + return; + + setup_all_disks(disk_state); +} + +static void disk_teardown_once(libxl__remus_disk_state *disk_state) +{ + int i, rc = 0; + const libxl__remus_disk_type *disk_type; + + /* Convenience aliases */ + libxl__egc *egc = disk_state->remus_state->egc; + libxl__domain_suspend_state *dss = disk_state->remus_state->dss; + + i = disk_state->curr_disk_id; + for (; i < disk_state->num_disks; i++) { + if (!disk_state->remus_disks[i]) + continue; + + disk_type = disk_state->remus_disks[i]->type; + + rc = disk_type->teardown(disk_state->remus_disks[i], + &disk_state->async_exec); + if (rc) + break; + + disk_state->remus_disks[i] = NULL; + } + + if (rc < 0 || i == disk_state->num_disks) + libxl__remus_disk_teardown_done(egc, dss); + + disk_state->curr_disk_id = i; +} + +void disk_teardown_script_cb(void *opaque) +{ + libxl__remus_disk_state *disk_state = opaque; + + disk_state->remus_disks[disk_state->curr_disk_id] = NULL; + disk_state->curr_disk_id++; + disk_teardown_once(disk_state); +} + +void libxl__remus_disk_teardown(libxl__egc *egc, + libxl__domain_suspend_state *dss) +{ + /* Convenience aliases */ + libxl__remus_disk_state *disk_state = dss->remus_state->disk_state; + + if (!disk_state) { + libxl__remus_disk_teardown_done(egc, dss); + return; + } + + libxl__ev_child_init(&disk_state->async_exec.child); + disk_state->async_exec.ao = dss->ao; + + disk_state->curr_disk_id = 0; + disk_teardown_once(disk_state); +} diff --git a/tools/libxl/libxl_remus_disk.h b/tools/libxl/libxl_remus_disk.h new file mode 100644 index 0000000..33b0e59 --- /dev/null +++ b/tools/libxl/libxl_remus_disk.h @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2014 FUJITSU LIMITED + * Author Lai Jiangshan <laijs@xxxxxxxxxxxxxx> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; version 2.1 only. with the special + * exception on linking described in file LICENSE. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + */ + +#ifndef LIBXL_REMUS_DISK_H +#define LIBXL_REMUS_DISK_H + +typedef struct libxl__remus_disk { + const struct libxl_device_disk *disk; + const struct libxl__remus_disk_type *type; + void *opaque; + void *disk_state; + + /* + * the asynchronous function will fill it, we will output the + * script name when the script fails + */ + const char *disk_script; +} libxl__remus_disk; + +typedef struct libxl__remus_disk_type { + /* checkpointing */ + int (*postsuspend)(libxl__remus_disk *remus_disk); + int (*preresume)(libxl__remus_disk *remus_disk); + int (*commit)(libxl__remus_disk *remus_disk); + + /* + * Return value: + * 1: the disk is not this type or the script is still running + * 0: the disk is this type + * -1: error + */ + int (*match)(libxl__domain_suspend_state *dss, + const libxl_device_disk *disk, + libxl_async_exec *async_exec, + void *disk_state); + + /* + * This is synchronous callback. Return value: + * 0: setup is done + * -1: error + * + */ + int (*setup)(libxl__remus_disk *remus_disk); + + /* + * Return value: + * 1: the script is still running + * 0: the script is done + * -1: error + */ + int (*teardown)(libxl__remus_disk *remus_disk, + libxl_async_exec *async_exec); + + /* the size of the private data */ + int size; +} libxl__remus_disk_type; + +/* used for asynchronous API */ +extern void disk_match_script_cb(void *disk_state, int status); +extern void disk_teardown_script_cb(void *disk_state); + +#endif -- 1.7.4.4 _______________________________________________ Xen-devel mailing list Xen-devel@xxxxxxxxxxxxx http://lists.xen.org/xen-devel
|
Lists.xenproject.org is hosted with RackSpace, monitoring our |