[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[Xen-devel] [PATCH 6/7] remus: implement remus replicated checkpointing disk



Signed-off-by: Lai Jiangshan <laijs@xxxxxxxxxxxxxx>
Signed-off-by: Wen Congyang <wency@xxxxxxxxxxxxxx>
---
 tools/libxl/Makefile           |    1 +
 tools/libxl/libxl_dom.c        |   19 +++-
 tools/libxl/libxl_internal.h   |   16 +++
 tools/libxl/libxl_remus.c      |   18 +++
 tools/libxl/libxl_remus_disk.c |  285 ++++++++++++++++++++++++++++++++++++++++
 tools/libxl/libxl_remus_disk.h |   74 +++++++++++
 6 files changed, 411 insertions(+), 2 deletions(-)
 create mode 100644 tools/libxl/libxl_remus_disk.c
 create mode 100644 tools/libxl/libxl_remus_disk.h

diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
index 670a2bc..b040a79 100644
--- a/tools/libxl/Makefile
+++ b/tools/libxl/Makefile
@@ -53,6 +53,7 @@ LIBXL_OBJS-y += libxl_nonetbuffer.o
 endif
 
 LIBXL_OBJS-y += libxl_remus.o
+LIBXL_OBJS-y += libxl_remus_disk.o
 
 LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o
 LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_arm.o
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index 1272bf6..ebaece2 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -1437,9 +1437,14 @@ static void 
remus_domain_suspend_callback_common_done(libxl__egc *egc,
 
     STATE_AO_GC(dss->ao);
 
-    /* REMUS TODO: Issue disk checkpoint reqs. */
     if (!remus_state->netbuf_state || !ok) goto out;
 
+    /* Issue disk checkpoint reqs. */
+    if (libxl__remus_disk_postsuspend(remus_state)) {
+        ok = 0;
+        goto out;
+    }
+
     /* The domain was suspended successfully. Start a new network
      * buffer for the next epoch. If this operation fails, then act
      * as though domain suspend failed -- libxc exits its infinite
@@ -1463,7 +1468,10 @@ static int libxl__remus_domain_resume_callback(void 
*data)
     if (libxl__domain_resume(gc, dss->domid, /* Fast Suspend */1))
         return 0;
 
-    /* REMUS TODO: Deal with disk. */
+    /* Deal with disk. */
+    if (libxl__remus_disk_preresume(dss->remus_state))
+        return 0;
+
     return 1;
 }
 
@@ -1506,6 +1514,13 @@ static void remus_checkpoint_dm_saved(libxl__egc *egc,
         goto out;
     }
 
+    rc = libxl__remus_disk_commit(remus_state);
+    if (rc) {
+        LOG(ERROR, "Failed to commit disk state"
+            " Terminating Remus..");
+        goto out;
+    }
+
     if (remus_state->netbuf_state) {
         rc = libxl__remus_netbuf_release_prev_epoch(gc, dss->domid,
                                                     remus_state);
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 4e07969..7c30d9a 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -2446,6 +2446,9 @@ typedef struct libxl__remus_state {
     void *netbuf_state;
     /* used for checkpoint */
     libxl__ev_time timeout;
+
+    /* Opaque context containing disk related stuff */
+    void *disk_state;
 } libxl__remus_state;
 
 _hidden int libxl__netbuffer_enabled(libxl__gc *gc);
@@ -2474,6 +2477,19 @@ _hidden int
 libxl__remus_netbuf_release_prev_epoch(libxl__gc *gc, uint32_t domid,
                                        libxl__remus_state *remus_state);
 
+_hidden int libxl__remus_disk_postsuspend(libxl__remus_state *state);
+_hidden int libxl__remus_disk_preresume(libxl__remus_state *state);
+_hidden int libxl__remus_disk_commit(libxl__remus_state *state);
+_hidden void libxl__remus_disk_setup(libxl__egc *egc,
+                                     libxl__domain_suspend_state *dss);
+_hidden void libxl__remus_disk_setup_done(libxl__egc *egc,
+                                          libxl__domain_suspend_state *dss,
+                                          int rc);
+_hidden void libxl__remus_disk_teardown(libxl__egc *egc,
+                                        libxl__domain_suspend_state *dss);
+_hidden void libxl__remus_disk_teardown_done(libxl__egc *egc,
+                                             libxl__domain_suspend_state *dss);
+
 _hidden void libxl__remus_setup_initiate(libxl__egc *egc,
                                          libxl__domain_suspend_state *dss);
 
diff --git a/tools/libxl/libxl_remus.c b/tools/libxl/libxl_remus.c
index 78652d2..9eab98d 100644
--- a/tools/libxl/libxl_remus.c
+++ b/tools/libxl/libxl_remus.c
@@ -47,6 +47,18 @@ void libxl__remus_netbuf_setup_done(libxl__egc *egc,
                                     libxl__domain_suspend_state *dss,
                                     int rc)
 {
+    if (rc) {
+        libxl__remus_setup_done(egc, dss, rc);
+        return;
+    }
+
+    libxl__remus_disk_setup(egc, dss);
+}
+
+void libxl__remus_disk_setup_done(libxl__egc *egc,
+                                   libxl__domain_suspend_state *dss,
+                                   int rc)
+{
     libxl__remus_setup_done(egc, dss, rc);
 }
 
@@ -72,5 +84,11 @@ static void libxl__remus_teardown_done(libxl__egc *egc,
 void libxl__remus_netbuf_teardown_done(libxl__egc *egc,
                                        libxl__domain_suspend_state *dss)
 {
+    libxl__remus_disk_teardown(egc, dss);
+}
+
+void libxl__remus_disk_teardown_done(libxl__egc *egc,
+                                     libxl__domain_suspend_state *dss)
+{
     libxl__remus_teardown_done(egc, dss);
 }
diff --git a/tools/libxl/libxl_remus_disk.c b/tools/libxl/libxl_remus_disk.c
new file mode 100644
index 0000000..ca3e879
--- /dev/null
+++ b/tools/libxl/libxl_remus_disk.c
@@ -0,0 +1,285 @@
+/*
+ * Copyright (C) 2014 FUJITSU LIMITED
+ * Author Lai Jiangshan <laijs@xxxxxxxxxxxxxx>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include "libxl_osdeps.h" /* must come before any other headers */
+
+#include "libxl_internal.h"
+
+#include "libxl_remus_disk.h"
+
+typedef struct libxl__remus_disk_state {
+    libxl__remus_state *remus_state;
+    libxl_device_disk *disks;
+    struct libxl__remus_disk **remus_disks;
+    uint32_t num_disks;
+    uint32_t curr_disk_id;
+    uint32_t curr_disktype_id;
+
+    libxl_async_exec async_exec;
+} libxl__remus_disk_state;
+
+/*** checkpoint disks states and callbacks ***/
+static const libxl__remus_disk_type *remus_disk_types[] =
+{
+};
+
+int libxl__remus_disk_postsuspend(libxl__remus_state *remus_state)
+{
+    int i;
+    int rc = 0;
+    libxl__remus_disk *remus_disk;
+
+    /* Convenience aliases */
+    libxl__remus_disk_state *disk_state = remus_state->disk_state;
+
+    for (i = 0; rc == 0 && i < disk_state->num_disks; i++) {
+        remus_disk = disk_state->remus_disks[i];
+        rc = remus_disk->type->postsuspend(remus_disk);
+    }
+
+    return rc;
+}
+
+int libxl__remus_disk_preresume(libxl__remus_state *remus_state)
+{
+    int i;
+    int rc = 0;
+    libxl__remus_disk *remus_disk;
+
+    /* Convenience aliases */
+    libxl__remus_disk_state *disk_state = remus_state->disk_state;
+
+    for (i = 0; rc == 0 && i < disk_state->num_disks; i++) {
+        remus_disk = disk_state->remus_disks[i];
+        rc = remus_disk->type->preresume(remus_disk);
+    }
+
+    return rc;
+}
+
+int libxl__remus_disk_commit(libxl__remus_state *remus_state)
+{
+    int i;
+    int rc = 0;
+    libxl__remus_disk *remus_disk;
+
+    /* Convenience aliases */
+    libxl__remus_disk_state *disk_state = remus_state->disk_state;
+
+    for (i = 0; rc == 0 && i < disk_state->num_disks; i++) {
+        remus_disk = disk_state->remus_disks[i];
+        rc = remus_disk->type->commit(remus_disk);
+    }
+
+    return rc;
+}
+
+static void alloc_remus_disk(libxl__domain_suspend_state *dss,
+                             libxl__remus_disk **remus_disk,
+                             const libxl_device_disk *disk,
+                             const libxl__remus_disk_type *disk_type)
+{
+    STATE_AO_GC(dss->ao);
+    libxl__remus_disk *new_disk;
+
+    new_disk = libxl__zalloc(gc, sizeof(libxl__remus_disk) + disk_type->size);
+    new_disk->type = disk_type;
+    new_disk->disk = disk;
+    new_disk->disk_state = dss->remus_state->disk_state;
+    new_disk->opaque = &new_disk[1];
+
+    *remus_disk = new_disk;
+}
+
+static void setup_all_disks(libxl__remus_disk_state *disk_state)
+{
+    int i, rc = 0;
+    libxl__remus_disk *remus_disk;
+
+    /* Convenience aliases */
+    libxl__egc *egc = disk_state->remus_state->egc;
+    libxl__domain_suspend_state *dss = disk_state->remus_state->dss;
+
+    for (i = 0; i < disk_state->num_disks; i++) {
+        remus_disk = disk_state->remus_disks[i];
+        rc = remus_disk->type->setup(remus_disk);
+        if (rc) {
+            libxl__remus_disk_setup_done(egc, dss, rc);
+            return;
+        }
+    }
+
+    libxl__remus_disk_setup_done(egc, dss, 0);
+}
+
+static int disk_match_once(libxl__remus_disk_state *disk_state)
+{
+    int i, j, rc = 0;
+    const libxl__remus_disk_type *disk_type;
+    const libxl_device_disk *disk;
+
+    /* Convenience aliases */
+    libxl__egc *egc = disk_state->remus_state->egc;
+    libxl__domain_suspend_state *dss = disk_state->remus_state->dss;
+
+    i = disk_state->curr_disk_id;
+    j = disk_state->curr_disktype_id;
+    for (; i < disk_state->num_disks; i++) {
+        disk = &disk_state->disks[i];
+        for (; j < ARRAY_SIZE(remus_disk_types); j++) {
+            disk_type = remus_disk_types[j];
+
+            rc = disk_type->match(dss, disk, &disk_state->async_exec,
+                                  disk_state);
+            if (rc)
+                goto out;
+
+            if (rc == 0) {
+                alloc_remus_disk(dss, &disk_state->remus_disks[i],
+                                 disk, disk_type);
+                break;
+            }
+        }
+        j = 0;
+    }
+
+out:
+    if (rc < 0)
+        libxl__remus_disk_setup_done(egc, dss, rc);
+    else if (i < disk_state->num_disks) {
+        disk_state->curr_disk_id = i;
+        disk_state->curr_disktype_id = j;
+    }
+
+    return rc;
+}
+
+void disk_match_script_cb(void *opaque, int status)
+{
+    libxl__remus_disk_state *disk_state = opaque;
+    int rc;
+
+    /* Convenience aliases */
+    int curr_disktype_id = disk_state->curr_disktype_id;
+    int curr_disk_id = disk_state->curr_disk_id;
+    libxl_device_disk *disk = &disk_state->disks[disk_state->curr_disk_id];
+    const libxl__remus_disk_type *disk_type = 
remus_disk_types[curr_disktype_id];
+    libxl__egc *egc = disk_state->remus_state->egc;
+    libxl__domain_suspend_state *dss = disk_state->remus_state->dss;
+
+    if (!status) {
+        alloc_remus_disk(dss, &disk_state->remus_disks[curr_disk_id],
+                         disk, disk_type);
+        disk_state->curr_disk_id++;
+        disk_state->curr_disktype_id = 0;
+    } else {
+        disk_state->curr_disktype_id++;
+        if (++disk_state->curr_disk_id > ARRAY_SIZE(remus_disk_types)) {
+            /* no disktype matches with this disk */
+            libxl__remus_disk_setup_done(egc, dss, ERROR_FAIL);
+            return;
+        }
+    }
+
+    rc = disk_match_once(disk_state);
+    if (rc)
+        return;
+
+    setup_all_disks(disk_state);
+}
+
+void libxl__remus_disk_setup(libxl__egc *egc, libxl__domain_suspend_state *dss)
+{
+    int rc, num_disks;
+    libxl__remus_disk_state *disk_state = NULL;
+
+    STATE_AO_GC(dss->ao);
+
+    GCNEW(disk_state);
+
+    dss->remus_state->disk_state = disk_state;
+    libxl__ev_child_init(&disk_state->async_exec.child);
+    disk_state->async_exec.ao = dss->ao;
+    disk_state->remus_state = dss->remus_state;
+
+    disk_state->disks = libxl_device_disk_list(CTX, dss->domid, &num_disks);
+    disk_state->num_disks = num_disks;
+    GCNEW_ARRAY(disk_state->remus_disks, num_disks);
+
+    disk_state->curr_disk_id = 0;
+    disk_state->curr_disktype_id = 0;
+    rc = disk_match_once(disk_state);
+    if (rc)
+        return;
+
+    setup_all_disks(disk_state);
+}
+
+static void disk_teardown_once(libxl__remus_disk_state *disk_state)
+{
+    int i, rc = 0;
+    const libxl__remus_disk_type *disk_type;
+
+    /* Convenience aliases */
+    libxl__egc *egc = disk_state->remus_state->egc;
+    libxl__domain_suspend_state *dss = disk_state->remus_state->dss;
+
+    i = disk_state->curr_disk_id;
+    for (; i < disk_state->num_disks; i++) {
+        if (!disk_state->remus_disks[i])
+            continue;
+
+        disk_type = disk_state->remus_disks[i]->type;
+
+        rc = disk_type->teardown(disk_state->remus_disks[i],
+                                 &disk_state->async_exec);
+        if (rc)
+            break;
+
+        disk_state->remus_disks[i] = NULL;
+    }
+
+    if (rc < 0 || i == disk_state->num_disks)
+        libxl__remus_disk_teardown_done(egc, dss);
+
+    disk_state->curr_disk_id = i;
+}
+
+void disk_teardown_script_cb(void *opaque)
+{
+    libxl__remus_disk_state *disk_state = opaque;
+
+    disk_state->remus_disks[disk_state->curr_disk_id] = NULL;
+    disk_state->curr_disk_id++;
+    disk_teardown_once(disk_state);
+}
+
+void libxl__remus_disk_teardown(libxl__egc *egc,
+                                libxl__domain_suspend_state *dss)
+{
+    /* Convenience aliases */
+    libxl__remus_disk_state *disk_state = dss->remus_state->disk_state;
+
+    if (!disk_state) {
+        libxl__remus_disk_teardown_done(egc, dss);
+        return;
+    }
+
+    libxl__ev_child_init(&disk_state->async_exec.child);
+    disk_state->async_exec.ao = dss->ao;
+
+    disk_state->curr_disk_id = 0;
+    disk_teardown_once(disk_state);
+}
diff --git a/tools/libxl/libxl_remus_disk.h b/tools/libxl/libxl_remus_disk.h
new file mode 100644
index 0000000..33b0e59
--- /dev/null
+++ b/tools/libxl/libxl_remus_disk.h
@@ -0,0 +1,74 @@
+/*
+ * Copyright (C) 2014 FUJITSU LIMITED
+ * Author Lai Jiangshan <laijs@xxxxxxxxxxxxxx>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#ifndef LIBXL_REMUS_DISK_H
+#define LIBXL_REMUS_DISK_H
+
+typedef struct libxl__remus_disk {
+    const struct libxl_device_disk *disk;
+    const struct libxl__remus_disk_type *type;
+    void *opaque;
+    void *disk_state;
+
+    /*
+     * the asynchronous function will fill it, we will output the
+     * script name when the script fails
+     */
+    const char *disk_script;
+} libxl__remus_disk;
+
+typedef struct libxl__remus_disk_type {
+    /* checkpointing */
+    int (*postsuspend)(libxl__remus_disk *remus_disk);
+    int (*preresume)(libxl__remus_disk *remus_disk);
+    int (*commit)(libxl__remus_disk *remus_disk);
+
+    /*
+     * Return value:
+     *   1: the disk is not this type or the script is still running
+     *   0: the disk is this type
+     *  -1: error
+     */
+    int (*match)(libxl__domain_suspend_state *dss,
+                 const libxl_device_disk *disk,
+                 libxl_async_exec *async_exec,
+                 void *disk_state);
+
+    /*
+     * This is synchronous callback. Return value:
+     *  0: setup is done
+     * -1: error
+     *
+     */
+    int (*setup)(libxl__remus_disk *remus_disk);
+
+    /*
+     * Return value:
+     *   1: the script is still running
+     *   0: the script is done
+     *  -1: error
+     */
+    int (*teardown)(libxl__remus_disk *remus_disk,
+                    libxl_async_exec *async_exec);
+
+    /* the size of the private data */
+    int size;
+} libxl__remus_disk_type;
+
+/* used for asynchronous API */
+extern void disk_match_script_cb(void *disk_state, int status);
+extern void disk_teardown_script_cb(void *disk_state);
+
+#endif
-- 
1.7.4.4


_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
http://lists.xen.org/xen-devel


 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.