[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[Xen-devel] [PATCH V9 05/12] remus: remus device core and APIs to setup/teardown



---
 tools/libxl/Makefile             |   2 +
 tools/libxl/libxl.c              |   7 +-
 tools/libxl/libxl_dom.c          |   4 +-
 tools/libxl/libxl_internal.h     |  28 ++++
 tools/libxl/libxl_remus.c        |  41 ++++++
 tools/libxl/libxl_remus_device.c | 311 +++++++++++++++++++++++++++++++++++++++
 tools/libxl/libxl_remus_device.h |  97 ++++++++++++
 7 files changed, 486 insertions(+), 4 deletions(-)
 create mode 100644 tools/libxl/libxl_remus.c
 create mode 100644 tools/libxl/libxl_remus_device.c
 create mode 100644 tools/libxl/libxl_remus_device.h

diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
index a29c505..8398386 100644
--- a/tools/libxl/Makefile
+++ b/tools/libxl/Makefile
@@ -52,6 +52,8 @@ else
 LIBXL_OBJS-y += libxl_nonetbuffer.o
 endif
 
+LIBXL_OBJS-y += libxl_remus.o libxl_remus_device.o
+
 LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o
 LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_arm.o
 
diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index 30b0b06..e3eca6e 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -741,7 +741,12 @@ int libxl_domain_remus_start(libxl_ctx *ctx, 
libxl_domain_remus_info *info,
 
     assert(info);
 
-    /* TBD: Remus setup - i.e. attach qdisc, enable disk buffering, etc */
+    GCNEW(dss->remus_state);
+
+    /* convenience shorthand */
+    libxl__remus_state *remus_state = dss->remus_state;
+    remus_state->dss = dss;
+    remus_state->egc = egc;
 
     /* Point of no return */
     libxl__domain_suspend(egc, dss);
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index 661999c..fc0c136 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -766,8 +766,6 @@ int libxl__toolstack_restore(uint32_t domid, const uint8_t 
*buf,
 
 /*==================== Domain suspend (save) ====================*/
 
-static void domain_suspend_done(libxl__egc *egc,
-                        libxl__domain_suspend_state *dss, int rc);
 static void domain_suspend_callback_common_done(libxl__egc *egc,
                                 libxl__domain_suspend_state *dss, int ok);
 static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
@@ -1716,7 +1714,7 @@ static void save_device_model_datacopier_done(libxl__egc 
*egc,
     dss->save_dm_callback(egc, dss, our_rc);
 }
 
-static void domain_suspend_done(libxl__egc *egc,
+void domain_suspend_done(libxl__egc *egc,
                         libxl__domain_suspend_state *dss, int rc)
 {
     STATE_AO_GC(dss->ao);
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 33b62a2..421ae24 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -2457,8 +2457,35 @@ typedef struct libxl__logdirty_switch {
     libxl__ev_time timeout;
 } libxl__logdirty_switch;
 
+typedef struct libxl__remus_state {
+    libxl__domain_suspend_state *dss;
+    libxl__egc *egc;
+
+    /* private */
+    int saved_rc;
+    /* Opaque context containing device related stuff */
+    void *device_state;
+} libxl__remus_state;
+
 _hidden int libxl__netbuffer_enabled(libxl__gc *gc);
 
+_hidden void domain_suspend_done(libxl__egc *egc,
+                                 libxl__domain_suspend_state *dss,
+                                 int rc);
+
+_hidden void libxl__remus_setup_done(libxl__egc *egc,
+                                     libxl__domain_suspend_state *dss,
+                                     int rc);
+
+_hidden void libxl__remus_device_setup(libxl__egc *egc,
+                                       libxl__domain_suspend_state *dss);
+
+_hidden void libxl__remus_teardown_done(libxl__egc *egc,
+                                        libxl__domain_suspend_state *dss);
+
+_hidden void libxl__remus_device_teardown(libxl__egc *egc,
+                                          libxl__domain_suspend_state *dss);
+
 struct libxl__domain_suspend_state {
     /* set by caller of libxl__domain_suspend */
     libxl__ao *ao;
@@ -2470,6 +2497,7 @@ struct libxl__domain_suspend_state {
     int live;
     int debug;
     const libxl_domain_remus_info *remus;
+    libxl__remus_state *remus_state;
     /* private */
     libxl__ev_evtchn guest_evtchn;
     int guest_evtchn_lockfd;
diff --git a/tools/libxl/libxl_remus.c b/tools/libxl/libxl_remus.c
new file mode 100644
index 0000000..05af451
--- /dev/null
+++ b/tools/libxl/libxl_remus.c
@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2014
+ * Author Shriram Rajagopalan <rshriram@xxxxxxxxx>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include "libxl_osdeps.h" /* must come before any other headers */
+
+#include "libxl_internal.h"
+
+/*----- remus setup/teardown code -----*/
+
+void libxl__remus_setup_done(libxl__egc *egc,
+                             libxl__domain_suspend_state *dss,
+                             int rc)
+{
+    STATE_AO_GC(dss->ao);
+    if (!rc) {
+        libxl__domain_suspend(egc, dss);
+        return;
+    }
+
+    LOG(ERROR, "Remus: failed to setup device for guest with domid %u",
+        dss->domid);
+    domain_suspend_done(egc, dss, rc);
+}
+
+void libxl__remus_teardown_done(libxl__egc *egc,
+                                libxl__domain_suspend_state *dss)
+{
+    dss->callback(egc, dss, dss->remus_state->saved_rc);
+}
diff --git a/tools/libxl/libxl_remus_device.c b/tools/libxl/libxl_remus_device.c
new file mode 100644
index 0000000..6e7d0d5
--- /dev/null
+++ b/tools/libxl/libxl_remus_device.c
@@ -0,0 +1,311 @@
+/*
+ * Copyright (C) 2014
+ * Author: Lai Jiangshan <laijs@xxxxxxxxxxxxxx>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include "libxl_osdeps.h" /* must come before any other headers */
+
+#include "libxl_internal.h"
+#include "libxl_remus_device.h"
+
+typedef struct libxl__remus_device_state {
+    /* nic */
+    libxl_device_nic *nics;
+    int num_nics;
+
+    /* disk */
+    libxl_device_disk *disks;
+    int num_disks;
+
+    int num_devices;
+    libxl__remus_device **dev;
+    libxl_async_exec async_exec;
+    libxl__remus_state *remus_state;
+    bool setup;
+    int curr_dev_id;
+    int curr_devtype_id;
+} libxl__remus_device_state;
+
+static libxl__remus_device_type *device_types[] = {
+};
+
+static void init_async_exec(libxl__remus_device_state *dev_state,
+                            void (*finish_cb)(void *opaque, int status))
+{
+    /*
+     * The callback may touch finish_cb/opaque to do something after
+     * the script is done.
+     */
+    dev_state->async_exec.ao = dev_state->remus_state->dss->ao;
+    dev_state->async_exec.finish_cb = finish_cb;
+    dev_state->async_exec.opaque = dev_state;
+}
+
+static void libxl__remus_teardown_cleanup(libxl__egc *egc,
+                                          libxl__domain_suspend_state *dss)
+{
+    int i;
+    libxl__remus_device_type *dev_type;
+
+    /* clean device_types */
+    for (i = 0; i < ARRAY_SIZE(device_types); i++) {
+        dev_type = device_types[i];
+        dev_type->destroy(dev_type);
+    }
+
+    libxl__remus_teardown_done(egc, dss);
+}
+
+static void dev_setup_teardown_script_cb(void *opaque, int status);
+static void dev_setup_teardown_once(libxl__remus_device_state *dev_state)
+{
+    int i, rc = REMUS_OK;
+    const libxl__remus_device_type *dev_type;
+
+    /* Convenience aliases */
+    libxl__egc *egc = dev_state->remus_state->egc;
+    libxl__domain_suspend_state *dss = dev_state->remus_state->dss;
+    const bool setup = dev_state->setup;
+
+    i = dev_state->curr_dev_id;
+    for (; i < dev_state->num_devices; i++) {
+        if (!setup && !dev_state->dev[i])
+            continue;
+
+        dev_type = dev_state->dev[i]->dev_type;
+        init_async_exec(dev_state, dev_setup_teardown_script_cb);
+
+        if (setup)
+            rc = dev_type->setup(dev_state->dev[i],
+                                 &dev_state->async_exec);
+        else
+            rc = dev_type->teardown(dev_state->dev[i],
+                                    &dev_state->async_exec);
+
+        /* ignore teardown error to teardown as many devices as possible */
+        if (rc == REMUS_INPROGRESS || (setup && rc == REMUS_FAIL))
+            break;
+
+        if (!setup)
+            dev_state->dev[i] = NULL;
+    }
+
+    if (rc == REMUS_FAIL || i == dev_state->num_devices) {
+        if (setup)
+            libxl__remus_setup_done(egc, dss, rc);
+        else
+            libxl__remus_teardown_cleanup(egc, dss);
+    }
+
+    dev_state->curr_dev_id = i;
+}
+
+static void dev_setup_teardown_script_cb(void *opaque, int status)
+{
+    libxl__remus_device_state *dev_state = opaque;
+
+    /* Convenience aliases */
+    libxl__egc *egc = dev_state->remus_state->egc;
+    libxl__domain_suspend_state *dss = dev_state->remus_state->dss;
+    const bool setup = dev_state->setup;
+
+    /* ignore teardown error to teardown as many devices as possible */
+    if (status == REMUS_FAIL && setup) {
+        libxl__remus_setup_done(egc, dss, ERROR_FAIL);
+        return;
+    }
+
+    if (!setup)
+        dev_state->dev[dev_state->curr_dev_id] = NULL;
+    dev_state->curr_dev_id++;
+    dev_setup_teardown_once(dev_state);
+}
+
+static void setup_all_devices(libxl__remus_device_state *dev_state)
+{
+    dev_state->curr_dev_id = 0;
+    dev_state->setup = true;
+    dev_setup_teardown_once(dev_state);
+}
+
+static void alloc_remus_dev(libxl__remus_device_state *dev_state,
+                            libxl__remus_device **remus_dev,
+                            const void *libxl_device,
+                            const libxl__remus_device_type *dev_type)
+{
+    STATE_AO_GC(dev_state->remus_state->dss->ao);
+    libxl__remus_device *new_dev;
+    int dev_id = dev_state->curr_dev_id;
+
+    if (dev_id >= dev_state->num_nics)
+        dev_id -= dev_state->num_nics;
+    new_dev = libxl__zalloc(gc, dev_type->size);
+    new_dev->dev_id = dev_id;
+    new_dev->dev_type = dev_type;
+    new_dev->libxl_device = libxl_device;
+
+    *remus_dev = new_dev;
+}
+
+static void dev_match_script_cb(void *opaque, int status);
+static int dev_match_once(libxl__remus_device_state *dev_state)
+{
+    int i, j, rc = REMUS_OK;
+    const libxl__remus_device_type *dev_type;
+    const void *libxl_device;
+    int device_type;
+
+    /* Convenience aliases */
+    libxl__egc *egc = dev_state->remus_state->egc;
+    libxl__domain_suspend_state *dss = dev_state->remus_state->dss;
+
+    i = dev_state->curr_dev_id;
+    j = dev_state->curr_devtype_id;
+    for (; i < dev_state->num_devices; i++) {
+        if (i >= dev_state->num_nics) {
+            libxl_device = &dev_state->disks[i - dev_state->num_nics];
+            device_type = REMUS_DISK;
+        } else {
+            libxl_device = &dev_state->nics[i];
+            device_type = REMUS_NIC;
+        }
+
+        for (; j < ARRAY_SIZE(device_types); j++) {
+            dev_type = device_types[j];
+            init_async_exec(dev_state, dev_match_script_cb);
+
+            rc = dev_type->match(dev_type, libxl_device, device_type,
+                                 &dev_state->async_exec);
+            if (rc == REMUS_INPROGRESS || rc == REMUS_FAIL)
+                goto out;
+
+            if (rc == REMUS_OK) {
+                alloc_remus_dev(dev_state, &dev_state->dev[i],
+                                libxl_device, dev_type);
+                break;
+            }
+        }
+
+        if (j == ARRAY_SIZE(device_types)) {
+            /* no devtype matches with this dev */
+            rc = ERROR_FAIL;
+            goto out;
+        }
+        j = 0;
+    }
+
+out:
+    if (rc == REMUS_FAIL)
+        libxl__remus_setup_done(egc, dss, rc);
+    else if (i < dev_state->num_devices) {
+        dev_state->curr_dev_id = i;
+        dev_state->curr_devtype_id = j;
+    }
+
+    return rc;
+}
+
+static void dev_match_script_cb(void *opaque, int status)
+{
+    libxl__remus_device_state *dev_state = opaque;
+    int rc;
+    const void *libxl_device;
+
+    /* Convenience aliases */
+    int curr_devtype_id = dev_state->curr_devtype_id;
+    int curr_dev_id = dev_state->curr_dev_id;
+    const libxl__remus_device_type *dev_type = device_types[curr_devtype_id];
+    libxl__egc *egc = dev_state->remus_state->egc;
+    libxl__domain_suspend_state *dss = dev_state->remus_state->dss;
+
+    if (curr_dev_id >= dev_state->num_nics)
+        libxl_device = &dev_state->disks[curr_dev_id - dev_state->num_nics];
+    else
+        libxl_device = &dev_state->nics[curr_dev_id];
+
+    if (status == REMUS_FAIL) {
+        libxl__remus_setup_done(egc, dss, ERROR_FAIL);
+        return;
+    } else if (status == REMUS_OK) {
+        alloc_remus_dev(dev_state, &dev_state->dev[curr_dev_id],
+                        libxl_device, dev_type);
+        dev_state->curr_dev_id++;
+        dev_state->curr_devtype_id = 0;
+    } else {
+        if (++dev_state->curr_devtype_id >= ARRAY_SIZE(device_types)) {
+            /* no devtype matches with this dev */
+            libxl__remus_setup_done(egc, dss, ERROR_FAIL);
+            return;
+        }
+    }
+
+    rc = dev_match_once(dev_state);
+    if (rc)
+        return;
+
+    setup_all_devices(dev_state);
+}
+
+void libxl__remus_device_setup(libxl__egc *egc,
+                               libxl__domain_suspend_state *dss)
+{
+    int i, rc;
+    libxl__remus_device_state *dev_state = NULL;
+    libxl__remus_device_type *dev_type;
+
+    STATE_AO_GC(dss->ao);
+
+    GCNEW(dev_state);
+
+    dss->remus_state->device_state = dev_state;
+    libxl__ev_child_init(&dev_state->async_exec.child);
+    dev_state->remus_state = dss->remus_state;
+
+    for (i = 0; i < ARRAY_SIZE(device_types); i++) {
+        dev_type = device_types[i];
+        if (dev_type->init(dev_type, dss->remus_state)) {
+            libxl__remus_setup_done(egc, dss, ERROR_FAIL);
+            return;
+        }
+    }
+
+    /* TBD: Remus setup - i.e. attach qdisc, enable disk buffering, etc */
+
+    GCNEW_ARRAY(dev_state->dev, dev_state->num_devices);
+
+    dev_state->curr_dev_id = 0;
+    dev_state->curr_devtype_id = 0;
+    rc = dev_match_once(dev_state);
+    if (rc)
+        return;
+
+    setup_all_devices(dev_state);
+}
+
+void libxl__remus_device_teardown(libxl__egc *egc,
+                                  libxl__domain_suspend_state *dss)
+{
+    /* Convenience aliases */
+    libxl__remus_device_state *dev_state = dss->remus_state->device_state;
+
+    if (!dev_state) {
+        libxl__remus_teardown_done(egc, dss);
+        return;
+    }
+
+    libxl__ev_child_init(&dev_state->async_exec.child);
+
+    dev_state->curr_dev_id = 0;
+    dev_state->setup = false;
+    dev_setup_teardown_once(dev_state);
+}
diff --git a/tools/libxl/libxl_remus_device.h b/tools/libxl/libxl_remus_device.h
new file mode 100644
index 0000000..d8d16ff
--- /dev/null
+++ b/tools/libxl/libxl_remus_device.h
@@ -0,0 +1,97 @@
+/*
+ * Copyright (C) 2014
+ * Author: Lai Jiangshan <laijs@xxxxxxxxxxxxxx>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#ifndef LIBXL_REMUS_DEVICE_H
+#define LIBXL_REMUS_DEVICE_H
+
+typedef struct libxl__remus_device libxl__remus_device;
+typedef struct libxl__remus_device_type libxl__remus_device_type;
+
+enum {
+    REMUS_NIC,
+    REMUS_DISK,
+};
+
+/* Return value of the callback */
+enum {
+    REMUS_FAIL = ERROR_FAIL,    /* -3 */
+    REMUS_OK = 0,
+    REMUS_INPROGRESS,
+    REMUS_NOT_SUPPORT,
+};
+
+struct libxl__remus_device_type {
+    int (*init)(libxl__remus_device_type *self,
+                libxl__remus_state *remus_state);
+    void (*destroy)(libxl__remus_device_type *self);
+    void *data;
+
+    /*
+     * checkpointing callbacks, don't execute any script in it.
+     */
+    int (*postsuspend)(libxl__remus_device *dev);
+    int (*preresume)(libxl__remus_device *dev);
+    int (*commit)(libxl__remus_device *dev);
+
+    /*
+     * libxl_device:
+     *   REMUS_NIC: libxl_device_nic
+     *  REMUS_DISK: libxl_device_disk
+     *
+     * Return value:
+     *   2: the device is not this type
+     *   1: the script is still running
+     *   0: the device is this type
+     *  -3: error
+     *
+     * If the callback execute a script, pass the return value via finish_cb.
+     */
+    int (*match)(const libxl__remus_device_type *self,
+                 const void *libxl_device, int devcie_type,
+                 libxl_async_exec *async_exec);
+
+    /*
+     * Return value:
+     *   1: the script is still running
+     *   0: no script is executed
+     *  -3: error
+     *
+     * If the callback execute a script, pass the return value via finish_cb.
+     */
+    int (*setup)(libxl__remus_device *remus_dev,
+                 libxl_async_exec *async_exec);
+
+    /*
+     * Return value:
+     *   1: the script is still running
+     *   0: no script is executed
+     *  -3: error
+     *
+     * If the callback execute a script, pass the return value via finish_cb.
+     */
+    int (*teardown)(libxl__remus_device *remus_dev,
+                    libxl_async_exec *async_exec);
+
+    /* the size of libxl__remus_device */
+    int size;
+};
+
+struct libxl__remus_device {
+    int dev_id;
+    const void *libxl_device;
+    const libxl__remus_device_type *dev_type;
+};
+
+#endif
-- 
1.8.3.2


_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
http://lists.xen.org/xen-devel


 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.