[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[Xen-devel] [PATCH 11/10] libxl: Asynchronous/long-running operation infrastructure



Provide a new set of machinery for writing public libxl functions
which may take a long time.  The application gets to decide whether
they want the function to be synchronous, or whether they'd prefer to
get a callback, or an event, when the operation is complete.

User(s) of this machinery will be introduced in later patch(es).

Signed-off-by: Ian Jackson <ian.jackson@xxxxxxxxxxxxx>
---
 tools/libxl/libxl.h          |   50 ++++++++++++
 tools/libxl/libxl_event.c    |  183 ++++++++++++++++++++++++++++++++++++++++++
 tools/libxl/libxl_internal.h |  105 ++++++++++++++++++++++++
 tools/libxl/libxl_types.idl  |    4 +
 4 files changed, 342 insertions(+), 0 deletions(-)

diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 5396ba8..e8fed73 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -235,8 +235,58 @@ enum {
     ERROR_NOT_READY = -11,
     ERROR_OSEVENT_REG_FAIL = -12,
     ERROR_BUFFERFULL = -13,
+    ERROR_ASYNC_INPROGRESS = -14,
 };
 
+
+/*
+ * Some libxl operations can take a long time.  These functions take a
+ * parameter to control their concurrency:
+ *     libxl_asyncop_how *ao_how
+ *
+ * If ao_how==NULL, the function will be synchronous.
+ *
+ * If ao_how!=NULL, the function will set the operation going, and
+ * if this is successful will return ERROR_ASYNCH_INPROGRESS.
+ *
+ * If ao_how->callback!=NULL, the callback will be called when the
+ * operation completes.  The same rules as for libxl_event_hooks
+ * apply, including the reentrancy rules and the possibility of
+ * "disaster", except that libxl calls ao_how->callback instead of
+ * libxl_event_hooks.event_occurs.
+ *
+ * If ao_how->callback==NULL, a libxl_event will be generated which
+ * can be obtained from libxl_event_wait or libxl_event_check.  The
+ * event will have type OPERATION_COMPLETE (which is not used
+ * elsewhere).
+ *
+ * Note that it is possible for an asynchronous operation which is to
+ * result in a callback to complete during its initiating function
+ * call.  In this case the initating function will return
+ * ERROR_ASYNCH_INPROGRESS, even though by the time it returns the
+ * operation is complete and the callback has already happened.
+ *
+ * The application must set and use ao_how->for_event (which will be
+ * copied into libxl_event.for_user) or ao_how->for_callback (passed
+ * to the callback) to determine which operation finished, and it must
+ * of course check the rc value for errors.
+ *
+ * *ao_how does not need to remain valid after the initiating function
+ * returns.
+ *
+ * Callbacks may occur on any thread in which the application calls
+ * libxl.
+ */
+
+typedef struct {
+    void (*callback)(libxl_ctx *ctx, int rc, void *for_callback);
+    union {
+        libxl_ev_user for_event; /* used if callback==NULL */
+        void *for_callback; /* passed to callback */
+    } u;
+} libxl_asyncop_how;
+
+
 #define LIBXL_VERSION 0
 
 typedef struct {
diff --git a/tools/libxl/libxl_event.c b/tools/libxl/libxl_event.c
index 84aa489..c594494 100644
--- a/tools/libxl/libxl_event.c
+++ b/tools/libxl/libxl_event.c
@@ -770,10 +770,21 @@ static void egc_run_callbacks(libxl__egc *egc)
 {
     libxl__gc *gc = &egc->gc;
     libxl_event *ev, *ev_tmp;
+
     LIBXL_TAILQ_FOREACH_SAFE(ev, &egc->occurred_for_callback, link, ev_tmp) {
         LIBXL_TAILQ_REMOVE(&egc->occurred_for_callback, ev, link);
         CTX->event_hooks->event_occurs(CTX->event_hooks_user, ev);
     }
+
+    libxl__ao *ao, *ao_tmp;
+    LIBXL_TAILQ_FOREACH_SAFE(ao, &egc->aos_for_callback,
+                             entry_for_callback, ao_tmp) {
+        LIBXL_TAILQ_REMOVE(&egc->aos_for_callback, ao, entry_for_callback);
+        ao->how.callback(CTX, ao->rc, ao->how.u.for_callback);
+        ao->notified = 1;
+        if (!ao->in_initiator)
+            libxl__ao__destroy(libxl__gc_owner(&egc->gc), ao);
+    }
 }
 
 void libxl__egc_cleanup(libxl__egc *egc)
@@ -1060,6 +1071,178 @@ int libxl_event_wait(libxl_ctx *ctx, libxl_event 
**event_r,
     return rc;
 }
 
+
+
+/*
+ * The two possible state flow of an ao:
+ *
+ * Completion before initiator return:
+ *
+ *     Initiator thread                       Possible other threads
+ *
+ *   * ao_create allocates memory and
+ *     initialises the struct
+ *
+ *   * the initiator function does its
+ *     work, setting up various internal
+ *     asynchronous operations -----------> * asynchronous operations
+ *                                            start to take place and
+ *                                            might cause ao completion
+ *                                                |
+ *   * initiator calls ao_complete:               |
+ *     - if synchronous, run event loop           |
+ *       until the ao completes                   |
+ *                              - ao completes on some thread
+ *                              - completing thread releases the lock
+ *                     <--------------'
+ *     - ao_complete takes the lock
+ *     - destroy the ao
+ *
+ *
+ * Completion after initiator return (asynch. only):
+ *
+ *
+ *     Initiator thread                       Possible other threads
+ *
+ *   * ao_create allocates memory and
+ *     initialises the struct
+ *
+ *   * the initiator function does its
+ *     work, setting up various internal
+ *     asynchronous operations -----------> * asynchronous operations
+ *                                            start to take place and
+ *                                            might cause ao completion
+ *                                                |
+ *   * initiator calls ao_complete:               |
+ *     - observes event not net done,             |
+ *     - returns to caller                        |
+ *                                                |
+ *                              - ao completes on some thread
+ *                              - generate the event or call the callback
+ *                              - destroy the ao
+ */
+
+void libxl__ao__destroy(libxl_ctx *ctx, libxl__ao *ao) {
+    if (!ao) return;
+    if (ao->poller) libxl__poller_put(ctx, ao->poller);
+    ao->magic = LIBXL__AO_MAGIC_DESTROYED;
+    libxl__gc_cleanup(&ao->gc);
+    free(ao);
+}
+
+void libxl__ao_abort(libxl__ao *ao) {
+    AO_GC;
+    assert(ao->magic == LIBXL__AO_MAGIC);
+    assert(ao->in_initiator);
+    assert(!ao->complete);
+    libxl__ao__destroy(CTX, ao);
+}
+
+void libxl__ao_complete(libxl__egc *egc, libxl__ao *ao, int rc) {
+    assert(ao->magic == LIBXL__AO_MAGIC);
+    assert(!ao->complete);
+    ao->complete = 1;
+    ao->rc = rc;
+
+    if (ao->poller) {
+        assert(ao->in_initiator);
+        libxl__poller_wakeup(egc, ao->poller);
+    } else if (ao->how.callback) {
+        LIBXL_TAILQ_INSERT_TAIL(&egc->aos_for_callback, ao, 
entry_for_callback);
+    } else {
+        libxl_event *ev;
+        ev = NEW_EVENT(egc, OPERATION_COMPLETE, ao->domid);
+        if (ev) {
+            ev->for_user = ao->how.u.for_event;
+            ev->u.operation_complete.rc = ao->rc;
+            libxl__event_occurred(egc, ev);
+        }
+        ao->notified = 1;
+    }
+    if (!ao->in_initiator && ao->notified)
+        libxl__ao__destroy(libxl__gc_owner(&egc->gc), ao);
+}
+
+libxl__ao *libxl__ao_create(libxl_ctx *ctx, uint32_t domid,
+                            const libxl_asyncop_how *how) {
+    libxl__ao *ao;
+
+    ao = calloc(sizeof(*ao),1);
+    if (!ao) goto out;
+
+    ao->magic = LIBXL__AO_MAGIC;
+    ao->in_initiator = 1;
+    ao->poller = 0;
+    ao->domid = domid;
+    LIBXL_INIT_GC(ao->gc, ctx);
+
+    if (how) {
+        ao->how = *how;
+    } else {
+        ao->poller = libxl__poller_get(ctx);
+        if (!ao->poller) goto out;
+    }
+    return ao;
+
+ out:
+    if (ao) libxl__ao__destroy(ctx, ao);
+    return NULL;
+}
+
+int libxl__ao_inprogress(libxl__ao *ao) {
+    AO_GC;
+    int rc;
+
+    /* We use a fresh gc, so that we can free things
+     * each time round the loop. */
+    assert(ao->magic == LIBXL__AO_MAGIC);
+    assert(ao->in_initiator);
+
+    if (ao->poller) {
+        /* Caller wants it done synchronously. */
+        libxl__egc egc;
+        LIBXL_INIT_EGC(egc,CTX);
+
+        for (;;) {
+            assert(ao->magic == LIBXL__AO_MAGIC);
+
+            if (ao->complete) {
+                rc = ao->rc;
+                ao->notified = 1;
+                break;
+            }
+
+            rc = eventloop_iteration(&egc,ao->poller);
+            if (rc) {
+                /* Oh dear, this is quite unfortunate. */
+                LIBXL__LOG(CTX, LIBXL__LOG_ERROR, "Error waiting for"
+                           " event during long-running operation (rc=%d)", rc);
+                sleep(1);
+                /* It's either this or return ERROR_I_DONT_KNOW_WHETHER
+                 * _THE_THING_YOU_ASKED_FOR_WILL_BE_DONE_LATER_WHEN
+                 * _YOU_DIDNT_EXPECT_IT, since we don't have any kind of
+                 * cancellation ability. */
+            }
+
+            CTX_UNLOCK;
+            libxl__egc_cleanup(&egc);
+            CTX_LOCK;
+        }
+    } else {
+        rc = ERROR_ASYNC_INPROGRESS;
+    }
+
+    ao->in_initiator = 0;
+
+    if (ao->notified) {
+        assert(ao->complete);
+        libxl__ao__destroy(CTX,ao);
+    }
+
+    return rc;
+}
+
+
 /*
  * Local variables:
  * mode: C
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 1fc3487..e0ff15c 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -112,6 +112,7 @@ _hidden void libxl__log(libxl_ctx *ctx, xentoollog_level 
msglevel, int errnoval,
 
 typedef struct libxl__gc libxl__gc;
 typedef struct libxl__egc libxl__egc;
+typedef struct libxl__ao libxl__ao;
 
 typedef struct libxl__ev_fd libxl__ev_fd;
 typedef void libxl__ev_fd_callback(libxl__egc *egc, libxl__ev_fd *ev,
@@ -216,6 +217,10 @@ struct libxl__poller {
      * releasing the ctx lock and going into poll; when it comes out
      * of poll it will take the poller off the pollers_event list.
      *
+     * A thread which is waiting for completion of a synchronous ao
+     * will allocate a poller and record it in the ao, so that other
+     * threads can wake it up.
+     *
      * When a thread is done with a poller it should put it onto
      * pollers_idle, where it can be reused later.
      *
@@ -322,6 +327,21 @@ struct libxl__egc {
     /* for event-generating functions only */
     struct libxl__gc gc;
     struct libxl__event_list occurred_for_callback;
+    LIBXL_TAILQ_HEAD(, libxl__ao) aos_for_callback;
+};
+
+#define LIBXL__AO_MAGIC              0xA0FACE00ul
+#define LIBXL__AO_MAGIC_DESTROYED    0xA0DEAD00ul
+
+struct libxl__ao {
+    uint32_t magic;
+    unsigned in_initiator:1, complete:1, notified:1;
+    int rc;
+    libxl__gc gc;
+    libxl_asyncop_how how;
+    libxl__poller *poller;
+    uint32_t domid;
+    LIBXL_TAILQ_ENTRY(libxl__ao) entry_for_callback;
 };
 
 #define LIBXL_INIT_GC(gc,ctx) do{               \
@@ -1116,6 +1136,91 @@ _hidden void libxl__egc_cleanup(libxl__egc *egc);
 
 
 /*
+ * Machinery for asynchronous operations ("ao")
+ *
+ * All "slow" functions (includes anything that might block on a
+ * guest or an external script) need to use the asynchronous
+ * operation ("ao") machinery.  The function should take a parameter
+ * const libxl_asyncop_how *ao_how and must start with a call to
+ * AO_INITIATOR_ENTRY.  These functions MAY NOT be called from
+ * outside libxl, because they can cause reentrancy callbacks.
+ *
+ * No functions called internally within libxl should ever return
+ * ERROR_ASYNCH_INPROGRESS.
+ *
+ * Lifecycle of an ao:
+ *
+ * - Created by libxl__ao_create (or the AO_CREATE convenience macro).
+ *
+ * - After creation, can be used by code which implements
+ *   the operation as follows:
+ *      - the ao's gc, for allocating memory for the lifetime
+ *        of the operation (possibly with the help of the AO_GC
+ *        macro to introduce the gc into scope)
+ *      - the ao itself may be passed about to sub-functions
+ *        so that they can stash it away etc.
+ *      - in particular, the ao pointer must be stashed in some
+ *        per-operation structure which is also passed as a user
+ *        pointer to the internal event generation request routines
+ *        libxl__evgen_FOO, so that at some point a CALLBACK will be
+ *        made when the operation is complete.
+ *
+ * - If initiation is successful, the initiating function needs
+ *   to run libxl__ao_inprogress right before unlocking and
+ *   returning, and return whatever it returns (AO_INPROGRESS macro).
+ *
+ * - If the initiation is unsuccessful, the initiating function must
+ *   call libxl__ao_abort before unlocking and returning whatever
+ *   error code is appropriate (AO_ABORT macro).
+ *
+ * - Later, some callback function, whose callback has been requested
+ *   directly or indirectly, should call libxl__ao_complete (with the
+ *   ctx locked, as it will generally already be in any event callback
+ *   function).  This must happen exactly once for each ao (and not if
+ *   the ao has been destroyed, obviously), and it may not happen
+ *   until libxl__ao_inprogress has been called on the ao.
+ *
+ * - NOTE RE MULTIPLE GCS
+ */
+
+#define AO_CREATE(ctx, domid, ao_how)                           \
+    libxl__ao *ao = libxl__ao_create(ctx, domid, ao_how);       \
+    if (!ao) return ERROR_NOMEM;                                \
+    AO_GC;                                                      \
+    CTX_LOCK;
+
+#define AO_INPROGRESS do{                                       \
+        int ao_inprogress_rc = libxl__ao_inprogress(ao);        \
+        CTX_UNLOCK;                                             \
+        return ao_inprogress_rc;                                \
+   }while(0)
+        
+
+#define AO_ABORT(rc) do{                        \
+        assert(rc);                             \
+        assert(rc != ERROR_ASYNC_INPROGRESS);   \
+        libxl__ao_abort(ao);                    \
+        CTX_UNLOCK;                             \
+        return (rc);                            \
+    }while(0)
+
+#define AO_GC                                   \
+    libxl__gc *const gc = &ao->gc
+
+
+_hidden libxl__ao *libxl__ao_create(libxl_ctx*, uint32_t domid,
+                                    const libxl_asyncop_how*);
+_hidden int libxl__ao_inprogress(libxl__ao *ao);
+_hidden void libxl__ao_abort(libxl__ao *ao);
+_hidden void libxl__ao_complete(libxl__egc *egc, libxl__ao *ao, int rc);
+  /* All of these MUST be called with the ctx locked.
+   * libxl__ao_wait MUST be called with the ctx locked exactly once.
+   */
+
+_hidden void libxl__ao__destroy(libxl_ctx*, libxl__ao *ao);
+  /* for use by ao machinery ONLY */
+
+/*
  * Convenience macros.
  */
 
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index 6728479..eff4d43 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -394,6 +394,7 @@ libxl_event_type = Enumeration("event_type", [
     (1, "DOMAIN_SHUTDOWN"),
     (2, "DOMAIN_DESTROY"),
     (3, "DISK_EJECT"),
+    (4, "OPERATION_COMPLETE"),
     ])
 
 libxl_ev_user = Number("libxl_ev_user")
@@ -417,4 +418,7 @@ libxl_event = Struct("event",[
                                         ("vdev", string),
                                         ("disk", libxl_device_disk),
                                  ])),
+           ("operation_complete", Struct(None, [
+                                        ("rc", integer),
+                                 ])),
            ]))])
-- 
1.7.2.5


_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel


 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.