[Xen-devel] [PATCH 20/25] libxl: provide progress reporting for long-running operations

[Ian Jackson <ian.jackson@xxxxxxxxxxxxx>]

This will be used for reporting, during domain creation, that the
console is ready.

Signed-off-by: Ian Jackson <ian.jackson@xxxxxxxxxxxxx>

Changes since v7:
 * If aop->how.callback, actually add the aop to the for_callback list (!)
 * Document the threadsafety of aop's, and make appropriate cross-references.
 * Allocate the actual aop from its thread's egc; do not free it.
 * Remove pointless code motion of libxl__ao_create.
 * Minor formatting fixes.
---
 tools/libxl/libxl.h          |   45 ++++++++++++++++++++++++++
 tools/libxl/libxl_event.c    |   72 ++++++++++++++++++++++++++++++++++++++++--
 tools/libxl/libxl_internal.h |   35 ++++++++++++++++++++
 3 files changed, 149 insertions(+), 3 deletions(-)

diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index b497430..7bd6231 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -435,6 +435,51 @@ typedef struct {
     } u;
 } libxl_asyncop_how;
 
+/*
+ * Some more complex asynchronous operations can report intermediate
+ * progress.  How this is to be reported is controlled, for each
+ * function, by a parameter
+ *    libxl_asyncprogress_how *aop_FOO_how;
+ * for each kind of progress FOO supported by that function.  Each
+ * such kind of progress is associated with an event type.
+ *
+ * The function description will document whether, when, and how
+ * many times, the intermediate progress will be reported, and
+ * what the corresponding event type(s) are.
+ *
+ * If aop_FOO_how==NULL, intermediate progress reports are discarded.
+ *
+ * If aop_FOO_how->callback==NULL, intermediate progress reports
+ * generate libxl events which can be obtained from libxl_event_wait
+ * or libxl_event_check.
+ *
+ * If aop_FOO_how->callback!=NULL, libxl will report intermediate
+ * progress by calling callback(ctx, &event, for_callback).
+ *
+ * The rules for these events are otherwise the same as those for
+ * ordinary events.  The reentrancy and threading rules for the
+ * callback are the same as those for ao completion callbacks.
+ *
+ * Note that the callback, if provided, is responsible for freeing
+ * the event.
+ *
+ * If callbacks are requested, they will be made, and returned, before
+ * the long-running libxl operation is considered finished (so if the
+ * long-running libxl operation was invoked with ao_how==NULL then any
+ * callbacks will occur strictly before the long-running operation
+ * returns).  However, the callbacks may occur on any thread.
+ *
+ * In general, otherwise, no promises are made about the relative
+ * order of callbacks in a multithreaded program.  In particular
+ * different callbacks relating to the same long-running operation may
+ * be delivered out of order.
+ */
+
+typedef struct {
+    void (*callback)(libxl_ctx *ctx, libxl_event*, void *for_callback);
+    libxl_ev_user for_event; /* always used */
+    void *for_callback; /* passed to callback */
+} libxl_asyncprogress_how;
 
 #define LIBXL_VERSION 0
 
diff --git a/tools/libxl/libxl_event.c b/tools/libxl/libxl_event.c
index 7e8d002..02deea5 100644
--- a/tools/libxl/libxl_event.c
+++ b/tools/libxl/libxl_event.c
@@ -898,18 +898,29 @@ void libxl__event_disaster(libxl__egc *egc, const char 
*msg, int errnoval,
 static void egc_run_callbacks(libxl__egc *egc)
 {
     /*
-     * The callbacks must happen with the ctx unlocked.
-     * See the comment near #define EGC_GC in libxl_internal.h and
-     * those in the definitions of libxl__egc and libxl__ao.
+     * The callbacks must happen with the ctx unlocked.  See the
+     * comment near #define EGC_GC in libxl_internal.h and those in
+     * the definitions of libxl__egc, libxl__ao and libxl__aop.
      */
     EGC_GC;
     libxl_event *ev, *ev_tmp;
+    libxl__aop_occurred *aop, *aop_tmp;
 
     LIBXL_TAILQ_FOREACH_SAFE(ev, &egc->occurred_for_callback, link, ev_tmp) {
         LIBXL_TAILQ_REMOVE(&egc->occurred_for_callback, ev, link);
         CTX->event_hooks->event_occurs(CTX->event_hooks_user, ev);
     }
 
+    LIBXL_TAILQ_FOREACH_SAFE(aop, &egc->aops_for_callback, entry, aop_tmp) {
+        LIBXL_TAILQ_REMOVE(&egc->aops_for_callback, aop, entry);
+        aop->how->callback(CTX, aop->ev, aop->how->for_callback);
+
+        CTX_LOCK;
+        aop->ao->progress_reports_outstanding--;
+        libxl__ao_complete_check_progress_reports(egc, aop->ao);
+        CTX_UNLOCK;
+    }
+
     libxl__ao *ao, *ao_tmp;
     LIBXL_TAILQ_FOREACH_SAFE(ao, &egc->aos_for_callback,
                              entry_for_callback, ao_tmp) {
@@ -1292,6 +1303,7 @@ void libxl__ao_abort(libxl__ao *ao)
     assert(ao->magic == LIBXL__AO_MAGIC);
     assert(ao->in_initiator);
     assert(!ao->complete);
+    assert(!ao->progress_reports_outstanding);
     libxl__ao__destroy(CTX, ao);
 }
 
@@ -1302,6 +1314,24 @@ void libxl__ao_complete(libxl__egc *egc, libxl__ao *ao, 
int rc)
     ao->complete = 1;
     ao->rc = rc;
 
+    libxl__ao_complete_check_progress_reports(egc, ao);
+}
+
+void libxl__ao_complete_check_progress_reports(libxl__egc *egc, libxl__ao *ao)
+{
+    /*
+     * We don't consider an ao complete if it has any outstanding
+     * callbacks.  These callbacks might be outstanding on other
+     * threads, queued up in the other threads' egc's.  Those threads
+     * will, after making the callback, take out the lock again,
+     * decrement progress_reports_outstanding, and call us again.
+     */
+
+    assert(ao->progress_reports_outstanding >= 0);
+
+    if (!ao->complete || ao->progress_reports_outstanding)
+        return;
+
     if (ao->poller) {
         assert(ao->in_initiator);
         if (!ao->constructing)
@@ -1351,6 +1381,7 @@ libxl__ao *libxl__ao_create(libxl_ctx *ctx, uint32_t 
domid,
     return NULL;
 }
 
+
 int libxl__ao_inprogress(libxl__ao *ao)
 {
     AO_GC;
@@ -1408,6 +1439,41 @@ int libxl__ao_inprogress(libxl__ao *ao)
 }
 
 
+/* progress reporting */
+
+/* The application indicates a desire to ignore events by passing NULL
+ * for how.  But we want to copy *how.  So we have this dummy function
+ * whose address is stored in callback if the app passed how==NULL. */
+static void dummy_asyncprogress_callback_ignore
+  (libxl_ctx *ctx, libxl_event *ev, void *for_callback) { }
+
+void libxl__ao_progress_gethow(libxl_asyncprogress_how *in_state,
+                               const libxl_asyncprogress_how *from_app) {
+    if (from_app)
+        *in_state = *from_app;
+    else
+        in_state->callback = dummy_asyncprogress_callback_ignore;
+}
+
+void libxl__ao_progress_report(libxl__egc *egc, libxl__ao *ao,
+        const libxl_asyncprogress_how *how, libxl_event *ev)
+{
+    ev->for_user = how->for_event;
+    if (how->callback == dummy_asyncprogress_callback_ignore) {
+        /* ignore */
+    } else if (how->callback) {
+        libxl__aop_occurred *aop = libxl__zalloc(&egc->gc, sizeof(*aop));
+        ao->progress_reports_outstanding++;
+        aop->ao = ao;
+        aop->ev = ev;
+        aop->how = how;
+        LIBXL_TAILQ_INSERT_TAIL(&egc->aops_for_callback, aop, entry);
+    } else {
+        libxl__event_occurred(egc, ev);
+    }
+}
+
+
 /*
  * Local variables:
  * mode: C
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 26e0713..d9dfd6d 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -118,6 +118,7 @@ _hidden void libxl__log(libxl_ctx *ctx, xentoollog_level 
msglevel, int errnoval,
 typedef struct libxl__gc libxl__gc;
 typedef struct libxl__egc libxl__egc;
 typedef struct libxl__ao libxl__ao;
+typedef struct libxl__aop_occurred libxl__aop_occurred;
 
 _hidden void libxl__alloc_failed(libxl_ctx *, const char *func,
                          size_t nmemb, size_t size) __attribute__((noreturn));
@@ -374,6 +375,21 @@ struct libxl__egc {
     struct libxl__gc gc;
     struct libxl__event_list occurred_for_callback;
     LIBXL_TAILQ_HEAD(, libxl__ao) aos_for_callback;
+    LIBXL_TAILQ_HEAD(, libxl__aop_occurred) aops_for_callback;
+};
+
+struct libxl__aop_occurred {
+    /*
+     * An aop belongs to, and may be accessed only on, the thread
+     * which created it.  It normally lives in that thread's egc.
+     *
+     * While an aop exists, it corresponds to one refcount in
+     * ao->progress_reports_outstanding, preventing ao destruction.
+     */
+    LIBXL_TAILQ_ENTRY(libxl__aop_occurred) entry;
+    libxl__ao *ao;
+    libxl_event *ev;
+    const libxl_asyncprogress_how *how;
 };
 
 #define LIBXL__AO_MAGIC              0xA0FACE00ul
@@ -396,6 +412,7 @@ struct libxl__ao {
      */
     uint32_t magic;
     unsigned constructing:1, in_initiator:1, complete:1, notified:1;
+    int progress_reports_outstanding;
     int rc;
     libxl__gc gc;
     libxl_asyncop_how how;
@@ -1393,6 +1410,7 @@ libxl__device_model_version_running(libxl__gc *gc, 
uint32_t domid);
         LIBXL_INIT_GC((egc).gc,ctx);                    \
         LIBXL_TAILQ_INIT(&(egc).occurred_for_callback); \
         LIBXL_TAILQ_INIT(&(egc).aos_for_callback);      \
+        LIBXL_TAILQ_INIT(&(egc).aops_for_callback);     \
     } while(0)
 
 _hidden void libxl__egc_cleanup(libxl__egc *egc);
@@ -1439,6 +1457,9 @@ _hidden void libxl__egc_cleanup(libxl__egc *egc);
  *        pointer to the internal event generation request routines
  *        libxl__evgen_FOO, so that at some point a CALLBACK will be
  *        made when the operation is complete.
+ *      - if the operation provides progress reports, the aop_how(s)
+ *        must be copied into the per-operation structure using
+ *        libxl__ao_progress_gethow.
  *
  * - If initiation is successful, the initiating function needs
  *   to run libxl__ao_inprogress right before unlocking and
@@ -1448,6 +1469,10 @@ _hidden void libxl__egc_cleanup(libxl__egc *egc);
  *   call libxl__ao_abort before unlocking and returning whatever
  *   error code is appropriate (AO_ABORT macro).
  *
+ * - If the operation supports progress reports, it may generate
+ *   suitable events with NEW_EVENT and report them with
+ *   libxl__ao_progress_report (with the ctx locked).
+ *
  * - Later, some callback function, whose callback has been requested
  *   directly or indirectly, should call libxl__ao_complete (with the
  *   ctx locked, as it will generally already be in any event callback
@@ -1503,8 +1528,18 @@ _hidden int libxl__ao_inprogress(libxl__ao *ao); /* 
temporarily unlocks */
 _hidden void libxl__ao_abort(libxl__ao *ao);
 _hidden void libxl__ao_complete(libxl__egc *egc, libxl__ao *ao, int rc);
 
+/* Can be called at any time.  Use is essential for any aop user. */
+_hidden void libxl__ao_progress_gethow(libxl_asyncprogress_how *in_state,
+                                       const libxl_asyncprogress_how 
*from_app);
+
+/* Must be called with the ctx locked.  Will fill in ev->for_user,
+ * so caller need not do that. */
+_hidden void libxl__ao_progress_report(libxl__egc *egc, libxl__ao *ao,
+   const libxl_asyncprogress_how *how, libxl_event *ev /* consumed */);
+
 /* For use by ao machinery ONLY */
 _hidden void libxl__ao__destroy(libxl_ctx*, libxl__ao *ao);
+_hidden void libxl__ao_complete_check_progress_reports(libxl__egc*, 
libxl__ao*);
 
 
 /*
-- 
1.7.2.5


_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
http://lists.xen.org/xen-devel