[Minios-devel] [UNIKRAFT PATCH 11/23] plat/xen: Paravirtualization context - Switch interrupted thread to a previously yielding thread

[Costin Lupu <costin.lupu@xxxxxxxxx>]

If switching an interrupted thread to a previously yielding thread, the
scheduled thread registers have to be pushed on the stack in order to mimic the
interrupt context save done before calling the interrupt routine. This is what
pv_ctx_emulate_exception_stack() function does.

Considering that newly created threads may be considered "interrupted" before
they start to run (see 'plat: Hardware context - Add trampoline for starting
threads' commit for more details), the exception stack emulation has to be
performed in this case too. However, not all threads start this way (e.g. idle
threads), so the initialization callback signature has to be extended in order
to parameterize this behaviour.

Signed-off-by: Costin Lupu <costin.lupu@xxxxxxxxx>
---
 include/uk/plat/thread.h        |  5 +++--
 lib/uksched/include/uk/thread.h |  3 ++-
 lib/uksched/sched.c             |  4 ++--
 lib/uksched/thread.c            |  5 +++--
 plat/common/sw_ctx.c            |  6 ++++--
 plat/common/thread.c            |  4 ++--
 plat/common/x86/hw_ctx.c        |  6 ++++--
 plat/xen/x86/pv_ctx.c           | 35 +++++++++++++++++++++++++++++++--
 8 files changed, 53 insertions(+), 15 deletions(-)

diff --git a/include/uk/plat/thread.h b/include/uk/plat/thread.h
index f2573195..53b44f59 100644
--- a/include/uk/plat/thread.h
+++ b/include/uk/plat/thread.h
@@ -39,6 +39,7 @@
 #define __UKPLAT_THREAD_H__
 
 #include <stdlib.h>
+#include <stdbool.h>
 #include <uk/essentials.h>
 #include <uk/assert.h>
 
@@ -51,7 +52,7 @@ enum ukplat_ctx_type {
 struct uk_alloc;
 
 typedef void  (*ukplat_ctx_init_func_t)
-               (void *ctx, unsigned long sp);
+               (void *ctx, unsigned long sp, bool start_interrupted);
 typedef void  (*ukplat_ctx_start_func_t)
                (void *ctx);
 typedef void  (*ukplat_ctx_switch_func_t)
@@ -81,7 +82,7 @@ struct thread_context {
 struct thread_context *
 ukplat_thread_ctx_create(struct ukplat_ctx_callbacks *cbs,
                struct uk_alloc *allocator, unsigned long sp,
-               unsigned long tlsp);
+               unsigned long tlsp, bool start_interrupted);
 void ukplat_thread_ctx_destroy(struct uk_alloc *allocator,
                struct thread_context *ctx);
 
diff --git a/lib/uksched/include/uk/thread.h b/lib/uksched/include/uk/thread.h
index 3f1e6cab..cf10b438 100644
--- a/lib/uksched/include/uk/thread.h
+++ b/lib/uksched/include/uk/thread.h
@@ -113,7 +113,8 @@ struct uk_thread *uk_thread_current(void)
 int uk_thread_init(struct uk_thread *thread,
                struct ukplat_ctx_callbacks *cbs, struct uk_alloc *allocator,
                const char *name, void *stack, void *tls,
-               void (*function)(void *), void *arg);
+               void (*function)(void *), void *arg,
+               bool start_interrupted);
 void uk_thread_fini(struct uk_thread *thread,
                struct uk_alloc *allocator);
 void uk_thread_block_timeout(struct uk_thread *thread, __nsec nsec);
diff --git a/lib/uksched/sched.c b/lib/uksched/sched.c
index a2505479..66a7656b 100644
--- a/lib/uksched/sched.c
+++ b/lib/uksched/sched.c
@@ -180,7 +180,7 @@ void uk_sched_idle_init(struct uk_sched *sched,
 
        rc = uk_thread_init(idle,
                        &sched->plat_ctx_cbs, sched->allocator,
-                       "Idle", stack, tls, function, NULL);
+                       "Idle", stack, tls, function, NULL, false);
        if (rc)
                goto out_crash;
 
@@ -217,7 +217,7 @@ struct uk_thread *uk_sched_thread_create(struct uk_sched 
*sched,
 
        rc = uk_thread_init(thread,
                        &sched->plat_ctx_cbs, sched->allocator,
-                       name, stack, tls, function, arg);
+                       name, stack, tls, function, arg, true);
        if (rc)
                goto err;
 
diff --git a/lib/uksched/thread.c b/lib/uksched/thread.c
index 6262f8c6..53a4ba48 100644
--- a/lib/uksched/thread.c
+++ b/lib/uksched/thread.c
@@ -83,7 +83,8 @@ struct _reent *__getreent(void)
 int uk_thread_init(struct uk_thread *thread,
                struct ukplat_ctx_callbacks *cbs, struct uk_alloc *allocator,
                const char *name, void *stack, void *tls,
-               void (*function)(void *), void *arg)
+               void (*function)(void *), void *arg,
+               bool start_interrupted)
 {
        unsigned long sp;
 
@@ -98,7 +99,7 @@ int uk_thread_init(struct uk_thread *thread,
 
        /* Call platform specific setup. */
        thread->ctx = ukplat_thread_ctx_create(cbs, allocator, sp,
-                       (uintptr_t)ukarch_tls_pointer(tls));
+                       (uintptr_t)ukarch_tls_pointer(tls), start_interrupted);
        if (thread->ctx == NULL)
                return -1;
 
diff --git a/plat/common/sw_ctx.c b/plat/common/sw_ctx.c
index 99074398..cc7ad9f9 100644
--- a/plat/common/sw_ctx.c
+++ b/plat/common/sw_ctx.c
@@ -40,7 +40,8 @@
 #include <uk/assert.h>
 #include <sw_ctx.h>
 
-static void  sw_ctx_init(void *ctx, unsigned long sp);
+static void  sw_ctx_init(void *ctx, unsigned long sp,
+               bool start_interrupted __unused);
 static void  sw_ctx_start(void *ctx) __noreturn;
 static void  sw_ctx_switch(void *prevctx, void *nextctx);
 
@@ -50,7 +51,8 @@ static void  sw_ctx_switch(void *prevctx, void *nextctx);
  */
 extern void asm_thread_starter(void);
 
-static void sw_ctx_init(void *ctx, unsigned long sp)
+static void sw_ctx_init(void *ctx, unsigned long sp,
+               bool start_interrupted __unused)
 {
        struct sw_ctx *sw_ctx;
 
diff --git a/plat/common/thread.c b/plat/common/thread.c
index 4535193d..c0c4377a 100644
--- a/plat/common/thread.c
+++ b/plat/common/thread.c
@@ -52,7 +52,7 @@
 struct thread_context *ukplat_thread_ctx_create(
                struct ukplat_ctx_callbacks *cbs,
                struct uk_alloc *allocator, unsigned long sp,
-               unsigned long tlsp)
+               unsigned long tlsp, bool start_interrupted)
 {
        struct thread_context *thread_ctx;
        void *ctx, *extregs;
@@ -78,7 +78,7 @@ struct thread_context *ukplat_thread_ctx_create(
        uk_pr_debug("Allocated %lu bytes for thread ctx at %p\n", sz, ctx);
 
        /* Initialize underlying context */
-       cbs->init_cb(ctx, sp);
+       cbs->init_cb(ctx, sp, start_interrupted);
 
        /* Initialize extregs area: zero out, then save a valid layout to it */
        extregs = (void *) ALIGN_UP(((__uptr) ctx + cbs->ctx_size),
diff --git a/plat/common/x86/hw_ctx.c b/plat/common/x86/hw_ctx.c
index fd46fbe3..0b36ed5c 100644
--- a/plat/common/x86/hw_ctx.c
+++ b/plat/common/x86/hw_ctx.c
@@ -43,7 +43,8 @@
 #include <kvm-x86/traps.h>
 #include <hw_ctx.h>
 
-static void  hw_ctx_init(void *ctx, unsigned long sp);
+static void  hw_ctx_init(void *ctx, unsigned long sp,
+               bool start_interrupted __unused);
 static void  hw_ctx_start(void *ctx) __noreturn;
 static void  hw_ctx_switch(void *prevctx, void *nextctx);
 
@@ -51,7 +52,8 @@ extern void asm_thread_starter(void);
 extern void asm_thread_starter_trampoline(void);
 
 /* Architecture specific setup of thread creation */
-void hw_ctx_init(void *ctx, unsigned long sp)
+void hw_ctx_init(void *ctx, unsigned long sp,
+               bool start_interrupted __unused)
 {
        struct hw_ctx *hw_ctx;
 
diff --git a/plat/xen/x86/pv_ctx.c b/plat/xen/x86/pv_ctx.c
index f3f1a79b..9203e6de 100644
--- a/plat/xen/x86/pv_ctx.c
+++ b/plat/xen/x86/pv_ctx.c
@@ -43,13 +43,29 @@
 #include "pda.h"
 
 
-static void  pv_ctx_init(void *ctx, unsigned long sp);
+static void  pv_ctx_init(void *ctx, unsigned long sp, bool start_interrupted);
 static void  pv_ctx_start(void *ctx) __noreturn;
 static void  pv_ctx_switch(void *prevctx, void *nextctx);
 
 extern void asm_thread_starter(void);
 
-static void pv_ctx_init(void *ctx, unsigned long sp)
+static void pv_ctx_emulate_exception_stack(struct pv_ctx *ctx)
+{
+       unsigned long old_sp = ctx->sw_ctx.sp;
+
+       /* Setup the exception stack */
+       ukarch_stack_push(&ctx->sw_ctx.sp, __KERNEL_SS);
+       ukarch_stack_push(&ctx->sw_ctx.sp, old_sp);
+       ukarch_stack_push(&ctx->sw_ctx.sp, X86_EFLAGS_IF);
+       ukarch_stack_push(&ctx->sw_ctx.sp, __KERNEL_CS | 3);
+       ukarch_stack_push(&ctx->sw_ctx.sp, ctx->sw_ctx.ip);
+
+       /* Setup the rest of the generic registers */
+       for (int i = 0; i < (int) (OFFSETOF_REGS_RIP / sizeof(long)); i++)
+               ukarch_stack_push(&ctx->sw_ctx.sp, 0);
+}
+
+static void pv_ctx_init(void *ctx, unsigned long sp, bool start_interrupted)
 {
        struct pv_ctx *pv_ctx;
 
@@ -59,6 +75,13 @@ static void pv_ctx_init(void *ctx, unsigned long sp)
        pv_ctx->sw_ctx.sp = sp;
        pv_ctx->sw_ctx.ip = (unsigned long) asm_thread_starter;
        pv_ctx->interrupted = false;
+
+       if (start_interrupted) {
+               pv_ctx->interrupted = true;
+               pv_ctx_emulate_exception_stack(pv_ctx);
+
+       } else
+               pv_ctx->interrupted = false;
 }
 
 extern void asm_ctx_start(unsigned long sp, unsigned long ip) __noreturn;
@@ -90,6 +113,14 @@ static void pv_ctx_switch(void *prevctx, void *nextctx)
                        /* switching interrupted to interrupted */
                        next_hw_ctx->interrupted = false;
 
+               else
+                       /* switching interrupted to yielding */
+                       /*
+                        * TODO optimization:
+                        * - only the exception stack should be enough
+                        */
+                       pv_ctx_emulate_exception_stack(next_hw_ctx);
+
                /* switch interrupted stack in PDA */
                prev_hw_ctx->sw_ctx.sp = cpu0_pda.userstackptr;
                cpu0_pda.userstackptr = next_hw_ctx->sw_ctx.sp;
-- 
2.20.1


_______________________________________________
Minios-devel mailing list
Minios-devel@xxxxxxxxxxxxxxxxxxxx
https://lists.xenproject.org/mailman/listinfo/minios-devel