|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-devel] [PATCH RFC 20/20] tools: expose postcopy live migration support in libxl and xl
- Add a 'memory_strategy' parameter to libxl_domain_live_migrate(),
which specifies how the remainder of the memory migration should be
approached after the iterative precopy phase is completed.
- Plug this parameter into the libxl migration precopy policy
implementation.
- Add --postcopy to xl migrate, and skip the xl-level handshaking at
both sides when postcopy migration occurs.
Signed-off-by: Joshua Otto <jtotto@xxxxxxxxxxxx>
---
tools/libxl/libxl.h | 6 ++++-
tools/libxl/libxl_dom_save.c | 19 ++++++-------
tools/libxl/libxl_domain.c | 9 ++++---
tools/libxl/libxl_internal.h | 1 +
tools/xl/xl.h | 7 ++++-
tools/xl/xl_cmdtable.c | 5 +++-
tools/xl/xl_migrate.c | 63 +++++++++++++++++++++++++++++++++++++++-----
tools/xl/xl_vmcontrol.c | 8 ++++--
8 files changed, 94 insertions(+), 24 deletions(-)
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 51e8760..3a2f7ea 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -1401,7 +1401,7 @@ int libxl_domain_live_migrate(libxl_ctx *ctx, uint32_t
domid, int send_fd,
int flags, /* LIBXL_SUSPEND_* */
unsigned int precopy_iterations,
unsigned int precopy_dirty_threshold,
- int recv_fd,
+ int recv_fd, int memory_strategy,
bool *postcopy_transitioned, /* OUT */
const libxl_asyncop_how *ao_how)
LIBXL_EXTERNAL_CALLERS_ONLY;
@@ -1409,6 +1409,10 @@ int libxl_domain_live_migrate(libxl_ctx *ctx, uint32_t
domid, int send_fd,
#define LIBXL_LM_PRECOPY_ITERATIONS_DEFAULT 5
#define LIBXL_LM_DIRTY_THRESHOLD_DEFAULT 50
+#define LIBXL_LM_MEMORY_STOP_AND_COPY 0
+#define LIBXL_LM_MEMORY_POSTCOPY 1
+#define LIBXL_LM_MEMORY_DEFAULT LIBXL_LM_MEMORY_STOP_AND_COPY
+
/* @param suspend_cancel [from xenctrl.h:xc_domain_resume( @param fast )]
* If this parameter is true, use co-operative resume. The guest
* must support this.
diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
index 9e565ae..9d5d435 100644
--- a/tools/libxl/libxl_dom_save.c
+++ b/tools/libxl/libxl_dom_save.c
@@ -333,18 +333,19 @@ int
libxl__save_emulator_xenstore_data(libxl__domain_save_state *dss,
* the precopy phase of live migrations, and is responsible for deciding when
* the precopy phase should terminate and what should be done next.
*/
-static int libxl__save_live_migration_simple_precopy_policy(
- struct precopy_stats stats, void *user)
+static int libxl__save_live_migration_precopy_policy(struct precopy_stats
stats,
+ void *user)
{
libxl__save_helper_state *shs = user;
libxl__domain_save_state *dss = shs->caller_state;
- if (stats.dirty_count >= 0 &&
- stats.dirty_count <= dss->precopy_dirty_threshold)
- return XGS_POLICY_STOP_AND_COPY;
-
- if (stats.iteration >= dss->precopy_iterations)
- return XGS_POLICY_STOP_AND_COPY;
+ if ((stats.dirty_count >= 0 &&
+ stats.dirty_count <= dss->precopy_dirty_threshold) ||
+ (stats.iteration >= dss->precopy_iterations)) {
+ return (dss->memory_strategy == LIBXL_LM_MEMORY_POSTCOPY)
+ ? XGS_POLICY_POSTCOPY
+ : XGS_POLICY_STOP_AND_COPY;
+ }
return XGS_POLICY_CONTINUE_PRECOPY;
}
@@ -452,7 +453,7 @@ void libxl__domain_save(libxl__egc *egc,
libxl__domain_save_state *dss)
libxl__save_live_migration_postcopy_transition_callback;
}
- callbacks->precopy_policy =
libxl__save_live_migration_simple_precopy_policy;
+ callbacks->precopy_policy = libxl__save_live_migration_precopy_policy;
callbacks->switch_qemu_logdirty =
libxl__domain_suspend_common_switch_qemu_logdirty;
dss->sws.ao = dss->ao;
diff --git a/tools/libxl/libxl_domain.c b/tools/libxl/libxl_domain.c
index ea778a6..feec293 100644
--- a/tools/libxl/libxl_domain.c
+++ b/tools/libxl/libxl_domain.c
@@ -489,6 +489,7 @@ static void domain_suspend_cb(libxl__egc *egc,
static int do_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
unsigned int precopy_iterations,
unsigned int precopy_dirty_threshold, int recv_fd,
+ int memory_strategy,
bool *postcopy_transitioned,
const libxl_asyncop_how *ao_how)
{
@@ -510,7 +511,8 @@ static int do_domain_suspend(libxl_ctx *ctx, uint32_t
domid, int fd, int flags,
dss->domid = domid;
dss->fd = fd;
dss->recv_fd = recv_fd;
- dss->postcopy_transitioned = postcopy_resumed_remotely;
+ dss->memory_strategy = memory_strategy;
+ dss->postcopy_transitioned = postcopy_transitioned;
dss->type = type;
dss->live = flags & LIBXL_SUSPEND_LIVE;
dss->debug = flags & LIBXL_SUSPEND_DEBUG;
@@ -536,12 +538,13 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid,
int fd, int flags,
return do_domain_suspend(ctx, domid, fd, flags,
LIBXL_LM_PRECOPY_ITERATIONS_DEFAULT,
LIBXL_LM_DIRTY_THRESHOLD_DEFAULT, -1,
- NULL, ao_how);
+ LIBXL_LM_MEMORY_DEFAULT, NULL, ao_how);
}
int libxl_domain_live_migrate(libxl_ctx *ctx, uint32_t domid, int send_fd,
int flags, unsigned int precopy_iterations,
unsigned int precopy_dirty_threshold, int
recv_fd,
+ int memory_strategy,
bool *postcopy_transitioned,
const libxl_asyncop_how *ao_how)
{
@@ -553,7 +556,7 @@ int libxl_domain_live_migrate(libxl_ctx *ctx, uint32_t
domid, int send_fd,
flags |= LIBXL_SUSPEND_LIVE;
return do_domain_suspend(ctx, domid, send_fd, flags, precopy_iterations,
- precopy_dirty_threshold, recv_fd,
+ precopy_dirty_threshold, recv_fd, memory_strategy,
postcopy_transitioned, ao_how);
}
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 0a7c0d1..209cee5 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -3313,6 +3313,7 @@ struct libxl__domain_save_state {
int fd;
int fdfl; /* original flags on fd */
int recv_fd;
+ int memory_strategy;
bool *postcopy_transitioned;
libxl_domain_type type;
int live;
diff --git a/tools/xl/xl.h b/tools/xl/xl.h
index aa95b77..279c716 100644
--- a/tools/xl/xl.h
+++ b/tools/xl/xl.h
@@ -48,6 +48,7 @@ struct domain_create {
bool userspace_colo_proxy;
int migrate_fd; /* -1 means none */
int send_back_fd; /* -1 means none */
+ bool *postcopy_resumed;
char **migration_domname_r; /* from malloc */
};
@@ -66,7 +67,6 @@ static const char migrate_permission_to_go[]=
"domain is yours, you are cleared to unpause";
static const char migrate_report[]=
"my copy unpause results are as follows";
-#endif
/* followed by one byte:
* 0: everything went well, domain is running
@@ -76,6 +76,11 @@ static const char migrate_report[]=
* from target to source
*/
+static const char migrate_postcopy_sync[]=
+ "postcopy migration completed successfully";
+
+#endif
+
#define XL_MANDATORY_FLAG_JSON (1U << 0) /* config data is in JSON format */
#define XL_MANDATORY_FLAG_STREAMv2 (1U << 1) /* stream is v2 */
#define XL_MANDATORY_FLAG_ALL (XL_MANDATORY_FLAG_JSON | \
diff --git a/tools/xl/xl_cmdtable.c b/tools/xl/xl_cmdtable.c
index 6df66fb..7bd2d1b 100644
--- a/tools/xl/xl_cmdtable.c
+++ b/tools/xl/xl_cmdtable.c
@@ -169,7 +169,10 @@ struct cmd_spec cmd_table[] = {
"--precopy-iterations Perform at most this many iterations of the
precopy\n"
" memory migration loop before suspending the
domain.\n"
"--precopy-threshold If fewer than this many pages are dirty at the end
of a\n"
- " copy round, exit the precopy loop and suspend the
domain."
+ " copy round, exit the precopy loop and suspend the
domain.\n"
+ "--postcopy At the end of the iterative precopy phase,
transition to a\n"
+ " postcopy memory migration rather than performing a
stop-and-copy\n"
+ " migration of the outstanding dirty pages.\n"
},
{ "restore",
&main_restore, 0, 1,
diff --git a/tools/xl/xl_migrate.c b/tools/xl/xl_migrate.c
index 1ffc32b..43c7d8e 100644
--- a/tools/xl/xl_migrate.c
+++ b/tools/xl/xl_migrate.c
@@ -179,7 +179,8 @@ static void migrate_do_preamble(int send_fd, int recv_fd,
pid_t child,
static void migrate_domain(uint32_t domid, const char *rune, int debug,
const char *override_config_file,
unsigned int precopy_iterations,
- unsigned int precopy_dirty_threshold)
+ unsigned int precopy_dirty_threshold,
+ int memory_strategy)
{
pid_t child = -1;
int rc;
@@ -210,18 +211,32 @@ static void migrate_domain(uint32_t domid, const char
*rune, int debug,
flags |= LIBXL_SUSPEND_DEBUG;
rc = libxl_domain_live_migrate(ctx, domid, send_fd, flags,
precopy_iterations, precopy_dirty_threshold,
- recv_fd, &postcopy_transitioned, NULL);
- assert(!postcopy_transitioned);
-
+ recv_fd, memory_strategy,
+ &postcopy_transitioned, NULL);
if (rc) {
fprintf(stderr, "migration sender: libxl_domain_suspend failed"
" (rc=%d)\n", rc);
- if (rc == ERROR_GUEST_TIMEDOUT)
+ if (postcopy_transitioned)
+ goto failed_postcopy;
+ else if (rc == ERROR_GUEST_TIMEDOUT)
goto failed_suspend;
else
goto failed_resume;
}
+ /* No need for additional ceremony if we already resumed the guest as part
+ * of a postcopy live migration. */
+ if (postcopy_transitioned) {
+ /* It doesn't matter if something happens to the pipe after we get to
+ * this point - we only bother to synchronize here for tidiness. */
+ migrate_read_fixedmessage(recv_fd, migrate_postcopy_sync,
+ sizeof(migrate_postcopy_sync),
+ "postcopy sync", rune);
+ libxl_domain_destroy(ctx, domid, 0);
+ fprintf(stderr, "Migration successful.\n");
+ exit(EXIT_SUCCESS);
+ }
+
//fprintf(stderr, "migration sender: Transfer complete.\n");
// Should only be printed when debugging as it's a bit messy with
// progress indication.
@@ -320,6 +335,21 @@ static void migrate_domain(uint32_t domid, const char
*rune, int debug,
close(send_fd);
migration_child_report(recv_fd);
exit(EXIT_FAILURE);
+
+ failed_postcopy:
+ if (common_domname) {
+ xasprintf(&away_domname, "%s--postcopy-inconsistent", common_domname);
+ libxl_domain_rename(ctx, domid, common_domname, away_domname);
+ }
+
+ fprintf(stderr,
+ "** Migration failed during memory postcopy **\n"
+ "It's possible that the guest has executed/is executing at the destination,\n"
+ " so resuming it here now may be unsafe.\n");
+
+ close(send_fd);
+ migration_child_report(recv_fd);
+ exit(EXIT_FAILURE);
}
static void migrate_receive(int debug, int daemonize, int monitor,
@@ -333,6 +363,7 @@ static void migrate_receive(int debug, int daemonize, int
monitor,
int rc, rc2;
char rc_buf;
char *migration_domname;
+ bool postcopy_resumed;
struct domain_create dom_info;
signal(SIGPIPE, SIG_IGN);
@@ -352,6 +383,7 @@ static void migrate_receive(int debug, int daemonize, int
monitor,
dom_info.paused = 1;
dom_info.migrate_fd = recv_fd;
dom_info.send_back_fd = send_fd;
+ dom_info.postcopy_resumed = &postcopy_resumed;
dom_info.migration_domname_r = &migration_domname;
dom_info.checkpointed_stream = checkpointed;
dom_info.colo_proxy_script = colo_proxy_script;
@@ -414,6 +446,18 @@ static void migrate_receive(int debug, int daemonize, int
monitor,
break;
}
+ /* No need for additional ceremony if we already resumed the guest as part
+ * of a postcopy live migration. */
+ if (postcopy_resumed) {
+ libxl_write_exactly(ctx, send_fd, migrate_postcopy_sync,
+ sizeof(migrate_postcopy_sync),
+ "migration ack stream", "postcopy sync");
+ fprintf(stderr, "migration target: Domain started successsfully.\n");
+ libxl_domain_rename(ctx, domid, migration_domname, common_domname);
+ exit(EXIT_SUCCESS);
+ }
+
+
fprintf(stderr, "migration target: Transfer complete,"
" requesting permission to start domain.\n");
@@ -545,12 +589,14 @@ int main_migrate(int argc, char **argv)
char *host;
int opt, daemonize = 1, monitor = 1, debug = 0, pause_after_migration = 0;
int precopy_iterations = LIBXL_LM_PRECOPY_ITERATIONS_DEFAULT,
- precopy_dirty_threshold = LIBXL_LM_DIRTY_THRESHOLD_DEFAULT;
+ precopy_dirty_threshold = LIBXL_LM_DIRTY_THRESHOLD_DEFAULT,
+ memory_strategy = LIBXL_LM_MEMORY_DEFAULT;
static struct option opts[] = {
{"debug", 0, 0, 0x100},
{"live", 0, 0, 0x200},
{"precopy-iterations", 1, 0, 'i'},
{"precopy-threshold", 1, 0, 'd'},
+ {"postcopy", 0, 0, 0x400},
COMMON_LONG_OPTS
};
@@ -591,6 +637,9 @@ int main_migrate(int argc, char **argv)
case 0x200: /* --live */
/* ignored for compatibility with xm */
break;
+ case 0x400: /* --postcopy */
+ memory_strategy = LIBXL_LM_MEMORY_POSTCOPY;
+ break;
}
domid = find_domain(argv[optind]);
@@ -622,7 +671,7 @@ int main_migrate(int argc, char **argv)
}
migrate_domain(domid, rune, debug, config_filename, precopy_iterations,
- precopy_dirty_threshold);
+ precopy_dirty_threshold, memory_strategy);
return EXIT_SUCCESS;
}
diff --git a/tools/xl/xl_vmcontrol.c b/tools/xl/xl_vmcontrol.c
index 47ba9f3..62e09c1 100644
--- a/tools/xl/xl_vmcontrol.c
+++ b/tools/xl/xl_vmcontrol.c
@@ -655,6 +655,7 @@ int create_domain(struct domain_create *dom_info)
const char *config_source = NULL;
const char *restore_source = NULL;
int migrate_fd = dom_info->migrate_fd;
+ bool *postcopy_resumed = dom_info->postcopy_resumed;
bool config_in_json;
int i;
@@ -675,6 +676,9 @@ int create_domain(struct domain_create *dom_info)
int restoring = (restore_file || (migrate_fd >= 0));
+ if (postcopy_resumed)
+ *postcopy_resumed = false;
+
libxl_domain_config_init(&d_config);
if (restoring) {
@@ -882,8 +886,8 @@ start:
ret = libxl_domain_create_restore(ctx, &d_config,
&domid, restore_fd,
- send_back_fd, NULL, ¶ms,
- 0, autoconnect_console_how);
+ send_back_fd, postcopy_resumed,
+ ¶ms, 0, autoconnect_console_how);
libxl_domain_restore_params_dispose(¶ms);
--
2.7.4
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
https://lists.xen.org/xen-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |