[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[Xen-devel] [PATCH] RFC: initial libxl support for xenpaging


  • To: xen-devel@xxxxxxxxxxxxxxxxxxx
  • From: Olaf Hering <olaf@xxxxxxxxx>
  • Date: Thu, 16 Feb 2012 08:47:01 +0100
  • Delivery-date: Thu, 16 Feb 2012 07:47:59 +0000
  • List-id: Xen developer discussion <xen-devel.lists.xensource.com>

# HG changeset patch
# User Olaf Hering <olaf@xxxxxxxxx>
# Date 1329378376 -3600
# Node ID d368cf36d66c1e8df60bd0a4868c171b6a929edc
# Parent  bf0a7c205687857a8f8d3bd3841654ed61828193
RFC: initial libxl support for xenpaging

After the previous discussion about integration of xenpaging into xl/libxl it
was not clear to me wether my proposal as a whole or only parts of it were
rejected. So here is my current understanding of the comments I received.

Add initial support to libxl to start xenpaging for a HVM guest.
These are the considerations:
- a knob in domU.cfg is needed to start xenpaging
- xenpaging needs a target in KiB in "memory/target-tot_pages"
 -> the knob should be the target value in MiB: mem_target_paging=NUM
    if the value is 0, xenpaging is not started

- an cmdline interface is needed to adjust "memory/target-tot_pages" at runtime
 -> it was suggested to use 'xl mem-set' which should adjust both
    "memory/target" and "memory/target-tot_pages" at the same time
 -> maybe another cmdline interface should be 'xl mem-target-paging' which
    adjusts "memory/target-tot_pages", and maybe 'xl mem-target-balloon' which
    adjusts "memory/target"

- libxl starts xenpaging with at least two cmdline options which specifys the
  pagefile to use and the dom_id. An optional "xenpaging_file=path" specifies
  the pagefile name. Optional additional cmdline options for xenpaging can be
  specified with a domU.cfg option "xenpaging_extra=[ 'opt', 'opt' ]"

- currently maxmem= + memory= and mem_target_paging= can not be used because
  paging for a PoD guest is not implemented. This does not affect ballooning
  within the guest.

- I have some ideas to add runtime tuneables for xenpaging. Should there be a
  "xl xenpaging_ctrl tuneable_name value" command, or should it be done with a
  new tool xenpaging_ctrl? If the latter, the proposed 'xl mem-target-*'
  commands are not needed and this new helper could also adjust
  "memory/target-tot_pages".




The patch below is just a forward port of my previous version. It adds three
new config options, no xl or other changes:
mem_target_paging=<int>, the amount of memory in MiB for the guest
xenpaging_file=<string>, pagefile to use (optional)
xenpaging_extra=[ 'string', 'string' ], optional cmdline args for xenpaging

If 'mem_target_paging=' is not specified in config file, xenpaging will not 
start.
If 'xenpaging_file=' is not specified in config file,
/var/lib/xen/xenpaging/<domain_name>.<domaind_id>.paging is used.

Signed-off-by: Olaf Hering <olaf@xxxxxxxxx>

diff -r bf0a7c205687 -r d368cf36d66c tools/libxl/libxl.h
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -322,6 +322,7 @@ int libxl_init_build_info(libxl_ctx *ctx
 typedef int (*libxl_console_ready)(libxl_ctx *ctx, uint32_t domid, void *priv);
 int libxl_domain_create_new(libxl_ctx *ctx, libxl_domain_config *d_config, 
libxl_console_ready cb, void *priv, uint32_t *domid);
 int libxl_domain_create_restore(libxl_ctx *ctx, libxl_domain_config *d_config, 
libxl_console_ready cb, void *priv, uint32_t *domid, int restore_fd);
+int libxl__create_xenpaging(libxl_ctx *ctx, libxl_domain_config *d_config, 
uint32_t domid, char *path);
 void libxl_domain_config_dispose(libxl_domain_config *d_config);
 int libxl_domain_suspend(libxl_ctx *ctx, libxl_domain_suspend_info *info,
                           uint32_t domid, int fd);
diff -r bf0a7c205687 -r d368cf36d66c tools/libxl/libxl_create.c
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -457,6 +457,132 @@ static int store_libxl_entry(libxl__gc *
         libxl_device_model_version_to_string(b_info->device_model_version));
 }
 
+static int create_xenpaging(libxl__gc *gc, char *dom_name, uint32_t domid,
+                            libxl_domain_build_info *b_info)
+{
+    libxl__spawner_starting *buf_starting;
+    libxl_string_list xpe = b_info->u.hvm.xenpaging_extra;
+    int i, rc;
+    char *logfile;
+    int logfile_w, null, need_pagefile;
+    char *path, *dom_path, *value;
+    char **args;
+    char *xp;
+    flexarray_t *xp_args;
+    libxl_ctx *ctx = libxl__gc_owner(gc);
+
+    /* Nothing to do */
+    if (!b_info->tot_memkb)
+        return 0;
+
+    /* Check if paging is already enabled */
+    dom_path = libxl__xs_get_dompath(gc, domid);
+    if (!dom_path ) {
+        rc = ERROR_NOMEM;
+        goto out;
+    }
+    path = libxl__sprintf(gc, "%s/xenpaging/state", dom_path);
+    if (!path ) {
+        rc = ERROR_NOMEM;
+        goto out;
+    }
+    value = xs_read(ctx->xsh, XBT_NULL, path, NULL);
+    rc = value && strcmp(value, "running") == 0;
+    free(value);
+    /* Already running, nothing to do */
+    if (rc)
+        return 0;
+
+    /* Check if xenpaging is present */
+    xp = libxl__abs_path(gc, "xenpaging", libxl_libexec_path());
+    if (access(xp, X_OK) < 0) {
+        LIBXL__LOG_ERRNO(ctx, LIBXL__LOG_ERROR, "%s is not executable", xp);
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+    /* Initialise settings for child */
+    buf_starting = calloc(sizeof(*buf_starting), 1);
+    if (!buf_starting) {
+        rc = ERROR_NOMEM;
+        goto out;
+    }
+    buf_starting->domid = domid;
+    buf_starting->dom_path = dom_path;
+    buf_starting->pid_path = "xenpaging/xenpaging-pid";
+    buf_starting->for_spawn = calloc(sizeof(libxl__spawn_starting), 1);
+    if (!buf_starting->for_spawn) {
+        rc = ERROR_NOMEM;
+        goto out;
+    }
+
+    /* Assemble arguments for xenpaging */
+    xp_args = flexarray_make(8, 1);
+    if (!xp_args) {
+        rc = ERROR_NOMEM;
+        goto out;
+    }
+    /* Set executable path */
+    flexarray_append(xp_args, xp);
+
+    /* Search pagefile option in extra flags */
+    need_pagefile = 1;
+    for (i = 0; xpe && xpe[i]; i++) {
+        if (strcmp(xpe[i], "-f") == 0) {
+            need_pagefile = 0;
+            break;
+        }
+    }
+    /* Append pagefile option if its not in extra flags */
+    if (need_pagefile) {
+        flexarray_append(xp_args, "-f");
+        if (b_info->u.hvm.xenpaging_file)
+            flexarray_append(xp_args, b_info->u.hvm.xenpaging_file);
+        else
+            flexarray_append(xp_args, libxl__sprintf(gc, "%s/%s.%u.paging",
+                             libxl_xenpaging_dir_path(), dom_name, domid));
+    }
+
+    /* Set maximum amount of memory xenpaging should handle */
+    flexarray_append(xp_args, "-m");
+    flexarray_append(xp_args, libxl__sprintf(gc, "%d", b_info->max_memkb));
+
+    /* Append extra args for pager */
+    for (i = 0; xpe && xpe[i]; i++)
+        flexarray_append(xp_args, xpe[i]);
+    /* Append domid for pager */
+    flexarray_append(xp_args, "-d");
+    flexarray_append(xp_args, libxl__sprintf(gc, "%u", domid));
+    flexarray_append(xp_args, NULL);
+    args = (char **) flexarray_contents(xp_args);
+
+    /* Initialise logfile */
+    libxl_create_logfile(ctx, libxl__sprintf(gc, "xenpaging-%s", dom_name),
+                         &logfile);
+    logfile_w = open(logfile, O_WRONLY|O_CREAT, 0644);
+    free(logfile);
+    null = open("/dev/null", O_RDONLY);
+
+    /* Spawn the child */
+    rc = libxl__spawn_spawn(gc, buf_starting->for_spawn, "xenpaging",
+                            libxl_spawner_record_pid, buf_starting);
+    if (rc < 0)
+        goto out_close;
+    if (!rc) { /* inner child */
+        setsid();
+        /* Finally run xenpaging */
+        libxl__exec(null, logfile_w, logfile_w, xp, args);
+    }
+    rc = libxl__spawn_confirm_offspring_startup(gc, 5, "xenpaging", path,
+                                                "running", buf_starting);
+out_close:
+    close(null);
+    close(logfile_w);
+    free(args);
+out:
+    return rc;
+}
+
 static int do_domain_create(libxl__gc *gc, libxl_domain_config *d_config,
                             libxl_console_ready cb, void *priv,
                             uint32_t *domid_out, int restore_fd)
@@ -633,6 +759,16 @@ static int do_domain_create(libxl__gc *g
             goto error_out;
     }
 
+    if (d_config->c_info.type == LIBXL_DOMAIN_TYPE_HVM) {
+        ret = create_xenpaging(gc, d_config->c_info.name, domid,
+                              &d_config->b_info);
+        if (ret) {
+            LIBXL__LOG_ERRNO(ctx, LIBXL__LOG_ERROR,
+                      "Failed to start xenpaging.\n");
+            goto error_out;
+        }
+    }
+
     *domid_out = domid;
     return 0;
 
diff -r bf0a7c205687 -r d368cf36d66c tools/libxl/libxl_dom.c
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -127,7 +127,7 @@ int libxl__build_post(libxl__gc *gc, uin
     if (info->cpuid != NULL)
         libxl_cpuid_set(ctx, domid, info->cpuid);
 
-    ents = libxl__calloc(gc, 12 + (info->max_vcpus * 2) + 2, sizeof(char *));
+    ents = libxl__calloc(gc, 14 + (info->max_vcpus * 2) + 2, sizeof(char *));
     ents[0] = "memory/static-max";
     ents[1] = libxl__sprintf(gc, "%d", info->max_memkb);
     ents[2] = "memory/target";
@@ -140,9 +140,11 @@ int libxl__build_post(libxl__gc *gc, uin
     ents[9] = libxl__sprintf(gc, "%"PRIu32, state->store_port);
     ents[10] = "store/ring-ref";
     ents[11] = libxl__sprintf(gc, "%lu", state->store_mfn);
+    ents[12] = "memory/target-tot_pages";
+    ents[13] = libxl__sprintf(gc, "%d", info->tot_memkb);
     for (i = 0; i < info->max_vcpus; i++) {
-        ents[12+(i*2)]   = libxl__sprintf(gc, "cpu/%d/availability", i);
-        ents[12+(i*2)+1] = (i && info->cur_vcpus && !(info->cur_vcpus & (1 << 
i)))
+        ents[14+(i*2)]   = libxl__sprintf(gc, "cpu/%d/availability", i);
+        ents[14+(i*2)+1] = (i && info->cur_vcpus && !(info->cur_vcpus & (1 << 
i)))
                             ? "offline" : "online";
     }
 
diff -r bf0a7c205687 -r d368cf36d66c tools/libxl/libxl_memory.txt
--- a/tools/libxl/libxl_memory.txt
+++ b/tools/libxl/libxl_memory.txt
@@ -1,28 +1,28 @@
 /* === Domain memory breakdown: HVM guests ==================================
                            
-             +  +----------+                                     +            
-             |  | shadow   |                                     |            
-             |  +----------+                                     |            
-    overhead |  | extra    |                                     |            
-             |  | external |                                     |            
-             |  +----------+                          +          |            
-             |  | extra    |                          |          |            
-             |  | internal |                          |          |            
-             +  +----------+                +         |          | footprint  
-             |  | video    |                |         |          |            
-             |  +----------+  +    +        |         | xen      |            
-             |  |          |  |    |        | actual  | maximum  |            
-             |  |          |  |    |        | target  |          |            
-             |  | guest    |  |    | build  |         |          |            
-             |  |          |  |    | start  |         |          |            
-      static |  |          |  |    |        |         |          |            
-     maximum |  +----------+  |    +        +         +          +            
-             |  |          |  |                                               
-             |  |          |  |                                               
-             |  | balloon  |  | build                                         
-             |  |          |  | maximum                                       
-             |  |          |  |                                               
-             +  +----------+  +                                               
+             +  +----------+                                                 + 
           
+             |  | shadow   |                                                 | 
           
+             |  +----------+                                                 | 
           
+    overhead |  | extra    |                                                 | 
           
+             |  | external |                                                 | 
           
+             |  +----------+                                      +          | 
           
+             |  | extra    |                                      |          | 
           
+             |  | internal |                                      |          | 
           
+             +  +----------+                            +         |          | 
footprint  
+             |  | video    |                            |         |          | 
           
+             |  +----------+  +           +    +        |         | xen      | 
           
+             |  |          |  | guest OS  |    |        | actual  | maximum  | 
           
+             |  | guest    |  | real RAM  |    |        | target  |          | 
           
+             |  |          |  |           |    | build  |         |          | 
           
+             |  |----------+  +           |    | start  +         |          | 
           
+      static |  | paging   |              |    |                  |          | 
           
+     maximum |  +----------+              |    +                  +          + 
           
+             |  |          |              |                                    
           
+             |  |          |              |                                    
           
+             |  | balloon  |              | build                              
           
+             |  |          |              | maximum                            
           
+             |  |          |              |                                    
           
+             +  +----------+              +                                    
           
                 
                 
     extra internal = LIBXL_MAXMEM_CONSTANT
@@ -34,6 +34,17 @@
     libxl_domain_setmaxmem -> xen maximum
     libxl_set_memory_target -> actual target
                 
+    build maximum = RAM as seen inside the virtual machine
+                    Guest OS has to configure itself for this amount of memory
+                    Increase/Decrease via memory hotplug of virtual hardware.
+                    xl mem-max
+    build start   = RAM usable by the guest OS
+                    Guest OS sees balloon driver as memory hog
+                    Increase/Decrease via commands to the balloon driver
+                    xl mem-set
+    actual target = RAM allocated for the guest
+                    Increase/Decrease via commands to paging daemon
+                    xl mem-paging_target (?)
                 
  === Domain memory breakdown: PV guests ==================================
                 
diff -r bf0a7c205687 -r d368cf36d66c tools/libxl/libxl_types.idl
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -207,6 +207,7 @@ libxl_domain_build_info = Struct("domain
     ("tsc_mode",        libxl_tsc_mode),
     ("max_memkb",       uint32),
     ("target_memkb",    uint32),
+    ("tot_memkb",       uint32),
     ("video_memkb",     uint32),
     ("shadow_memkb",    uint32),
     ("disable_migrate", bool),
@@ -240,6 +241,8 @@ libxl_domain_build_info = Struct("domain
                                        ("vpt_align", bool),
                                        ("timer_mode", libxl_timer_mode),
                                        ("nested_hvm", bool),
+                                       ("xenpaging_file", string),
+                                       ("xenpaging_extra", libxl_string_list),
                                        ("no_incr_generationid", bool),
                                        ("nographic",        bool),
                                        ("stdvga",           bool),
diff -r bf0a7c205687 -r d368cf36d66c tools/libxl/xl_cmdimpl.c
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -508,6 +508,28 @@ vcpp_out:
     return rc;
 }
 
+static void parse_xenpaging_extra(const XLU_Config *config, libxl_string_list 
*xpe)
+{
+    XLU_ConfigList *args;
+    libxl_string_list l;
+    const char *val;
+    int nr_args = 0, i;
+
+    if (xlu_cfg_get_list(config, "xenpaging_extra", &args, &nr_args, 1))
+        return;
+
+    l = xmalloc(sizeof(char*)*(nr_args + 1));
+    if (!l)
+        return;
+
+    l[nr_args] = NULL;
+    for (i = 0; i < nr_args; i++) {
+        val = xlu_cfg_get_listitem(args, i);
+        l[i] = val ? strdup(val) : NULL;
+    }
+    *xpe = l;
+}
+
 static void parse_config_data(const char *configfile_filename_report,
                               const char *configfile_data,
                               int configfile_len,
@@ -629,6 +651,9 @@ static void parse_config_data(const char
     if (!xlu_cfg_get_long (config, "maxmem", &l, 0))
         b_info->max_memkb = l * 1024;
 
+    if (!xlu_cfg_get_long (config, "mem_target_paging", &l, 0))
+        b_info->tot_memkb = l * 1024;
+
     if (xlu_cfg_get_string (config, "on_poweroff", &buf, 0))
         buf = "destroy";
     if (!parse_action_on_shutdown(buf, &d_config->on_poweroff)) {
@@ -747,6 +772,10 @@ static void parse_config_data(const char
 
         if (!xlu_cfg_get_long (config, "nestedhvm", &l, 0))
             b_info->u.hvm.nested_hvm = l;
+
+        xlu_cfg_replace_string (config, "xenpaging_file", 
&b_info->u.hvm.xenpaging_file, 0);
+        parse_xenpaging_extra(config, &b_info->u.hvm.xenpaging_extra);
+
         break;
     case LIBXL_DOMAIN_TYPE_PV:
     {
diff -r bf0a7c205687 -r d368cf36d66c tools/xenpaging/xenpaging.c
--- a/tools/xenpaging/xenpaging.c
+++ b/tools/xenpaging/xenpaging.c
@@ -39,6 +39,8 @@
 
 /* Defines number of mfns a guest should use at a time, in KiB */
 #define WATCH_TARGETPAGES "memory/target-tot_pages"
+/* Defines path to startup confirmation */
+#define WATCH_STARTUP "xenpaging/state"
 static char *watch_target_tot_pages;
 static char *dom_path;
 static char watch_token[16];
@@ -845,6 +847,20 @@ static int evict_pages(struct xenpaging 
     return num;
 }
 
+static void xenpaging_confirm_startup(struct xenpaging *paging)
+{
+    xc_interface *xch = paging->xc_handle;
+    char *path;
+    int len;
+
+    len = asprintf(&path, "%s/%s", dom_path, WATCH_STARTUP);
+    if ( len < 0 )
+        return;
+    DPRINTF("confirming startup in %s\n", path);
+    xs_write(paging->xs_handle, XBT_NULL, path, "running", strlen("running"));
+    free(path);
+}
+
 int main(int argc, char *argv[])
 {
     struct sigaction act;
@@ -880,6 +896,9 @@ int main(int argc, char *argv[])
     /* listen for page-in events to stop pager */
     create_page_in_thread(paging);
 
+    /* Confirm startup to caller */
+    xenpaging_confirm_startup(paging);
+
     /* Swap pages in and out */
     while ( 1 )
     {

_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel


 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.