[Xen-devel] [PATCH] [RFC]: xenpaging: add command to to flush qemu buffer cache via VMCALL

[Olaf Hering <olaf@xxxxxxxxx>]

# HG changeset patch
# User Olaf Hering <olaf@xxxxxxxxx>
# Date 1330272880 -3600
# Node ID e17afed13a35989bdf422b75d44843bfd2fda0f4
# Parent  3ace34b89ebe3bdd6afb0bca85b0c1270f99c1bb
[RFC]: xenpaging: add command to to flush qemu buffer cache via VMCALL

Is the approach taken in this patch ok?
With the debug output in my patch I get output like pasted below. It
means that xenpaging set qemu_mapcache_invalidate very often before a
VMCALL occoured.

What triggers a VMCALL anyway? It looks like its not something one can
depend on. From xenpagings PoV its not required that the command reaches
qemu right away, but it should be processed "soon".
Could send_invalidate_req() be called from another exit reason (or from
another place perhaps)?

Olaf


....
(XEN) irq.c:350: Dom1 callback via changed to PCI INTx Dev 0x03 IntA
(XEN) hvm_do_hypercall: 0 1 0
(XEN) io.c:155:d1 send_invalidate_req
(XEN) hvm_do_hypercall: 0 1 0
(XEN) io.c:155:d1 send_invalidate_req
(XEN) hvm_do_hypercall: 3049 0 0
(XEN) io.c:155:d1 send_invalidate_req
(XEN) hvm_do_hypercall: 4562 0 0
(XEN) io.c:155:d1 send_invalidate_req
(XEN) hvm_do_hypercall: 6 0 0
(XEN) io.c:155:d1 send_invalidate_req
(XEN) hvm_do_hypercall: 6 0 0
(XEN) io.c:155:d1 send_invalidate_req
(XEN) hvm_do_hypercall: 14 0 0
(XEN) io.c:155:d1 send_invalidate_req
(XEN) hvm_do_hypercall: 598 0 0
(XEN) io.c:155:d1 send_invalidate_req
(XEN) hvm_do_hypercall: 1 0 0
(XEN) io.c:155:d1 send_invalidate_req
(XEN) hvm_do_hypercall: 3 0 0
(XEN) io.c:155:d1 send_invalidate_req
(XEN) hvm_do_hypercall: 161 0 0
(XEN) io.c:155:d1 send_invalidate_req
(XEN) hvm_do_hypercall: 1 0 0
(XEN) io.c:155:d1 send_invalidate_req
(XEN) hvm_do_hypercall: 4860 0 0
(XEN) io.c:155:d1 send_invalidate_req
(XEN) hvm_do_hypercall: 1 0 0
(XEN) io.c:155:d1 send_invalidate_req
(XEN) hvm_do_hypercall: 799 0 0
(XEN) io.c:155:d1 send_invalidate_req
(XEN) hvm_do_hypercall: 5 0 0
(XEN) io.c:155:d1 send_invalidate_req
....



....
qemu will just keep mapping pages and not release them, which causes
problems for the memory pager (since the page is mapped, it won't get
paged out).  When the pager has trouble finding a page to page out, it
asks qemu to flush its buffer, which releases all the page mappings.
This makes it possible to find  pages to swap out agian.

This change removes the code which writes "flush-cache" to xenstore
because the corresponding change for qemu-xen-traditional was never
applied, and it also will not work with qemu-xen-upstream.

Instead of sending commands via xenstore, use the existing
IOREQ_TYPE_INVALIDATE command for HVM guests. A simple memory_op just
sets qemu_mapcache_invalidate flag which in turn causes the hypervisor
to send the actual IOREQ_TYPE_INVALIDATE during next VMEXIT_VMCALL.

Signed-off-by: Olaf Hering <olaf@xxxxxxxxx>

diff -r 3ace34b89ebe -r e17afed13a35 tools/libxc/xc_mem_paging.c
--- a/tools/libxc/xc_mem_paging.c
+++ b/tools/libxc/xc_mem_paging.c
@@ -93,6 +93,14 @@ int xc_mem_paging_load(xc_interface *xch
     return rc;
 }
 
+int xc_mem_paging_flush(xc_interface *xch, domid_t domain_id)
+{
+    return xc_mem_event_memop(xch, domain_id,
+                                XENMEM_paging_op_ioemu_flush,
+                                XENMEM_paging_op,
+                                0, NULL);
+}
+
 
 /*
  * Local variables:
diff -r 3ace34b89ebe -r e17afed13a35 tools/libxc/xenctrl.h
--- a/tools/libxc/xenctrl.h
+++ b/tools/libxc/xenctrl.h
@@ -1902,6 +1902,7 @@ int xc_mem_paging_evict(xc_interface *xc
 int xc_mem_paging_prep(xc_interface *xch, domid_t domain_id, unsigned long 
gfn);
 int xc_mem_paging_load(xc_interface *xch, domid_t domain_id, 
                         unsigned long gfn, void *buffer);
+int xc_mem_paging_flush(xc_interface *xch, domid_t domain_id);
 
 int xc_mem_access_enable(xc_interface *xch, domid_t domain_id,
                         void *shared_page, void *ring_page);
diff -r 3ace34b89ebe -r e17afed13a35 tools/xenpaging/xenpaging.c
--- a/tools/xenpaging/xenpaging.c
+++ b/tools/xenpaging/xenpaging.c
@@ -62,15 +62,11 @@ static void close_handler(int sig)
     unlink_pagefile();
 }
 
-static void xenpaging_mem_paging_flush_ioemu_cache(struct xenpaging *paging)
+static int xenpaging_mem_paging_flush_ioemu_cache(struct xenpaging *paging)
 {
-    struct xs_handle *xsh = paging->xs_handle;
+    xc_interface *xch = paging->xc_handle;
     domid_t domain_id = paging->mem_event.domain_id;
-    char path[80];
-
-    sprintf(path, "/local/domain/0/device-model/%u/command", domain_id);
-
-    xs_write(xsh, XBT_NULL, path, "flush-cache", strlen("flush-cache")); 
+    return xc_mem_paging_flush(xch, domain_id);
 }
 
 static int xenpaging_wait_for_event_or_timeout(struct xenpaging *paging)
@@ -768,7 +764,12 @@ static int evict_victim(struct xenpaging
             if ( num_paged_out != paging->num_paged_out )
             {
                 DPRINTF("Flushing qemu cache\n");
-                xenpaging_mem_paging_flush_ioemu_cache(paging);
+                ret = xenpaging_mem_paging_flush_ioemu_cache(paging);
+                if ( ret < 0 )
+                {
+                    PERROR("Flushing qemu cache\n");
+                    goto out;
+                }
                 num_paged_out = paging->num_paged_out;
             }
             ret = ENOSPC;
diff -r 3ace34b89ebe -r e17afed13a35 xen/arch/x86/hvm/hvm.c
--- a/xen/arch/x86/hvm/hvm.c
+++ b/xen/arch/x86/hvm/hvm.c
@@ -2997,6 +2997,7 @@ static long hvm_memory_op(int cmd, XEN_G
         return -ENOSYS;
     case XENMEM_decrease_reservation:
         rc = do_memory_op(cmd, arg);
+        current->domain->arch.hvm_domain.inv_hvm++;
         current->domain->arch.hvm_domain.qemu_mapcache_invalidate = 1;
         return rc;
     }
@@ -3088,6 +3089,7 @@ static long hvm_memory_op_compat32(int c
         return -ENOSYS;
     case XENMEM_decrease_reservation:
         rc = compat_memory_op(cmd, arg);
+        current->domain->arch.hvm_domain.inv_hvm32++;
         current->domain->arch.hvm_domain.qemu_mapcache_invalidate = 1;
         return rc;
     }
@@ -3246,7 +3248,17 @@ int hvm_do_hypercall(struct cpu_user_reg
     if ( unlikely(curr->domain->arch.hvm_domain.qemu_mapcache_invalidate) &&
          test_and_clear_bool(curr->domain->arch.hvm_domain.
                              qemu_mapcache_invalidate) )
+    {
+        printk("%s: %u %u %u\n",
+            __func__,
+            curr->domain->arch.hvm_domain.inv_paging,
+            curr->domain->arch.hvm_domain.inv_hvm,
+            curr->domain->arch.hvm_domain.inv_hvm32);
+        curr->domain->arch.hvm_domain.inv_paging = 
+        curr->domain->arch.hvm_domain.inv_hvm = 
+        curr->domain->arch.hvm_domain.inv_hvm32 = 0;
         return HVM_HCALL_invalidate;
+    }
 
     return HVM_HCALL_completed;
 }
diff -r 3ace34b89ebe -r e17afed13a35 xen/arch/x86/hvm/io.c
--- a/xen/arch/x86/hvm/io.c
+++ b/xen/arch/x86/hvm/io.c
@@ -152,6 +152,7 @@ void send_invalidate_req(void)
     struct vcpu *v = current;
     ioreq_t *p = get_ioreq(v);
 
+    gdprintk(XENLOG_DEBUG,"%s\n", __func__);
     if ( p->state != STATE_IOREQ_NONE )
     {
         gdprintk(XENLOG_ERR, "WARNING: send invalidate req with something "
diff -r 3ace34b89ebe -r e17afed13a35 xen/arch/x86/mm/mem_paging.c
--- a/xen/arch/x86/mm/mem_paging.c
+++ b/xen/arch/x86/mm/mem_paging.c
@@ -53,6 +53,15 @@ int mem_paging_memop(struct domain *d, x
     }
     break;
 
+    case XENMEM_paging_op_ioemu_flush:
+    {
+        /* Flush qemu cache on next VMEXIT_VMCALL */
+        d->arch.hvm_domain.inv_paging++;
+        d->arch.hvm_domain.qemu_mapcache_invalidate = 1;
+        return 0;
+    }
+    break;
+
     default:
         return -ENOSYS;
         break;
diff -r 3ace34b89ebe -r e17afed13a35 xen/include/asm-x86/hvm/domain.h
--- a/xen/include/asm-x86/hvm/domain.h
+++ b/xen/include/asm-x86/hvm/domain.h
@@ -95,6 +95,9 @@ struct hvm_domain {
     bool_t                 mem_sharing_enabled;
     bool_t                 qemu_mapcache_invalidate;
     bool_t                 is_s3_suspended;
+    unsigned int inv_paging;
+    unsigned int inv_hvm;
+    unsigned int inv_hvm32;
 
     union {
         struct vmx_domain vmx;
diff -r 3ace34b89ebe -r e17afed13a35 xen/include/public/memory.h
--- a/xen/include/public/memory.h
+++ b/xen/include/public/memory.h
@@ -322,6 +322,7 @@ typedef struct xen_pod_target xen_pod_ta
 #define XENMEM_paging_op_nominate           0
 #define XENMEM_paging_op_evict              1
 #define XENMEM_paging_op_prep               2
+#define XENMEM_paging_op_ioemu_flush        3
 
 #define XENMEM_access_op                    21
 #define XENMEM_access_op_resume             0

_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
http://lists.xen.org/xen-devel