# HG changeset patch
# User Juergen Gross <juergen.gross@xxxxxxxxxxxxxx>
# Date 1332404507 -3600
# Node ID 932dc3987e3dac816d51e49861ab2ff21e8228b5
# Parent  12d8bb0e02edf68b4a0526a2989a660218535fcb
Avoid vcpu migration of paused vcpus

Currently offlining a cpu will migrate all vcpus which were active on that
cpu to another one, possibly breaking existing cpu affinities.

In case of an ACPI state change the cpus are taken offline and online later
(if not poweroff) while all domains are paused. There is no reason to
migrate the vcpus during offlining the cpus, as the cpus will be available
again when the domains are being unpaused.

This patch defers the migration check in case of paused vcpus or domains
by adding vcpu_arouse() to wake up a vcpu and check whether it must be
migrated to another cpu.

Signed-off-by: Juergen Gross <juergen.gross@xxxxxxxxxxxxxx>

diff -r 12d8bb0e02ed -r 932dc3987e3d xen/common/domain.c
--- a/xen/common/domain.c	Thu Mar 22 09:21:44 2012 +0100
+++ b/xen/common/domain.c	Thu Mar 22 09:21:47 2012 +0100
@@ -734,7 +734,7 @@ void vcpu_unpause(struct vcpu *v)
 void vcpu_unpause(struct vcpu *v)
 {
     if ( atomic_dec_and_test(&v->pause_count) )
-        vcpu_wake(v);
+        vcpu_arouse(v);
 }
 
 void domain_pause(struct domain *d)
@@ -755,7 +755,7 @@ void domain_unpause(struct domain *d)
 
     if ( atomic_dec_and_test(&d->pause_count) )
         for_each_vcpu( d, v )
-            vcpu_wake(v);
+            vcpu_arouse(v);
 }
 
 void domain_pause_by_systemcontroller(struct domain *d)
diff -r 12d8bb0e02ed -r 932dc3987e3d xen/common/schedule.c
--- a/xen/common/schedule.c	Thu Mar 22 09:21:44 2012 +0100
+++ b/xen/common/schedule.c	Thu Mar 22 09:21:47 2012 +0100
@@ -525,6 +525,47 @@ void vcpu_force_reschedule(struct vcpu *
     }
 }
 
+static int vcpu_chk_affinity(struct vcpu *v, unsigned int cpu)
+{
+    cpumask_t online_affinity;
+    struct cpupool *c;
+
+    c = v->domain->cpupool;
+    cpumask_and(&online_affinity, v->cpu_affinity, c->cpu_valid);
+    if ( cpumask_empty(&online_affinity) &&
+         cpumask_test_cpu(cpu, v->cpu_affinity) )
+    {
+        printk("Breaking vcpu affinity for domain %d vcpu %d\n",
+                v->domain->domain_id, v->vcpu_id);
+        cpumask_setall(v->cpu_affinity);
+    }
+
+    return ( !cpumask_test_cpu(cpu, c->cpu_valid) && (v->processor == cpu) );
+}
+
+void vcpu_arouse(struct vcpu *v)
+{
+    unsigned long flags;
+
+    if ( atomic_read(&v->pause_count) ||
+         atomic_read(&v->domain->pause_count) )
+        return;
+
+    vcpu_schedule_lock_irqsave(v, flags);
+
+    if ( unlikely(vcpu_chk_affinity(v, v->processor) && (v != current)) )
+    {
+        set_bit(_VPF_migrating, &v->pause_flags);
+        vcpu_schedule_unlock_irqrestore(v, flags);
+        vcpu_migrate(v);
+        return;
+    }
+
+    vcpu_schedule_unlock_irqrestore(v, flags);
+
+    vcpu_wake(v);
+}
+
 /*
  * This function is used by cpu_hotplug code from stop_machine context
  * and from cpupools to switch schedulers on a cpu.
@@ -534,7 +575,6 @@ int cpu_disable_scheduler(unsigned int c
     struct domain *d;
     struct vcpu *v;
     struct cpupool *c;
-    cpumask_t online_affinity;
     int    ret = 0;
 
     c = per_cpu(cpupool, cpu);
@@ -547,34 +587,33 @@ int cpu_disable_scheduler(unsigned int c
         {
             vcpu_schedule_lock_irq(v);
 
-            cpumask_and(&online_affinity, v->cpu_affinity, c->cpu_valid);
-            if ( cpumask_empty(&online_affinity) &&
-                 cpumask_test_cpu(cpu, v->cpu_affinity) )
+            if ( likely(!atomic_read(&v->pause_count) &&
+                        !atomic_read(&d->pause_count)) )
             {
-                printk("Breaking vcpu affinity for domain %d vcpu %d\n",
-                        v->domain->domain_id, v->vcpu_id);
-                cpumask_setall(v->cpu_affinity);
-            }
+                if ( vcpu_chk_affinity(v, cpu) )
+                {
+                    set_bit(_VPF_migrating, &v->pause_flags);
+                    vcpu_schedule_unlock_irq(v);
+                    vcpu_sleep_nosync(v);
+                    vcpu_migrate(v);
+                }
+                else
+                {
+                    vcpu_schedule_unlock_irq(v);
+                }
 
-            if ( v->processor == cpu )
-            {
-                set_bit(_VPF_migrating, &v->pause_flags);
-                vcpu_schedule_unlock_irq(v);
-                vcpu_sleep_nosync(v);
-                vcpu_migrate(v);
+                /*
+                 * A vcpu active in the hypervisor will not be migratable.
+                 * The caller should try again after releasing and reaquiring
+                 * all locks.
+                 */
+                if ( v->processor == cpu )
+                    ret = -EAGAIN;
             }
             else
             {
                 vcpu_schedule_unlock_irq(v);
             }
-
-            /*
-             * A vcpu active in the hypervisor will not be migratable.
-             * The caller should try again after releasing and reaquiring
-             * all locks.
-             */
-            if ( v->processor == cpu )
-                ret = -EAGAIN;
         }
 
         domain_update_node_affinity(d);
diff -r 12d8bb0e02ed -r 932dc3987e3d xen/include/xen/sched.h
--- a/xen/include/xen/sched.h	Thu Mar 22 09:21:44 2012 +0100
+++ b/xen/include/xen/sched.h	Thu Mar 22 09:21:47 2012 +0100
@@ -523,6 +523,7 @@ void sched_tick_suspend(void);
 void sched_tick_suspend(void);
 void sched_tick_resume(void);
 void vcpu_wake(struct vcpu *d);
+void vcpu_arouse(struct vcpu *d);    /* like vcpu_wake, check migrate */
 void vcpu_sleep_nosync(struct vcpu *d);
 void vcpu_sleep_sync(struct vcpu *d);