[PATCH v5 1/7] xen: introduce hardware domain create flag

[Jason Andryuk <jason.andryuk@xxxxxxx>]

From: "Daniel P. Smith" <dpsmith@xxxxxxxxxxxxxxxxxxxx>

Add and use a new internal create domain flag to specify the hardware
domain.  This removes the hardcoding of domid 0 as the hardware domain.

This allows more flexibility with domain creation.

The assignment of d->cdf is moved later so CDF_hardware is added for the
late_hwdom case.  Also old_hwdom has the flag removed to reflect the
change.

Signed-off-by: Daniel P. Smith <dpsmith@xxxxxxxxxxxxxxxxxxxx>
Signed-off-by: Jason Andryuk <jason.andryuk@xxxxxxx>
Reviewed-by: Jan Beulich <jbeulich@xxxxxxxx>
---
v5:
Restore CDF_hardware to old_hwdom on error
Add Jan's R-b
Rebase bd->domid addition

v4:
Move d->cdf assignment later
Remove Jan's R-b

v3:
Or-in CDF_hardware for late hwdom case
Add Jan's R-b

v2:
() around binary &
Only allow late_hwdom for dom0
---
 xen/arch/arm/domain_build.c |  2 +-
 xen/arch/x86/setup.c        |  3 ++-
 xen/common/domain.c         | 19 +++++++++++++++----
 xen/include/xen/domain.h    |  2 ++
 4 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/xen/arch/arm/domain_build.c b/xen/arch/arm/domain_build.c
index 634333cdde..b8f282ff10 100644
--- a/xen/arch/arm/domain_build.c
+++ b/xen/arch/arm/domain_build.c
@@ -2369,7 +2369,7 @@ void __init create_dom0(void)
         .max_maptrack_frames = -1,
         .grant_opts = XEN_DOMCTL_GRANT_version(opt_gnttab_max_version),
     };
-    unsigned int flags = CDF_privileged;
+    unsigned int flags = CDF_privileged | CDF_hardware;
     int rc;
 
     /* The vGIC for DOM0 is exactly emulating the hardware GIC */
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index 3c257f0bad..24b36c1a59 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -1011,7 +1011,8 @@ static struct domain *__init create_dom0(struct boot_info 
*bi)
 
     /* Create initial domain.  Not d0 for pvshim. */
     bd->domid = get_initial_domain_id();
-    d = domain_create(bd->domid, &dom0_cfg, pv_shim ? 0 : CDF_privileged);
+    d = domain_create(bd->domid, &dom0_cfg,
+                      pv_shim ? 0 : CDF_privileged | CDF_hardware);
     if ( IS_ERR(d) )
         panic("Error creating d%u: %ld\n", bd->domid, PTR_ERR(d));
 
diff --git a/xen/common/domain.c b/xen/common/domain.c
index 585fd726a9..abf1969e60 100644
--- a/xen/common/domain.c
+++ b/xen/common/domain.c
@@ -804,9 +804,6 @@ struct domain *domain_create(domid_t domid,
     d->domain_id = domid;
     d->unique_id = get_unique_id();
 
-    /* Holding CDF_* internal flags. */
-    d->cdf = flags;
-
     /* Debug sanity. */
     ASSERT(is_system_domain(d) ? config == NULL : config != NULL);
 
@@ -820,15 +817,25 @@ struct domain *domain_create(domid_t domid,
     d->is_privileged = flags & CDF_privileged;
 
     /* Sort out our idea of is_hardware_domain(). */
-    if ( domid == 0 || domid == hardware_domid )
+    if ( (flags & CDF_hardware) || domid == hardware_domid )
     {
         if ( hardware_domid < 0 || hardware_domid >= DOMID_FIRST_RESERVED )
             panic("The value of hardware_dom must be a valid domain ID\n");
 
+        /* late_hwdom is only allowed for dom0. */
+        if ( hardware_domain && hardware_domain->domain_id )
+            return ERR_PTR(-EINVAL);
+
         old_hwdom = hardware_domain;
         hardware_domain = d;
+        flags |= CDF_hardware;
+        if ( old_hwdom )
+            old_hwdom->cdf &= ~CDF_hardware;
     }
 
+    /* Holding CDF_* internal flags. */
+    d->cdf = flags;
+
     TRACE_TIME(TRC_DOM0_DOM_ADD, d->domain_id);
 
     lock_profile_register_struct(LOCKPROF_TYPE_PERDOM, d, domid);
@@ -973,7 +980,11 @@ struct domain *domain_create(domid_t domid,
 
     d->is_dying = DOMDYING_dead;
     if ( hardware_domain == d )
+    {
+        if ( old_hwdom )
+            old_hwdom->cdf |= CDF_hardware;
         hardware_domain = old_hwdom;
+    }
     atomic_set(&d->refcnt, DOMAIN_DESTROYED);
 
     sched_destroy_domain(d);
diff --git a/xen/include/xen/domain.h b/xen/include/xen/domain.h
index a34daa7d10..e10baf2615 100644
--- a/xen/include/xen/domain.h
+++ b/xen/include/xen/domain.h
@@ -53,6 +53,8 @@ domid_t get_initial_domain_id(void);
 #else
 #define CDF_staticmem            0
 #endif
+/* This is the hardware domain.  Only 1 allowed. */
+#define CDF_hardware             (1U << 3)
 
 #define is_domain_direct_mapped(d) ((d)->cdf & CDF_directmap)
 #define is_domain_using_staticmem(d) ((d)->cdf & CDF_staticmem)
-- 
2.49.0