|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-API] [PATCH 1 of 4] CA-35397: add a simple cache of per-domain data in the ballooning daemon
# HG changeset patch
# User David Scott <dave.scott@xxxxxxxxxxxxx>
# Date 1259589188 0
# Node ID c90daafe028b57f2b597c4a8961355e23770ff93
# Parent 91eb1c3b26e7c10b249c0a2a2197587abb0c9a92
CA-35397: add a simple cache of per-domain data in the ballooning daemon.
The cache stores:
* values of xenstore keys written only by the ballooning daemon
* values of maxmem (set only by the ballooning daemon)
but not
* values of dynamic-min, dynamic-max, written by the toolstack.
Signed-off-by: David Scott <dave.scott@xxxxxxxxxxxxx>
diff -r 91eb1c3b26e7 -r c90daafe028b ocaml/xenops/squeeze_xen.ml
--- a/ocaml/xenops/squeeze_xen.ml Wed Nov 25 16:01:11 2009 -0800
+++ b/ocaml/xenops/squeeze_xen.ml Mon Nov 30 13:53:08 2009 +0000
@@ -25,13 +25,14 @@
let debug = Squeeze.debug
let error = Squeeze.error
-let initial_reservation_path dom_path = dom_path ^
"/memory/initial-reservation"
-let target_path dom_path = dom_path ^ "/memory/target"
-let dynamic_min_path dom_path = dom_path ^ "/memory/dynamic-min"
-let dynamic_max_path dom_path = dom_path ^ "/memory/dynamic-max"
-let feature_balloon_path dom_path = dom_path ^ "/control/feature-balloon"
-let memory_offset_path dom_path = dom_path ^ "/memory/memory-offset"
-let uncooperative_path dom_path = dom_path ^ "/memory/uncooperative"
+let _initial_reservation = "/memory/initial-reservation" (* immutable *)
+let _target = "/memory/target" (* immutable *)
+let _feature_balloon = "/control/feature-balloon" (* immutable *)
+let _memory_offset = "/memory/memory-offset" (* immutable *)
+let _uncooperative = "/memory/uncooperative" (* mutable: written
by us *)
+
+let _dynamic_min = "/memory/dynamic-min" (* mutable: written
by domain manager *)
+let _dynamic_max = "/memory/dynamic-max" (* mutable: written
by domain manager *)
let ( ** ) = Int64.mul
let ( +* ) = Int64.add
@@ -42,70 +43,156 @@
let low_mem_emergency_pool = 1L ** mib (** Same as xen commandline *)
-let exists xs path = try ignore(xs.Xs.read path); true with Xb.Noent -> false
-
-let xs_read xs path = try xs.Xs.read path with Xb.Noent as e -> begin debug
"xenstore-read %s returned ENOENT" path; raise e end
-
(** Return the extra amount we always add onto maxmem *)
-let xen_max_offset_kib di =
- let maxmem_mib = if di.Xc.hvm_guest then Memory.HVM.xen_max_offset_mib else
Memory.Linux.xen_max_offset_mib in
+let xen_max_offset_kib hvm =
+ let maxmem_mib = if hvm then Memory.HVM.xen_max_offset_mib else
Memory.Linux.xen_max_offset_mib in
Memory.kib_of_mib maxmem_mib
-let domain_setmaxmem_noexn xc domid target_kib =
- let di = Xc.domain_getinfo xc domid in
- let maxmem_kib = xen_max_offset_kib di +* target_kib in
- try
- let existing_maxmem_kib = Xc.pages_to_kib (Int64.of_nativeint
di.Xc.max_memory_pages) in
- if existing_maxmem_kib <> maxmem_kib then begin
- debug "Xc.domain_setmaxmem domid=%d target=%Ld max=%Ld" domid target_kib
maxmem_kib;
- Xc.domain_setmaxmem xc domid maxmem_kib
- end
- with e ->
- (* someone probably just destroyed the domain *)
- error "Xc.domain_setmaxmem domid=%d max=%Ld: %s" domid maxmem_kib
(Printexc.to_string e)
-
-let set_target_noexn xs dom_path target_kib =
- let path = target_path dom_path in
- try
- Xs.transaction xs
- (fun t ->
- (* make sure no-one deletes the tree *)
- let existing_target_kib = Int64.of_string (t.Xst.read path) in
- if existing_target_kib <> target_kib then begin
- debug "xenstore-write %s = %Ld" path target_kib;
- t.Xst.write path (Int64.to_string target_kib)
- end
- )
- with e ->
- (* someone probably just destroyed the domain *)
- error "xenstore-write %s = %Ld: %s" path target_kib (Printexc.to_string e)
+(** Cache of per-domain info, to avoid repeated per-domain queries *)
+module Domain = struct
+ type per_domain = { path: string;
+ hvm: bool;
+ mutable maxmem: int64;
+ keys: (string, string option)
Hashtbl.t }
+ type t = (int, per_domain) Hashtbl.t
+
+ let cache = Hashtbl.create 10
+ let get_per_domain (xc, xs) domid =
+ if Hashtbl.mem cache domid
+ then Hashtbl.find cache domid
+ else
+ let di = Xc.domain_getinfo xc domid in
+ let maxmem = Xc.pages_to_kib (Int64.of_nativeint di.Xc.max_memory_pages)
in
+ let d = { path = xs.Xs.getdomainpath domid;
+ hvm = di.Xc.hvm_guest;
+ maxmem = maxmem;
+ keys = Hashtbl.create 10 } in
+ Hashtbl.replace cache domid d;
+ d
+ let get_hvm cnx domid = (get_per_domain cnx domid).hvm
+ let get_maxmem cnx domid = (get_per_domain cnx domid).maxmem
+ let set_maxmem_noexn (xc, xs) domid m =
+ let per_domain = get_per_domain (xc, xs) domid in
+ if per_domain.maxmem <> m then begin
+ debug "Xc.domain_setmaxmem domid=%d max=%Ld" domid m;
+ try
+ Xc.domain_setmaxmem xc domid m;
+ per_domain.maxmem <- m
+ with e ->
+ error "Xc.domain_setmaxmem domid=%d max=%Ld: %s" domid m
(Printexc.to_string e)
+ end
-let set_uncooperative_noexn xs dom_path x =
- try
- if x
- then
- Xs.transaction xs
- (fun t ->
- (* make sure no-one deletes the tree *)
- ignore (t.Xst.read dom_path);
- t.Xst.write (uncooperative_path dom_path) ""
- )
- else xs.Xs.rm (uncooperative_path dom_path)
- with e ->
- error "set_uncooperative %s %b: %s" dom_path x (Printexc.to_string e)
+ (** Read a particular domain's key, using the cache *)
+ let read (xc, xs) domid key =
+ let per_domain = get_per_domain (xc, xs) domid in
+ let x =
+ if Hashtbl.mem per_domain.keys key
+ then Hashtbl.find per_domain.keys key
+ else begin
+ let x = (try Some (xs.Xs.read (per_domain.path ^ key))
+ with Xb.Noent -> None) in
+ Hashtbl.replace per_domain.keys key x;
+ x
+ end in
+ match x with
+ | Some y -> y
+ | None ->
+ debug "xenstore-read %s%s returned ENOENT" per_domain.path
key;
+ raise Xb.Noent
-let get_uncooperative_noexn xs dom_path = exists xs (uncooperative_path
dom_path)
+ (** Read a particular domain's key from xenstore, for when we believe the
cache is out of date *)
+ let read_nocache (xc, xs) domid key =
+ let per_domain = get_per_domain (xc, xs) domid in
+ if Hashtbl.mem per_domain.keys key then Hashtbl.remove per_domain.keys
key;
+ read (xc, xs) domid key
+ (** Write a new (key, value) pair into a domain's directory in xenstore.
Don't write anything
+ if the domain's directory doesn't exist. Don't throw exceptions. *)
-let set_memory_offset_noexn xs dom_path offset_kib =
- try
- Xs.transaction xs
- (fun t ->
- (* Make sure the domain still exists *)
- ignore (xs.Xs.read dom_path);
- t.Xst.write (memory_offset_path dom_path) (Int64.to_string offset_kib)
- )
- with e ->
- error "set_memory_offset_noexn %s %Ld: %s" dom_path offset_kib
(Printexc.to_string e)
+ let write_noexn (xc, xs) domid key value =
+ let per_domain = get_per_domain (xc, xs) domid in
+ if not(Hashtbl.mem per_domain.keys key) || Hashtbl.find per_domain.keys
key <> (Some value) then begin
+ try
+ Xs.transaction xs
+ (fun t ->
+ (* Fail if the directory has been deleted *)
+ ignore (xs.Xs.read per_domain.path);
+ t.Xst.write (per_domain.path ^ key) value
+ );
+ Hashtbl.replace per_domain.keys key (Some value);
+ with e ->
+ (* Log but don't throw an exception *)
+ error "xenstore-write %d %s = %s failed: %s" domid key value
(Printexc.to_string e)
+ end
+ (** Returns true if the key exists, false otherwise *)
+ let exists (xc, xs) domid key = try ignore(read (xc, xs) domid key); true
with Xb.Noent -> false
+ (** Delete the key. Don't throw exceptions. *)
+ let rm_noexn (xc, xs) domid key =
+ let per_domain = get_per_domain (xc, xs) domid in
+ Hashtbl.replace per_domain.keys key None;
+ try
+ xs.Xs.rm (per_domain.path ^ key)
+ with e ->
+ error "xenstore-rm %d %s: %s" domid key (Printexc.to_string e)
+
+ (** {High-level functions} *)
+
+ (** Set a domain's memory target. Don't throw an exception if the domain has
been destroyed. *)
+ let set_target_noexn cnx domid target_kib =
+ write_noexn cnx domid _target (Int64.to_string target_kib)
+
+ (** Get a domain's memory target. Throws Xb.Noent if the domain has been
destroyed *)
+ let get_target cnx domid =
+ Int64.of_string (read cnx domid _target)
+
+ (** Mark a domain as uncooperative. Don't throw an exception if the domain
has been destroyed. *)
+ let set_uncooperative_noexn cnx domid x =
+ if x
+ then write_noexn cnx domid _uncooperative ""
+ else rm_noexn cnx domid _uncooperative
+
+ (** Query a domain's uncooperative status. Throws Xb.Noent if the domain has
been destroyed *)
+ let get_uncooperative_noexn cnx domid = exists cnx domid _uncooperative
+
+ (** Set a domain's memory-offset. Don't throw an exception if the domain has
been destroyed *)
+ let set_memory_offset_noexn cnx domid offset_kib =
+ write_noexn cnx domid _memory_offset (Int64.to_string offset_kib)
+
+ (** Query a domain's memory-offset. Throws Xb.Noent if the domain has been
destroyed *)
+ let get_memory_offset cnx domid =
+ Int64.of_string (read cnx domid _memory_offset)
+
+ (** Set a domain's maxmem. Don't throw an exception if the domain has been
destroyed *)
+ let set_maxmem_noexn cnx domid target_kib =
+ let maxmem_kib = xen_max_offset_kib (get_hvm cnx domid) +* target_kib in
+ set_maxmem_noexn cnx domid maxmem_kib
+
+ (** Return true if feature_balloon has been advertised *)
+ let get_feature_balloon (xc, xs) domid =
+ (* Cache the presence of this key when it appears but always read
through when it hasn't yet *)
+ let per_domain = get_per_domain (xc, xs) domid in
+ if Hashtbl.mem per_domain.keys _feature_balloon && Hashtbl.find
per_domain.keys _feature_balloon <> None
+ then true
+ else (try ignore (read_nocache (xc, xs) domid _feature_balloon); true
with Xb.Noent -> false)
+
+ (** Query a domain's initial reservation. Throws Xb.Noent if the domain has
been destroyed *)
+ let get_initial_reservation cnx domid =
+ Int64.of_string (read cnx domid _initial_reservation)
+
+ (** Query a domain's dynamic_min. Throws Xb.Noent if the domain has been
destroyed *)
+ let get_dynamic_min cnx domid =
+ Int64.of_string (read_nocache cnx domid _dynamic_min)
+
+ (** Query a domain's dynamic_max. Throws Xb.Noent if the domain has been
destroyed *)
+ let get_dynamic_max cnx domid =
+ Int64.of_string (read_nocache cnx domid _dynamic_max)
+
+ (** Expire old domain information from the cache *)
+ let gc live_domids =
+ let cached_domids = Hashtbl.fold (fun d _ ds -> d::ds) cache [] in
+ let to_remove = set_difference cached_domids live_domids in
+ List.iter (Hashtbl.remove cache) to_remove
+end
+
(** Record when the domain was last co-operative *)
@@ -127,13 +214,12 @@
) host.Squeeze.domains
(** Update all the flags in xenstore *)
-let update_cooperative_flags xs =
+let update_cooperative_flags cnx =
let now = Unix.gettimeofday () in
Hashtbl.iter (fun domid last_time ->
- let dom_path = xs.Xs.getdomainpath domid in
- let old_value = get_uncooperative_noexn xs dom_path in
+ let old_value = Domain.get_uncooperative_noexn cnx domid in
let new_value = now -. last_time > 20. in
- if old_value <> new_value then set_uncooperative_noexn xs
dom_path new_value
+ if old_value <> new_value then Domain.set_uncooperative_noexn
cnx domid new_value
)
when_domain_was_last_cooperative
@@ -171,30 +257,30 @@
and total_pages_kib = Xc.pages_to_kib (Int64.of_nativeint
physinfo.Xc.total_pages) in
let free_mem_kib = free_pages_kib +* scrub_pages_kib in
+ let cnx = xc, xs in
let domains = List.concat
(List.map
(fun di ->
try
- let path = xs.Xs.getdomainpath
di.Xc.domid in
let memory_actual_kib = Xc.pages_to_kib
(Int64.of_nativeint di.Xc.total_memory_pages) in
(* dom0 is special for some reason *)
let memory_max_kib = if di.Xc.domid = 0
then 0L else Xc.pages_to_kib (Int64.of_nativeint di.Xc.max_memory_pages) in
(* Misc other stuff appears in
max_memory_pages *)
- let memory_max_kib = max 0L
(memory_max_kib -* (xen_max_offset_kib di)) in
- let can_balloon = exists xs
(feature_balloon_path path) in
+ let memory_max_kib = max 0L
(memory_max_kib -* (xen_max_offset_kib di.Xc.hvm_guest)) in
+ let can_balloon =
Domain.get_feature_balloon cnx di.Xc.domid in
(* Once the domain tells us it can
balloon, we assume it's not currently ballooning and
record the offset between
memory_actual and target. We assume this is constant over the
lifetime of the domain. *)
- let offset_kib =
+ let offset_kib : int64 =
if not can_balloon then 0L
else begin
try
- Int64.of_string (xs_read xs
(memory_offset_path path))
+ Domain.get_memory_offset cnx
di.Xc.domid
with Xb.Noent ->
- let target_kib = Int64.of_string
(xs_read xs (target_path path)) in
+ let target_kib =
Domain.get_target cnx di.Xc.domid in
let offset_kib =
memory_actual_kib -* target_kib in
debug "domid %d just exposed
feature-balloon; calibrating total_pages offset = %Ld KiB" di.Xc.domid
offset_kib;
- set_memory_offset_noexn xs path
offset_kib;
+ Domain.set_memory_offset_noexn
cnx di.Xc.domid offset_kib;
offset_kib
end in
let memory_actual_kib =
memory_actual_kib -* offset_kib in
@@ -218,7 +304,7 @@
(* If the domain has yet to expose it's
feature-balloon flag then we assume it is using at least its
"initial-reservation". *)
if not can_balloon then begin
- let initial_reservation_kib =
Int64.of_string (xs_read xs (initial_reservation_path path)) in
+ let initial_reservation_kib =
Domain.get_initial_reservation cnx di.Xc.domid in
(* memory_actual_kib is memory
which xen has accounted to this domain. We bump this up to
the "initial-reservation"
and compute how much memory to subtract from the host's free
memory *)
@@ -233,13 +319,13 @@
} ]
end else begin
- let target_kib =
Int64.of_string (xs_read xs (target_path path)) in
+ let target_kib =
Domain.get_target cnx di.Xc.domid in
(* min and max are written
separately; if we notice they *)
(* are missing set them both to
the target for now. *)
let min_kib, max_kib =
try
- Int64.of_string
(xs_read xs (dynamic_min_path path)),
- Int64.of_string
(xs_read xs (dynamic_max_path path))
+ Domain.get_dynamic_min cnx
di.Xc.domid,
+ Domain.get_dynamic_max cnx
di.Xc.domid
with _ ->
target_kib, target_kib
in
@@ -252,7 +338,7 @@
end
with
| Xb.Noent ->
- (* useful debug message is printed by the
xs_read function *)
+ (* useful debug message is printed by the
Domain.read* functions *)
[]
| e ->
debug "Skipping domid %d: %s"
@@ -270,7 +356,9 @@
let host = Squeeze.make_host ~domains ~free_mem_kib:(Int64.sub
free_mem_kib !reserved_kib) in
update_cooperative_table host;
- update_cooperative_flags xs;
+ update_cooperative_flags cnx;
+
+ Domain.gc (List.map (fun di -> di.Xc.domid) domain_infolist);
Printf.sprintf "F%Ld S%Ld R%Ld T%Ld" free_pages_kib scrub_pages_kib
!reserved_kib total_pages_kib, host
@@ -278,18 +366,17 @@
let execute_action ~xc ~xs action =
try
let domid = action.Squeeze.action_domid in
- let path = xs.Xs.getdomainpath domid in
let target_kib = action.Squeeze.new_target_kib in
if target_kib < 0L
then failwith "Proposed target is negative (domid %d): %Ld"
domid target_kib;
- let di = Xc.domain_getinfo xc domid in
- let memory_max_kib = Xc.pages_to_kib (Int64.of_nativeint
di.Xc.max_memory_pages) in
+ let cnx = (xc, xs) in
+ let memory_max_kib = Domain.get_maxmem cnx domid in
if target_kib > memory_max_kib then begin
- domain_setmaxmem_noexn xc domid target_kib;
- set_target_noexn xs path target_kib;
+ Domain.set_maxmem_noexn cnx domid target_kib;
+ Domain.set_target_noexn cnx domid target_kib;
end else begin
- set_target_noexn xs path target_kib;
- domain_setmaxmem_noexn xc domid target_kib;
+ Domain.set_target_noexn cnx domid target_kib;
+ Domain.set_maxmem_noexn cnx domid target_kib;
end
with e ->
debug "Failed to reset balloon target (domid: %d) (target:
%Ld): %s"
1 file changed, 176 insertions(+), 89 deletions(-)
ocaml/xenops/squeeze_xen.ml | 265 ++++++++++++++++++++++++++++---------------
Attachment:
xen-api.hg-4.patch _______________________________________________ xen-api mailing list xen-api@xxxxxxxxxxxxxxxxxxx http://lists.xensource.com/mailman/listinfo/xen-api
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |