# HG changeset patch
# User David Scott <dave.scott@xxxxxxxxxxxxx>
# Date 1282565147 -3600
# Node ID 23fa063db91d4eae0179fb29179002a85a75cf31
# Parent  acfa0e8405cb12be60262655c8f21d97284e1a3b
CP-1883: Allow raw VDI import to receive an export-like chunked encoding.

The raw VDI import HTTP handler currently assumes the whole disk is being uploaded at once. Instead we add a 'chunked' mode which allows arbitrary-sized disk blocks to be selectively uploaded.

Signed-off-by: David Scott <dave.scott@xxxxxxxxxxxxx>

diff -r acfa0e8405cb -r 23fa063db91d ocaml/xapi/OMakefile
--- a/ocaml/xapi/OMakefile	Mon Aug 23 13:03:21 2010 +0100
+++ b/ocaml/xapi/OMakefile	Mon Aug 23 13:05:47 2010 +0100
@@ -73,6 +73,7 @@
 	console \
 	xen_helpers \
 	importexport \
+	sparse_encoding \
 	create_storage \
 	create_networks \
 	xapi_fist \
diff -r acfa0e8405cb -r 23fa063db91d ocaml/xapi/import_raw_vdi.ml
--- a/ocaml/xapi/import_raw_vdi.ml	Mon Aug 23 13:03:21 2010 +0100
+++ b/ocaml/xapi/import_raw_vdi.ml	Mon Aug 23 13:05:47 2010 +0100
@@ -20,9 +20,13 @@
 
 open Http
 open Importexport
+open Sparse_encoding
 open Unixext
 open Pervasiveext
 
+let receive_chunks (s: Unix.file_descr) (fd: Unix.file_descr) = 
+	Chunk.fold (fun () -> Chunk.write fd) () s
+
 let vdi_of_req ~__context (req: request) = 
 	let vdi = 
 		if List.mem_assoc "vdi" req.Http.query
@@ -37,6 +41,7 @@
   Xapi_http.with_context "Importing raw VDI" req s
     (fun __context ->
       let vdi = vdi_of_req ~__context req in
+      let chunked = List.mem_assoc "chunked" req.Http.query in
       try
 	match req.transfer_encoding, req.content_length with
 	| Some "chunked", _ ->
@@ -56,7 +61,9 @@
 		      finally 
 			(fun () -> 
 			   try
-			     Unixext.copy_file ~limit:len s fd;
+			     if chunked
+			     then receive_chunks s fd
+			     else ignore(Unixext.copy_file ~limit:len s fd);
 			     Unixext.fsync fd
 			   with Unix.Unix_error(Unix.EIO, _, _) ->
 			     raise (Api_errors.Server_error (Api_errors.vdi_io_error, ["Device I/O errors"]))
diff -r acfa0e8405cb -r 23fa063db91d ocaml/xapi/importexport.ml
--- a/ocaml/xapi/importexport.ml	Mon Aug 23 13:03:21 2010 +0100
+++ b/ocaml/xapi/importexport.ml	Mon Aug 23 13:05:47 2010 +0100
@@ -212,3 +212,4 @@
 			 Helpers.log_exn_continue "executing cleanup action" (action __context rpc) session_id) x
 	 )
     )
+
diff -r acfa0e8405cb -r 23fa063db91d ocaml/xapi/sparse_encoding.ml
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/ocaml/xapi/sparse_encoding.ml	Mon Aug 23 13:05:47 2010 +0100
@@ -0,0 +1,122 @@
+(*
+ * Copyright (C) 2010 Citrix Systems Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *)
+(** Utility functions for reading and writing disk blocks to/from a network stream.
+ * @group Import and Export
+ *)
+
+module Unmarshal = struct
+	let int64 (s, offset) = 
+		let (<<) a b = Int64.shift_left a b
+		and (||) a b = Int64.logor a b in
+		let a = Int64.of_int (int_of_char (s.[offset + 0])) 
+		and b = Int64.of_int (int_of_char (s.[offset + 1])) 
+		and c = Int64.of_int (int_of_char (s.[offset + 2])) 
+		and d = Int64.of_int (int_of_char (s.[offset + 3]))
+		and e = Int64.of_int (int_of_char (s.[offset + 4]))
+		and f = Int64.of_int (int_of_char (s.[offset + 5]))
+		and g = Int64.of_int (int_of_char (s.[offset + 6]))
+		and h = Int64.of_int (int_of_char (s.[offset + 7])) in
+		(a << 0) || (b << 8) || (c << 16) || (d << 24) || (e << 32) || (f << 40) || (g << 48) || (h << 56), 
+		(s, offset + 8)
+	let int32 (s, offset) = 
+		let (<<) a b = Int32.shift_left a b
+		and (||) a b = Int32.logor a b in
+		let a = Int32.of_int (int_of_char (s.[offset + 0])) 
+		and b = Int32.of_int (int_of_char (s.[offset + 1])) 
+		and c = Int32.of_int (int_of_char (s.[offset + 2])) 
+		and d = Int32.of_int (int_of_char (s.[offset + 3])) in
+		(a << 0) || (b << 8) || (c << 16) || (d << 24), (s, offset + 4)
+end
+
+module Marshal = struct
+	let int64 x = 
+		let (>>) a b = Int64.shift_right_logical a b
+		and (&&) a b = Int64.logand a b in
+		let a = (x >> 0) && 0xffL 
+		and b = (x >> 8) && 0xffL
+		and c = (x >> 16) && 0xffL
+		and d = (x >> 24) && 0xffL
+		and e = (x >> 32) && 0xffL
+		and f = (x >> 40) && 0xffL
+		and g = (x >> 48) && 0xffL
+		and h = (x >> 56) && 0xffL in
+		let result = String.make 8 '\000' in
+		result.[0] <- char_of_int (Int64.to_int a);
+		result.[1] <- char_of_int (Int64.to_int b);
+		result.[2] <- char_of_int (Int64.to_int c);
+		result.[3] <- char_of_int (Int64.to_int d);
+		result.[4] <- char_of_int (Int64.to_int e);
+		result.[5] <- char_of_int (Int64.to_int f);
+		result.[6] <- char_of_int (Int64.to_int g);
+		result.[7] <- char_of_int (Int64.to_int h);
+		result
+	let int32 x = 
+		let (>>) a b = Int32.shift_right_logical a b
+		and (&&) a b = Int32.logand a b in
+		let a = (x >> 0) && 0xffl 
+		and b = (x >> 8) && 0xffl
+		and c = (x >> 16) && 0xffl
+		and d = (x >> 24) && 0xffl in
+		let result = String.make 4 '\000' in
+		result.[0] <- char_of_int (Int32.to_int a);
+		result.[1] <- char_of_int (Int32.to_int b);
+		result.[2] <- char_of_int (Int32.to_int c);
+		result.[3] <- char_of_int (Int32.to_int d);
+		result
+
+end
+
+module Chunk = struct
+	(** Represents an single block of data to write *)
+	type t = {
+		start: int64;
+		data: string;
+	}
+
+	let really_write fd offset buf off len = 
+		ignore(Unix.LargeFile.lseek fd offset Unix.SEEK_SET);
+		let n = Unix.write fd buf off len in
+		if n < len 
+		then failwith "Short write: attempted to write %d bytes at %Ld, only wrote %d" len offset n
+
+	(** Writes a single block of data to the output device *)
+	let write fd x = really_write fd x.start x.data 0 (String.length x.data)
+
+	(** Reads a type t from a file descriptor *)
+	let unmarshal fd = 
+		let buf = String.make 12 '\000' in
+		Unixext.really_read fd buf 0 (String.length buf);
+		let stream = (buf, 0) in
+		let start, stream = Unmarshal.int64 stream in
+		let len, stream = Unmarshal.int32 stream in
+		let payload = String.make (Int32.to_int len) '\000' in
+		Unixext.really_read fd payload 0 (String.length payload);
+		{ start = start; data = payload }
+
+	(** Writes a type t from a file descriptor *)
+	let marshal fd x = 
+		let start' = Marshal.int64 x.start in
+		let len' = Marshal.int32 (Int32.of_int (String.length x.data)) in
+		really_write fd 0L start' 0 (String.length start');
+		really_write fd 8L len' 0 (String.length len');
+		really_write fd 12L x.data 0 (String.length x.data)
+
+	(** Fold [f] across all ts unmarshalled from [fd] *)
+	let rec fold f init fd = 
+		let x = unmarshal fd in
+		if x.data = "" 
+		then init
+		else fold f (f init x) fd
+end
+