Re: [Xen-devel] [PATCH] add ssl/tls support to relocation

[Zhigang Wang <zhigang.x.wang@xxxxxxxxxx>]

Thanks Keir,

I reorganized the code and did a little more test, it just works.

But maybe the best way is to patch pyOpenSSL, or use python built-in
SSL support after python 2.5 in the future.

And maybe someone (me ;-)) can rewrite the migration protocol based on
a more robust framework.

Someone already uses it to make papers:
http://www.eecs.umich.edu/techreports/cse/2007/CSE-TR-539-07.pdf

comments & testing are welcome.

thanks

zhigang

Keir Fraser wrote:
> On 8/5/08 13:55, "Zhigang Wang" <zhigang.x.wang@xxxxxxxxxx> wrote:
> 
>> After further investigation, I find that I didn't get relocation using
>> ssl/tls: the read/write to the pyOpenSSL socket.fileno() will communicate
>> without data encrypted.
> 
> Need to be merged with current unstable tip (which is at least changeset
> 17589). Note also that 17577 has already made OpenSSL optional, and with
> less code movement than your approach.
> 
>  -- Keir
> 
> 
> 
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@xxxxxxxxxxxxxxxxxxx
> http://lists.xensource.com/xen-devel

Fix relocation ssl/tls support

 * Make a wrapper of read/write sock.fileno().

 * Makes pyOpenSSL an optional package.

 * Implement reference:
   http://twistedmatrix.com/trac/browser/trunk/twisted/internet/tcp.py

Signed-off-by: Zhigang Wang <zhigang.x.wang@xxxxxxxxxx>

diff -Nura xen-unstable.orig/tools/python/xen/web/connection.py 
xen-unstable/tools/python/xen/web/connection.py
--- xen-unstable.orig/tools/python/xen/web/connection.py        2008-05-12 
16:43:51.000000000 +0800
+++ xen-unstable/tools/python/xen/web/connection.py     2008-05-13 
13:25:19.000000000 +0800
@@ -18,12 +18,18 @@
 #============================================================================
 
 import sys
+import os
 import threading
 import socket
 import fcntl
 
 from errno import EAGAIN, EINTR, EWOULDBLOCK
 
+try:
+    from OpenSSL import SSL
+except ImportError:
+    pass
+
 from xen.xend.XendLogging import log
 
 """General classes to support server and client sockets, without
@@ -115,6 +121,163 @@
             self.close()
 
 
+class SSLSocketServerConnection(SocketServerConnection):
+    """An SSL aware accepted connection to a server.
+
+    As pyOpenSSL SSL.Connection fileno() method just retrieve the file
+    descriptor number for the underlying socket, direct read/write to the file
+    descriptor will result no data encrypted.
+    
+    recv2fd() and fd2send() are simple wrappers for functions who need direct
+    read/write to a file descriptor rather than a socket like object.
+    
+    To use recv2fd(), you can create a pipe and start a thread to transfer all
+    received data to one end of the pipe, then read from the other end:
+    
+    p2cread, p2cwrite = os.pipe()
+    threading.Thread(target=connection.recv2fd, args=(sock, p2cwrite)).start()
+    os.read(p2cread, 1024)
+    
+    To use fd2send():
+    
+    p2cread, p2cwrite = os.pipe()
+    threading.Thread(target=connection.fd2send, args=(sock, p2cread)).start()
+    os.write(p2cwrite, "data")
+    """
+
+    def __init__(self, sock, protocol_class):
+        SocketServerConnection.__init__(self, sock, protocol_class)
+
+
+    def main(self):
+        try:
+            while True:
+                try:
+                    data = self.sock.recv(BUFFER_SIZE)
+                    if data == "":
+                        break
+                    if self.protocol.dataReceived(data):
+                        break
+                except socket.error, ex:
+                    if ex.args[0] not in (EWOULDBLOCK, EAGAIN, EINTR):
+                        break
+                except (SSL.WantReadError, SSL.WantWriteError, \
+                        SSL.WantX509LookupError):
+                    # The operation did not complete; the same I/O method
+                    # should be called again.
+                    continue
+                except SSL.ZeroReturnError:
+                    # The SSL Connection has been closed.
+                    break
+                except SSL.SysCallError, (retval, desc):
+                    if ((retval == -1 and desc == "Unexpected EOF")
+                        or retval > 0):
+                        # The SSL Connection is lost.
+                        break
+                    log.debug("SSL SysCallError:%d:%s" % (retval, desc))
+                    break
+                except SSL.Error, e:
+                    # other SSL errors
+                    log.debug("SSL Error:%s" % e)
+                    break
+        finally:
+            try:
+                self.sock.close()
+            except:
+                pass
+
+
+    def recv2fd(sock, fd):
+        try:
+            while True:
+                try:
+                    data = sock.recv(BUFFER_SIZE)
+                    if data == "":
+                        break
+                    count = 0
+                    while count < len(data):
+                        try:
+                            nbytes = os.write(fd, data[count:])
+                            count += nbytes
+                        except os.error, ex:
+                            if ex.args[0] not in (EWOULDBLOCK, EAGAIN, EINTR):
+                                raise
+                except socket.error, ex:
+                    if ex.args[0] not in (EWOULDBLOCK, EAGAIN, EINTR):
+                        break
+                except (SSL.WantReadError, SSL.WantWriteError, \
+                        SSL.WantX509LookupError):
+                    # The operation did not complete; the same I/O method
+                    # should be called again.
+                    continue
+                except SSL.ZeroReturnError:
+                    # The SSL Connection has been closed.
+                    break
+                except SSL.SysCallError, (retval, desc):
+                    if ((retval == -1 and desc == "Unexpected EOF")
+                        or retval > 0):
+                        # The SSL Connection is lost.
+                        break
+                    log.debug("SSL SysCallError:%d:%s" % (retval, desc))
+                    break
+                except SSL.Error, e:
+                    # other SSL errors
+                    log.debug("SSL Error:%s" % e)
+                    break
+        finally:
+            try:
+                sock.close()
+                os.close(fd)
+            except:
+                pass
+
+    recv2fd = staticmethod(recv2fd)
+
+    def fd2send(sock, fd):
+        try:
+            while True:
+                try:
+                    data = os.read(fd, BUFFER_SIZE)
+                    if data == "":
+                        break
+                    count = 0
+                    while count < len(data):
+                        try:
+                            nbytes = sock.send(data[count:])
+                            count += nbytes
+                        except socket.error, ex:
+                            if ex.args[0] not in (EWOULDBLOCK, EAGAIN, EINTR):
+                                raise
+                        except (SSL.WantReadError, SSL.WantWriteError, \
+                                SSL.WantX509LookupError):
+                            # The operation did not complete; the same I/O 
method
+                            # should be called again.
+                            continue
+                        except SSL.ZeroReturnError:
+                            # The SSL Connection has been closed.
+                            raise
+                        except SSL.SysCallError, (retval, desc):
+                            if not (retval == -1 and data == ""):
+                                # errors when writing empty strings are 
expected
+                                # and can be ignored
+                                log.debug("SSL SysCallError:%d:%s" % (retval, 
desc))
+                                raise
+                        except SSL.Error, e:
+                            # other SSL errors
+                            log.debug("SSL Error:%s" % e)
+                            raise
+                except os.error, ex:
+                    if ex.args[0] not in (EWOULDBLOCK, EAGAIN, EINTR):
+                        break
+        finally:
+            try:
+                sock.close()
+                os.close(fd)
+            except:
+                pass
+
+    fd2send = staticmethod(fd2send)
+
 def hostAllowed(addrport, hosts_allowed):
     if hosts_allowed is None:
         return True
diff -Nura xen-unstable.orig/tools/python/xen/web/tcp.py 
xen-unstable/tools/python/xen/web/tcp.py
--- xen-unstable.orig/tools/python/xen/web/tcp.py       2008-05-12 
16:43:51.000000000 +0800
+++ xen-unstable/tools/python/xen/web/tcp.py    2008-05-12 17:03:49.000000000 
+0800
@@ -88,6 +88,7 @@
         ctx.use_certificate_file(self.ssl_cert_file)
         sock = SSL.Connection(ctx,
                               socket.socket(socket.AF_INET, 
socket.SOCK_STREAM))
+        sock.set_accept_state()
         sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
 
         # SO_REUSEADDR does not always ensure that we do not get an address
@@ -104,3 +105,14 @@
                 else:
                     raise
 
+
+    def acceptConnection(self, sock, addrport):
+        addr = addrport[0]
+        if connection.hostAllowed(addrport, self.hosts_allow):
+            connection.SSLSocketServerConnection(sock, self.protocol_class)
+        else:
+            try:
+                sock.close()
+            except:
+                pass
+
diff -Nura xen-unstable.orig/tools/python/xen/xend/server/relocate.py 
xen-unstable/tools/python/xen/xend/server/relocate.py
--- xen-unstable.orig/tools/python/xen/xend/server/relocate.py  2008-05-12 
16:43:52.000000000 +0800
+++ xen-unstable/tools/python/xen/xend/server/relocate.py       2008-05-13 
14:26:23.000000000 +0800
@@ -17,10 +17,12 @@
 #============================================================================
 
 import re
+import os
 import sys
 import StringIO
+import threading
 
-from xen.web import protocol, tcp, unix
+from xen.web import protocol, tcp, unix, connection
 
 from xen.xend import sxp
 from xen.xend import XendDomain
@@ -116,6 +118,24 @@
             log.error(name + ": no transport")
             raise XendError(name + ": no transport")
 
+    def op_sslreceive(self, name, _):
+        if self.transport:
+            self.send_reply(["ready", name])
+            p2cread, p2cwrite = os.pipe()
+            
threading.Thread(target=connection.SSLSocketServerConnection.recv2fd,
+                             args=(self.transport.sock, p2cwrite)).start()
+            try:
+                XendDomain.instance().domain_restore_fd(p2cread,
+                                                        relocating=True)
+            except:
+                os.close(p2cread)
+                os.close(p2cwrite)
+                self.send_error()
+                self.close()
+        else:
+            log.error(name + ": no transport")
+            raise XendError(name + ": no transport")
+
 
 def listenRelocation():
     xoptions = XendOptions.instance()
diff -Nura xen-unstable.orig/tools/python/xen/xend/XendDomain.py 
xen-unstable/tools/python/xen/xend/XendDomain.py
--- xen-unstable.orig/tools/python/xen/xend/XendDomain.py       2008-05-12 
16:43:52.000000000 +0800
+++ xen-unstable/tools/python/xen/xend/XendDomain.py    2008-05-13 
11:18:34.000000000 +0800
@@ -1293,25 +1293,56 @@
         if port == 0:
             port = xoptions.get_xend_relocation_port()
 
-        try:
-            tls = xoptions.get_xend_relocation_tls()
-            if tls:
-                from OpenSSL import SSL
+        tls = xoptions.get_xend_relocation_tls()
+        if tls:
+            from OpenSSL import SSL
+            from xen.web import connection
+            try:
                 ctx = SSL.Context(SSL.SSLv23_METHOD)
-                sock = SSL.Connection(ctx, socket.socket(socket.AF_INET, 
socket.SOCK_STREAM))
+                sock = SSL.Connection(ctx,
+                           socket.socket(socket.AF_INET, socket.SOCK_STREAM))
                 sock.set_connect_state()
-            else:
+                sock.connect((dst, port))
+                sock.send("sslreceive\n")
+                sock.recv(80)
+            except SSL.Error, err:
+                raise XendError("SSL error: %s" % err)
+            except socket.error, err:
+                raise XendError("can't connect: %s" % err)
+
+            p2cread, p2cwrite = os.pipe()
+            
threading.Thread(target=connection.SSLSocketServerConnection.fd2send,
+                             args=(sock, p2cread)).start()
+
+            try:
+                XendCheckpoint.save(p2cwrite, dominfo, True, live, dst,
+                                    node=node)
+            finally:
+                sock.shutdown()
+                sock.close()
+
+            os.close(p2cread)
+            os.close(p2cwrite)
+        else:
+            try:
                 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-            sock.connect((dst, port))
-        except socket.error, err:
-            raise XendError("can't connect: %s" % err[1])
-
-        sock.send("receive\n")
-        sock.recv(80)
-        try:
-            XendCheckpoint.save(sock.fileno(), dominfo, True, live, dst, 
node=node)
-        finally:
-            sock.close()
+                # When connecting to our ssl enabled relocation server using a
+                # plain socket, send will success but recv will block. Add a
+                # 30 seconds timeout to raise a socket.timeout exception to
+                # inform the client.
+                sock.settimeout(30.0)
+                sock.connect((dst, port))
+                sock.send("receive\n")
+                sock.recv(80)
+                sock.settimeout(None)
+            except socket.error, err:
+                raise XendError("can't connect: %s" % err)
+
+            try:
+                XendCheckpoint.save(sock.fileno(), dominfo, True, live,
+                                    dst, node=node)
+            finally:
+                sock.close()
 
     def domain_save(self, domid, dst, checkpoint=False):
         """Start saving a domain to file.

_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel