[Xen-devel] [XTF PATCH] xtf-runner: fix two synchronisation issues

[Wei Liu <wei.liu2@xxxxxxxxxx>]

There were two synchronisation issues for the old code:

1. There was no guarantee that guest console was ready before "xl
   console" invocation.
2. There was no guarantee that runner wouldn't not exit before all test
   guests were gone.

For xtf-runner to be suitable to run in an automatic testing system like
osstest, we need to eliminate those two issues, otherwise we risk seeing
failures that are caused by races.

To fix these two issues:

1. Use "xl create -F" to force the process that creates a guest to
   stay until the guest is dead. With this we can wait for that
   particular subprocess in runner to make sure the guest is gone.
2. Poll xenstore guest console node to make sure console is available
   before "xl console" invocation.

To avoid the runner loops forever in that console checking loop, use
SIGALRM to make it time out after 5 seconds.

Redirect output that we're not interested in to DEVNULL.

Signed-off-by: Wei Liu <wei.liu2@xxxxxxxxxx>
---
 xtf-runner | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 50 insertions(+), 7 deletions(-)

diff --git a/xtf-runner b/xtf-runner
index 66b2fb7..3296cdf 100755
--- a/xtf-runner
+++ b/xtf-runner
@@ -7,7 +7,7 @@ xtf-runner - A utility for enumerating and running XTF tests.
 Currently assumes the presence and availability of the `xl` toolstack.
 """
 
-import sys, os, os.path as path
+import sys, os, os.path as path, signal
 
 from optparse import OptionParser
 from subprocess import Popen, PIPE, call as subproc_call, check_output
@@ -17,6 +17,11 @@ try:
 except ImportError:
     import simplejson as json
 
+try:
+    from subprocess import DEVNULL
+except ImportError:
+    DEVNULL = open(os.devnull, 'wb')
+
 # All results of a test, keep in sync with C code report.h.
 # Note that warning is not a result on its own.
 all_results = ('SUCCESS', 'SKIP', 'ERROR', 'FAILURE')
@@ -41,6 +46,10 @@ all_environments = pv_environments + hvm_environments
 class RunnerError(Exception):
     """ Errors relating to xtf-runner itself """
 
+def sigalrm_handler(signum, frame):
+    """ Signal handler for SIGALRM """
+    raise RunnerError("Operation timed out")
+
 def open_test_info():
     """ Open and collate each test-info.json """
 
@@ -132,24 +141,53 @@ def list_tests(args):
 
         print name
 
+# A list of processes that we need to wait until they exits.
+wait_list = []
 
 def run_test(test):
     """ Run a specific test """
 
+    console_path = '/local/domain/0/backend/console'
+    # Setup watch for console
+    cmd = ['xenstore-watch', console_path]
+    print "Executing '%s'" % (" ".join(cmd), )
+    xs_watch = Popen(cmd, stdout = PIPE, stderr = PIPE)
+
     _, _, name = test.split('-', 2)
 
     cfg = path.join("tests", name, test + ".cfg")
 
-    cmd = ['xl', 'create', '-p', cfg]
+    cmd = ['xl', 'create', '-Fp', cfg]
 
     print "Executing '%s'" % (" ".join(cmd), )
-    rc = subproc_call(cmd)
-    if rc:
-        raise RunnerError("Failed to create VM")
+    wait = Popen(cmd, stdout = DEVNULL, stderr = DEVNULL)
+    wait_list.append(wait)
+
+    # Wait up to 5 seconds for console to show up
+    signal.alarm(5)
+    while True:
+        l = xs_watch.stdout.readline()
+        domid = l[len(console_path)+1:].split('/')[0]
+        if domid == '': continue
+
+        cmd = ['xenstore-read', '/local/domain/'+domid+'/name']
+        print "Executing '%s'" % (" ".join(cmd), )
+        gname = check_output(cmd).splitlines()[0]
+        if gname != test: continue
+
+        cmd = ['xenstore-read', '/local/domain/'+domid+'/console/tty']
+        print "Executing '%s'" % (" ".join(cmd), )
+        con = check_output(cmd).splitlines()[0]
+        if con != '': break
+
+    # Tear down watch and timer
+    signal.alarm(0)
+    xs_watch.kill()
 
     cmd = ['xl', 'console', test]
     print "Executing '%s'" % (" ".join(cmd), )
     console = Popen(cmd, stdout = PIPE)
+    wait_list.append(console)
 
     cmd = ['xl', 'unpause', test]
     print "Executing '%s'" % (" ".join(cmd), )
@@ -327,12 +365,17 @@ def main():
     opts, args = parser.parse_args()
 
     if opts.list_tests:
-        return list_tests(args)
+        ret = list_tests(args)
     else:
-        return run_tests(args)
+        ret = run_tests(args)
+
+    for child in wait_list: child.wait()
+
+    return ret
 
 
 if __name__ == "__main__":
+    signal.signal(signal.SIGALRM, sigalrm_handler)
     try:
         sys.exit(main())
     except RunnerError, e:
-- 
2.1.4


_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
https://lists.xen.org/xen-devel