[Xen-devel] [OSSTEST PATCH RFC 11/14] Introduce ts-xtf-run

[Wei Liu <wei.liu2@xxxxxxxxxx>]

This is the main script for running XTF.  It will first perform
selftest, and then run each XTF test case as a substep.

It does the following things:

1. Run self tests for individual environment and record the result.
2. Collect tests according to available environments.
3. Run the collected tests one by one.

The script may exit early if it detects the test host is down or
xtf-runner returns non-recognisable exit code.

Signed-off-by: Wei Liu <wei.liu2@xxxxxxxxxx>
---
 ts-xtf-run | 139 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 139 insertions(+)
 create mode 100755 ts-xtf-run

diff --git a/ts-xtf-run b/ts-xtf-run
new file mode 100755
index 0000000..596ebc4
--- /dev/null
+++ b/ts-xtf-run
@@ -0,0 +1,139 @@
+#!/usr/bin/perl -w
+# This is part of "osstest", an automated testing framework for Xen.
+# Copyright (C) 2016 Citrix Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use strict qw(vars);
+use Osstest;
+use POSIX;
+use Osstest::TestSupport;
+
+tsreadconfig();
+
+our $ho = selecthost('host');
+
+our $runner;
+our @tests;
+our @all_environments;
+our @environments;
+
+# XTF results (runner returned numeric values) and OSStest results:
+#
+# SUCCESS(0)  -> pass
+# SKIP(3)     -> skip
+# ERROR(4)    -> fail
+# FAILURE(5)  -> fail
+#
+# All other return codes are mapped to fail.
+
+sub xtf_result_to_osstest_result ($) {
+    my ($xret) = @_;
+
+    return "pass" if $xret == 0;
+    return "skip" if $xret == 3;
+    return "fail";
+}
+
+# Check if the xtf result is within pre-defined range
+# See xtf-runner --help
+sub xtf_result_defined ($) {
+    my ($xret) = @_;
+    return $xret >= 0 && $xret <= 5;
+}
+
+sub prep () {
+    my $xtfdir = get_runvar('xtfdir', $r{xtfbuildjob});
+    $runner = "$xtfdir/xtf-runner";
+}
+
+sub get_environments () {
+    my $output = target_cmd_output_root($ho, <<END);
+        $runner --list --environments
+END
+    foreach my $e (split('\n', $output)) {
+        push @all_environments, $e
+    }
+}
+
+# Call the runner on one test case, generate a substep for it in final test
+# report. Return runner exit code to caller. May exit the script early if
+# things go really bad.
+sub do_one_test ($) {
+    my ($name) = @_;
+    my $tid = "xtf/$name";
+    my $cmd = "xtf-runner $name";
+
+    substep_start($tid, $cmd);
+    my $output = target_cmd_output_root($ho, <<END, 600);
+        $runner $name 1>&2; echo \$\?
+END
+    my ($ret) = $output =~ /(\d+)/;
+
+    # If the host doesn't respond after a test case, always make this substep
+    # fail and exit the script early with 0
+    my $msg = target_ping_check_up($ho);
+    if ($msg) {
+        logm("$msg");
+        substep_finish($tid, "fail");
+        exit 0;
+    }
+
+    # If xtf result can't be recognised, always make this substep fail and exit
+    # the script early with status 0.
+    if (!xtf_result_defined($ret)) {
+        substep_finish($tid, "fail");
+        exit 0;
+    }
+
+    substep_finish($tid, xtf_result_to_osstest_result($ret));
+
+    return $ret;
+}
+
+# Run selftest for each environment, record the ones that are
+# funtional to get maximum coverage.
+sub do_selftest () {
+    foreach my $e (sort @all_environments) {
+        my $output = target_cmd_output_root($ho, <<END);
+            $runner --list $e selftest
+END
+        my $ret = do_one_test($output);
+        push @environments, $e if $ret == 0;
+    }
+}
+
+sub get_tests_list () {
+    foreach my $e (sort @environments) {
+        my $output = target_cmd_output_root($ho, <<END);
+            $runner --list $e --all --host
+END
+        foreach my $t (split('\n', $output)) {
+            push @tests, $t
+        }
+    }
+}
+
+sub do_all_tests () {
+    foreach my $t (sort @tests) {
+        do_one_test($t);
+    }
+}
+
+prep();
+get_environments();
+do_selftest();
+get_tests_list();
+do_all_tests();
+exit 0;
-- 
2.1.4


_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
https://lists.xen.org/xen-devel