[Xen-devel] [PATCH OSSTEST v2 5/5] ms-flights-summary: Produce an HTML report of all active flights

[Ian Campbell <ian.campbell@xxxxxxxxxx>]

From: Ian Campbell <ijc@xxxxxxxxxxxxxx>

Jobs are categorised by a new ->Job field. This is added by
ts-hosts-alllocate-Executive and propagated byu the planner after
recent patches. It contains $flight.$job.

Jobs which do not include this are anonymous and are listed
separately.

Jobs which the plan describes as "(preparing)" (which I think means they
are waiting for another job to regroove build host for sharing) are
currently classified as anonymous until they are done preparing, at
which point they become correctly classified again. I can't see how
to figure out the correct ->{Job} at this point in the planner.

TODO: Hook up to cron (via for-maintjobs.git) or otherwise arrange for
this to be run when the plan is updated.

Signed-off-by: Ian Campbell <ijc@xxxxxxxxxxxxxx>
---
Example output:
http://xenbits.xen.org/people/ianc/tmp/fsummary-v2.html

This could surely use better Perl and produce better output, however
I'm sending it now because it would be useful for further development
if some or all of the preceding patches could go into production and
this serves as an example of why I think I want them.

v2:
  - Get the plan from the queue daemon.
  - Do not parse ->Info, instead expect a new ->Job field
  - Handle multiple resources for a job.
---
 ms-flights-summary | 315 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 ms-planner         |   1 +
 2 files changed, 316 insertions(+)
 create mode 100755 ms-flights-summary

diff --git a/ms-flights-summary b/ms-flights-summary
new file mode 100755
index 0000000..76eb4b1
--- /dev/null
+++ b/ms-flights-summary
@@ -0,0 +1,315 @@
+#!/usr/bin/perl
+
+use strict qw(vars refs);
+
+use Osstest;
+use Osstest::Executive;
+
+use JSON;
+use POSIX;
+use Data::Dumper;
+use HTML::Entities;
+
+# $flights{NNNN} = {
+#     Nr => NNNN,
+#     Stats => {
+#         Pass => <NRPASS>,
+#         Fail => <NRFAIL>,
+#         ...
+#     },
+#     Started => ...,
+#     Blessing => ...,
+#     Branch => ...,
+#     Intended => ...,
+#     ExpectedEnd => ...,
+#     Jobs => {
+#         JNAME1 => {
+#             Reso => {
+#                 RNAME1 => {
+#                     Start => ...,
+#                     End => ...,
+#                 },
+#                 RNAME2 => { ... },
+#             },
+#             OverallTime => { Start => ..., End => ... },
+#             Receipe => ...,
+#             Status => pass|fail|...,
+#         },
+#         JNAME2 => { ... },
+#     },
+# };
+our %flights;
+
+# As for $flights{NNN}{Jobs} but each Job only has the Reso and
+# OverallTime keys
+our %anon_jobs;
+
+our $plan;
+
+sub get_current_plan () {
+    my $qserv = tcpconnect_queuedaemon();
+    my $jplan = get_plan($qserv);
+    $plan= from_json($jplan);
+    #print STDERR Dumper($plan);
+}
+
+# Find all the flights referenced by an Event and insert into %flights.
+sub enumerate_flights() {
+    while (my ($reso,$evts) = each %{ $plan->{Events} }) {
+       foreach my $evt ( @{$evts} ) {
+           next unless $evt->{Type} =~ m/^(Start|End)$/;
+           next unless $evt->{Job};
+           $evt->{Job} =~ m/^([0-9]+)\.(.*)/ or die;
+
+           my $f = $1;
+
+           next if $flights{$f};
+
+           my $flightinfo= $dbh_tests->selectrow_hashref(<<END);
+           SELECT started,blessing,branch,intended FROM flights
+               WHERE flight=$f
+END
+            $flights{$f}= { Nr => $f,
+                           Stats => {},
+                           Jobs => {} };
+            foreach my $fld (qw(started blessing branch intended)) {
+                $flights{$f}->{ucfirst($fld)}= $flightinfo->{$fld};
+           };
+           $flights{$f}->{ExpectedEnd} = 0;
+       }
+    }
+}
+
+# Enumerate all jobs of every known flight and populate the
+# corresponding ->{Jobs}.
+sub enumerate_jobs($) {
+    my ($f) = @_;
+
+    $f->{Jobs} = {};
+
+    my $jobs= $dbh_tests->selectall_arrayref(<<END, { Slice => {} });
+        SELECT job,recipe,status FROM jobs
+            WHERE flight=$f->{Nr}
+END
+
+    for my $row (@{$jobs}) {
+       $f->{Jobs}{$row->{job}} =
+       {
+           Status => $row->{status},
+           Recipe => $row->{recipe},
+           Reso => {},
+       };
+    }
+}
+
+# Gather statistics around number of pass/fail/etc jobs in each
+# flight.
+sub gather_stats($) {
+    my ($f) = @_;
+
+    my $stats= $dbh_tests->selectall_arrayref(<<END);
+        SELECT status,COUNT(*) FROM jobs
+           WHERE flight=$f->{Nr}
+            GROUP BY status
+END
+    for my $row (@{$stats}) {
+       my ($stat,$count) = @$row;
+       $f->{Stats}{ucfirst($stat)} = $count;
+    }
+}
+
+sub add_event($$$$) {
+    my ($job,$reso,$type,$time) = @_;
+
+    die unless $type =~ m/^(Start|End)/;
+
+    $job->{OverallTime} //= {};
+
+    $job->{Reso}{$reso} //= {};
+    $job->{Reso}{$reso}{$type} = $time;
+
+    my $cmp = $type eq "Start" ?
+       sub { $_[0] < $_[1] ? $_[0] : $_[1] } :
+       sub { $_[0] > $_[1] ? $_[0] : $_[1] };
+
+    $job->{OverallTime}{$type} //= $time;
+
+    $job->{OverallTime}{$type} =
+       $cmp->($time, $job->{OverallTime}{$type});
+}
+
+# Walk all events in the plan and update the corresponding flight/job
+# with the timespan. Events relating to unknown jobs are added to
+# %anon_jobs.
+sub gather_events() {
+    while (my ($reso,$evts) = each %{ $plan->{Events} }) {
+       foreach my $evt ( @{$evts} ) {
+           my ($f,$job);
+           next unless $evt->{Type} =~ m/^(Start|End)$/;
+           if ( $evt->{Job} ) {
+               my $j;
+               $evt->{Job} =~ m/^([0-9]+)\.(.*)/ or die;
+               ($fnum,$j) = ($1,$2);
+               goto anon unless $flights{$fnum};
+               $f = $flights{$fnum};
+               goto anon unless $f->{Jobs}{$j};
+               $job = $f->{Jobs}{$j};
+           } else {
+ anon:
+               # The JSON version of the plan does not include
+               # $evt->{Info}, the best we have got is the resource
+               # itself.
+               $anon_jobs{$reso} //= { Reso => {} };
+               $job = $anon_jobs{$reso}
+           }
+
+           my $time = $plan->{Start} + $evt->{Time};
+
+           add_event($job, $reso, $evt->{Type}, $time);
+
+           if ($f && $evt->{Type} eq "End") {
+               $f->{ExpectedEnd} =
+                   ($evt->{Time} > $f->{ExpectedEnd}) ?
+                   $evt->{Time} : $f->{ExpectedEnd}
+           }
+       }
+    }
+}
+
+###########
+
+csreadconfig();
+
+get_current_plan();
+
+enumerate_flights();
+
+foreach my $f (keys %flights) {
+    enumerate_jobs($flights{$f});
+    gather_stats($flights{$f});
+}
+
+gather_events();
+
+############
+
+my @cols = ("Job",
+           #"Recipe",
+           "Status", "Resource", "Scheduled");#, "Start", "End", "Host");
+
+sub fmttime($)
+{
+    my ($t) = @_;
+    return strftime("%Y-%b-%d %a %H:%M:%S", gmtime $t);
+    #return strftime("%+", gmtime $t);
+}
+
+sub cols_hdr() {
+    printf("<tr>\n");
+    printf(" <th align='left'>%s</th>\n", encode_entities($_)) foreach @cols;
+    printf("</tr>\n");
+}
+sub flight_hdr($) {
+    my $text = encode_entities(shift);
+    printf("<tr><td colspan=%d><b>$text</b></td></tr>\n", scalar @cols);
+}
+sub job_cell($) {
+    my $text = encode_entities(shift);
+    printf(" <td><b>$text</b></td>\n");
+}
+sub rawcell($) {
+    my $text = shift;
+    printf(" <td>$text</td>\n");
+}
+sub cell($) {
+    rawcell(encode_entities(shift));
+}
+
+sub fmt_timespan($) {
+    my ($s) = @_;
+
+    if ( $s->{Start} && $s->{End} ) {
+       return join(" - ", map { fmttime($s->{$_}) } qw(Start End));
+    } else {
+       return "";
+    }
+}
+
+sub row($$$;$$) {
+    my ($omitjob,$job,$status, $reso,$tspan) = @_;
+
+    print "<tr>\n";
+
+    if ($omitjob) {
+       print "<td></td>" foreach (1..2);
+    } else {
+       job_cell($job);
+       cell($status // "???");
+    }
+    rawcell($reso) if $reso;
+    cell(fmt_timespan($tspan)) if $tspan;
+
+    print "</tr>\n";
+}
+
+sub do_one_job($$) {
+    my ($job,$info) = @_;
+
+    my $nrreso = scalar keys %{ $info->{Reso} };
+
+#    my $prinfo = sub {
+#      job_cell($job);
+#      cell($info->{Status} // "???");
+#    };
+
+#    my $row = sub {
+#      print "<tr>\n";
+#      $prinfo->();
+#      $prinfo = sub { print "<td></td>" foreach (1..2) };
+#      rawcell($_[0].$_[1]) if $_[1];
+#      cell(fmt_timespan($_[2])) if $_[2];
+#      print "</tr>\n";
+#    };
+
+    if ($nrreso == 0) {
+       row(0,$job,$info->{Status});
+    } else {
+       my $pfx = '';
+       my $omitjob = 0;
+       if ($nrreso > 1) {
+           row(0, $job, $info->{Status}, "Overall", $info->{OverallTime});
+           $omitjob = 1;
+           $pfx = "&mdash;";
+       }
+       while (my ($reso,$rinf) = each %{ $info->{Reso} }) {
+           row($omitjob, $job, $info->{Status}, $pfx.encode_entities($reso), 
$rinf)
+       }
+    }
+}
+
+printf("<p>Based on plan at ".fmttime($plan->{Start})."</p>\n");
+
+printf("<table>\n");
+
+foreach my $f (sort keys %flights) {
+    my $fi = $flights{$f};
+    flight_hdr("Flight: $f [$fi->{Branch} $fi->{Intended}]");
+    flight_hdr("Started: ".fmttime($fi->{Started}));
+    flight_hdr("Expected end: ".fmttime($fi->{ExpectedEnd}));
+    flight_hdr(
+       join("; ", map { "$_: $fi->{Stats}{$_}" } (keys %{$fi->{Stats}}))
+       );
+    cols_hdr();
+    do_one_job($_, $fi->{Jobs}{$_})
+       foreach sort { $a cmp $b } keys %{$fi->{Jobs}};
+    print("<tr><td>&nbsp</td></tr>\n");
+}
+print "\n";
+
+flight_hdr("Anonymous Jobs");
+cols_hdr();
+foreach my $j (sort keys %anon_jobs) {
+    do_one_job($j, $anon_jobs{$j});
+}
+
+print "</table>\n";
diff --git a/ms-planner b/ms-planner
index f38f05b..35d430b 100755
--- a/ms-planner
+++ b/ms-planner
@@ -289,6 +289,7 @@ END
            $info= "rogue task $arow->{subtask}";
        }
        $plan->{Allocations}{$reskey}= {
+           # Can we find a Job here?
             Task => $arow->{owntaskid},
            Info => $info,
            Start => $plan->{Start},
-- 
2.1.4


_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
http://lists.xen.org/xen-devel