Francois Gouget : winetest/build-patterns: Highlight recent changes in the failure modes.

[Alexandre Julliard]

Module: tools
Branch: master
Commit: 9f40728cd785c04965642292963833d532070c6a
URL:    https://source.winehq.org/git/tools.git/?a=commit;h=9f40728cd785c04965642292963833d532070c6a

Author: Francois Gouget <fgouget at codeweavers.com>
Date:   Mon May  3 14:59:00 2021 +0200

winetest/build-patterns: Highlight recent changes in the failure modes.

Highlight the new failure modes and put the test units that would
otherwise go into the 'has always been failing' list in their own list.
This should help catch new failures that pile on an already failing
test unit.

Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard at winehq.org>

---

 winetest/build-patterns | 53 ++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 50 insertions(+), 3 deletions(-)

diff --git a/winetest/build-patterns b/winetest/build-patterns
index a84e2a7..ea9cd4c 100755
--- a/winetest/build-patterns
+++ b/winetest/build-patterns
@@ -260,6 +260,10 @@ my %reports;
 #   True if the test had enough successful results after the last failure to be
 #   considered fixed.
 #
+# - newmodes
+#   A hashtable of new failure modes, that its of status values present in the
+#   last $patternbuilds build results that are not present in older builds.
+#
 # - testreports
 #   A hashtable mapping report directory names to objects storing the results
 #   for that test and report combination. Each testreport object has the
@@ -695,6 +699,11 @@ sub get_status_html($$)
     return ("?", "", "unknown status $status", "report", "");
 }
 
+sub cmpstatus
+{
+    return ($a =~ /^[0-9]+$/ and $b =~ /^[0-9]+$/) ? $a <=> $b : $a cmp $b;
+}
+
 sub write_patterns_list($$$$)
 {
     my ($html, $pagereports, $mainpage, $list) = @_;
@@ -720,6 +729,17 @@ sub write_patterns_list($$$$)
         compute_set_colors($test->{colors});
 
         print $html "<div class='test'>\n";
+        if (%{$test->{newmodes}})
+        {
+            print $html "New failure modes:";
+            foreach my $status (sort cmpstatus keys %{$test->{newmodes}})
+            {
+                my ($symbol, $class, $title, $_link, $attrs) = get_status_html($status, $test->{colors});
+                print $html " <span class='pat$class'$attrs title='$title'><b><i>$symbol</i></b></span>";
+            }
+            print $html "\n";
+        }
+
         foreach my $reportdir (@sortedreports)
         {
             next if (!$pagereports->{$reportdir});
@@ -727,6 +747,7 @@ sub write_patterns_list($$$$)
             next if (!$testreport->{failed});
             print $html "<div class='pattern'>";
 
+            my $has_newmode;
             my ($range_symbol, $range_count) = ("", 0);
             my ($range_start, $range_end, $range_title);
             foreach my $build (@sortedbuilds)
@@ -773,6 +794,11 @@ sub write_patterns_list($$$$)
                         $attrs .= sprintf " href='%s/%s/%s.html#%s'",
                                   $build->{name}, $reportdir, $link, $dll;
                     }
+                    if ($test->{newmodes}->{$status})
+                    {
+                        $symbol = "<b><i>$symbol</i></b>";
+                        $has_newmode = 1;
+                    }
                 }
 
                 if ($range_symbol eq $symbol)
@@ -815,7 +841,9 @@ sub write_patterns_list($$$$)
                              date_range($range_start, $range_end),
                              $range_title, $range_symbol x $range_count;
             }
-            print $html "</div> $reportdir\n";
+            my $label = $reportdir;
+            $label = "<b>$label</b>" if ($has_newmode);
+            print $html "</div> $label\n";
         }
         print $html "</div></div>\n";
     }
@@ -848,6 +876,9 @@ EOF
         recent   => {title => "Recent failures",
                      desc => "have recent failures (that is failures that did not happen for the $patternbuilds oldest builds).",
                      testnames => []},
+        newmode  => {title => "New failure modes",
+                    desc => "failed in new ways in the $patternbuilds most recent builds.",
+                    testnames => []},
         regular  => {title => "Regular failures",
                      desc => "had failures.",
                      testnames => []},
@@ -862,6 +893,7 @@ EOF
         my $first = @sortedbuilds;
         my $last = -1;
         my $fixed = 1;
+        my ($oldfailures, $newfailures) = ({}, {});
         foreach my $reportdir (keys %$pagereports)
         {
             my $testreport = $test->{testreports}->{$reportdir};
@@ -876,8 +908,16 @@ EOF
                           $testreport->{first};
             $last = $replast if ($replast > $last);
 
-            foreach my $status (values %{$testreport->{status}})
+            for my $i (0.. at sortedbuilds-1)
             {
+                my $build = $sortedbuilds[$i];
+                my $status = $testreport->{status}->{$build->{name}};
+                next if (!defined $status);
+                next if ($status eq "skipped");
+
+                my $failures = ($i < @sortedbuilds - $patternbuilds) ? $oldfailures : $newfailures;
+                $failures->{$status}++;
+
                 next if ($status !~ /^[0-9]+$/);
                 next if ($status eq "258"); # timeouts have their own color
                 $test->{colors}->{$status} = undef;
@@ -885,9 +925,16 @@ EOF
         }
         next if ($last == -1); # no report had a pattern of interest
 
+        $test->{newmodes} = {};
+        foreach my $status (keys %$newfailures)
+        {
+            next if ($oldfailures->{$status});
+            $test->{newmodes}->{$status} = 1;
+        }
         $fixed = 1 if (($test->{fixed} || 0) >= $fixed_threshold);
         my $listid = ($fixed or $last < @sortedbuilds - $patternbuilds) ? "old" :
                      ($first > $patternbuilds) ? "recent" :
+                     %{$test->{newmodes}} ? "newmode" :
                      "regular";
         push @{$lists{$listid}->{testnames}}, $testname;
     }
@@ -895,7 +942,7 @@ EOF
     # Generate the lists index (and up test unit links)
     print $html "<h1>$title</h1>\n";
     print $html "<ul>\n";
-    my @listids = ("recent", "regular", "old");
+    my @listids = ("recent", "newmode", "regular", "old");
     my $prevunit = "";
     foreach my $listid (@listids)
     {