[tools] winetest/build-patterns: Add a failed test units pattern.
Francois Gouget
fgouget at codeweavers.com
Thu May 13 05:24:06 CDT 2021
This pattern shows the number of failed test units for each test
configuration. This allows detecting when results get worse after
a VM update, or when a test configuration gets worse results than
its peers.
Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
---
winetest/build-patterns | 121 +++++++++++++++++++++++++++-------------
1 file changed, 83 insertions(+), 38 deletions(-)
diff --git a/winetest/build-patterns b/winetest/build-patterns
index f6704bd00..c9fcb6638 100755
--- a/winetest/build-patterns
+++ b/winetest/build-patterns
@@ -31,12 +31,19 @@ sub BEGIN
}
unshift @INC, $1 if ($0 =~ m=^(/.*)/[^/]+$=);
}
-use vars qw/$workdir $gitdir $gitweb $patternbuilds $fixed_threshold/;
+use vars qw/$workdir $gitdir $gitweb @groups $patternbuilds $fixed_threshold/;
require "winetest.conf";
my $name0=$0;
$name0 =~ s+^.*/++;
+# Map dissect's IDs to the platform groups
+my %idmap;
+foreach my $group (@groups)
+{
+ map { $idmap{$_} = $group } @{$group->{ids}};
+}
+
#
# Common helpers
@@ -249,6 +256,16 @@ my %reports;
# - name
# The uniquely identifying test name in the form 'module:unit'.
#
+# - id
+# The HTML id for linking to this test.
+#
+# - title
+# The test's HTML title.
+#
+# - desc
+# An optional chunk of HTML shown above the test pattern which can be used
+# to provide additional information about the test.
+#
# - source
# The source file for that test unit.
#
@@ -310,7 +327,7 @@ my %reports;
# - commits
# A hashtable of the commit objects, indexed first buy build name and then
# by commit id.
-my %tests;
+my %tests = ( 0 => { name => "0", id => "summary", newmodes => {} } );
# A set containing the test units source files.
my %testsources;
@@ -360,7 +377,9 @@ foreach my $build (@sortedbuilds)
if (!$test)
{
$tests{$testname} = $test = {
+ id => $testname,
name => $testname,
+ title => "<a href='tests/$testname.html'>$testname</a>",
source => $source,
};
$testsources{$source} = 1;
@@ -377,6 +396,10 @@ foreach my $build (@sortedbuilds)
next;
}
$test->{testreports}->{$reportdir}->{status}->{$build->{name}} = $status;
+
+ # Count failed test units
+ next if ($status =~ /^(?:missing.|skipped|stub)/);
+ $tests{0}->{testreports}->{$reportdir}->{status}->{$build->{name}}++;
}
}
}
@@ -882,7 +905,15 @@ sub write_pattern_line($$$)
else
{
($symbol, $class, $title, my $link, $attrs) = get_status_html($status, $test->{colors});
- if ($link eq "t")
+ if ($test->{name} eq "0")
+ {
+ $tag = "a";
+ my $platform = $reports{$reportdir}->{platform};
+ my $group = $idmap{$platform} || $idmap{unknown};
+ $attrs .= sprintf " href='%s/index_%s.html'",
+ $build->{name}, $group->{name};
+ }
+ elsif ($link eq "t")
{
$tag = "a";
$attrs .= sprintf " href='%s/%s/%s.html'",
@@ -1042,8 +1073,11 @@ sub write_pattern($$$)
my ($html, $test, $pagereports) = @_;
compute_set_colors($test->{colors});
- write_newmodes_line($html, $test) if (%{$test->{newmodes}});
+ print $html $test->{desc} if (defined $test->{desc});
+
+ print $html "<div class='test'>\n";
+ write_newmodes_line($html, $test) if (%{$test->{newmodes}});
foreach my $reportdir (@sortedreports)
{
next if (!$pagereports->{$reportdir});
@@ -1051,6 +1085,7 @@ sub write_pattern($$$)
write_pattern_line($html, $test, $reportdir);
}
write_commits($html, $test) if (%{$test->{commits}});
+ print $html "</div>\n";
}
sub write_patterns_list($$$$)
@@ -1063,8 +1098,8 @@ sub write_patterns_list($$$$)
my $testname = $testnames->[$i];
my $test = $tests{$testname};
- print $html "<div class='testfile' id='$testname'>\n";
- print $html "<div class='updownbar'><a href='tests/$testname.html'>$testname</a>";
+ print $html "<div class='testfile' id='$test->{id}'>\n";
+ print $html "<div class='updownbar'>$test->{title}";
print $html "<div class='ralign'>";
if ($mainpage)
{
@@ -1078,9 +1113,8 @@ sub write_patterns_list($$$$)
print $html "</div></div>\n";
- print $html "<div class='test'>\n";
write_pattern($html, $test, $pagereports);
- print $html "</div></div>\n";
+ print $html "</div>\n";
}
}
@@ -1114,30 +1148,12 @@ sub write_patterns_page($$$)
</head>
<body>
<div class="navbar"><a href="..">index</a> | Failure patterns: <a href="patterns.html">all</a> | <a href="patterns-tb-win.html">testbot windows</a> | <a href="patterns-tb-wine.html">testbot wine</a></div>
-
-<div class='testfile' id='summary'>
-<div class='updownbar'>$title</div>
-<p>The purpose of this page is to:</p>
-<ul>
- <li>Help detect new test failures, that is test / Wine regressions.</li>
- <li>Help diagnose failures by showing when they are specific to a set of Windows versions, a locale, a day of the week, etc.</li>
-</ul>
-<p>The patterns on this page read as follows:</p>
-<ul>
- <li>Each line corresponds to a test configuration sorted by platform and tag name.</li>
- <li>Test configurations with no failure for a given test are omitted from that test's pattern.</li>
- <li>Each column corresponds to a WineTest build from oldest to newest.</li>
- <li>Lines with a _N suffix are when WineTest was run more than once for a test configuration + build combination. They are rare and thus hang at the bottom of the pattern to not disrupt it.</li>
- <li>Each cell is a single letter describing the result for that test. See the tooltips for a description. Click on the letter to go to that result page.</li>
- <li>Each result is also color coded to make the patterns pop out. There is one color per type of result (crash , timeout, etc.), and one per number of failures (to help detect changes).</li>
- <li>Failure modes that have not been seen before (test that newly crashes, new failure count, etc.) are highlighted, as well as the configuration names in which it appears.</li>
- <li>Below the patterns is a line showing which builds have potentially related commits if any.
-</ul>
EOF
# Show the test units with recent failures first.
# Also use these lists to link the test units to each other.
my %lists = (
+ summary => {testnames => [ 0 ]},
recent => {title => "Recent failures",
desc => "recently started failing (the $patternbuilds oldest builds had no failure).",
testnames => []},
@@ -1194,6 +1210,7 @@ EOF
}
}
next if ($last == -1); # no report had a pattern of interest
+ next if ($testname eq "0");
$test->{newmodes} = {};
foreach my $status (keys %$newfailures)
@@ -1221,23 +1238,51 @@ EOF
}
# Generate the lists index (and up test unit links)
- print $html "<p>The patterns are grouped according to the likelyhood that they contain new failures:</p>\n";
- print $html "<ul>\n";
- my @listids = ("recent", "newmode", "regular", "old");
+ $tests{0}->{title} = $title;
+ $tests{0}->{desc} = "
+<p>The purpose of this page is to:</p>
+<ul>
+ <li>Help detect new test failures, that is test / Wine regressions.</li>
+ <li>Help diagnose failures by showing when they are specific to a set of Windows versions, a locale, a day of the week, etc.</li>
+ <li>Help detect test configuration changes that increase the number of failed test units.</li>
+ <li>Help identify test configurations that get worse results than their peers.</li>
+</ul>
+
+<p>The patterns on this page read as follows:</p>
+<ul>
+ <li>Each line corresponds to a test configuration sorted by platform and tag name.</li>
+ <li>Test configurations with no failure for a given test are omitted from that test's pattern.</li>
+ <li>Each column corresponds to a WineTest build from oldest to newest.</li>
+ <li>Lines with a _N suffix are when WineTest was run more than once for a test configuration + build combination. They are rare and thus hang at the bottom of the pattern to not disrupt it.</li>
+ <li>Each cell is a single letter describing the result for that test. See the tooltips for a description. Click on the letter to go to that result page.</li>
+ <li>Each result is also color coded to make the patterns pop out. There is one color per type of result (crash , timeout, etc.), and one per number of failures (to help detect changes).</li>
+ <li>Failure modes that have not been seen before (test that newly crashes, new failure count, etc.) are highlighted, as well as the configuration names in which it appears.</li>
+ <li>Below the patterns is a line showing which builds have potentially related commits if any.
+</ul>
+
+<p>The patterns are grouped according to the likelyhood that they contain new failures:</p>
+<ul>";
+
+ my @listids = ("summary", "recent", "newmode", "regular", "old");
my ($prevunit, $nolist) = ("", 1);
foreach my $listid (@listids)
{
my $list = $lists{$listid};
my $count = @{$list->{testnames}};
next if (!$count);
- print $html "<li><a href='#$listid'>$count tests</a> $list->{desc}</li>\n";
- $nolist = 0;
$list->{prevunit} = $prevunit;
- $prevunit = $list->{testnames}->[-1];
+ $prevunit = $tests{$list->{testnames}->[-1]}->{id};
+
+ next if (!$list->{desc});
+ $tests{0}->{desc} .= "<li><a href='#$listid'>$count tests</a> $list->{desc}</li>\n";
+ $nolist = 0;
+ }
+ $tests{0}->{desc} .= "<p>No failures. Don't change anything!</p>\n" if ($nolist);
+ $tests{0}->{desc} .= "</ul>";
+ if (!$nolist)
+ {
+ $tests{0}->{desc} .= "<p>Unlike the other patterns on this page, the one below shows the number of failed test units for each configuration.</p>";
}
- print $html "<p>No failures. Don't change anything!</p>\n" if ($nolist);
- print $html "</ul>\n";
- print $html "</div>\n";
# Link the last test unit of each list to the first one of the next list
my $nextunit;
@@ -1247,7 +1292,7 @@ EOF
my $count = @{$list->{testnames}};
next if (!$count);
$list->{nextunit} = $nextunit;
- $nextunit = $list->{testnames}->[0];
+ $nextunit = $tests{$list->{testnames}->[0]}->{id};
}
# Generate the test unit lists themselves
@@ -1257,7 +1302,7 @@ EOF
my $list = $lists{$listid};
my $count = @{$list->{testnames}};
next if (!$count);
- print $html "<h2 id='$listid'>$list->{title}</h2>\n";
+ print $html "<h2 id='$listid'>$list->{title}</h2>\n" if ($list->{title});
write_patterns_list($html, $pagereports, $mainpage, $list);
}
--
2.20.1
More information about the wine-devel
mailing list