Francois Gouget : winetest/build-patterns: Detect fixed failures from the last failure day.
Alexandre Julliard
julliard at winehq.org
Fri Apr 30 15:44:59 CDT 2021
Module: tools
Branch: master
Commit: 1528f95b83cd13e118e2609cfc3752063ef22ae7
URL: https://source.winehq.org/git/tools.git/?a=commit;h=1528f95b83cd13e118e2609cfc3752063ef22ae7
Author: Francois Gouget <fgouget at codeweavers.com>
Date: Fri Apr 30 14:57:14 2021 +0200
winetest/build-patterns: Detect fixed failures from the last failure day.
Instead of analyzing the failures of a single report across time, this
analyzes the results of the last build with failures across reports.
This is then compared to the failure-free results of the next days to
determine the probability that the failure has been fixed.
This allows detecting when short-lived issues that impacted many test
configurations are fixed.
Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard at winehq.org>
---
winetest/build-patterns | 78 +++++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 76 insertions(+), 2 deletions(-)
diff --git a/winetest/build-patterns b/winetest/build-patterns
index 6a14fc3..460c17c 100755
--- a/winetest/build-patterns
+++ b/winetest/build-patterns
@@ -252,6 +252,14 @@ my %reports;
# - colors
# A hashtable of colors indexed by failure count.
#
+# - last
+# The @sortedbuilds index of the most recent build for which a report has a
+# failure. -1 if there is none.
+#
+# - fixed
+# True if the test had enough successful results after the last failure to be
+# considered fixed.
+#
# - testreports
# A hashtable mapping report directory names to objects storing the results
# for that test and report combination. Each testreport object has the
@@ -389,7 +397,7 @@ my @sortedreports = sort cmpreports keys %reports;
#
-# Analyze single-report patterns
+# Analyze single-report and single-build patterns
#
sub fail_type($)
@@ -434,6 +442,9 @@ sub get_fix_probability($$$)
foreach my $testname (keys %tests)
{
my $test = $tests{$testname};
+ $test->{last} = -1;
+ $test->{fixed} = 1;
+
foreach my $reportdir (@sortedreports)
{
my $testreport = $test->{testreports}->{$reportdir};
@@ -498,9 +509,18 @@ foreach my $testname (keys %tests)
$testreport->{failruns} += $testreport->{fixedruns} + 1;
$testreport->{fixedruns} = 0;
}
+ if ($test->{last} < $testreport->{last})
+ {
+ $test->{last} = $testreport->{last};
+ }
+
}
next if (!$testreport->{failed});
- next if (!$testreport->{fixedruns});
+ if (!$testreport->{fixedruns})
+ {
+ $test->{fixed} = 0;
+ next;
+ }
if ($testreport->{failtype} eq "random")
{
@@ -518,7 +538,60 @@ foreach my $testname (keys %tests)
# it is fixed.
$testreport->{fixed} = 1;
}
+ $test->{fixed} = 0 if ($testreport->{fixed} < $fixed_threshold);
+ }
+ if ($test->{last} == @sortedbuilds-1)
+ {
+ $test->{fixed} = 0;
+ next;
+ }
+
+ # No need for further analysis if this test unit seems fixed already
+ next if ($test->{fixed});
+ next if ($test->{last} < @sortedbuilds - $patternbuilds);
+
+ # Analyze the pattern formed by the results from the build of the latest
+ # failure: sometimes a commit will cause failures in many reports and is
+ # fixed soon after (e.g. next day). In such a case analysing the reports
+ # individually does not provide enough evidence that the bug is fixed until
+ # many days later. But analysing the results on the last failure build can
+ # yield evidence of a high failure rate so that the fix is obvious in the
+ # next few days.
+ my $lastbuild = $sortedbuilds[$test->{last}];
+ my ($failures, $failruns, $fixedruns);
+ foreach my $reportdir (keys %{$test->{testreports}})
+ {
+ my $testreport = $test->{testreports}->{$reportdir};
+ next if (!$testreport->{failed});
+
+ my $status = $testreport->{status}->{$lastbuild->{name}};
+ if (!defined $status and
+ $lastbuild->{hasreport}->{$reportdir} and
+ $lastbuild->{hastest}->{$testname})
+ {
+ $failruns++;
+ next;
+ }
+
+ my $failtype = fail_type($status);
+ next if ($failtype ne "random");
+ $failures++;
+ $failruns++;
+
+ for my $i ($test->{last}+1.. at sortedbuilds-1)
+ {
+ my $build = $sortedbuilds[$i];
+ $status = $testreport->{status}->{$build->{name}};
+ if (!defined $status and
+ $build->{hasreport}->{$reportdir} and
+ $build->{hastest}->{$testname})
+ {
+ $fixedruns++;
+ }
+ }
}
+ next if (!$fixedruns); # no evidence of the issue being fixed
+ $test->{fixed} = get_fix_probability($failures, $failruns, $fixedruns);
}
@@ -809,6 +882,7 @@ EOF
}
next if ($last == -1); # no report had a pattern of interest
+ $fixed = 1 if (($test->{fixed} || 0) >= $fixed_threshold);
my $listid = ($fixed or $last < @sortedbuilds - $patternbuilds) ? "old" :
($first > $patternbuilds) ? "recent" :
"regular";
More information about the wine-cvs
mailing list