[tools] testbot/TestWTBS: Allow mission-specific directives, e.g. for test=module.
Francois Gouget
fgouget at codeweavers.com
Mon Feb 1 05:02:45 CST 2021
A different set of tests are run depending on whether a VM's mission
is test=test (the default) or test=module. Being unable to distinguish
these two cases prevents checking that the right set of test units is
being run in many cases.
So add support for the wine:build, wine:test, wine:module categories to
allow providing specific checks for the test=build/test/module cases
respectively.
Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
---
testbot/tests/TestWTBS | 184 +++++++++++++++++++++++++++++------------
1 file changed, 131 insertions(+), 53 deletions(-)
diff --git a/testbot/tests/TestWTBS b/testbot/tests/TestWTBS
index bc4e0d9ba..f6ca54e84 100755
--- a/testbot/tests/TestWTBS
+++ b/testbot/tests/TestWTBS
@@ -48,6 +48,7 @@ use WineTestBot::Config; # For $PatchesMailingList
use WineTestBot::Jobs;
use WineTestBot::Log;
use WineTestBot::LogUtils;
+use WineTestBot::Missions;
use WineTestBot::Patches;
use WineTestBot::VMs;
@@ -190,13 +191,15 @@ Property names are of the form 'category.name' where the category is one of:
- build for checks to perform on the build task.
- win32, win64 for checks to perform on the Windows 32- or 64-bit test results
respectively.
-- win for the checks to perform on both the 32- and 64-bit test results; that
- is equivalent to duplicating the directive for the win32 and win64
- categories.
-- wine for the checks to perform on the Wine test results.
+- win for the checks to perform on both the 32- and 64-bit test results. This
+ is equivalent to duplicating the directive for win32 and win64.
+- wine:build, wine:test, wine:module for checks to perform on test results of
+ missions with test=build, test=test and test=module respectively.
+- wine for the checks to perform on the Wine test results. This is equivalent
+ to duplicating the directive for the wine:build, wine:test and wine:module.
- tests for checks to perform on all test results, that is equivalent to
- duplicating the directive for the win32, win64 and wine categories (but not
- build since it does not run the tests).
+ duplicating the directive for the win and wine categories (but not build
+ since it does not run the tests).
A corollary is that any property which is documented as being valid for a
category can also be specified for any of its subcategories. So for instance if
@@ -208,8 +211,9 @@ sub DumpTestInfo($)
{
my ($TestInfo) = @_;
- foreach my $Category ("webpatch", "patch", "job", "tasks", "tests", "win",
- "build", "win32", "win64", "wine")
+ foreach my $Category ("webpatch", "patch", "job", "tasks", "tests", "build",
+ "win", "win32", "win64",
+ "wine", "wine:build", "wine:test", "wine:module")
{
WineTestBot::LogUtils::_WriteLogErrorsToFh(*STDERR, $TestInfo->{$Category});
WineTestBot::LogUtils::_DumpErrors($Category, $TestInfo->{$Category});
@@ -293,14 +297,15 @@ sub LoadTestInfo($)
# Split up the information to jobs, tasks, etc.
my $TestInfo = {
webpatch => {}, patch => {}, job => {},
- tasks => {}, tests => {}, win => {},
- build => {}, win32 => {}, win64 => {}, wine => {},
+ tasks => {}, tests => {}, build => {},
+ win => {}, win32 => {}, win64 => {},
+ wine => {}, "wine:build" => {}, "wine:test" => {}, "wine:module" => {},
};
my $HasTestInfo;
foreach my $Entry (keys %{$RawInfo})
{
my $Field = lcfirst($Entry);
- if ($Field =~ s/^(webpatch|patch|job|tasks|tests|win|build|win32|win64|wine)\.//)
+ if ($Field =~ s/^(webpatch|patch|job|tasks|tests|build|win|win32|win64|wine|wine:build|wine:test|wine:module)\.//)
{
my $TaskType = $1;
$TestInfo->{$TaskType}->{$Field} = $RawInfo->{$Entry};
@@ -314,7 +319,7 @@ sub LoadTestInfo($)
foreach my $RawGroupName (@{$RawInfo->{ErrGroupNames}})
{
my $GroupName = lcfirst($RawGroupName);
- if ($GroupName =~ s/^(tasks|build|tests|win|win32|win64|wine)\.(report|log|testbot)\.//)
+ if ($GroupName =~ s/^(tasks|tests|build|win|win32|win64|wine|wine:build|wine:test|wine:module)\.(report|log|testbot)\.//)
{
my $ErrInfo = ($TestInfo->{$1}->{"$2.errors"} ||= {});
push @{$ErrInfo->{ErrGroupNames}}, $GroupName;
@@ -338,11 +343,16 @@ sub LoadTestInfo($)
SetDefault($TestInfo, "tasks", "Status", "completed");
SetDefault($TestInfo, "tasks", "HasTask", 1);
}
+ if (defined $TestInfo->{"wine:build"}->{TestUnits})
+ {
+ fail("wine:build.TestUnits should not be set");
+ }
# Then propagate the defaults
foreach my $Pair (["tasks", ["build", "tests"]],
["tests", ["win", "wine"]],
- ["win", ["win32", "win64"]])
+ ["win", ["win32", "win64"]],
+ ["wine", ["wine:test", "wine:module"]])
{
my ($Src, $TaskTypes) = @$Pair;
foreach my $Field (keys %{$TestInfo->{$Src}})
@@ -368,41 +378,51 @@ sub LoadTestInfo($)
}
}
}
+ # Reset the inherited wine:build.TestUnits since it makes no sense
+ delete $TestInfo->{"wine:build"}->{TestUnits};
# Automatically check the Task: lines in simple cases
- # Make sure no test is run for build tasks
- my $GrepV = ($TestInfo->{build}->{"report.GrepV"} ||= []);
- push @$GrepV, '.';
- $GrepV = ($TestInfo->{build}->{"log.GrepV"} ||= []);
- push @$GrepV, '^Task: tests$';
- if (($TestInfo->{build}->{Status} || "") =~ /^bad(?:build|patch)$/ or
- $TestInfo->{build}->{HasTimeout})
- {
- push @$GrepV, '^Task: ok$';
- }
- else
+ foreach my $Build ("build", "wine:build")
{
- my $Grep = ($TestInfo->{build}->{"log.Grep"} ||= []);
- push @$Grep, '^Task: ok$';
+ # Make sure no test is run for build tasks
+ my $GrepV = ($TestInfo->{$Build}->{"report.GrepV"} ||= []);
+ push @$GrepV, '.';
+ $GrepV = ($TestInfo->{$Build}->{"log.GrepV"} ||= []);
+ push @$GrepV, '^Task: tests$';
+
+ if (($TestInfo->{$Build}->{Status} || "") =~ /^bad(?:build|patch)$/ or
+ $TestInfo->{$Build}->{HasTimeout})
+ {
+ push @$GrepV, '^Task: ok$';
+ }
+ else
+ {
+ my $Grep = ($TestInfo->{$Build}->{"log.Grep"} ||= []);
+ push @$Grep, '^Task: ok$';
+ }
}
- # Note: Depending on where the timeout occurs, the wine task log may or
- # may not have a 'Task: ok' line.
- if (($TestInfo->{wine}->{Status} || "") eq "completed" and
- !$TestInfo->{wine}->{HasTimeout})
+ foreach my $Test ("test", "module")
{
- my $Grep = ($TestInfo->{wine}->{"log.Grep"} ||= []);
- if (CheckValue($TestInfo->{wine}->{TestUnits}))
+ # Note: Depending on where the timeout occurs, the wine task log may or
+ # may not have a 'Task: ok' line.
+ if (($TestInfo->{"wine:$Test"}->{Status} || "") eq "completed" and
+ !$TestInfo->{"wine:$Test"}->{HasTimeout})
{
- push @$Grep, '^Task: tests$';
+ my $Grep = ($TestInfo->{"wine:$Test"}->{"log.Grep"} ||= []);
+ if (CheckValue($TestInfo->{"wine:$Test"}->{TestUnits}))
+ {
+ push @$Grep, '^Task: tests$';
+ }
+ push @$Grep, '^Task: ok$';
}
- push @$Grep, '^Task: ok$';
}
# Validate and fix the Grep* fields
foreach my $GrepType ("Grep", "GrepV")
{
- foreach my $TaskType ("tasks", "tests", "build", "win", "win32", "win64", "wine")
+ foreach my $TaskType ("tasks", "tests", "build", "win", "win32", "win64",
+ "wine", "wine:build", "wine:test", "wine:module")
{
foreach my $LogType ("report", "log", "testbot")
{
@@ -426,15 +446,19 @@ sub LoadTestInfo($)
CheckValue($TestInfo->{job}->{Status}) and
!SkipCheck($TestInfo->{win32}->{TestFailures}) and
!SkipCheck($TestInfo->{win64}->{TestFailures}) and
- !SkipCheck($TestInfo->{wine}->{TestFailures}))
+ !SkipCheck($TestInfo->{"wine:test"}->{TestFailures}) and
+ !SkipCheck($TestInfo->{"wine:module"}->{TestFailures}))
{
my $Status = ($TestInfo->{job}->{Status} ne 'completed' or
$TestInfo->{win32}->{TestFailures} or
$TestInfo->{win64}->{TestFailures} or
- $TestInfo->{wine}->{TestFailures} or
+ $TestInfo->{"wine:test"}->{TestFailures} or
+ $TestInfo->{"wine:module"}->{TestFailures} or
$TestInfo->{win32}->{HasTimeout} or
$TestInfo->{win64}->{HasTimeout} or
- $TestInfo->{wine}->{HasTimeout}) ? "Failed" : "OK";
+ $TestInfo->{"wine:build"}->{HasTimeout} or
+ $TestInfo->{"wine:test"}->{HasTimeout} or
+ $TestInfo->{"wine:module"}->{HasTimeout}) ? "Failed" : "OK";
SetDefault($TestInfo, "webpatch", "Status", $Status);
}
@@ -828,9 +852,30 @@ sub CheckTask($$$$)
$TestUnits->{$TaskType}->{"*skipped*"} = 1;
}
+ # Assume the VM's Missions field has not changed since the tests were run
+ my ($ErrMessage, $Missions) = ParseMissionStatement($Task->Missions);
+ if (@$Missions != 1)
+ {
+ fail(TaskKeyStr($Task) ." has an invalid missions field: ". $Task->Missions);
+ return;
+ }
+
+ my %ReportTypes;
+ if ($TaskType =~ /^win/)
+ {
+ foreach my $Mission (@{$Missions->[0]->{Missions}})
+ {
+ my $ReportName = GetMissionBaseName($Mission) .".report";
+ my $MissionType = $TaskType;
+ $MissionType .= ":". ($Mission->{test} || "test") if ($TaskType eq "wine");
+ $ReportTypes{$ReportName} = $MissionType;
+ }
+ }
+
my $CheckTimeouts = ($Task->Status eq "completed" and
CheckValue($TaskInfo->{HasTimeout}));
+ my $ExpectedFailures;
my ($ReportCount, $TimeoutCount, $NewFailures) = (0, 0, 0);
foreach my $LogName (@{GetLogFileNames($Task->GetDir())})
{
@@ -838,15 +883,33 @@ sub CheckTask($$$$)
my $LogInfo = LoadLogErrors($LogPath);
$NewFailures += $LogInfo->{NewCount} || 0;
+ # Get the mission-specific "wine:xxx" report directives
+ my $MissionType = $ReportTypes{$LogName} || $TaskType;
+ my $MissionInfo = $TestInfo->{$MissionType};
my $LogType = "log";
if ($LogName =~ /\.report$/)
{
- $ReportCount++;
$LogType = "report";
+ $ReportCount++;
+
+ ok($ReportTypes{$LogName}, "Check that $LogName is expected");
+
if ($TaskType eq "wine")
{
my $ReportTestUnits = GetReportTestUnits($LogPath);
- map { $TestUnits->{wine}->{$_} = 1 } (keys %$ReportTestUnits);
+ map { $TestUnits->{$MissionType}->{$_} = 1 } (keys %$ReportTestUnits);
+ }
+
+ if (CheckValue($MissionInfo->{TestFailures}))
+ {
+ ok(($LogInfo->{ErrCount} || 0) <= $MissionInfo->{TestFailures},
+ "Check Failures of $LogName in task ". TaskKeyStr($Task))
+ or diag("report error count = ", ($LogInfo->{ErrCount} || 0), ", expected at most $MissionInfo->{TestFailures}");
+ $ExpectedFailures += $MissionInfo->{TestFailures};
+ }
+ else
+ {
+ $ExpectedFailures = undef;
}
}
elsif ($LogName =~ /^testbot\./)
@@ -854,24 +917,35 @@ sub CheckTask($$$$)
$LogType = "testbot";
}
- if ($TaskInfo->{"$LogType.errors"} or $CheckTimeouts)
+ if ($MissionInfo->{"$LogType.errors"} or $CheckTimeouts)
{
- my $HasTimeout = CheckLogErrors($LogInfo, $TaskInfo->{"$LogType.errors"},
+ my $HasTimeout = CheckLogErrors($LogInfo, $MissionInfo->{"$LogType.errors"},
TaskKeyStr($Task) ."/$LogName",
- $TaskInfo->{HasTimeout});
+ $MissionInfo->{HasTimeout});
$TimeoutCount++ if ($HasTimeout);
}
- GrepTaskLog($Task, $LogName, $TaskInfo, "$LogType.");
+ GrepTaskLog($Task, $LogName, $MissionInfo, "$LogType.");
}
if ($CheckTimeouts)
{
ok($TimeoutCount >= $ReportCount, "Expecting 1+ timeout per report: $TimeoutCount timeouts, $ReportCount reports");
}
- if ($Task->Status eq "completed" and CheckValue($TaskInfo->{TestFailures}))
+ if ($Task->Status eq "completed")
{
- # Scale the expected TestFailures count with the number of times the test
- # was run, i.e. $ReportCount, or take it as is if no report is available.
- is($Task->TestFailures, $TaskInfo->{TestFailures} * ($ReportCount || 1), "Check Failures of task ". TaskKeyStr($Task));
+ if (defined $ExpectedFailures)
+ {
+ is($Task->TestFailures, $ExpectedFailures, "Check Failures of task ". TaskKeyStr($Task));
+ }
+ elsif (CheckValue($TaskInfo->{TestFailures}) and !$ReportCount)
+ {
+ # Scale the expected TestFailures count with the number of times the test
+ # was run, i.e. $ReportCount, or take it as is if no report is available.
+ is($Task->TestFailures, $TaskInfo->{TestFailures}, "Check Failures of task ". TaskKeyStr($Task));
+ }
+ # else there are reports for which we cannot check the TestFailures count.
+ # In particular this can happen if the test failure count can be checked
+ # for test=test but not for test=module. Then whether TestFailures can be
+ # checked or not depends on the missions mix.
}
return $NewFailures;
}
@@ -973,7 +1047,7 @@ sub CheckJobTree($;$)
$CheckedJobs{$JobId} = 1;
my ($HasTask, $HasNewFailures);
- my $TestUnits = { wine => {} };
+ my $TestUnits = {};
my $Email = $Emails{$JobId};
my %FailedVMs;
@@ -1059,12 +1133,16 @@ sub CheckJobTree($;$)
}
next if ($TestUnits->{$Type}->{"*skipped*"});
- if (CheckValue($TypeInfo->{TestUnits}))
+ my @MissionTypes = $Type eq "wine" ? ("wine:test", "wine:module") : ($Type);
+ foreach my $MissionType (@MissionTypes)
{
- foreach my $TestUnit (split / +/, $TypeInfo->{TestUnits})
+ my $MissionInfo = $TestInfo->{$MissionType};
+ next if (!CheckValue($MissionInfo->{TestUnits}));
+
+ foreach my $TestUnit (split / +/, $MissionInfo->{TestUnits})
{
- ok($TestUnits->{$Type}->{$TestUnit}, "Check that $TestUnit was tested by $Type VMs for job $JobId")
- or diag("TestUnits=", join(" ", sort keys %{$TestUnits->{$Type}}));
+ ok($TestUnits->{$MissionType}->{$TestUnit}, "Check that $TestUnit was tested by $MissionType VMs for job $JobId")
+ or diag("TestUnits=", join(" ", sort keys %{$TestUnits->{$MissionType}}));
}
}
}
--
2.20.1
More information about the wine-devel
mailing list