Francois Gouget : testbot/TestWTBS: Allow verifying the presence of specific errors.

[Alexandre Julliard]

Module: tools
Branch: master
Commit: 1b3cd386fb5b51525336b721c4027c9042394da8
URL:    https://source.winehq.org/git/tools.git/?a=commit;h=1b3cd386fb5b51525336b721c4027c9042394da8

Author: Francois Gouget <fgouget at codeweavers.com>
Date:   Wed Jan 20 13:57:00 2021 +0100

testbot/TestWTBS: Allow verifying the presence of specific errors.

This allows verifying that the TestBot detected the errors issued by the
test suite; or reported the expected consistency issues.

For instance:
----- TestWTBS -----
g 0 tests.report.Report validation errors
n 0 kernel32:comm has unaccounted for failure messages
----- TestWTBS -----

Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard at winehq.org>

---

 testbot/tests/TestWTBS | 114 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 114 insertions(+)

diff --git a/testbot/tests/TestWTBS b/testbot/tests/TestWTBS
index 6e1e7ed..af43060 100755
--- a/testbot/tests/TestWTBS
+++ b/testbot/tests/TestWTBS
@@ -297,6 +297,24 @@ sub LoadTestInfo($)
       fail("$FileName: $Entry is not a valid property name");
     }
   }
+  foreach my $RawGroupName (@{$RawInfo->{ErrGroupNames}})
+  {
+    my $GroupName = lcfirst($RawGroupName);
+    if ($GroupName =~ s/^(tasks|build|tests|win|win32|win64|wine)\.(log|report)\.//)
+    {
+      my $ErrInfo = ($TestInfo->{$1}->{"$2.errors"} ||= {});
+      push @{$ErrInfo->{ErrGroupNames}}, $GroupName;
+      my $ErrGroup = $RawInfo->{ErrGroups}->{$RawGroupName};
+      $ErrInfo->{ErrGroups}->{$GroupName} = $ErrGroup;
+      $ErrInfo->{ErrCount} += @{$ErrGroup->{Errors}};
+      $ErrInfo->{NewCount} += $ErrGroup->{NewCount} || 0;
+      $HasTestInfo = 1;
+    }
+    else
+    {
+      fail("$FileName: $RawGroupName is not a valid group name");
+    }
+  }
   return undef if (!$HasTestInfo);
 
   # Fill in some useful defaults
@@ -345,6 +363,95 @@ sub IsMailingListJob($)
   return $Job->Remarks =~ /^\[\Q$PatchesMailingList\E\] /;
 }
 
+=pod
+=item <tasks.(log|report).groupname>
+
+Verifies the presence of new errors in the specified error log or report
+of the tasks in the specified category. For this, list the group and errors
+that are expected to appear; where the group name is prefixed by the task
+category and either 'log' or 'report'.
+
+For instance:
+g 0 tests.report.kernel32
+n 0 comm.c:2210: Test failed: WTBS Simulate an unreported test failure
+g 0 tests.report.Report validation errors
+n 0 kernel32:comm has unaccounted for failure messages
+
+This checks that the 'comm.c' failure appears in the 'kernel32' group
+of the reports of all test tasks; that the TestBot issues a report consistency
+failure as well; and that no other new error is present. If no preexisting
+errors are expected, that is if test unit produces no failure when run outside
+the test suite, then one could also issue the following check:
+
+p tests.TestFailures 2
+
+=cut
+
+sub CheckLogErrors($$$)
+{
+  my ($LogInfo, $RefInfo, $LogKey) = @_;
+
+  foreach my $GroupName (@{$LogInfo->{ErrGroupNames}})
+  {
+    my $LogGroup = $LogInfo->{ErrGroups}->{$GroupName};
+
+    my $RefGroup = $RefInfo->{ErrGroups}->{$GroupName};
+    if ($RefGroup)
+    {
+      my $Diff = Algorithm::Diff->new($LogGroup->{Errors}, $RefGroup->{Errors},
+                                      { keyGen => \&WineTestBot::LogUtils::_GetLineKey });
+      my ($LogIndex, $RefIndex) = (0, 0);
+      while ($Diff->Next())
+      {
+        my $SameCount = $Diff->Same();
+        if ($SameCount)
+        {
+          $LogIndex += $SameCount;
+          $RefIndex += $SameCount;
+        }
+        else
+        {
+          # Check for extra new errors in the log
+          my $ExtraCount = $Diff->Items(1);
+          foreach (1..$ExtraCount)
+          {
+            next if (!$LogGroup->{IsNew}->[$LogIndex]);
+            fail("$LogKey has an unexpected new error in $GroupName: $LogGroup->{Errors}->[$LogIndex]");
+            $LogIndex++;
+          }
+          # And for missing expected errors
+          my $MissingCount = $Diff->Items(2);
+          foreach (1..$MissingCount)
+          {
+            fail("$LogKey is missing an error in $GroupName: $RefGroup->{Errors}->[$RefIndex]");
+            $RefIndex++;
+          }
+        }
+      }
+    }
+    else
+    {
+      # Check that the log's extra errors are old errors
+      foreach my $ErrIndex (0..@{$LogGroup->{Errors}} - 1)
+      {
+        next if (!$LogGroup->{IsNew}->[$ErrIndex]);
+        fail("$LogKey has an unexpected new error in $GroupName: $LogGroup->{Errors}->[$ErrIndex]");
+      }
+    }
+  }
+  foreach my $GroupName (@{$RefInfo->{ErrGroupNames}})
+  {
+    my $LogGroup = $LogInfo->{ErrGroups}->{$GroupName};
+    next if ($LogGroup);
+
+    my $RefGroup = $RefInfo->{ErrGroups}->{$GroupName};
+    foreach my $Error (@{$RefGroup->{Errors}})
+    {
+      fail("$LogKey is missing an error in $GroupName: $Error");
+    }
+  }
+}
+
 =pod
 
 =item <job.Status>
@@ -378,9 +485,16 @@ sub CheckTask($$$)
     my $LogPath = $Task->GetDir() ."/$LogName";
     my $LogInfo = LoadLogErrors($LogPath);
 
+    my $LogType = "log";
     if ($LogName =~ /\.report$/)
     {
       $ReportCount++;
+      $LogType = "report";
+    }
+    if ($TaskInfo->{"$LogType.errors"})
+    {
+      CheckLogErrors($LogInfo, $TaskInfo->{"$LogType.errors"},
+                     TaskKeyStr($Task) ."/$LogName");
     }
   }
   if ($Task->Status eq "completed" and CheckValue($TaskInfo->{TestFailures}))