Francois Gouget : testbot/LogUtils: Deduplicate the latest WineTest reports.

[Alexandre Julliard]

Module: tools
Branch: master
Commit: 8c6d8e66595d376adbe3cb1f199dbb69331739a8
URL:    https://source.winehq.org/git/tools.git/?a=commit;h=8c6d8e66595d376adbe3cb1f199dbb69331739a8

Author: Francois Gouget <fgouget at codeweavers.com>
Date:   Mon Feb 24 04:45:23 2020 +0100

testbot/LogUtils: Deduplicate the latest WineTest reports.

There is no need to keep old logs if they don't contain errors that are
already present in the latest one. It cuts down on the number of reports
that test results need to be compared to to detect new failures.

Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard at winehq.org>

---

 testbot/bin/UpdateTaskLogs          | 21 +++++---
 testbot/lib/WineTestBot/LogUtils.pm | 95 +++++++++++++++++++++++++++++++++++--
 2 files changed, 107 insertions(+), 9 deletions(-)

diff --git a/testbot/bin/UpdateTaskLogs b/testbot/bin/UpdateTaskLogs
index 49f3c2e..94adf42 100755
--- a/testbot/bin/UpdateTaskLogs
+++ b/testbot/bin/UpdateTaskLogs
@@ -210,6 +210,9 @@ sub BuildErrorsCache($$$$;$)
   return CreateLogErrorsCache($LogInfo, $Task);
 }
 
+# The list of files in latest/. This includes files deleted for redundancy.
+my %LatestFiles;
+
 sub DoUpdateLatestReport($$$)
 {
   my ($Task, $ReportName, $SrcReportPath) = @_;
@@ -218,6 +221,8 @@ sub DoUpdateLatestReport($$$)
   my $SrcIsRef = ($SrcReportPath =~ /-job\d+-/);
   my $RefReportName = $SrcIsRef ? basename($SrcReportPath) :
                                   $Task->GetRefReportName($ReportName);
+  return 0 if ($LatestFiles{$RefReportName});
+  $LatestFiles{$RefReportName} = 1;
 
   my $Rc = 0;
   my $LatestReportPath = "$DataDir/latest/$RefReportName";
@@ -519,14 +524,18 @@ sub ProcessLatestReports()
     {
       $Rc += Delete("$LatestReportPath.errors", "orphaned");
     }
-    elsif (!-f "$LatestReportPath.errors")
+    else
     {
-      # Build the missing .errors file
-      my $ErrMessage = BuildErrorsCache("$DataDir/latest", $RefReportName, 1, 0);
-      if (defined $ErrMessage)
+      $LatestFiles{$RefReportName} = 1;
+      if (!-f "$LatestReportPath.errors")
       {
-        Error "$ErrMessage\n";
-        $Rc = 1;
+        # Build the missing .errors file
+        my $ErrMessage = BuildErrorsCache("$DataDir/latest", $RefReportName, 1, 0);
+        if (defined $ErrMessage)
+        {
+          Error "$ErrMessage\n";
+          $Rc = 1;
+        }
       }
     }
   }
diff --git a/testbot/lib/WineTestBot/LogUtils.pm b/testbot/lib/WineTestBot/LogUtils.pm
index f7b6f42..e309753 100644
--- a/testbot/lib/WineTestBot/LogUtils.pm
+++ b/testbot/lib/WineTestBot/LogUtils.pm
@@ -1028,9 +1028,9 @@ sub MarkAllErrorsAsNew($)
 
 =item C<_GetLineKey()>
 
-This is a helper for TagNewErrors(). It reformats the log lines so they
-can meaningfully be compared to the reference log even if line numbers change,
-etc.
+This is a helper for _DeduplicateLatestReport() and TagNewErrors(). It
+reformats the log lines so they can meaningfully be compared to the reference
+log even if line numbers change, etc.
 
 =back
 =cut
@@ -1244,6 +1244,92 @@ sub SnapshotLatestReport($$)
   return \@ErrMessages;
 }
 
+sub _IsReportRedundant($$)
+{
+  my ($RefInfo, $LogInfo) = @_;
+
+  return undef if (($RefInfo->{ErrCount} || 0) < ($LogInfo->{ErrCount} || 0));
+  return 1 if (($LogInfo->{ErrCount} || 0) == 0);
+
+  foreach my $GroupName (@{$LogInfo->{ErrGroupNames}})
+  {
+    my $LogGroup = $LogInfo->{ErrGroups}->{$GroupName};
+    my $RefGroup = $RefInfo->{ErrGroups}->{$GroupName};
+    return undef if (!$RefGroup);
+
+    my $Diff = Algorithm::Diff->new($RefGroup->{Errors}, $LogGroup->{Errors},
+                                    { keyGen => \&_GetLineKey });
+    while ($Diff->Next())
+    {
+      # Same() > 0 => Items(2) > 0 despite there being no new lines
+      if (!$Diff->Same() and $Diff->Items(2) > 0)
+      {
+        # This old log has errors not present in the latest one so keep it
+        return undef;
+      }
+    }
+  }
+
+  return 1;
+}
+
+sub _DeduplicateLatestReport($)
+{
+  my ($RefReportName) = @_;
+
+  my $LatestGlob = $RefReportName;
+  $LatestGlob =~ s/-job\d+-/-job*-/;
+  my @LatestPaths = glob("$DataDir/latest/$LatestGlob");
+  return undef if (@LatestPaths <= 1);
+
+  my $RefReportPath = "$DataDir/latest/$RefReportName";
+  my $RefInfo = LoadLogErrors($RefReportPath);
+  return $RefInfo->{BadLog} if (defined $RefInfo->{BadLog});
+
+  my %LatestAges;
+  foreach my $LogPath (@LatestPaths)
+  {
+    $LatestAges{$LogPath} = -M $LogPath || 0;
+  }
+  my $RefAge = $LatestAges{$RefReportPath};
+
+  my $ErrMessage;
+  my $ReportRE = $RefReportName;
+  $ReportRE =~ s/^([a-zA-Z0-9_]+)-job\d+-([a-zA-Z0-9_]+)$/\\Q$1\\E-job[0-9]+-\\Q$2/;
+  foreach my $LogPath (sort { $LatestAges{$a} <=> $LatestAges{$b} } @LatestPaths)
+  {
+    my $LogName = basename($LogPath);
+    next if ($LogName eq $RefReportName);
+    next if ($LogName !~ /^($ReportRE)$/);
+    $LogName = $1; # untaint
+    $LogPath = "$DataDir/latest/$LogName";
+
+    my $LogInfo = LoadLogErrors($LogPath);
+    if (defined $LogInfo->{BadLog})
+    {
+      # Take note of the error but continue to try deduplicating
+      $ErrMessage = $LogInfo->{BadLog};
+      next;
+    }
+    if ($RefAge < $LatestAges{$LogPath})
+    {
+      if (_IsReportRedundant($RefInfo, $LogInfo))
+      {
+        unlink $LogPath, "$LogPath.errors";
+      }
+    }
+    else
+    {
+      if (_IsReportRedundant($LogInfo, $RefInfo))
+      {
+        unlink $RefReportPath, "$RefReportPath.errors";
+        last;
+      }
+    }
+  }
+  return $ErrMessage;
+}
+
 sub UpdateLatestReport($$)
 {
   my ($RefReportName, $SrcReportPath) = @_;
@@ -1262,6 +1348,9 @@ sub UpdateLatestReport($$)
     }
   }
 
+  my $ErrMessage = _DeduplicateLatestReport($RefReportName);
+  push @ErrMessages, $ErrMessage if (defined $ErrMessage);
+
   return \@ErrMessages;
 }