[1/2] testbot: Integrate the WineTestBot with the patches website.
Francois Gouget
fgouget at codeweavers.com
Wed Mar 13 10:03:36 CDT 2013
This lets the WineTestBot create jobs for each patch sent to wine-patches.
However for now the part that sends the results back to the patches website is disabled to preserve compatibility with the old WineTestBot (see WineSendLog.pl).
---
This takes advantage of the fact that the patches website and the
WineTestBot server are going to be running on the same machine now to
simplify the way they interact. So instead of exchanging emails and
grabbing patches via http they now simply work off of a shared
directory. At some point they may get even more integrated.
testbot/bin/Engine.pl | 65 --------------
testbot/bin/WinePatchesWebGet.pl | 108 ------------------------
testbot/bin/WinePatchesWebNotify.pl | 55 ------------
testbot/bin/WinePatchesWebSubmit.pl | 92 ++++++++++++++++++++
testbot/bin/WineSendLog.pl | 92 ++++++++++----------
testbot/doc/INSTALL.txt | 21 ++---
testbot/lib/WineTestBot/Config.pm | 4 +-
testbot/lib/WineTestBot/ConfigLocalTemplate.pl | 4 -
testbot/lib/WineTestBot/Engine/Notify.pm | 20 +----
9 files changed, 146 insertions(+), 315 deletions(-)
delete mode 100755 testbot/bin/WinePatchesWebGet.pl
delete mode 100755 testbot/bin/WinePatchesWebNotify.pl
create mode 100755 testbot/bin/WinePatchesWebSubmit.pl
diff --git a/testbot/bin/Engine.pl b/testbot/bin/Engine.pl
index 3d8f159..3f9c9b4 100755
--- a/testbot/bin/Engine.pl
+++ b/testbot/bin/Engine.pl
@@ -348,66 +348,6 @@ sub HandleWinePatchMLSubmission
return "1OK";
}
-sub HandleWinePatchWebNotification
-{
- # Validate file name
- if ($_[0] !~ m/([0-9a-fA-F]{32}_patchnotification)/)
- {
- return "0Invalid file name";
- }
- my $FullMessageFileName = "$DataDir/staging/$1";
-
- my $LatestWebPatchId;
- if (open(NOTIFICATION, "<$FullMessageFileName"))
- {
- my $Line;
- while (defined($Line = <NOTIFICATION>) && ! defined($LatestWebPatchId))
- {
- if ($Line =~ m/^The latest patch is (\d+)$/)
- {
- $LatestWebPatchId = $1;
- }
- }
- close(NOTIFICATION);
- }
-
- unlink($FullMessageFileName);
-
- if (! defined($LatestWebPatchId))
- {
- return "0No patch id found in message";
- }
-
- my $MaxExistingWebPatchId = 0;
- foreach my $Patch (@{CreatePatches()->GetItems()})
- {
- if ($MaxExistingWebPatchId < $Patch->WebPatchId)
- {
- $MaxExistingWebPatchId = $Patch->WebPatchId;
- }
- }
-
- if ($MaxExistingWebPatchId < $LatestWebPatchId)
- {
- $ActiveBackEnds{'WineTestBot'}->PrepareForFork();
- my $Pid = fork;
- if (!defined $Pid)
- {
- LogMsg "Unable to fork for WinePatchesWebGet.pl: $!\n";
- }
- elsif (!$Pid)
- {
- WineTestBot::Log::SetupRedirects();
- exec("$BinDir/WinePatchesWebGet.pl " . ($MaxExistingWebPatchId + 1) .
- " " . $LatestWebPatchId) or
- LogMsg "Unable to exec WinePatchesWebGet.pl: $!\n";
- exit(1);
- }
- }
-
- return "1OK";
-}
-
sub HandleWinePatchWebSubmission
{
# Validate file name
@@ -489,7 +429,6 @@ my %Handlers=(
"taskcomplete" => \&HandleTaskComplete,
"vmstatuschange" => \&HandleVMStatusChange,
"winepatchmlsubmission" => \&HandleWinePatchMLSubmission,
- "winepatchwebnotification" => \&HandleWinePatchWebNotification,
"winepatchwebsubmission" => \&HandleWinePatchWebSubmission,
);
@@ -566,10 +505,6 @@ sub SafetyNet
{
HandleWinePatchMLSubmission($DirEntry);
}
- elsif ($DirEntry =~ m/[0-9a-fA-F]{32}_patchnotification/)
- {
- HandleWinePatchWebNotification($DirEntry);
- }
}
}
diff --git a/testbot/bin/WinePatchesWebGet.pl b/testbot/bin/WinePatchesWebGet.pl
deleted file mode 100755
index 85f7986..0000000
--- a/testbot/bin/WinePatchesWebGet.pl
+++ /dev/null
@@ -1,108 +0,0 @@
-#!/usr/bin/perl -Tw
-#
-# Retrieve the latest patches from http://source.winehq.org/patches and submit
-# them for testing. See also WinePatchesWebNotify.pl.
-#
-# Copyright 2009 Ge van Geldorp
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
-
-use strict;
-
-sub BEGIN
-{
- if ($0 !~ m=^/=)
- {
- # Turn $0 into an absolute path so it can safely be used in @INC
- require Cwd;
- $0 = Cwd::cwd() . "/$0";
- }
- if ($0 =~ m=^(/.*)/[^/]+/[^/]+$=)
- {
- $::RootDir = $1;
- unshift @INC, "$::RootDir/lib";
- }
-}
-
-use LWP::UserAgent;
-use HTTP::Request;
-use HTTP::Response;
-use HTTP::Status;
-use WineTestBot::Config;
-use WineTestBot::Log;
-use WineTestBot::Utils;
-use WineTestBot::Engine::Notify;
-
-my ($StartWebPatchId, $EndWebPatchId) = @ARGV;
-if (! $StartWebPatchId || ! $EndWebPatchId)
-{
- die "Usage: WinePatchesWebGet.pl <StartWebPatchId> <EndWebPatchId>";
-}
-
-if ($StartWebPatchId =~ m/^(\d+)$/)
-{
- $StartWebPatchId = $1;
-}
-else
-{
- die "WinePatchesWebGet: Invalid StartWebPatchId $StartWebPatchId";
-}
-if ($EndWebPatchId =~ m/^(\d+)$/)
-{
- $EndWebPatchId = $1;
-}
-else
-{
- die "WinePatchesWebGet: Invalid EndWebPatchId $EndWebPatchId";
-}
-if ($EndWebPatchId < $StartWebPatchId)
-{
- die "WinePatchesWebGet: EndWebPatchId $EndWebPatchId shouldn't be smaller than StartWebPatchId $StartWebPatchId";
-}
-
-my $BaseURL = "http://source.winehq.org/patches/data";
-
-my $UA = LWP::UserAgent->new();
-$UA->agent("WineTestBot");
-
-foreach my $WebPatchId ($StartWebPatchId..$EndWebPatchId)
-{
- my $Request = HTTP::Request->new(GET => "$BaseURL/$WebPatchId");
- my $Response = $UA->request($Request);
- if ($Response->code != RC_OK)
- {
- LogMsg "Unexpected HTTP response code ", $Response->code, "\n";
- exit 1;
- }
- my $FileNameRandomPart = GenerateRandomString(32);
- while (-e "$DataDir/staging/${FileNameRandomPart}_patch_$WebPatchId")
- {
- $FileNameRandomPart = GenerateRandomString(32);
- }
- my $StagingFileName = "$DataDir/staging/${FileNameRandomPart}_patch_$WebPatchId";
- if (! open STAGINGFILE, ">$StagingFileName")
- {
- LogMsg "Can't create staging file $StagingFileName: $!\n";
- exit 1;
- }
- print STAGINGFILE $Response->decoded_content();
- close STAGINGFILE;
-
- WinePatchWebSubmission("${FileNameRandomPart}_patch_$WebPatchId", $WebPatchId);
-
- LogMsg "Retrieved patch $WebPatchId\n";
-}
-
-exit;
diff --git a/testbot/bin/WinePatchesWebNotify.pl b/testbot/bin/WinePatchesWebNotify.pl
deleted file mode 100755
index 0fd0318..0000000
--- a/testbot/bin/WinePatchesWebNotify.pl
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/usr/bin/perl -Tw
-#
-# Notifies WineTestBot that there are new patches to test on
-# http://source.winehq.org/patches.
-#
-# Copyright 2009 Ge van Geldorp
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
-
-use strict;
-
-sub BEGIN
-{
- if ($0 !~ m=^/=)
- {
- # Turn $0 into an absolute path so it can safely be used in @INC
- require Cwd;
- $0 = Cwd::cwd() . "/$0";
- }
- if ($0 =~ m=^(/.*)/[^/]+/[^/]+$=)
- {
- $::RootDir = $1;
- unshift @INC, "$::RootDir/lib";
- }
-}
-
-use File::Copy;
-use WineTestBot::Config;
-use WineTestBot::Utils;
-use WineTestBot::Engine::Notify;
-
-# Store the message in the staging dir
-my $FileNameRandomPart = GenerateRandomString(32);
-while (-e ("$DataDir/staging/${FileNameRandomPart}_patchnotification"))
-{
- $FileNameRandomPart = GenerateRandomString(32);
-}
-copy(\*STDIN, "$DataDir/staging/${FileNameRandomPart}_patchnotification");
-
-# Let the engine handle it
-WinePatchWebNotification("${FileNameRandomPart}_patchnotification");
-
-exit 0;
diff --git a/testbot/bin/WinePatchesWebSubmit.pl b/testbot/bin/WinePatchesWebSubmit.pl
new file mode 100755
index 0000000..d5b032a
--- /dev/null
+++ b/testbot/bin/WinePatchesWebSubmit.pl
@@ -0,0 +1,92 @@
+#!/usr/bin/perl -Tw
+#
+# Retrieve the latest patches from the patches website and submit them for
+# testing.
+#
+# Copyright 2009 Ge van Geldorp
+# Copyright 2013 Francois Gouget
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
+
+use strict;
+
+sub BEGIN
+{
+ if ($0 !~ m=^/=)
+ {
+ # Turn $0 into an absolute path so it can safely be used in @INC
+ require Cwd;
+ $0 = Cwd::cwd() . "/$0";
+ }
+ if ($0 =~ m=^(/.*)/[^/]+/[^/]+$=)
+ {
+ $::RootDir = $1;
+ unshift @INC, "$::RootDir/lib";
+ }
+}
+
+use File::Copy;
+use WineTestBot::Config;
+use WineTestBot::Log;
+use WineTestBot::Utils;
+use WineTestBot::Patches;
+use WineTestBot::Engine::Notify;
+
+my ($MaxCount) = @ARGV;
+if (defined $MaxCount)
+{
+ $MaxCount =~ m/^(\d+)$/;
+ $MaxCount = $1;
+}
+
+my $LastWebPatchId = 0;
+foreach my $Patch (@{CreatePatches()->GetItems()})
+{
+ my $WebPatchId = $Patch->WebPatchId;
+ if (defined $WebPatchId and $LastWebPatchId < $WebPatchId)
+ {
+ $LastWebPatchId = $WebPatchId;
+ }
+}
+
+while (1)
+{
+ $LastWebPatchId++;
+ my $NewPatch = "$DataDir/webpatches/$LastWebPatchId";
+ last if (!-f $NewPatch);
+
+ my $FileNameRandomPart = GenerateRandomString(32);
+ while (-e "$DataDir/staging/${FileNameRandomPart}_patch_$LastWebPatchId")
+ {
+ $FileNameRandomPart = GenerateRandomString(32);
+ }
+ my $StagingFileName = "$DataDir/staging/${FileNameRandomPart}_patch_$LastWebPatchId";
+
+ if (!copy($NewPatch, $StagingFileName))
+ {
+ LogMsg "Unable to copy '$NewPatch' to '$StagingFileName': $!\n";
+ exit 1;
+ }
+
+ WinePatchWebSubmission("${FileNameRandomPart}_patch_$LastWebPatchId", $LastWebPatchId);
+ LogMsg "Added wine-patches patch $LastWebPatchId\n";
+ if (defined $MaxCount)
+ {
+ $MaxCount--;
+ last if ($MaxCount == 0);
+ }
+}
+
+exit;
diff --git a/testbot/bin/WineSendLog.pl b/testbot/bin/WineSendLog.pl
index 515b6c8..ac522c1 100755
--- a/testbot/bin/WineSendLog.pl
+++ b/testbot/bin/WineSendLog.pl
@@ -446,68 +446,68 @@ EOF
close SENDMAIL;
}
- if (defined($PatchResultsEMail))
+ my $Patch = $Job->Patch;
+ if (0 and defined $Patch->WebPatchId and -d "$DataDir/webpatches")
{
- my $Patch = $Job->Patch;
- open (SENDMAIL, "|/usr/sbin/sendmail -oi -t -odq");
- print SENDMAIL "From: Marvin <$RobotEMail>\n";
- print SENDMAIL "To: $PatchResultsEMail\n";
- print SENDMAIL "X-TestBot-Results: ", $Patch->WebPatchId,
- ($Messages ? " failed\n" : " passed\n");
- print SENDMAIL "Subject: TestBot results for patch ", $Patch->WebPatchId,
- ($Messages ? ": Failed\n" : ": Passed\n");
- print SENDMAIL "\n";
- print SENDMAIL "--- BEGIN GENERAL ---\n";
- print SENDMAIL "Patch: ", $Patch->WebPatchId, "\n";
- print SENDMAIL "Patch-Subject: ", $Patch->Subject, "\n";
- print SENDMAIL "Test-Result: ", $Messages ? "Failed" : "Passed", "\n";
- print SENDMAIL "--- END GENERAL ---\n";
- print SENDMAIL "--- BEGIN NEW_ERRORS ---\n";
+ my $BaseName = "$DataDir/webpatches/" . $Patch->WebPatchId;
if ($Messages)
{
- print SENDMAIL $Messages;
+ if (open(my $testfail, ">", "$BaseName.testfail"))
+ {
+ print $testfail $Messages;
+ close($testfail);
+ }
+ else
+ {
+ LogMsg "Job " . $Job->Id . ": Unable to open '$BaseName.testfail' for writing: $!";
+ }
}
- print SENDMAIL "--- END NEW_ERRORS ---\n";
- print SENDMAIL "--- BEGIN FULL_LOGS ---\n";
- foreach my $Key (@SortedKeys)
+ if (open (my $result, ">", "$BaseName.testbot"))
{
- my $StepTask = $StepsTasks->GetItem($Key);
-
- print SENDMAIL "\n=== ", $StepTask->GetTitle(), " ===\n";
+ foreach my $Key (@SortedKeys)
+ {
+ my $StepTask = $StepsTasks->GetItem($Key);
+ print $result "\n=== ", $StepTask->GetTitle(), " ===\n";
- my $TaskDir = "$DataDir/jobs/" . $Job->Id . "/" . $StepTask->StepNo .
- "/" . $StepTask->TaskNo;
+ my $TaskDir = "$DataDir/jobs/" . $Job->Id . "/" . $StepTask->StepNo .
+ "/" . $StepTask->TaskNo;
- my $PrintSeparator = !1;
- if (open LOGFILE, "<$TaskDir/log")
- {
- my $Line;
- while (defined($Line = <LOGFILE>))
+ my $PrintSeparator = !1;
+ if (open(my $logfile, "<", "$TaskDir/log"))
{
- $Line =~ s/\s*$//;
- print SENDMAIL "$Line\n";
- $PrintSeparator = 1;
+ my $Line;
+ while (defined($Line = <$logfile>))
+ {
+ $Line =~ s/\s*$//;
+ print $result "$Line\n";
+ $PrintSeparator = 1;
+ }
+ close($logfile);
}
- close LOGFILE;
- }
- if (open ERRFILE, "<$TaskDir/err")
- {
- my $Line;
- while (defined($Line = <ERRFILE>))
+ if (open(my $errfile, "<", "$TaskDir/err"))
{
- if ($PrintSeparator)
+ my $Line;
+ while (defined($Line = <$errfile>))
{
- print SENDMAIL "\n";
- $PrintSeparator = !1;
+ if ($PrintSeparator)
+ {
+ print $result "\n";
+ $PrintSeparator = !1;
+ }
+ $Line =~ s/\s*$//;
+ print $result "$Line\n";
}
- $Line =~ s/\s*$//;
- print SENDMAIL "$Line\n";
+ close($errfile);
}
- close ERRFILE;
}
+ print $result "--- END FULL_LOGS ---\n";
+ close($result);
+ }
+ else
+ {
+ LogMsg "Job " . $Job->Id . ": Unable to open '$BaseName.testbot' for writing: $!";
}
- print SENDMAIL "--- END FULL_LOGS ---\n";
}
}
diff --git a/testbot/doc/INSTALL.txt b/testbot/doc/INSTALL.txt
index 86c0e79..c618c15 100644
--- a/testbot/doc/INSTALL.txt
+++ b/testbot/doc/INSTALL.txt
@@ -75,26 +75,15 @@ Janitorial tasks:
*/20 * * * * $HOME/tools/testbot/scripts/CheckWineTestBot.pl
Setup for Wine's patches site:
-- See http://source.winehq.org/patches/ for reference.
- Before trying to set this up make sure that you can submit patches
using the WineTestBot web interface and that the results are sane.
You may also want to run with $PatchFromEMailOverride set for a while
to spot any issue under load.
-- First bootstrap things with WinePatchesWebGet.pl. Go to the patches
- website and write down the id of the oldest and newest patches you
- want to inject into WineTestBot. Then run:
- ./bin/WinePatchesWebGet.pl OLD_PATCH_ID NEW_PATCH_ID
-
-- If that goes well you can then keep WineTestBot up to date by
- following the steps below.
-- Set up an email address to receive the patches site notifications
- and send that email address to the patches maintainer.
-- Arrange for the WinePatchesWebNotify.pl script to be fed the
- notification emails. One way to do so would be to use procmail and
- add the following lines to the $HOME/.procmailrc file:
- :0c:
- * ^Subject: New patches available at
- | $HOME/tools/testbot/bin/WinePatchesWebNotify.pl
+- In $HOME/tools/testbot/var, set up the webpatches symbolic link to point to
+ where the patches website stores its patches and test results. Typically:
+ ln -s $HOME/patches $HOME/tools/testbot/var/webpatches
+- Use a cron job to run WinePatchesWebSubmit.pl periodically. For instance:
+ */5 * * * * $HOME/tools/testbot/bin/WinePatchesWebSubmit.pl
Setup for Winetest updates:
- Use a cron job to run CheckForWinetestUpdate.pl periodically. For
diff --git a/testbot/lib/WineTestBot/Config.pm b/testbot/lib/WineTestBot/Config.pm
index bb1cabb..6de8b9f 100644
--- a/testbot/lib/WineTestBot/Config.pm
+++ b/testbot/lib/WineTestBot/Config.pm
@@ -30,7 +30,7 @@ use vars qw (@ISA @EXPORT @EXPORT_OK $UseSSL $LogDir $DataDir $BinDir
$WaitForToolsInVM $AdminEMail $RobotEMail $WinePatchToOverride
$WinePatchCc $SuiteTimeout $SingleTimeout
$BuildTimeout $ReconfigTimeout $OverheadTimeout $TagPrefix
- $ProjectName $PatchesMailingList $PatchResultsEMail $LDAPServer
+ $ProjectName $PatchesMailingList $LDAPServer
$LDAPBindDN $LDAPSearchBase $LDAPSearchFilter
$LDAPRealNameAttribute $LDAPEMailAttribute $AgentPort $Tunnel
$TunnelDefaults $JobPurgeDays $JobArchiveDays $WebHostName);
@@ -42,7 +42,7 @@ require Exporter;
$SleepAfterRevert $WaitForToolsInVM $AdminEMail $RobotEMail
$WinePatchToOverride $WinePatchCc $SuiteTimeout
$SingleTimeout $BuildTimeout $ReconfigTimeout $OverheadTimeout
- $TagPrefix $ProjectName $PatchesMailingList $PatchResultsEMail
+ $TagPrefix $ProjectName $PatchesMailingList
$LDAPServer $LDAPBindDN $LDAPSearchBase $LDAPSearchFilter
$LDAPRealNameAttribute $LDAPEMailAttribute $AgentPort $Tunnel
$TunnelDefaults $JobPurgeDays $JobArchiveDays $WebHostName);
diff --git a/testbot/lib/WineTestBot/ConfigLocalTemplate.pl b/testbot/lib/WineTestBot/ConfigLocalTemplate.pl
index 544ac3e..260d79e 100644
--- a/testbot/lib/WineTestBot/ConfigLocalTemplate.pl
+++ b/testbot/lib/WineTestBot/ConfigLocalTemplate.pl
@@ -46,10 +46,6 @@ $WineTestBot::Config::WinePatchToOverride = $WineTestBot::Config::AdminEMail;
# the wine-devel mailing list.
$WineTestBot::Config::WinePatchCc = "";
-# Email address to send the results to for integration with the Wine Patches
-# web site
-$WineTestBot::Config::PatchResultsEMail = undef;
-
# Host name of the web interface
$WineTestBot::Config::WebHostName = undef;
diff --git a/testbot/lib/WineTestBot/Engine/Notify.pm b/testbot/lib/WineTestBot/Engine/Notify.pm
index b836751..c0793d5 100644
--- a/testbot/lib/WineTestBot/Engine/Notify.pm
+++ b/testbot/lib/WineTestBot/Engine/Notify.pm
@@ -36,8 +36,7 @@ require Exporter;
@EXPORT = qw(&PingEngine &JobSubmit &JobStatusChange &JobCancel &JobRestart
&TaskComplete &VMStatusChange
&ExpectWinetestUpdate &FoundWinetestUpdate
- &WinePatchMLSubmission &WinePatchWebNotification
- &WinePatchWebSubmission &GetScreenshot);
+ &WinePatchMLSubmission &WinePatchWebSubmission &GetScreenshot);
@EXPORT_OK = qw($RunningInEngine);
@@ -233,23 +232,6 @@ sub WinePatchMLSubmission
return substr($Reply, 1);
}
-sub WinePatchWebNotification
-{
- my $FileName = $_[0];
-
- my $Reply = SendCmdReceiveReply("winepatchwebnotification $FileName\n");
- if (length($Reply) < 1)
- {
- return "Unrecognized reply received from engine";
- }
- if (substr($Reply, 0, 1) eq "1")
- {
- return undef;
- }
-
- return substr($Reply, 1);
-}
-
sub WinePatchWebSubmission
{
my ($FileName, $WebPatchId) = @_;
--
1.7.10.4
More information about the wine-patches
mailing list