[PATCH 2/4] testbot: Always notify the administrator when putting a VM offline.
Francois Gouget
fgouget at codeweavers.com
Tue Sep 10 04:07:57 CDT 2019
Include the reason why the VM was put offline in the notification email.
Add NotifyAdministrator() to the Utils module to simplify sending the
notification.
Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
---
testbot/bin/LibvirtTool.pl | 25 +--------------
testbot/bin/WineRunBuild.pl | 6 +++-
testbot/bin/WineRunReconfig.pl | 6 +++-
testbot/bin/WineRunTask.pl | 6 +++-
testbot/bin/WineRunWineTest.pl | 6 +++-
testbot/lib/WineTestBot/Engine/Scheduler.pm | 5 +++
testbot/lib/WineTestBot/Utils.pm | 34 ++++++++++++++++++++-
testbot/lib/WineTestBot/VMs.pm | 5 +++
8 files changed, 64 insertions(+), 29 deletions(-)
diff --git a/testbot/bin/LibvirtTool.pl b/testbot/bin/LibvirtTool.pl
index 4000c147..e348834a 100755
--- a/testbot/bin/LibvirtTool.pl
+++ b/testbot/bin/LibvirtTool.pl
@@ -43,6 +43,7 @@ $Name0 =~ s+^.*/++;
use WineTestBot::Config;
use WineTestBot::Log;
+use WineTestBot::Utils;
use WineTestBot::VMs;
my $Debug;
@@ -58,30 +59,6 @@ sub Error(@)
LogMsg @_;
}
-sub NotifyAdministrator($$)
-{
- my ($Subject, $Body) = @_;
-
- if (open(my $fh, "|/usr/sbin/sendmail -oi -t -odq"))
- {
- LogMsg "Notifying administrator: $Subject\n";
- print $fh <<"EOF";
-From: $RobotEMail
-To: $AdminEMail
-Subject: $Subject
-
-$Body
-EOF
- close($fh);
- }
- else
- {
- LogMsg "Could not send administrator notification: $!\n";
- LogMsg " Subject: $Subject\n";
- LogMsg " Body: $Body\n";
- }
-}
-
#
# Setup and command line processing
diff --git a/testbot/bin/WineRunBuild.pl b/testbot/bin/WineRunBuild.pl
index fdcf770b..1d57c440 100755
--- a/testbot/bin/WineRunBuild.pl
+++ b/testbot/bin/WineRunBuild.pl
@@ -280,10 +280,14 @@ sub FatalTAError($$)
my $IsPoweredOn = $VM->GetDomain()->IsPoweredOn();
if (!defined $IsPoweredOn)
{
- # The VM host is not accessible anymore so mark the VM as offline and
+ # The VM host is not accessible anymore so put the VM offline and
# requeue the task. This does not count towards the task's tries limit
# since neither the VM nor the task are at fault.
Error("$ErrMessage\n");
+ NotifyAdministrator("Putting the ". $VM->Name ." VM offline",
+ "A TestAgent operation to the ". $VM->Name ." VM failed:\n".
+ "\n$ErrMessage\n".
+ "So the VM has been put offline and the TestBot will try to regain access to it.");
WrapUpAndExit('queued');
}
diff --git a/testbot/bin/WineRunReconfig.pl b/testbot/bin/WineRunReconfig.pl
index 44919fa6..3a0b3f3e 100755
--- a/testbot/bin/WineRunReconfig.pl
+++ b/testbot/bin/WineRunReconfig.pl
@@ -282,10 +282,14 @@ sub FatalTAError($$)
my $IsPoweredOn = $VM->GetDomain()->IsPoweredOn();
if (!defined $IsPoweredOn)
{
- # The VM host is not accessible anymore so mark the VM as offline and
+ # The VM host is not accessible anymore so put the VM offline and
# requeue the task. This does not count towards the task's tries limit
# since neither the VM nor the task are at fault.
Error("$ErrMessage\n");
+ NotifyAdministrator("Putting the ". $VM->Name ." VM offline",
+ "A TestAgent operation to the ". $VM->Name ." VM failed:\n".
+ "\n$ErrMessage\n".
+ "So the VM has been put offline and the TestBot will try to regain access to it.");
WrapUpAndExit('queued');
}
diff --git a/testbot/bin/WineRunTask.pl b/testbot/bin/WineRunTask.pl
index 6a718494..53e9e500 100755
--- a/testbot/bin/WineRunTask.pl
+++ b/testbot/bin/WineRunTask.pl
@@ -332,10 +332,14 @@ sub FatalTAError($$;$)
my $IsPoweredOn = $VM->GetDomain()->IsPoweredOn();
if (!defined $IsPoweredOn)
{
- # The VM host is not accessible anymore so mark the VM as offline and
+ # The VM host is not accessible anymore so put the VM offline and
# requeue the task. This does not count towards the task's tries limit
# since neither the VM nor the task are at fault.
Error("$ErrMessage\n");
+ NotifyAdministrator("Putting the ". $VM->Name ." VM offline",
+ "A TestAgent operation to the ". $VM->Name ." VM failed:\n".
+ "\n$ErrMessage\n".
+ "So the VM has been put offline and the TestBot will try to regain access to it.");
WrapUpAndExit('queued');
}
diff --git a/testbot/bin/WineRunWineTest.pl b/testbot/bin/WineRunWineTest.pl
index 74b974ae..508605b3 100755
--- a/testbot/bin/WineRunWineTest.pl
+++ b/testbot/bin/WineRunWineTest.pl
@@ -332,10 +332,14 @@ sub FatalTAError($$;$)
my $IsPoweredOn = $VM->GetDomain()->IsPoweredOn();
if (!defined $IsPoweredOn)
{
- # The VM host is not accessible anymore so mark the VM as offline and
+ # The VM host is not accessible anymore so put the VM offline and
# requeue the task. This does not count towards the task's tries limit
# since neither the VM nor the task are at fault.
Error("$ErrMessage\n");
+ NotifyAdministrator("Putting the ". $VM->Name ." VM offline",
+ "A TestAgent operation to the ". $VM->Name ." VM failed:\n".
+ "\n$ErrMessage\n".
+ "So the VM has been put offline and the TestBot will try to regain access to it.");
WrapUpAndExit('queued');
}
diff --git a/testbot/lib/WineTestBot/Engine/Scheduler.pm b/testbot/lib/WineTestBot/Engine/Scheduler.pm
index 1b42ea9a..39d23425 100644
--- a/testbot/lib/WineTestBot/Engine/Scheduler.pm
+++ b/testbot/lib/WineTestBot/Engine/Scheduler.pm
@@ -37,6 +37,7 @@ use WineTestBot::Engine::Events;
use WineTestBot::Jobs;
use WineTestBot::Log;
use WineTestBot::RecordGroups;
+use WineTestBot::Utils;
use WineTestBot::VMs;
@@ -277,6 +278,10 @@ sub _CheckAndClassifyVMs()
if ($Errors >= $MaxVMErrors)
{
$NewStatus = "maintenance";
+ NotifyAdministrator("Putting the $VMKey VM in maintenance mode",
+ "The last $Errors revert operations timed out.\n\n".
+ "No further operation will be attempted until an administrator has put\n".
+ "the VM back online.");
$Sched->{busyvms}->{$VMKey} = 1;
}
}
diff --git a/testbot/lib/WineTestBot/Utils.pm b/testbot/lib/WineTestBot/Utils.pm
index e28a34b8..83f60fa0 100644
--- a/testbot/lib/WineTestBot/Utils.pm
+++ b/testbot/lib/WineTestBot/Utils.pm
@@ -30,7 +30,8 @@ use Exporter 'import';
our @EXPORT = qw(MakeSecureURL SecureConnection GenerateRandomString
OpenNewFile CreateNewFile CreateNewLink CreateNewDir
DurationToString BuildEMailRecipient IsValidFileName
- BuildTag SanitizeTag LocaleName ShQuote ShArgv2Cmd);
+ BuildTag SanitizeTag LocaleName NotifyAdministrator
+ ShQuote ShArgv2Cmd);
use Fcntl;
@@ -276,6 +277,37 @@ sub BuildTag($;$)
}
+#
+# EMail helper
+#
+
+sub NotifyAdministrator($$)
+{
+ my ($Subject, $Body) = @_;
+
+ if (open(my $fh, "|/usr/sbin/sendmail -oi -t -odq"))
+ {
+ require WineTestBot::Log;
+ WineTestBot::Log::LogMsg("Notifying administrator: $Subject\n");
+ print $fh <<"EOF";
+From: $RobotEMail
+To: $AdminEMail
+Subject: $Subject
+
+$Body
+EOF
+ close($fh);
+ }
+ else
+ {
+ require WineTestBot::Log;
+ WineTestBot::Log::LogMsg("Could not send administrator notification: $!\n");
+ WineTestBot::Log::LogMsg(" Subject: $Subject\n");
+ WineTestBot::Log::LogMsg(" Body: $Body\n");
+ }
+}
+
+
#
# Shell helpers
#
diff --git a/testbot/lib/WineTestBot/VMs.pm b/testbot/lib/WineTestBot/VMs.pm
index 5d738e2a..78e066fe 100644
--- a/testbot/lib/WineTestBot/VMs.pm
+++ b/testbot/lib/WineTestBot/VMs.pm
@@ -452,6 +452,11 @@ sub Run($$$$$$)
# Reset the Status and ChildPid since the exec failed
$self->Status("offline");
+ NotifyAdministrator("Putting the ". $self->Name ." VM offline",
+ "Could not run $Tool on the ". $self->Name ." VM. The failed command is:\n\n".
+ ShArgv2Cmd(@$Args) ."\n\n".
+ "An administrator should investigate why the command failed but the\n".
+ "TestBot will still try to continue using the VM.\n");
$self->ChildDeadline(undef);
$self->ChildPid(undef);
my ($ErrProperty, $ErrMessage) = $self->Save();
--
2.20.1
More information about the wine-devel
mailing list