Francois Gouget : testbot/LibvirtTool: Allow cleanly powering off VMs.

[Alexandre Julliard]

Module: tools
Branch: master
Commit: 9fcde072a9be44154fecd49cdbe36430810e5c7a
URL:    https://source.winehq.org/git/tools.git/?a=commit;h=9fcde072a9be44154fecd49cdbe36430810e5c7a

Author: Francois Gouget <fgouget at codeweavers.com>
Date:   Mon Mar  1 16:28:07 2021 +0100

testbot/LibvirtTool: Allow cleanly powering off VMs.

If a snapshot has the '-off' suffix, try cleanly powering of the VM
after running the tests. This may avoid perma-crashing the GPU when
using PCI-passthrough.
Forcefully power of the VM as usual if it does not cleanly shuts down
within $WaitForShutdown seconds.

Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard at winehq.org>

---

 testbot/bin/LibvirtTool.pl        | 50 +++++++++++++++++++++++++++++++++++++--
 testbot/lib/WineTestBot/Config.pm |  9 ++++---
 2 files changed, 54 insertions(+), 5 deletions(-)

diff --git a/testbot/bin/LibvirtTool.pl b/testbot/bin/LibvirtTool.pl
index a2939e7..59adf2f 100755
--- a/testbot/bin/LibvirtTool.pl
+++ b/testbot/bin/LibvirtTool.pl
@@ -323,9 +323,55 @@ sub Monitor()
 
 sub PowerOff()
 {
-  # Power off VMs no matter what their initial status is
+  my $Domain = $VM->GetDomain();
+  if ($VM->IdleSnapshot =~ /-off$/ and $Domain->IsPoweredOn())
+  {
+    # Attempt to perform a clean shutdown.
+    # Use SetTime() to detect if privileged operations are allowed.
+    my $TA = $VM->GetAgent();
+    if (!$TA->SetTime())
+    {
+      # Not a fatal error. Try the next port in case the VM runs a privileged
+      # TestAgentd daemon there.
+      $TA->Disconnect();
+      $TA = $VM->GetAgent(1);
+      if (!$TA->SetTime())
+      {
+        Error "Unable to get a privileged TestAgent: ". $TA->GetLastError() ."\n";
+        $TA->Disconnect();
+        $TA = undef;
+      }
+    }
+    if ($TA)
+    {
+      my $Cmd = $VM->Type =~ /^win(?:32|64)$/ ?
+                ["shutdown.exe", "/p", "/d", "00:00"] :
+                ["/sbin/shutdown", "--poweroff", "now"];
+      Debug(Elapsed($Start), " Running @$Cmd\n");
+      if (!$TA->Run($Cmd, 0))
+      {
+        Error "Could not run @$Cmd: ". $TA->GetLastError() ."\n";
+      }
+      else
+      {
+        Debug(Elapsed($Start), " Waiting for the VM to power off\n");
+        my $Deadline = time() + $WaitForShutdown;
+        while ($Domain->IsPoweredOn())
+        {
+          if (time() >= $Deadline)
+          {
+            Error "Timed out waiting for ". $VM->Name ." to perform a clean shutdown. Forcefully shutting down now...\n";
+            last;
+          }
+          sleep(1);
+        }
+      }
+    }
+  }
+
+  # Power off the VM no matter what its initial status is
   $CurrentStatus = $VM->Status;
-  my $ErrMessage = $VM->GetDomain()->PowerOff();
+  my $ErrMessage = $Domain->PowerOff();
   FatalError("$ErrMessage\n") if (defined $ErrMessage);
 
   return ChangeStatus(undef, "off", "done");
diff --git a/testbot/lib/WineTestBot/Config.pm b/testbot/lib/WineTestBot/Config.pm
index 252c63c..fe4a27c 100644
--- a/testbot/lib/WineTestBot/Config.pm
+++ b/testbot/lib/WineTestBot/Config.pm
@@ -29,7 +29,8 @@ use vars qw (@ISA @EXPORT @EXPORT_OK $UseSSL $LogDir $DataDir $BinDir
              %RepoURLs $DbDataSource $DbUsername $DbPassword $MaxRevertingVMs
              $MaxRevertsWhileRunningVMs $MaxActiveVMs $MaxRunningVMs
              $MaxVMsWhenIdle $WaitForBoot $SleepAfterBoot $SleepAfterRevert
-             $VMToolTimeout $MaxVMErrors $MaxTaskTries $AdminEMail $RobotEMail
+             $VMToolTimeout $WaitForShutdown $MaxVMErrors $MaxTaskTries
+             $AdminEMail $RobotEMail
              $WinePatchToOverride $WinePatchCc
              $ReconfigBuildTimeout $ExeBuildTimeout $ExeModuleTimeout
              $WineBuildTimeout $WineModuleTimeout $TimeoutMargin
@@ -45,8 +46,8 @@ require Exporter;
 @EXPORT = qw($UseSSL $LogDir $DataDir $BinDir %RepoURLs
              $MaxRevertingVMs $MaxRevertsWhileRunningVMs $MaxActiveVMs
              $MaxRunningVMs $MaxVMsWhenIdle $WaitForBoot $SleepAfterBoot
-             $SleepAfterRevert
-             $VMToolTimeout $MaxVMErrors $MaxTaskTries $AdminEMail
+             $SleepAfterRevert $VMToolTimeout $WaitForShutdown
+             $MaxVMErrors $MaxTaskTries $AdminEMail
              $RobotEMail $WinePatchToOverride $WinePatchCc $SuiteTimeout
              $ReconfigBuildTimeout $ExeBuildTimeout $ExeModuleTimeout
              $WineBuildTimeout $WineModuleTimeout $TimeoutMargin
@@ -101,6 +102,8 @@ $SleepAfterRevert = 0;
 # How long to wait before considering a VM operation is stuck. In particular
 # for reverts this should take into account the time it may need to boot.
 $VMToolTimeout = 6 * 60;
+# How long to wait for the VM to perform a clean shutdown.
+$WaitForShutdown = 30;
 
 # Notify the administrator that a VM probably needs manual fixing after
 # $MaxVMErrors consecutive revert errors.