diff --git a/VERSION b/VERSION index 6ebad14888..711ee4f504 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.2 \ No newline at end of file +3.1.3 \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/ExecuteProfileCommand.cs b/src/VirtualClient/VirtualClient.Main/ExecuteProfileCommand.cs index aef26e512e..e121823ebb 100644 --- a/src/VirtualClient/VirtualClient.Main/ExecuteProfileCommand.cs +++ b/src/VirtualClient/VirtualClient.Main/ExecuteProfileCommand.cs @@ -17,6 +17,7 @@ namespace VirtualClient using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Newtonsoft.Json; + using VirtualClient.Common; using VirtualClient.Common.Contracts; using VirtualClient.Common.Extensions; using VirtualClient.Common.Telemetry; @@ -163,6 +164,11 @@ protected override async Task ExecuteAsync(string[] args, IServiceCollectio logger.LogMessage($"Platform.Initialize", telemetryContext); + // Defend long-running workloads against unattended-upgrades triggered SIGKILL on + // Ubuntu 24.04+ (needrestart auto-restart-on-upgrade is enabled by default). + // Best-effort; never fails VC startup. + await this.DisableLinuxAutoUpdatesAsync(logger, systemManagement, telemetryContext, cancellationToken); + if (this.Profiles?.Any() == true) { this.LogContextToConsole(dependencies); @@ -243,6 +249,94 @@ protected override async Task ExecuteAsync(string[] args, IServiceCollectio return exitCode; } + /// + /// On Ubuntu 24.04 and later, the default installation of the + /// needrestart + /// package (combined with the apt-daily-upgrade.timer systemd timer that fires every day + /// between 06:00 and 07:00 UTC) will automatically restart services whose shared libraries are + /// updated by unattended upgrades. For long-running workloads this manifests as the Virtual Client + /// process being killed mid-run, desynchronizing multi-VM experiments and invalidating results. + /// + /// This method is a best-effort mitigation that masks (and stops) the apt-daily timers and + /// services when running on Ubuntu 24.04 or newer. Failures are logged but never propagated + /// so that Virtual Client startup is unaffected. + /// + /// + protected virtual async Task DisableLinuxAutoUpdatesAsync( + ILogger logger, + ISystemManagement systemManagement, + EventContext telemetryContext, + CancellationToken cancellationToken) + { + if (systemManagement == null || systemManagement.Platform != PlatformID.Unix) + { + return; + } + + try + { + LinuxDistributionInfo distroInfo = await systemManagement.GetLinuxDistributionAsync(cancellationToken) + .ConfigureAwait(false); + + // The auto-restart-on-unattended-upgrade behavior has only been confirmed on Ubuntu 24.04+. + // Earlier Ubuntu releases and Debian do not auto-restart services in non-interactive mode + // and are not known to cause this issue in the field. + if (distroInfo?.LinuxDistribution != LinuxDistribution.Ubuntu + || !ExecuteProfileCommand.TryGetUbuntuMajorVersion(distroInfo.OperationSystemFullName, out int majorVersion) + || majorVersion < 24) + { + return; + } + + EventContext maskContext = telemetryContext.Clone() + .AddContext("linuxDistribution", distroInfo.LinuxDistribution.ToString()) + .AddContext("linuxDistributionFullName", distroInfo.OperationSystemFullName) + .AddContext("ubuntuMajorVersion", majorVersion); + + // Double-quoted -c argument is required: .NET Process argument tokenization follows + // Windows CommandLineToArgvW-style rules (double-quote grouping only), so single quotes + // would be passed literally to bash. + string bashArgs = "-c \"systemctl mask apt-daily.timer apt-daily-upgrade.timer apt-daily.service apt-daily-upgrade.service;" + + " systemctl stop apt-daily.timer apt-daily-upgrade.timer apt-daily.service apt-daily-upgrade.service;" + + " exit 0\""; + + using (IProcessProxy process = systemManagement.ProcessManager.CreateProcess("bash", bashArgs)) + { + await process.StartAndWaitAsync(cancellationToken).ConfigureAwait(false); + maskContext.AddContext("exitCode", process.ExitCode); + maskContext.AddContext("standardError", process.StandardError?.ToString()); + } + + logger.LogMessage($"{nameof(ExecuteProfileCommand)}.DisabledLinuxAutoUpdates", LogLevel.Information, maskContext); + } + catch (Exception exc) + { + // Never fail VC startup because of this mitigation. + logger.LogMessage( + $"{nameof(ExecuteProfileCommand)}.DisableLinuxAutoUpdatesFailed", + LogLevel.Warning, + telemetryContext.Clone().AddError(exc)); + } + } + + /// + /// Parses the Ubuntu major version (e.g. 22, 24) from the PRETTY_NAME string returned + /// by (e.g. "Ubuntu 24.04.1 LTS"). + /// Returns false when the name does not contain a parseable Ubuntu version. + /// + internal static bool TryGetUbuntuMajorVersion(string osFullName, out int majorVersion) + { + majorVersion = 0; + + if (string.IsNullOrWhiteSpace(osFullName)) + { + return false; + } + + Match match = Regex.Match(osFullName, @"Ubuntu\s+(\d+)\.", RegexOptions.IgnoreCase); + return match.Success && int.TryParse(match.Groups[1].Value, out majorVersion); + } + /// /// Downloads the profile from a remote location to the local profile downloads folder. /// diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPRATE.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPRATE.json index 852da775b1..9400ceece9 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPRATE.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPRATE.json @@ -41,14 +41,6 @@ } ], "Dependencies": [ - { - "Type": "ExecuteCommand", - "Parameters": { - "Scenario": "MaskAptDailyTimers", - "SupportedPlatforms": "linux-x64,linux-arm64", - "Command": "bash -c 'systemctl mask apt-daily.timer apt-daily-upgrade.timer apt-daily.service apt-daily-upgrade.service; systemctl stop apt-daily.timer apt-daily-upgrade.timer apt-daily.service apt-daily-upgrade.service; exit 0'" - } - }, { "Type": "ChocolateyInstallation", "Parameters": { diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPSPEED.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPSPEED.json index 9104d251c9..1a6c80b7c6 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPSPEED.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPSPEED.json @@ -41,14 +41,6 @@ } ], "Dependencies": [ - { - "Type": "ExecuteCommand", - "Parameters": { - "Scenario": "MaskAptDailyTimers", - "SupportedPlatforms": "linux-x64,linux-arm64", - "Command": "bash -c 'systemctl mask apt-daily.timer apt-daily-upgrade.timer apt-daily.service apt-daily-upgrade.service; systemctl stop apt-daily.timer apt-daily-upgrade.timer apt-daily.service apt-daily-upgrade.service; exit 0'" - } - }, { "Type": "ChocolateyInstallation", "Parameters": { diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTRATE.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTRATE.json index 13d7844874..5b3970ff09 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTRATE.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTRATE.json @@ -41,14 +41,6 @@ } ], "Dependencies": [ - { - "Type": "ExecuteCommand", - "Parameters": { - "Scenario": "MaskAptDailyTimers", - "SupportedPlatforms": "linux-x64,linux-arm64", - "Command": "bash -c 'systemctl mask apt-daily.timer apt-daily-upgrade.timer apt-daily.service apt-daily-upgrade.service; systemctl stop apt-daily.timer apt-daily-upgrade.timer apt-daily.service apt-daily-upgrade.service; exit 0'" - } - }, { "Type": "ChocolateyInstallation", "Parameters": { diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTSPEED.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTSPEED.json index 5e53cca496..183431d0fa 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTSPEED.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTSPEED.json @@ -41,14 +41,6 @@ } ], "Dependencies": [ - { - "Type": "ExecuteCommand", - "Parameters": { - "Scenario": "MaskAptDailyTimers", - "SupportedPlatforms": "linux-x64,linux-arm64", - "Command": "bash -c 'systemctl mask apt-daily.timer apt-daily-upgrade.timer apt-daily.service apt-daily-upgrade.service; systemctl stop apt-daily.timer apt-daily-upgrade.timer apt-daily.service apt-daily-upgrade.service; exit 0'" - } - }, { "Type": "ChocolateyInstallation", "Parameters": { diff --git a/src/VirtualClient/VirtualClient.UnitTests/ExecuteProfileCommandTests.cs b/src/VirtualClient/VirtualClient.UnitTests/ExecuteProfileCommandTests.cs index e69672c718..904d794906 100644 --- a/src/VirtualClient/VirtualClient.UnitTests/ExecuteProfileCommandTests.cs +++ b/src/VirtualClient/VirtualClient.UnitTests/ExecuteProfileCommandTests.cs @@ -498,6 +498,22 @@ public async Task RunProfileCommandSupportsParametersOnListInProfile_Scenario4_P Assert.AreEqual("conditionalA", profile.Parameters["Parameter4"].ToString()); } + [TestCase("Ubuntu 24.04.1 LTS", true, 24)] + [TestCase("Ubuntu 24.04 LTS", true, 24)] + [TestCase("Ubuntu 22.04.3 LTS", true, 22)] + [TestCase("Ubuntu 25.10", true, 25)] + [TestCase("ubuntu 24.04.1 lts", true, 24)] + [TestCase("Debian GNU/Linux 12 (bookworm)", false, 0)] + [TestCase("Ubuntu Noble Numbat (development branch)", false, 0)] + [TestCase("", false, 0)] + [TestCase(null, false, 0)] + public void TryGetUbuntuMajorVersionParsesExpectedValues(string prettyName, bool expectedSuccess, int expectedMajor) + { + bool actualSuccess = ExecuteProfileCommand.TryGetUbuntuMajorVersion(prettyName, out int actualMajor); + Assert.AreEqual(expectedSuccess, actualSuccess); + Assert.AreEqual(expectedMajor, actualMajor); + } + private class TestRunProfileCommand : ExecuteProfileCommand { public new PlatformExtensions Extensions