Check LocalDB create/start HRESULTs and retry failed starts

src/EfClassicLocalDb/EfClassicLocalDb.csproj

@@ -24,6 +24,7 @@
     <Compile Include="..\LocalDb\DbNamer.cs" />
     <Compile Include="..\LocalDb\DirectoryFinder.cs" />
     <Compile Include="..\LocalDb\ExceptionBuilder.cs" />
+    <Compile Include="..\LocalDb\StartRetry.cs" />
     <Compile Include="..\LocalDb\FileExtensions.cs" />
     <Compile Include="..\LocalDb\Guard.cs" />
     <Compile Include="..\LocalDb\LocalDbRegistryReader.cs" />

src/EfLocalDb/EfLocalDb.csproj

@@ -22,6 +22,7 @@
     <Compile Include="..\LocalDb\DirectoryCleaner.cs" />
     <Compile Include="..\LocalDb\InstanceMarker.cs" />
     <Compile Include="..\LocalDb\ExceptionBuilder.cs" />
+    <Compile Include="..\LocalDb\StartRetry.cs" />
     <Compile Include="..\LocalDb\FileExtensions.cs" />
     <Compile Include="..\LocalDb\Guard.cs" />
     <Compile Include="..\LocalDb\LocalDbRegistryReader.cs" />

src/LocalDb.Tests/StartRetryTests.cs

@@ -0,0 +1,86 @@
+[TestFixture]
+public class StartRetryTests
+{
+    [Test]
+    public void SucceedsFirstAttempt()
+    {
+        var starts = 0;
+        var recovers = 0;
+        StartRetry.Run(
+            "Instance",
+            () =>
+            {
+                starts++;
+                return 0;
+            },
+            () => false,
+            () => recovers++,
+            _ => { });
+        AreEqual(1, starts);
+        AreEqual(0, recovers);
+    }
+
+    [Test]
+    public void RecoversAndRetries()
+    {
+        var starts = 0;
+        var recovers = 0;
+        var sleeps = new List<TimeSpan>();
+        StartRetry.Run(
+            "Instance",
+            () =>
+            {
+                starts++;
+                return starts < 3 ? StartRetry.SqlServerStartupFailed : 0;
+            },
+            () => false,
+            () => recovers++,
+            sleeps.Add);
+        AreEqual(3, starts);
+        AreEqual(2, recovers);
+        AreEqual(new[] { TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2) }, sleeps);
+    }
+
+    [Test]
+    public void FailureButRunningIsSuccess()
+    {
+        var recovers = 0;
+        StartRetry.Run(
+            "Instance",
+            () => StartRetry.SqlServerStartupFailed,
+            () => true,
+            () => recovers++,
+            _ => { });
+        AreEqual(0, recovers);
+    }
+
+    [Test]
+    public Task ThrowsAfterMaxAttempts()
+    {
+        var starts = 0;
+        var exception = Throws<Exception>(
+            () => StartRetry.Run(
+                "Instance",
+                () =>
+                {
+                    starts++;
+                    return StartRetry.SqlServerStartupFailed;
+                },
+                () => false,
+                () => { },
+                _ => { }))!;
+        AreEqual(StartRetry.MaxAttempts, starts);
+        return Verify(exception.Message)
+            .Snapshot(
+                """
+                Failed to start LocalDB instance after 3 attempts.
+                name: Instance
+                HRESULT: 0x89C5010A
+
+                To cleanup perform the following actions:
+                 * Execute 'sqllocaldb stop Instance -k'
+                 * Execute 'sqllocaldb delete Instance'
+                 * Delete the instance data directory
+                """);
+    }
+}

src/LocalDb/ExceptionBuilder.cs

@@ -14,4 +14,33 @@ Failed to setup a LocalDB instance.
             """;
         return new(message, exception);
     }
+
+    public static Exception StartFailure(string name, int result, int attempts)
+    {
+        var message = $"""
+            Failed to start LocalDB instance after {attempts} attempts.
+            {nameof(name)}: {name}
+            HRESULT: {StartRetry.ToHex(result)}
+
+            To cleanup perform the following actions:
+             * Execute 'sqllocaldb stop {name} -k'
+             * Execute 'sqllocaldb delete {name}'
+             * Delete the instance data directory
+            """;
+        return new(message);
+    }
+
+    public static Exception CreateFailure(string name, int result)
+    {
+        var message = $"""
+            Failed to create LocalDB instance.
+            {nameof(name)}: {name}
+            HRESULT: {StartRetry.ToHex(result)}
+
+            To cleanup perform the following actions:
+             * Execute 'sqllocaldb delete {name}'
+             * Delete the instance data directory
+            """;
+        return new(message);
+    }
 }

src/LocalDb/LocalDbApi.cs

@@ -166,7 +166,13 @@ public static void StopAndDelete(string instanceName, ShutdownMode mode, TimeSpa
 
     public static void CreateInstance(string instanceName)
     {
-        createInstance(ApiVersion, instanceName, 0);
+        // Creating an instance that already exists (with the same version) succeeds
+        var result = createInstance(ApiVersion, instanceName, 0);
+        if (result < 0)
+        {
+            throw ExceptionBuilder.CreateFailure(instanceName, result);
+        }
+
         InstanceMarker.Mark(instanceName);
     }
 
@@ -177,7 +183,7 @@ public static State CreateAndStart(string instance)
         if (!localDbInstanceInfo.Exists)
         {
             CreateInstance(instance);
-            StartInstance(instance);
+            StartInstance(instance, recreateOnFailure: true);
             return State.NotExists;
         }
 
@@ -189,12 +195,30 @@ public static State CreateAndStart(string instance)
         return State.Running;
     }
 
-    public static void StartInstance(string instanceName)
+    // recreateOnFailure deletes and recreates the instance between attempts. Only pass true when
+    // the instance was just created, so no existing template is registered with it
+    public static void StartInstance(string instanceName, bool recreateOnFailure = false)
     {
-        var connection = new StringBuilder(MaxPath);
-        var size = connection.Capacity;
-
-        startInstance(instanceName, 0, connection, ref size);
+        StartRetry.Run(
+            instanceName,
+            () =>
+            {
+                // Starting an instance that is already running succeeds
+                var connection = new StringBuilder(MaxPath);
+                var size = connection.Capacity;
+                return startInstance(instanceName, 0, connection, ref size);
+            },
+            () => GetInstance(instanceName).IsRunning,
+            () =>
+            {
+                StopInstance(instanceName, ShutdownMode.KillProcess);
+                if (recreateOnFailure)
+                {
+                    DeleteInstance(instanceName);
+                    CreateInstance(instanceName);
+                }
+            },
+            Thread.Sleep);
         // also marked here, not just on create, so instances created before marking existed
         // are picked up the next time they are used
         InstanceMarker.Mark(instanceName);

src/LocalDb/StartRetry.cs

@@ -0,0 +1,48 @@
+static class StartRetry
+{
+    public const int MaxAttempts = 3;
+
+    // LOCALDB_ERROR_SQL_SERVER_STARTUP_FAILED. Seen on CI runners where the sqlservr process
+    // intermittently fails to start
+    public const int SqlServerStartupFailed = unchecked((int)0x89C5010A);
+
+    // Retries a failing LocalDBStartInstance. Between attempts recover is called so the caller can
+    // kill any half started process (and optionally recreate the instance). If another process
+    // started the instance in the meantime, that is treated as success.
+    public static void Run(
+        string instanceName,
+        Func<int> start,
+        Func<bool> isRunning,
+        Action recover,
+        Action<TimeSpan> sleep)
+    {
+        var result = 0;
+        for (var attempt = 1; attempt <= MaxAttempts; attempt++)
+        {
+            result = start();
+            if (result >= 0)
+            {
+                return;
+            }
+
+            if (isRunning())
+            {
+                return;
+            }
+
+            LocalDbLogging.Log($"Failed to start LocalDB instance `{instanceName}` (attempt {attempt} of {MaxAttempts}). HRESULT: {ToHex(result)}");
+
+            if (attempt == MaxAttempts)
+            {
+                break;
+            }
+
+            recover();
+            sleep(TimeSpan.FromSeconds(attempt));
+        }
+
+        throw ExceptionBuilder.StartFailure(instanceName, result, MaxAttempts);
+    }
+
+    public static string ToHex(int result) => $"0x{result:X8}";
+}

src/LocalDb/Wrapper.cs

@@ -358,7 +358,8 @@ void CleanStart()
         {
             FileExtensions.CleanDirectory(Directory);
             LocalDbApi.CreateInstance(instance);
-            LocalDbApi.StartInstance(instance);
+            // The directory was just cleaned, so recreating the instance on a failed start loses nothing
+            LocalDbApi.StartInstance(instance, recreateOnFailure: true);
             startupTask = CreateAndDetachTemplate(
                 timestamp,
                 buildTemplate,