fix: SSH connection pool concurrency and behavioral issues

- Remove srcBaseDir from connection key: workspaces on same host now
  share health tracking and control socket multiplexing
- Fix double markFailedByKey on timeout: add timedOut flag to prevent
  both timeout callback and on('close') from incrementing failures
- Add HEALTHY_TTL_MS (5 min): stale healthy connections get re-probed
  when network may have silently degraded
- Fix singleflighting test: actually test concurrent probes share
  one failure count instead of pre-marking healthy

Author: deansheather

src/node/runtime/sshConnectionPool.test.ts

@@ -58,7 +58,9 @@ describe("sshConnectionPool", () => {
       expect(getControlPath(config1)).not.toBe(getControlPath(config2));
     });
 
-    test("different srcBaseDirs produce different controlPaths", () => {
+    test("different srcBaseDirs produce same controlPaths (connection shared)", () => {
+      // srcBaseDir is intentionally excluded from connection key -
+      // workspaces on the same host share health tracking and multiplexing
       const config1: SSHRuntimeConfig = {
         host: "test.com",
         srcBaseDir: "/work1",
@@ -68,7 +70,7 @@ describe("sshConnectionPool", () => {
         srcBaseDir: "/work2",
       };
 
-      expect(getControlPath(config1)).not.toBe(getControlPath(config2));
+      expect(getControlPath(config1)).toBe(getControlPath(config2));
     });
 
     test("controlPath is in tmpdir with expected format", () => {
@@ -262,22 +264,22 @@ describe("SSHConnectionPool", () => {
     test("concurrent acquireConnection calls share same probe", async () => {
       const pool = new SSHConnectionPool();
       const config: SSHRuntimeConfig = {
-        host: "test.example.com",
+        host: "nonexistent.invalid.host.test",
         srcBaseDir: "/work",
       };
 
-      // Mark healthy to avoid actual probe
-      pool.markHealthy(config);
-
-      // Multiple concurrent calls should all succeed
-      const results = await Promise.all([
-        pool.acquireConnection(config),
-        pool.acquireConnection(config),
-        pool.acquireConnection(config),
+      // All concurrent calls should share the same probe and get same result
+      const results = await Promise.allSettled([
+        pool.acquireConnection(config, 1000),
+        pool.acquireConnection(config, 1000),
+        pool.acquireConnection(config, 1000),
       ]);
 
-      // All should resolve (no errors)
-      expect(results).toHaveLength(3);
+      // All should be rejected (connection fails)
+      expect(results.every((r) => r.status === "rejected")).toBe(true);
+
+      // Only 1 failure should be recorded (not 3) - proves singleflighting worked
+      expect(pool.getConnectionHealth(config)?.consecutiveFailures).toBe(1);
     });
   });
 });

src/node/runtime/sshConnectionPool.ts

@@ -58,6 +58,12 @@ export interface ConnectionHealth {
  */
 const BACKOFF_SCHEDULE = [1, 5, 10, 20, 40, 60];
 
+/**
+ * Time after which a "healthy" connection should be re-probed.
+ * Prevents stale health state when network silently degrades.
+ */
+const HEALTHY_TTL_MS = 5 * 60 * 1000; // 5 minutes
+
 /**
  * SSH Connection Pool
  *
@@ -91,10 +97,16 @@ export class SSHConnectionPool {
       );
     }
 
-    // Return immediately if known healthy
+    // Return immediately if known healthy and not stale
     if (health?.status === "healthy") {
-      log.debug(`SSH connection to ${config.host} is known healthy, skipping probe`);
-      return;
+      const age = Date.now() - (health.lastSuccess?.getTime() ?? 0);
+      if (age < HEALTHY_TTL_MS) {
+        log.debug(`SSH connection to ${config.host} is known healthy, skipping probe`);
+        return;
+      }
+      log.debug(
+        `SSH connection to ${config.host} health is stale (${Math.round(age / 1000)}s), re-probing`
+      );
     }
 
     // Check for inflight probe - singleflighting
@@ -243,7 +255,9 @@ export class SSHConnectionPool {
         stderr += data.toString();
       });
 
+      let timedOut = false;
       const timeout = setTimeout(() => {
+        timedOut = true;
         proc.kill("SIGKILL");
         const error = "SSH probe timed out";
         this.markFailedByKey(key, error);
@@ -252,6 +266,7 @@ export class SSHConnectionPool {
 
       proc.on("close", (code) => {
         clearTimeout(timeout);
+        if (timedOut) return; // Already handled by timeout
 
         if (code === 0) {
           this.markHealthyByKey(key);
@@ -304,11 +319,13 @@ export function getControlPath(config: SSHRuntimeConfig): string {
  * Includes local username to prevent cross-user socket collisions.
  */
 function makeConnectionKey(config: SSHRuntimeConfig): string {
+  // Note: srcBaseDir is intentionally excluded - connection identity is determined
+  // by user + host + port + key. This allows health tracking and multiplexing
+  // to be shared across workspaces on the same host.
   const parts = [
     os.userInfo().username, // Include local user to prevent cross-user collisions
     config.host,
     config.port?.toString() ?? "22",
-    config.srcBaseDir,
     config.identityFile ?? "default",
   ];
   return parts.join(":");