test(stack): wait on podman container removal instead of its rotating event log

packages/stack/README.md

@@ -158,7 +158,7 @@ Inputs that the composition binds or the owner fills from the stack credentials,
 
 To recompose a stopped stack, pass the IDs to keep in `reuseIds`. `composition.plan(services)` compares requested creations with every saved instance of the same kinds without contacting the owner or changing state. Each entry reports its instance `id`, `service`, whether it is a composition `member`, and a `change`: `unchanged`, `changed` with the differing config `paths`, or `incompatible` with the `paths` of an endpoint, artifact version, or PostgreSQL major-version change that the saved instance cannot adopt. Inputs that the composition or the stack credentials supply are ignored, an automatic API port is compared as the shared fixed port the composition would assign, and a PostgreSQL major alias matches its pinned version. Recomposing with `reuseIds` replaces a `changed` instance's configuration while keeping its identity, data, and ports.
 
-The whole-stack E2E suite covers the default lazy lifecycle and reopen, all-eager startup, idle and wake, and parallel stack isolation for native, Docker, and Podman runtimes, split into a `lifecycle` and an `idle-parallel` file per runtime. The Podman files need `SUPABASE_STACK_TEST_ENGINE=podman`; CI runs them, and the container integration suite, on rootless Podman 4.9 on Linux. macOS `podman machine` is not covered by CI. Run one runtime with a filter such as `pnpm --filter @supabase/stack test:e2e:run src/whole-stack.native` (matches both native files) or target one file, for example `src/whole-stack.docker.idle-parallel.e2e.test.ts`.
+The whole-stack E2E suite covers the default lazy lifecycle and reopen, all-eager startup, idle and wake, and parallel stack isolation for native, Docker, and Podman runtimes, split into a `lifecycle` and an `idle-parallel` file per runtime. The Podman files need `SUPABASE_STACK_TEST_ENGINE=podman`; CI runs them, and the container integration suite, on the Linux runner image's rootless Podman. macOS `podman machine` is not covered by CI. Run one runtime with a filter such as `pnpm --filter @supabase/stack test:e2e:run src/whole-stack.native` (matches both native files) or target one file, for example `src/whole-stack.docker.idle-parallel.e2e.test.ts`.
 
 For multiple instances or custom dependencies, configure members and bindings explicitly instead:
 

packages/stack/src/Sweep.integration.test.ts

@@ -174,12 +174,14 @@ it.live.skipIf(process.platform === "win32")(
         yield* state.save(saved(nextId, `${rootA}/next`, testEngine, "detached"));
 
         const since = Math.floor((yield* Clock.currentTimeMillis) / 1000) - 1;
+        const pending = new Set(["orphan", "deleted stack", "session stack", "dead owner record"]);
+        const done = (name: string) => Effect.tap(Effect.sync(() => pending.delete(name)));
         const swept = yield* Effect.all(
           [
-            awaitContainerRemoved(orphan, since),
-            awaitContainerRemoved(deleted, since),
-            awaitRemoval(rootA, sessionId),
-            awaitRemoval(path.join(rootA, deadId), "owner.json"),
+            awaitContainerRemoved(orphan, since).pipe(done("orphan")),
+            awaitContainerRemoved(deleted, since).pipe(done("deleted stack")),
+            awaitRemoval(rootA, sessionId).pipe(done("session stack")),
+            awaitRemoval(path.join(rootA, deadId), "owner.json").pipe(done("dead owner record")),
           ],
           { concurrency: "unbounded" },
         ).pipe(Effect.forkChild({ startImmediately: true }));
@@ -191,7 +193,11 @@ it.live.skipIf(process.platform === "win32")(
           Effect.timeoutOrElse({
             duration: "1 minute",
             orElse: () =>
-              Effect.fail(new SweepTestError({ message: "The next owner did not sweep orphans" })),
+              Effect.fail(
+                new SweepTestError({
+                  message: `The next owner did not sweep orphans; pending: ${[...pending].join(", ")}`,
+                }),
+              ),
           }),
         );
 

packages/stack/tests/engine-events.ts

@@ -20,90 +20,101 @@ const eventLines = Effect.fn("EngineEvents.eventLines")((args: ReadonlyArray<str
   }),
 );
 
-/**
- * Completes when the engine reports the container removed, including a removal that happened
- * after `sinceSeconds` and before the stream attached. Docker names that event `destroy`; Podman
- * names it `remove`.
- */
-export const awaitContainerRemoved = (id: string, sinceSeconds: number) =>
+/** Runs `podman wait` and resolves to its output, the container's exit code. */
+const podmanWait = Effect.fn("EngineEvents.podmanWait")((args: ReadonlyArray<string>) =>
   Effect.scoped(
     Effect.gen(function* () {
-      const lines = yield* eventLines(
-        testEngine === "docker"
-          ? [
-              "--since",
-              String(sinceSeconds),
-              "--filter",
-              `container=${id}`,
-              "--filter",
-              "event=destroy",
-              "--format",
-              "{{.Actor.ID}}",
-            ]
-          : [
-              "--since",
-              String(sinceSeconds),
-              "--filter",
-              `container=${id}`,
-              "--filter",
-              "event=remove",
-              "--format",
-              "{{.ID}}",
-            ],
+      const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
+      const wait = yield* spawner.spawn(
+        ChildProcess.make(testEngine, [...engineTarget.argv, "wait", ...args], {
+          stdin: "ignore",
+          stdout: "pipe",
+          stderr: "pipe",
+        }),
       );
-      const removed = yield* lines.pipe(
-        Stream.filter((line) => line.trim() === id),
-        Stream.runHead,
+      const [stdout, stderr, code] = yield* Effect.all(
+        [
+          Stream.mkString(Stream.decodeText(wait.stdout)),
+          Stream.mkString(Stream.decodeText(wait.stderr)),
+          wait.exitCode,
+        ],
+        { concurrency: "unbounded" },
       );
-      if (Option.isNone(removed))
-        return yield* new EngineEventsError({ message: `${testEngine} event stream ended` });
+      if (Number(code) !== 0)
+        return yield* new EngineEventsError({ message: `podman wait failed: ${stderr}` });
+      return stdout.trim();
     }),
-  );
+  ),
+);
+
+/**
+ * Completes once the engine has removed the container, including a removal that finished before
+ * the call. Docker's event stream replays from `sinceSeconds` and names the event `destroy`.
+ * Podman's file event logger drops unread events when it rotates the shared log, so Podman waits
+ * on the container itself, which also returns for a container that no longer exists.
+ */
+export const awaitContainerRemoved = (id: string, sinceSeconds: number) =>
+  testEngine === "podman"
+    ? podmanWait(["--condition", "removing", "--ignore", id]).pipe(Effect.asVoid)
+    : Effect.scoped(
+        Effect.gen(function* () {
+          const lines = yield* eventLines([
+            "--since",
+            String(sinceSeconds),
+            "--filter",
+            `container=${id}`,
+            "--filter",
+            "event=destroy",
+            "--format",
+            "{{.Actor.ID}}",
+          ]);
+          const removed = yield* lines.pipe(
+            Stream.filter((line) => line.trim() === id),
+            Stream.runHead,
+          );
+          if (Option.isNone(removed))
+            return yield* new EngineEventsError({ message: `${testEngine} event stream ended` });
+        }),
+      );
 
 /**
- * What the engine reports for a container stopped with SIGINT that exits cleanly. Docker's stop
- * sends the signal through the daemon, so it reports `kill` with the signal before `die`; Podman's
- * stop signals the container directly and reports only `died` with the exit code.
+ * What a container stopped with SIGINT that exits cleanly reports. Docker's stop sends the signal
+ * through the daemon, so its events show `kill` with the signal before `die`; Podman's stop signals
+ * the container directly, so only its exit code is observable.
  */
 export const cleanStopEvents: ReadonlyArray<string> =
-  testEngine === "docker" ? ["kill 2", "die 0"] : ["died 0"];
+  testEngine === "docker" ? ["kill 2", "die 0"] : ["exited 0"];
 
 /**
- * Subscribes to the container's stop events from `since` (replaying any logged before the stream
- * attaches) and resolves, once its terminal event arrives, to every line observed. Subscribing
- * before triggering the stop avoids Docker Desktop's intermittently empty bounded replays.
+ * Starts observing the container's stop and resolves, once it has exited, to what was observed.
+ * Docker subscribes to its stop events from `since`, replaying any logged before the stream
+ * attaches; subscribing before triggering the stop avoids Docker Desktop's intermittently empty
+ * bounded replays. Podman's file event logger drops unread events when it rotates the shared log,
+ * so Podman waits on the container's exit instead.
  */
 export const observeContainerStop = Effect.fn("EngineEvents.observeContainerStop")(function* (
   container: string,
   since: string,
 ) {
-  const lines = yield* eventLines(
-    testEngine === "docker"
-      ? [
-          "--since",
-          since,
-          "--filter",
-          `container=${container}`,
-          "--filter",
-          "event=kill",
-          "--filter",
-          "event=die",
-          "--format",
-          '{{.Action}} {{index .Actor.Attributes "signal"}}{{index .Actor.Attributes "exitCode"}}',
-        ]
-      : [
-          "--since",
-          since,
-          "--filter",
-          `container=${container}`,
-          "--filter",
-          "event=died",
-          "--format",
-          "{{.Status}} {{.ContainerExitCode}}",
-        ],
-  );
+  if (testEngine === "podman")
+    return yield* podmanWait([container]).pipe(
+      Effect.map((code): ReadonlyArray<string> => [`exited ${code}`]),
+      Effect.forkScoped,
+    );
+  const lines = yield* eventLines([
+    "--since",
+    since,
+    "--filter",
+    `container=${container}`,
+    "--filter",
+    "event=kill",
+    "--filter",
+    "event=die",
+    "--format",
+    '{{.Action}} {{index .Actor.Attributes "signal"}}{{index .Actor.Attributes "exitCode"}}',
+  ]);
   return yield* lines.pipe(
-    Stream.takeUntil((line) => line.startsWith(testEngine === "docker" ? "die " : "died ")),
+    Stream.takeUntil((line) => line.startsWith("die ")),
     Stream.runCollect,
     Effect.forkScoped,
   );