test(webapp): replace slow metadata replica guard with unit test (#4312)

Author: carderne

apps/webapp/test/metadataRouteReplicaLag.guard.test.ts

@@ -1,38 +1,17 @@
-// Property: the metadata GET loader resolves a live run on a replica+buffer double-miss via its primary
-// fallback, returning 200 + the run's metadata. Drives the REAL exported `loader` end-to-end:
-//   1. runStore.findRun(..., $replica)        → REPLICA (lagging) → null
-//   2. findRunByIdWithMollifierFallback(...)   → buffer MISS (null)
-//   3. runStore.findRunOnPrimary(...)          → owning PRIMARY → HIT
-// Store is REAL: a split RoutingRunStore over two testcontainer Postgres DBs, the owning
-// (legacy/control-plane) REPLICA frozen behind the shared laggingReplica. Only the loader's webapp
-// singletons (auth, $replica brand, mollifier buffer, route builder, logging) are stubbed.
-
-import { describe, expect, vi } from "vitest";
-import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers";
-import type { PrismaClient } from "@trigger.dev/database";
-import type { RunOpsPrismaClient } from "@internal/run-ops-database";
-import { PostgresRunStore, RoutingRunStore } from "@internal/run-store";
-import type { CreateRunInput } from "@internal/run-store";
-
-// vi.mock factories are hoisted above the imports, so anything they reference must be created inside
-// vi.hoisted. `holder.store` is filled in per-test with the REAL split router; the mocked
-// `~/v3/runStore.server` export is a stable Proxy that forwards every property access to it. The
-// branded `$replica` marker uses the global-registry symbol the run-store brands replicas with
-// (readReplicaClient.ts) — a branded client makes the routing store keep the read on the owning
-// REPLICA (no primary escalation), which under lag is exactly the miss the primary fallback recovers.
-const { holder } = vi.hoisted(() => {
-  const REPLICA_BRAND = Symbol.for("trigger.dev/run-store/read-replica");
-  return {
-    holder: {
-      store: undefined as unknown,
-      brandedReplica: { [REPLICA_BRAND]: true } as object,
-      environment: undefined as unknown,
-      bufferResult: null as unknown,
-    },
-  };
-});
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const { mocks } = vi.hoisted(() => ({
+  mocks: {
+    replica: {},
+    environment: { id: "env_meta", organizationId: "org_meta" },
+    authenticateApiRequest: vi.fn(),
+    findRun: vi.fn(),
+    findRunOnPrimary: vi.fn(),
+    findRunByIdWithMollifierFallback: vi.fn(),
+  },
+}));
 
-vi.mock("~/db.server", () => ({ prisma: {}, $replica: holder.brandedReplica }));
+vi.mock("~/db.server", () => ({ prisma: {}, $replica: mocks.replica }));
 vi.mock("~/env.server", () => ({
   env: {
     TASK_RUN_METADATA_MAXIMUM_SIZE: 256 * 1024,
@@ -41,42 +20,30 @@ vi.mock("~/env.server", () => ({
     TRIGGER_MOLLIFIER_METADATA_BACKOFF_STEP_MS: 10,
   },
 }));
-// The route module builds its action at import time via createActionApiRoute; stub the builder so the
-// heavy platform/auth middleware graph never evaluates. We drive the exported `loader` directly.
 vi.mock("~/services/routeBuilders/apiBuilder.server", () => ({
   createActionApiRoute: () => ({ action: vi.fn() }),
 }));
 vi.mock("~/services/apiAuth.server", () => ({
-  authenticateApiRequest: vi.fn(async () => ({ environment: holder.environment })),
+  authenticateApiRequest: mocks.authenticateApiRequest,
 }));
-// Inject the REAL split router. A stable Proxy keeps the named import binding constant while
-// forwarding every method to the per-test router set in holder.store.
 vi.mock("~/v3/runStore.server", () => ({
-  runStore: new Proxy(
-    {},
-    {
-      get(_target, prop) {
-        const store = holder.store as Record<string | symbol, unknown>;
-        if (!store) throw new Error("test bug: holder.store not initialised before loader ran");
-        const value = store[prop];
-        return typeof value === "function"
-          ? (value as (...a: unknown[]) => unknown).bind(store)
-          : value;
-      },
-    }
-  ),
+  runStore: {
+    findRun: mocks.findRun,
+    findRunOnPrimary: mocks.findRunOnPrimary,
+  },
 }));
-// Buffer fallback returns whatever the test set (null = buffer miss, the double-miss scenario).
 vi.mock("~/v3/mollifier/readFallback.server", () => ({
-  findRunByIdWithMollifierFallback: vi.fn(async () => holder.bufferResult),
+  findRunByIdWithMollifierFallback: mocks.findRunByIdWithMollifierFallback,
 }));
-// Action-only leaf; stub to keep the import graph light.
 vi.mock("~/v3/mollifier/applyMetadataMutation.server", () => ({
   applyMetadataMutationToBufferedRun: vi.fn(),
 }));
 vi.mock("~/services/metadata/updateMetadataInstance.server", () => ({
   updateMetadataService: { call: vi.fn(async () => undefined) },
 }));
+vi.mock("~/services/realtime/runChangeNotifierInstance.server", () => ({
+  publishChangeRecord: vi.fn(),
+}));
 vi.mock("~/v3/services/common.server", () => ({
   ServiceValidationError: class extends Error {},
 }));
@@ -86,176 +53,66 @@ vi.mock("~/services/logger.server", () => ({
 
 import { loader } from "~/routes/api.v1.runs.$runId.metadata";
 
-// A cuid (25 chars after `run_`) classifies LEGACY, so both the create and the friendlyId-keyed reads
-// route to the legacy (control-plane) store — the store that owns this run.
-const CUID_25 = "c".repeat(25);
+const friendlyId = "run_meta_live";
+const runWhere = { friendlyId, runtimeEnvironmentId: mocks.environment.id };
+const metadataSelect = { select: { metadata: true, metadataType: true } };
 
-async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) {
-  const organization = await prisma.organization.create({
-    data: { title: `Org ${suffix}`, slug: `org-${suffix}` },
-  });
-  const project = await prisma.project.create({
-    data: {
-      name: `Project ${suffix}`,
-      slug: `project-${suffix}`,
-      externalRef: `proj_${suffix}`,
-      organizationId: organization.id,
-    },
-  });
-  const environment = await prisma.runtimeEnvironment.create({
-    data: {
-      type: "DEVELOPMENT",
-      slug: "dev",
-      projectId: project.id,
-      organizationId: organization.id,
-      apiKey: `tr_dev_${suffix}`,
-      pkApiKey: `pk_dev_${suffix}`,
-      shortcode: `short_${suffix}`,
-    },
-  });
-  return { organization, project, environment };
-}
-
-function buildCreateRunInput(params: {
-  runId: string;
-  friendlyId: string;
-  organizationId: string;
-  projectId: string;
-  runtimeEnvironmentId: string;
-  metadata?: string;
-  metadataType?: string;
-}): CreateRunInput {
-  return {
-    data: {
-      id: params.runId,
-      engine: "V2",
-      status: "PENDING",
-      friendlyId: params.friendlyId,
-      runtimeEnvironmentId: params.runtimeEnvironmentId,
-      environmentType: "DEVELOPMENT",
-      organizationId: params.organizationId,
-      projectId: params.projectId,
-      taskIdentifier: "my-task",
-      payload: '{"hello":"world"}',
-      payloadType: "application/json",
-      metadata: params.metadata,
-      metadataType: params.metadataType,
-      context: { foo: "bar" },
-      traceContext: { trace: "ctx" },
-      traceId: "trace_1",
-      spanId: "span_1",
-      runTags: ["alpha"],
-      queue: "task/my-task",
-      isTest: false,
-      taskEventStore: "taskEvent",
-      depth: 0,
-      createdAt: new Date("2024-01-01T00:00:00.000Z"),
-    },
-    snapshot: {
-      engine: "V2",
-      executionStatus: "RUN_CREATED",
-      description: "Run was created",
-      runStatus: "PENDING",
-      environmentId: params.runtimeEnvironmentId,
-      environmentType: "DEVELOPMENT",
-      projectId: params.projectId,
-      organizationId: params.organizationId,
-    },
-  };
-}
-
-function buildRouterWithLaggingLegacyReplica(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) {
-  const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]);
-  const legacyStore = new PostgresRunStore({
-    prisma: prisma14,
-    readOnlyPrisma: legacyReplica.client,
-    schemaVariant: "legacy",
-  });
-  const newStore = new PostgresRunStore({
-    prisma: prisma17 as never,
-    readOnlyPrisma: prisma17 as never,
-    schemaVariant: "dedicated",
-  });
-  const router = new RoutingRunStore({ new: newStore, legacy: legacyStore });
-  return { router, legacyStore, legacyReplica };
-}
-
-async function callLoader(friendlyId: string) {
-  const response = await loader({
-    request: new Request("https://api.trigger.dev/api/v1/runs/" + friendlyId + "/metadata", {
+async function callLoader(runId = friendlyId) {
+  return (await loader({
+    request: new Request(`https://example.com/api/v1/runs/${runId}/metadata`, {
       headers: { Authorization: "Bearer tr_dev_meta" },
     }),
-    params: { runId: friendlyId },
+    params: { runId },
     context: {} as never,
-  });
-  return response as Response;
+  })) as Response;
 }
 
-describe("metadata GET loader under replica lag", () => {
-  heteroRunOpsPostgresTest(
-    "metadata GET loader resolves a live run on a replica and buffer double-miss",
-    async ({ prisma14, prisma17 }) => {
-      const { router, legacyStore, legacyReplica } = buildRouterWithLaggingLegacyReplica(
-        prisma14,
-        prisma17
-      );
-
-      const seed = await seedEnvironmentLegacy(prisma14, "meta");
-      const runId = `run_${CUID_25}`; // cuid → LEGACY-owned
-      const friendlyId = "run_meta_live";
-      const metadata = '{"phase":"one"}';
-      await legacyStore.createRun(
-        buildCreateRunInput({
-          runId,
-          friendlyId,
-          organizationId: seed.organization.id,
-          projectId: seed.project.id,
-          runtimeEnvironmentId: seed.environment.id,
-          metadata,
-          metadataType: "application/json",
-        })
-      );
-
-      // Wire the loader's module singletons: the real split router, the authenticated env, and a
-      // buffer that MISSES (the run has drained from the buffer to the primary but the replica has
-      // not caught up — the exact double-miss under test).
-      holder.store = router;
-      holder.environment = { id: seed.environment.id, organizationId: seed.organization.id };
-      holder.bufferResult = null;
-
-      const response = await callLoader(friendlyId);
-      const body = (await response.json()) as {
-        metadata?: unknown;
-        metadataType?: unknown;
-        error?: string;
-      };
-
-      // The replica WAS consulted (and, being frozen, missed) — proving the read genuinely went
-      // through the lagging replica and the recovery is the primary fallback, not a lucky replica hit.
-      expect(legacyReplica.wasHit()).toBe(true);
-
-      // The property: the loader re-reads the owning primary and returns the live run's metadata.
-      expect(response.status).toBe(200);
-      expect(body.metadata).toBe(metadata);
-      expect(body.metadataType).toBe("application/json");
-    }
-  );
+beforeEach(() => {
+  vi.clearAllMocks();
+  mocks.authenticateApiRequest.mockResolvedValue({ environment: mocks.environment });
+  mocks.findRun.mockResolvedValue(null);
+  mocks.findRunByIdWithMollifierFallback.mockResolvedValue(null);
+  mocks.findRunOnPrimary.mockResolvedValue(null);
+});
 
-  // Negative control: when the run truly does not exist anywhere (replica miss + buffer miss +
-  // primary miss), the loader must still 404. This pins the behavior to "recover a LIVE run" rather
-  // than "never 404".
-  heteroRunOpsPostgresTest(
-    "metadata GET loader 404s when the run is absent on the primary too",
-    async ({ prisma14, prisma17 }) => {
-      const { router } = buildRouterWithLaggingLegacyReplica(prisma14, prisma17);
-      const seed = await seedEnvironmentLegacy(prisma14, "meta_absent");
+describe("metadata GET loader under replica lag", () => {
+  it("resolves a live run after a replica and buffer double-miss", async () => {
+    const metadata = '{"phase":"one"}';
+    mocks.findRunOnPrimary.mockResolvedValue({
+      metadata,
+      metadataType: "application/json",
+    });
+
+    const response = await callLoader();
+
+    expect(mocks.findRun).toHaveBeenCalledWith(runWhere, metadataSelect, mocks.replica);
+    expect(mocks.findRunByIdWithMollifierFallback).toHaveBeenCalledWith({
+      runId: friendlyId,
+      environmentId: mocks.environment.id,
+      organizationId: mocks.environment.organizationId,
+    });
+    expect(mocks.findRunOnPrimary).toHaveBeenCalledWith(runWhere, metadataSelect);
+    expect(mocks.findRun.mock.invocationCallOrder[0]).toBeLessThan(
+      mocks.findRunByIdWithMollifierFallback.mock.invocationCallOrder[0]
+    );
+    expect(mocks.findRunByIdWithMollifierFallback.mock.invocationCallOrder[0]).toBeLessThan(
+      mocks.findRunOnPrimary.mock.invocationCallOrder[0]
+    );
+    await expect(response.json()).resolves.toEqual({
+      metadata,
+      metadataType: "application/json",
+    });
+    expect(response.status).toBe(200);
+  });
 
-      holder.store = router;
-      holder.environment = { id: seed.environment.id, organizationId: seed.organization.id };
-      holder.bufferResult = null;
+  it("returns 404 when the run is absent from the primary too", async () => {
+    const response = await callLoader("run_does_not_exist");
 
-      const response = await callLoader("run_does_not_exist");
-      expect(response.status).toBe(404);
-    }
-  );
+    expect(mocks.findRunOnPrimary).toHaveBeenCalledWith(
+      { friendlyId: "run_does_not_exist", runtimeEnvironmentId: mocks.environment.id },
+      metadataSelect
+    );
+    await expect(response.json()).resolves.toEqual({ error: "Run not found" });
+    expect(response.status).toBe(404);
+  });
 });