fix(tests): purge deleted tests, bill their storage, and record versions as loaded

apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx

@@ -582,7 +582,7 @@ interface EmbeddedTestActionsProps {
 
 /** Runs the test the tab shows, and opens its page. */
 function EmbeddedTestActions({ workspaceId, name }: EmbeddedTestActionsProps) {
-  const router = useRouter()
+  const openInternalLink = useOpenInternalLink()
   const testsEnabled = useFeatureFlag('workflow-tests')
   const detail = useWorkflowTest(workspaceId, name)
   const run = useTestRunAction({
@@ -612,7 +612,7 @@ function EmbeddedTestActions({ workspaceId, name }: EmbeddedTestActionsProps) {
         <Tooltip.Trigger asChild>
           <TabStripAction
             variant='subtle'
-            onClick={() => router.push(`/workspace/${workspaceId}/tests/${name}`)}
+            onClick={() => openInternalLink(`/workspace/${workspaceId}/tests/${name}`)}
             aria-label='Open test'
           >
             <SquareArrowUpRight className={RESOURCE_TAB_ICON_CLASS} />

apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/plus-menu-dropdown.tsx

@@ -80,11 +80,12 @@ function candidateKey({ type, item }: MentionCandidate): string {
 const MENTION_ONLY_RESOURCE_TYPES = new Set<MothershipResourceType>(['integration'])
 
 /**
- * Families an organization chat's workspace submenus leave out: the mention-only
- * ones, plus Browser and Terminal, which belong to this desktop rather than to a
- * workspace and so sit once after the workspaces.
+ * Families an organization chat's workspace submenus leave out: the composer's
+ * exclusions, the mention-only ones, plus Browser and Terminal, which belong to
+ * this desktop rather than to a workspace and so sit once after the workspaces.
  */
 const WORKSPACE_SUBMENU_EXCLUDED_TYPES: readonly MothershipResourceType[] = [
+  ...COMPOSER_EXCLUDED_TYPES,
   ...MENTION_ONLY_RESOURCE_TYPES,
   'browser',
   'terminal',

apps/sim/background/cleanup-soft-deletes.integration.ts

@@ -0,0 +1,75 @@
+/**
+ * Retention cleanup of deleted workflow tests against the disposable TEST_DATABASE_URL database:
+ * the foreign keys, the storage ledger, and the deletes all run for real.
+ */
+import { readTestDatabaseUrl } from '@sim/db/testing/test-infrastructure'
+import { generateId } from '@sim/utils/id'
+import postgres from 'postgres'
+import { afterAll, beforeAll, describe, expect, it } from 'vitest'
+import { runCleanupSoftDeletes } from '@/background/cleanup-soft-deletes'
+
+const control = postgres(readTestDatabaseUrl(), { max: 2, onnotice: () => {} })
+const userId = generateId()
+const workspaceId = generateId()
+const expired = '2020-01-01T00:00:00Z'
+
+async function seedTest(deletedAt: string | null, sizeBytes: number) {
+  const fileId = generateId()
+  const testId = generateId()
+  await control`INSERT INTO workspace_files (id, key, user_id, workspace_id, context, original_name, content_type, size_bytes, deleted_at)
+    VALUES (${fileId}, ${`test/${workspaceId}/${fileId}`}, ${userId}, ${workspaceId}, 'test',
+      ${`${testId}.test.js`}, 'text/javascript', ${sizeBytes}, ${deletedAt})`
+  await control`INSERT INTO workflow_test (id, workspace_id, name, title, body_file_id, source_hash, deleted_at)
+    VALUES (${testId}, ${workspaceId}, ${testId}, 'Fixture', ${fileId}, 'hash', ${deletedAt})`
+  await control`INSERT INTO workflow_test_run (id, test_id, workspace_id, version, triggered_by_actor)
+    VALUES (${generateId()}, ${testId}, ${workspaceId}, 'draft', ${control.json({ type: 'user' })})`
+  return { fileId, testId }
+}
+
+describe('Retention cleanup of deleted workflow tests', () => {
+  beforeAll(async () => {
+    await control`INSERT INTO "user" (id, name, email, email_verified, created_at, updated_at)
+      VALUES (${userId}, 'Test cleanup fixture', ${`${userId}@example.test`}, true, now(), now())`
+    await control`INSERT INTO user_stats (id, user_id, storage_used_bytes)
+      VALUES (${generateId()}, ${userId}, 100)`
+    await control`INSERT INTO workspace (id, name, owner_id, billed_account_user_id, storage_used_bytes)
+      VALUES (${workspaceId}, 'Test cleanup fixtures', ${userId}, ${userId}, 100)`
+  })
+
+  afterAll(async () => {
+    await control`DELETE FROM workspace WHERE id = ${workspaceId}`
+    await control`DELETE FROM "user" WHERE id = ${userId}`
+    await control.end()
+  })
+
+  it('purges an expired test with its source and releases the source bytes', async () => {
+    const deleted = await seedTest(expired, 40)
+    const live = await seedTest(null, 60)
+
+    await runCleanupSoftDeletes({
+      label: 'tests-integration',
+      plan: 'free',
+      retentionHours: 24,
+      workspaceIds: [workspaceId],
+    })
+
+    const tests = await control<
+      { id: string }[]
+    >`SELECT id FROM workflow_test WHERE workspace_id = ${workspaceId}`
+    expect(tests.map(({ id }) => id)).toEqual([live.testId])
+    const files = await control<
+      { id: string }[]
+    >`SELECT id FROM workspace_files WHERE workspace_id = ${workspaceId}`
+    expect(files.map(({ id }) => id)).toEqual([live.fileId])
+    const runs = await control`SELECT 1 FROM workflow_test_run WHERE test_id = ${deleted.testId}`
+    expect(runs).toHaveLength(0)
+    const [ws] = await control<
+      { storage_used_bytes: number }[]
+    >`SELECT storage_used_bytes::int FROM workspace WHERE id = ${workspaceId}`
+    expect(ws.storage_used_bytes).toBe(60)
+    const [stats] = await control<
+      { storage_used_bytes: number }[]
+    >`SELECT storage_used_bytes::int FROM user_stats WHERE user_id = ${userId}`
+    expect(stats.storage_used_bytes).toBe(60)
+  })
+})

apps/sim/background/cleanup-soft-deletes.test.ts

@@ -170,8 +170,11 @@ describe('cleanup soft deletes', () => {
     expect(dbChainMockFns.transaction.mock.invocationCallOrder[0]).toBeLessThan(
       mockReleaseWorkspaceFileVersionsForPurgeInTx.mock.invocationCallOrder[0]
     )
+    const fileDelete = dbChainMockFns.delete.mock.calls.findIndex(
+      ([table]) => table === schemaMock.workspaceFiles
+    )
     expect(mockReleaseWorkspaceFileVersionsForPurgeInTx.mock.invocationCallOrder[0]).toBeLessThan(
-      dbChainMockFns.delete.mock.invocationCallOrder[0]
+      dbChainMockFns.delete.mock.invocationCallOrder[fileDelete]
     )
   })
 

apps/sim/background/cleanup-soft-deletes.ts

@@ -9,6 +9,7 @@ import {
   userTableDefinitions,
   workflow,
   workflowMcpServer,
+  workflowTest,
   workspaceFile,
   workspaceFiles,
 } from '@sim/db/schema'
@@ -255,6 +256,9 @@ async function deleteExpiredLegacyWorkspaceFileRows(
   return result
 }
 
+/** Contexts whose bytes count toward the workspace's billed storage: a test's source is one. */
+const BILLED_FILE_CONTEXTS: readonly StorageContext[] = ['workspace', 'test']
+
 async function deleteExpiredUnbilledWorkspaceFileRows(
   rows: WorkspaceFileScope['multiContextRows'],
   retentionDate: Date,
@@ -263,7 +267,7 @@ async function deleteExpiredUnbilledWorkspaceFileRows(
   const result = { deleted: 0, failed: 0 }
   const rowsByContext = new Map<StorageContext, WorkspaceFileScope['multiContextRows']>()
   for (const row of rows) {
-    if (row.context === 'workspace') continue
+    if (BILLED_FILE_CONTEXTS.includes(row.context)) continue
     const bucket = rowsByContext.get(row.context)
     if (bucket) bucket.push(row)
     else rowsByContext.set(row.context, [row])
@@ -305,7 +309,7 @@ async function deleteExpiredBillableWorkspaceFileRows(
   const result = { deleted: 0, failed: 0 }
   const rowsByWorkspace = new Map<string, WorkspaceFileScope['multiContextRows']>()
   for (const row of rows) {
-    if (row.context !== 'workspace') continue
+    if (!BILLED_FILE_CONTEXTS.includes(row.context)) continue
     if (!row.workspaceId) {
       result.failed++
       logger.error(`[${label}/workspaceFiles] Billable row has no workspace attribution`, {
@@ -334,21 +338,24 @@ async function deleteExpiredBillableWorkspaceFileRows(
     for (const batch of chunkArray(workspaceRows, DEFAULT_DELETE_CHUNK_SIZE)) {
       try {
         const deletedCount = await db.transaction(async (tx) => {
-          await releaseWorkspaceFileVersionsForPurgeInTx(
-            tx,
-            batch.map(({ id }) => id),
-            retentionDate
-          )
+          const fileIds = batch.map(({ id }) => id)
+          await tx
+            .delete(workflowTest)
+            .where(
+              and(
+                inArray(workflowTest.bodyFileId, fileIds),
+                isNotNull(workflowTest.deletedAt),
+                lt(workflowTest.deletedAt, retentionDate)
+              )
+            )
+          await releaseWorkspaceFileVersionsForPurgeInTx(tx, fileIds, retentionDate)
           const deletedRows = await tx
             .delete(workspaceFiles)
             .where(
               and(
-                inArray(
-                  workspaceFiles.id,
-                  batch.map(({ id }) => id)
-                ),
+                inArray(workspaceFiles.id, fileIds),
                 eq(workspaceFiles.workspaceId, workspaceId),
-                eq(workspaceFiles.context, 'workspace'),
+                inArray(workspaceFiles.context, BILLED_FILE_CONTEXTS),
                 isNotNull(workspaceFiles.deletedAt),
                 lt(workspaceFiles.deletedAt, retentionDate)
               )

apps/sim/executor/execution/executor.ts

@@ -89,6 +89,7 @@ export class DAGExecutor {
   async execute(workflowId: string, triggerBlockId?: string): Promise<ExecutionResult> {
     await this.contextExtensions.testHooks?.enterWorkflow({
       workflowId,
+      deploymentVersionId: this.loadedDeploymentVersionId(workflowId),
       blocks: this.workflow.blocks.map((block) => ({
         id: block.id,
         name: block.metadata?.name ?? block.id,
@@ -284,6 +285,15 @@ export class DAGExecutor {
     return result
   }
 
+  /** The version this execution's state was loaded from, as its delegation authority records it. */
+  private loadedDeploymentVersionId(workflowId: string): string | null {
+    const current = this.contextExtensions.executorDelegationOrigin?.currentWorkflow
+    if (current?.workflowId !== workflowId) {
+      throw new Error(`Test run of workflow ${workflowId} has no loaded version`)
+    }
+    return current.mode === 'deployment' ? current.deploymentVersionId : null
+  }
+
   private restoreSavedIncomingEdges(dag: DAG, savedIncomingEdges?: Record<string, string[]>): void {
     if (!savedIncomingEdges) return
 

apps/sim/executor/execution/test-hooks.test.ts

@@ -79,6 +79,10 @@ function run(
       workspaceId: 'ws',
       executionId: 'exec-1',
       principal: PRINCIPAL,
+      executorDelegationOrigin: {
+        workflowId: 'wf',
+        currentWorkflow: { workflowId: 'wf', mode: 'draft' },
+      },
       testHooks: channel.hooks,
       ...(abortSignal ? { abortSignal } : {}),
     },

apps/sim/executor/execution/types.ts

@@ -288,12 +288,15 @@ export interface TestWorkflowBlock {
  * and spies against those blocks by name. A mocked block awaits `resolveMock` in place of its
  * handler; everything after the handler (normalization, redaction, logging, edges) runs as
  * usual. A mocked tool call awaits `resolveToolMock` in place of the tool. Child workflow
- * executions inherit the hooks.
+ * executions inherit the hooks, except a custom block's source workflow, which a test mocks as a
+ * whole, and a workflow an Agent calls as a tool, which runs without them.
  */
 export interface ExecutionTestHooks {
   /** `resolvedSecretTraceRegistry` redacts what this run sends back to the test. */
   enterWorkflow(workflow: {
     workflowId: string
+    /** The deployment this execution loaded; null when it loaded the draft. */
+    deploymentVersionId: string | null
     blocks: TestWorkflowBlock[]
     resolvedSecretTraceRegistry: ResolvedSecretTraceRegistry | undefined
   }): Promise<void>

apps/sim/lib/api/contracts/mothership-management-tools.ts

@@ -24,7 +24,7 @@ export const managementToolContracts = [
     route: 'sim',
     scope: 'all',
     description:
-      'Create and run the selected workspace\u2019s workflow tests. create takes a name, a one-line title for the concern, and an optional description, and returns tests/<name>.test.js; write the cases into that file with the file tools. A test file is plain vitest: import { describe, it, expect, vi } from "vitest" and { runWorkflow, mockBlock, mockTool, spyOnBlock } from "sim:test", one top-level describe, an it per case. runWorkflow(name, input) runs a workflow and returns { output } (pass { trigger: \"Trigger block name\" } as a third argument when it has several triggers); mockBlock(blockName) returns a vi.fn whose value replaces that block\u2019s output and records its inputs; mockTool(toolId) or mockTool(agentBlockName, toolId) answers an Agent\u2019s calls to that tool the same way while the model still runs, naming built-in tools by id (slack_message), MCP tools by server as mockTool({ mcp: "Server name", tool: "tool_name" }), and custom tools by title as mockTool({ customTool: "Title" }); .mockSampleOutput({ ...overrides }) on either mock returns a placeholder output shaped like the real one, with your fields merged in; await expect(value).toMatchRubric(rubric) asks a model judge for pass or fail. Every write is checked and refused if the file does not load. run takes a version (draft while editing, deployed before shipping), waits, and returns each file\u2019s failures with line numbers and messages. list and get report status; update changes title or description.',
+      'Create and run the selected workspace\u2019s workflow tests. create takes a name, a one-line title for the concern, and an optional description, and returns tests/<name>.test.js; write the cases into that file with the file tools. A test file is plain vitest: import { describe, it, expect, vi } from "vitest" and { runWorkflow, mockBlock, mockTool, spyOnBlock } from "sim:test", one top-level describe, an it per case. runWorkflow(name, input) runs a workflow and returns { output } (pass { trigger: \"Trigger block name\" } as a third argument when it has several triggers); mockBlock(blockName) returns a vi.fn whose value replaces that block\u2019s output and records its inputs; mockTool(toolId) or mockTool(agentBlockName, toolId) answers an Agent\u2019s calls to that tool the same way while the model still runs, naming built-in tools by id (slack_message), MCP tools by server as mockTool({ mcp: "Server name", tool: "tool_name" }), and custom tools by title as mockTool({ customTool: "Title" }); .mockSampleOutput({ ...overrides }) on either mock makes it answer with a placeholder output shaped like the real one, with your fields merged in; await expect(value).toMatchRubric(rubric) asks a model judge for pass or fail. Every write is checked and refused if the file does not load. run takes a version (draft while editing, deployed before shipping), waits, and returns each file\u2019s failures with line numbers and messages. list and get report status; update changes title or description.',
     inputSchema: mothershipTestsInputSchema,
   },
   {

apps/sim/lib/workflow-tests/application/run-tests.ts

@@ -55,7 +55,7 @@ async function executeTestRun(params: {
     const buffer = await fetchWorkspaceFileBuffer(file, { maxBytes: MAX_TEST_SOURCE_BYTES })
     const source = buffer.toString('utf-8')
     sourceHash = testSourceHash(source)
-    const { report, enteredWorkflowIds } = await runWorkflowTestFile({
+    const { report, entered } = await runWorkflowTestFile({
       principal,
       workspaceId: test.workspaceId,
       source,
@@ -69,9 +69,8 @@ async function executeTestRun(params: {
     )
     const ranAgainst = await readRanAgainst({
       executionIds,
-      enteredWorkflowIds,
+      entered,
       workspaceId: test.workspaceId,
-      version,
     })
     await completeWorkflowTestRun(runId, report, sourceHash, ranAgainst)
   } catch (error) {

apps/sim/lib/workflow-tests/repository.ts

@@ -142,77 +142,50 @@ export interface RanAgainstEntry {
   deploymentVersionId: string | null
   /** A `runWorkflow()` execution of it, whose snapshot is the workflow as it ran; null for a child. */
   executionId: string | null
-  /** For a draft, the workflow's `updatedAt` when the run ended; a later edit makes the run stale. */
+  /** For a draft, the workflow's `updatedAt` before the run loaded it; a later edit makes the run stale. */
   draftUpdatedAt: string | null
 }
 
+/** A workflow version a test run loaded, recorded as the executor entered it. */
+export type EnteredWorkflow = Omit<RanAgainstEntry, 'executionId'>
+
 type ExecutedWorkflow = Omit<RanAgainstEntry, 'draftUpdatedAt'>
 
+/** When each of the workspace's drafts last changed; a test run reads them before it loads any. */
+export async function readDraftTimestamps(workspaceId: string): Promise<Map<string, string>> {
+  const rows = await db
+    .select({ id: workflow.id, updatedAt: workflow.updatedAt })
+    .from(workflow)
+    .where(and(eq(workflow.workspaceId, workspaceId), isNull(workflow.archivedAt)))
+  return new Map(rows.map((row) => [row.id, row.updatedAt.toISOString()]))
+}
+
+const enteredKey = (entry: { workflowId: string; deploymentVersionId: string | null }) =>
+  `${entry.workflowId}:${entry.deploymentVersionId ?? 'draft'}`
+
 /**
- * Every workflow a run executed: the ones `runWorkflow()` started, from their execution logs, and
- * the children they called, which run inside their parent's execution and so log nothing of their
- * own. A child ran the same version as its run: its live deployment, or its draft.
+ * Every workflow version a run executed, as each was loaded: the ones `runWorkflow()` started,
+ * with the execution whose log holds their snapshot, and the children they called, which run
+ * inside their parent's execution and so log nothing of their own.
  */
 export async function readRanAgainst(params: {
   executionIds: string[]
-  enteredWorkflowIds: string[]
+  entered: EnteredWorkflow[]
   workspaceId: string
-  version: WorkflowTestVersion
 }): Promise<RanAgainstEntry[]> {
   const started = await readExecutedDeployments(params.executionIds, params.workspaceId)
-  const startedIds = new Set(started.map((entry) => entry.workflowId))
-  const childIds = params.enteredWorkflowIds.filter((id) => !startedIds.has(id))
-  const children = await readChildVersions(childIds, params.version)
-  const executed = [...started, ...children]
-  const drafts = executed.filter((entry) => entry.deploymentVersionId === null)
-  const updatedAt = new Map(
-    drafts.length === 0
-      ? []
-      : (
-          await db
-            .select({ id: workflow.id, updatedAt: workflow.updatedAt })
-            .from(workflow)
-            .where(
-              inArray(
-                workflow.id,
-                drafts.map((entry) => entry.workflowId)
-              )
-            )
-        ).map((row) => [row.id, row.updatedAt.toISOString()])
-  )
-  return executed.map((entry) => ({
-    ...entry,
-    draftUpdatedAt:
-      entry.deploymentVersionId === null ? (updatedAt.get(entry.workflowId) ?? null) : null,
-  }))
-}
-
-async function readChildVersions(
-  workflowIds: string[],
-  version: WorkflowTestVersion
-): Promise<ExecutedWorkflow[]> {
-  if (workflowIds.length === 0) return []
-  if (version === 'draft') {
-    return workflowIds.map((workflowId) => ({
-      workflowId,
-      deploymentVersionId: null,
-      executionId: null,
-    }))
-  }
-  const active = await db
-    .select({ workflowId: workflowDeploymentVersion.workflowId, id: workflowDeploymentVersion.id })
-    .from(workflowDeploymentVersion)
-    .where(
-      and(
-        inArray(workflowDeploymentVersion.workflowId, workflowIds),
-        eq(workflowDeploymentVersion.isActive, true)
-      )
-    )
-  return active.map((row) => ({
-    workflowId: row.workflowId,
-    deploymentVersionId: row.id,
-    executionId: null,
-  }))
+  const entered = new Map(params.entered.map((entry) => [enteredKey(entry), entry]))
+  const startedKeys = new Set(started.map(enteredKey))
+  return [
+    // A run that failed before its executor started logged an execution but loaded nothing.
+    ...started.map((entry) => ({
+      ...entry,
+      draftUpdatedAt: entered.get(enteredKey(entry))?.draftUpdatedAt ?? null,
+    })),
+    ...params.entered
+      .filter((entry) => !startedKeys.has(enteredKey(entry)))
+      .map((entry) => ({ ...entry, executionId: null })),
+  ]
 }
 
 /** The workflows and deployments these executions ran, from their execution logs. */