diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx index 02027b3a060..4b700b7863c 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx @@ -582,7 +582,7 @@ interface EmbeddedTestActionsProps { /** Runs the test the tab shows, and opens its page. */ function EmbeddedTestActions({ workspaceId, name }: EmbeddedTestActionsProps) { - const router = useRouter() + const openInternalLink = useOpenInternalLink() const testsEnabled = useFeatureFlag('workflow-tests') const detail = useWorkflowTest(workspaceId, name) const run = useTestRunAction({ @@ -612,7 +612,7 @@ function EmbeddedTestActions({ workspaceId, name }: EmbeddedTestActionsProps) { router.push(`/workspace/${workspaceId}/tests/${name}`)} + onClick={() => openInternalLink(`/workspace/${workspaceId}/tests/${name}`)} aria-label='Open test' > diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/plus-menu-dropdown.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/plus-menu-dropdown.tsx index d7ddfa8ef0c..56e54e485b4 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/plus-menu-dropdown.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/plus-menu-dropdown.tsx @@ -80,11 +80,12 @@ function candidateKey({ type, item }: MentionCandidate): string { const MENTION_ONLY_RESOURCE_TYPES = new Set(['integration']) /** - * Families an organization chat's workspace submenus leave out: the mention-only - * ones, plus Browser and Terminal, which belong to this desktop rather than to a - * workspace and so sit once after the workspaces. + * Families an organization chat's workspace submenus leave out: the composer's + * exclusions, the mention-only ones, plus Browser and Terminal, which belong to + * this desktop rather than to a workspace and so sit once after the workspaces. */ const WORKSPACE_SUBMENU_EXCLUDED_TYPES: readonly MothershipResourceType[] = [ + ...COMPOSER_EXCLUDED_TYPES, ...MENTION_ONLY_RESOURCE_TYPES, 'browser', 'terminal', diff --git a/apps/sim/background/cleanup-soft-deletes.integration.ts b/apps/sim/background/cleanup-soft-deletes.integration.ts new file mode 100644 index 00000000000..3c1d7b920ed --- /dev/null +++ b/apps/sim/background/cleanup-soft-deletes.integration.ts @@ -0,0 +1,75 @@ +/** + * Retention cleanup of deleted workflow tests against the disposable TEST_DATABASE_URL database: + * the foreign keys, the storage ledger, and the deletes all run for real. + */ +import { readTestDatabaseUrl } from '@sim/db/testing/test-infrastructure' +import { generateId } from '@sim/utils/id' +import postgres from 'postgres' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { runCleanupSoftDeletes } from '@/background/cleanup-soft-deletes' + +const control = postgres(readTestDatabaseUrl(), { max: 2, onnotice: () => {} }) +const userId = generateId() +const workspaceId = generateId() +const expired = '2020-01-01T00:00:00Z' + +async function seedTest(deletedAt: string | null, sizeBytes: number) { + const fileId = generateId() + const testId = generateId() + await control`INSERT INTO workspace_files (id, key, user_id, workspace_id, context, original_name, content_type, size_bytes, deleted_at) + VALUES (${fileId}, ${`test/${workspaceId}/${fileId}`}, ${userId}, ${workspaceId}, 'test', + ${`${testId}.test.js`}, 'text/javascript', ${sizeBytes}, ${deletedAt})` + await control`INSERT INTO workflow_test (id, workspace_id, name, title, body_file_id, source_hash, deleted_at) + VALUES (${testId}, ${workspaceId}, ${testId}, 'Fixture', ${fileId}, 'hash', ${deletedAt})` + await control`INSERT INTO workflow_test_run (id, test_id, workspace_id, version, triggered_by_actor) + VALUES (${generateId()}, ${testId}, ${workspaceId}, 'draft', ${control.json({ type: 'user' })})` + return { fileId, testId } +} + +describe('Retention cleanup of deleted workflow tests', () => { + beforeAll(async () => { + await control`INSERT INTO "user" (id, name, email, email_verified, created_at, updated_at) + VALUES (${userId}, 'Test cleanup fixture', ${`${userId}@example.test`}, true, now(), now())` + await control`INSERT INTO user_stats (id, user_id, storage_used_bytes) + VALUES (${generateId()}, ${userId}, 100)` + await control`INSERT INTO workspace (id, name, owner_id, billed_account_user_id, storage_used_bytes) + VALUES (${workspaceId}, 'Test cleanup fixtures', ${userId}, ${userId}, 100)` + }) + + afterAll(async () => { + await control`DELETE FROM workspace WHERE id = ${workspaceId}` + await control`DELETE FROM "user" WHERE id = ${userId}` + await control.end() + }) + + it('purges an expired test with its source and releases the source bytes', async () => { + const deleted = await seedTest(expired, 40) + const live = await seedTest(null, 60) + + await runCleanupSoftDeletes({ + label: 'tests-integration', + plan: 'free', + retentionHours: 24, + workspaceIds: [workspaceId], + }) + + const tests = await control< + { id: string }[] + >`SELECT id FROM workflow_test WHERE workspace_id = ${workspaceId}` + expect(tests.map(({ id }) => id)).toEqual([live.testId]) + const files = await control< + { id: string }[] + >`SELECT id FROM workspace_files WHERE workspace_id = ${workspaceId}` + expect(files.map(({ id }) => id)).toEqual([live.fileId]) + const runs = await control`SELECT 1 FROM workflow_test_run WHERE test_id = ${deleted.testId}` + expect(runs).toHaveLength(0) + const [ws] = await control< + { storage_used_bytes: number }[] + >`SELECT storage_used_bytes::int FROM workspace WHERE id = ${workspaceId}` + expect(ws.storage_used_bytes).toBe(60) + const [stats] = await control< + { storage_used_bytes: number }[] + >`SELECT storage_used_bytes::int FROM user_stats WHERE user_id = ${userId}` + expect(stats.storage_used_bytes).toBe(60) + }) +}) diff --git a/apps/sim/background/cleanup-soft-deletes.test.ts b/apps/sim/background/cleanup-soft-deletes.test.ts index b7c1bfeaace..ab23abbcb6a 100644 --- a/apps/sim/background/cleanup-soft-deletes.test.ts +++ b/apps/sim/background/cleanup-soft-deletes.test.ts @@ -170,8 +170,11 @@ describe('cleanup soft deletes', () => { expect(dbChainMockFns.transaction.mock.invocationCallOrder[0]).toBeLessThan( mockReleaseWorkspaceFileVersionsForPurgeInTx.mock.invocationCallOrder[0] ) + const fileDelete = dbChainMockFns.delete.mock.calls.findIndex( + ([table]) => table === schemaMock.workspaceFiles + ) expect(mockReleaseWorkspaceFileVersionsForPurgeInTx.mock.invocationCallOrder[0]).toBeLessThan( - dbChainMockFns.delete.mock.invocationCallOrder[0] + dbChainMockFns.delete.mock.invocationCallOrder[fileDelete] ) }) diff --git a/apps/sim/background/cleanup-soft-deletes.ts b/apps/sim/background/cleanup-soft-deletes.ts index ba955e7e2ab..a6ef69d9cfa 100644 --- a/apps/sim/background/cleanup-soft-deletes.ts +++ b/apps/sim/background/cleanup-soft-deletes.ts @@ -9,6 +9,7 @@ import { userTableDefinitions, workflow, workflowMcpServer, + workflowTest, workspaceFile, workspaceFiles, } from '@sim/db/schema' @@ -255,6 +256,9 @@ async function deleteExpiredLegacyWorkspaceFileRows( return result } +/** Contexts whose bytes count toward the workspace's billed storage: a test's source is one. */ +const BILLED_FILE_CONTEXTS: readonly StorageContext[] = ['workspace', 'test'] + async function deleteExpiredUnbilledWorkspaceFileRows( rows: WorkspaceFileScope['multiContextRows'], retentionDate: Date, @@ -263,7 +267,7 @@ async function deleteExpiredUnbilledWorkspaceFileRows( const result = { deleted: 0, failed: 0 } const rowsByContext = new Map() for (const row of rows) { - if (row.context === 'workspace') continue + if (BILLED_FILE_CONTEXTS.includes(row.context)) continue const bucket = rowsByContext.get(row.context) if (bucket) bucket.push(row) else rowsByContext.set(row.context, [row]) @@ -305,7 +309,7 @@ async function deleteExpiredBillableWorkspaceFileRows( const result = { deleted: 0, failed: 0 } const rowsByWorkspace = new Map() for (const row of rows) { - if (row.context !== 'workspace') continue + if (!BILLED_FILE_CONTEXTS.includes(row.context)) continue if (!row.workspaceId) { result.failed++ logger.error(`[${label}/workspaceFiles] Billable row has no workspace attribution`, { @@ -334,21 +338,24 @@ async function deleteExpiredBillableWorkspaceFileRows( for (const batch of chunkArray(workspaceRows, DEFAULT_DELETE_CHUNK_SIZE)) { try { const deletedCount = await db.transaction(async (tx) => { - await releaseWorkspaceFileVersionsForPurgeInTx( - tx, - batch.map(({ id }) => id), - retentionDate - ) + const fileIds = batch.map(({ id }) => id) + await tx + .delete(workflowTest) + .where( + and( + inArray(workflowTest.bodyFileId, fileIds), + isNotNull(workflowTest.deletedAt), + lt(workflowTest.deletedAt, retentionDate) + ) + ) + await releaseWorkspaceFileVersionsForPurgeInTx(tx, fileIds, retentionDate) const deletedRows = await tx .delete(workspaceFiles) .where( and( - inArray( - workspaceFiles.id, - batch.map(({ id }) => id) - ), + inArray(workspaceFiles.id, fileIds), eq(workspaceFiles.workspaceId, workspaceId), - eq(workspaceFiles.context, 'workspace'), + inArray(workspaceFiles.context, BILLED_FILE_CONTEXTS), isNotNull(workspaceFiles.deletedAt), lt(workspaceFiles.deletedAt, retentionDate) ) diff --git a/apps/sim/executor/execution/executor.ts b/apps/sim/executor/execution/executor.ts index 27c1b2f1822..93c12e6a96b 100644 --- a/apps/sim/executor/execution/executor.ts +++ b/apps/sim/executor/execution/executor.ts @@ -89,6 +89,7 @@ export class DAGExecutor { async execute(workflowId: string, triggerBlockId?: string): Promise { await this.contextExtensions.testHooks?.enterWorkflow({ workflowId, + deploymentVersionId: this.loadedDeploymentVersionId(workflowId), blocks: this.workflow.blocks.map((block) => ({ id: block.id, name: block.metadata?.name ?? block.id, @@ -284,6 +285,15 @@ export class DAGExecutor { return result } + /** The version this execution's state was loaded from, as its delegation authority records it. */ + private loadedDeploymentVersionId(workflowId: string): string | null { + const current = this.contextExtensions.executorDelegationOrigin?.currentWorkflow + if (current?.workflowId !== workflowId) { + throw new Error(`Test run of workflow ${workflowId} has no loaded version`) + } + return current.mode === 'deployment' ? current.deploymentVersionId : null + } + private restoreSavedIncomingEdges(dag: DAG, savedIncomingEdges?: Record): void { if (!savedIncomingEdges) return diff --git a/apps/sim/executor/execution/test-hooks.test.ts b/apps/sim/executor/execution/test-hooks.test.ts index 97dbdba0439..85134650631 100644 --- a/apps/sim/executor/execution/test-hooks.test.ts +++ b/apps/sim/executor/execution/test-hooks.test.ts @@ -79,6 +79,10 @@ function run( workspaceId: 'ws', executionId: 'exec-1', principal: PRINCIPAL, + executorDelegationOrigin: { + workflowId: 'wf', + currentWorkflow: { workflowId: 'wf', mode: 'draft' }, + }, testHooks: channel.hooks, ...(abortSignal ? { abortSignal } : {}), }, diff --git a/apps/sim/executor/execution/types.ts b/apps/sim/executor/execution/types.ts index a8cfcdb56f8..9d3ecd33162 100644 --- a/apps/sim/executor/execution/types.ts +++ b/apps/sim/executor/execution/types.ts @@ -288,12 +288,15 @@ export interface TestWorkflowBlock { * and spies against those blocks by name. A mocked block awaits `resolveMock` in place of its * handler; everything after the handler (normalization, redaction, logging, edges) runs as * usual. A mocked tool call awaits `resolveToolMock` in place of the tool. Child workflow - * executions inherit the hooks. + * executions inherit the hooks, except a custom block's source workflow, which a test mocks as a + * whole, and a workflow an Agent calls as a tool, which runs without them. */ export interface ExecutionTestHooks { /** `resolvedSecretTraceRegistry` redacts what this run sends back to the test. */ enterWorkflow(workflow: { workflowId: string + /** The deployment this execution loaded; null when it loaded the draft. */ + deploymentVersionId: string | null blocks: TestWorkflowBlock[] resolvedSecretTraceRegistry: ResolvedSecretTraceRegistry | undefined }): Promise diff --git a/apps/sim/lib/api/contracts/mothership-management-tools.ts b/apps/sim/lib/api/contracts/mothership-management-tools.ts index 21adcee8ed7..eec17efb3ae 100644 --- a/apps/sim/lib/api/contracts/mothership-management-tools.ts +++ b/apps/sim/lib/api/contracts/mothership-management-tools.ts @@ -24,7 +24,7 @@ export const managementToolContracts = [ route: 'sim', scope: 'all', description: - 'Create and run the selected workspace\u2019s workflow tests. create takes a name, a one-line title for the concern, and an optional description, and returns tests/.test.js; write the cases into that file with the file tools. A test file is plain vitest: import { describe, it, expect, vi } from "vitest" and { runWorkflow, mockBlock, mockTool, spyOnBlock } from "sim:test", one top-level describe, an it per case. runWorkflow(name, input) runs a workflow and returns { output } (pass { trigger: \"Trigger block name\" } as a third argument when it has several triggers); mockBlock(blockName) returns a vi.fn whose value replaces that block\u2019s output and records its inputs; mockTool(toolId) or mockTool(agentBlockName, toolId) answers an Agent\u2019s calls to that tool the same way while the model still runs, naming built-in tools by id (slack_message), MCP tools by server as mockTool({ mcp: "Server name", tool: "tool_name" }), and custom tools by title as mockTool({ customTool: "Title" }); .mockSampleOutput({ ...overrides }) on either mock returns a placeholder output shaped like the real one, with your fields merged in; await expect(value).toMatchRubric(rubric) asks a model judge for pass or fail. Every write is checked and refused if the file does not load. run takes a version (draft while editing, deployed before shipping), waits, and returns each file\u2019s failures with line numbers and messages. list and get report status; update changes title or description.', + 'Create and run the selected workspace\u2019s workflow tests. create takes a name, a one-line title for the concern, and an optional description, and returns tests/.test.js; write the cases into that file with the file tools. A test file is plain vitest: import { describe, it, expect, vi } from "vitest" and { runWorkflow, mockBlock, mockTool, spyOnBlock } from "sim:test", one top-level describe, an it per case. runWorkflow(name, input) runs a workflow and returns { output } (pass { trigger: \"Trigger block name\" } as a third argument when it has several triggers); mockBlock(blockName) returns a vi.fn whose value replaces that block\u2019s output and records its inputs; mockTool(toolId) or mockTool(agentBlockName, toolId) answers an Agent\u2019s calls to that tool the same way while the model still runs, naming built-in tools by id (slack_message), MCP tools by server as mockTool({ mcp: "Server name", tool: "tool_name" }), and custom tools by title as mockTool({ customTool: "Title" }); .mockSampleOutput({ ...overrides }) on either mock makes it answer with a placeholder output shaped like the real one, with your fields merged in; await expect(value).toMatchRubric(rubric) asks a model judge for pass or fail. Every write is checked and refused if the file does not load. run takes a version (draft while editing, deployed before shipping), waits, and returns each file\u2019s failures with line numbers and messages. list and get report status; update changes title or description.', inputSchema: mothershipTestsInputSchema, }, { diff --git a/apps/sim/lib/workflow-tests/application/run-tests.ts b/apps/sim/lib/workflow-tests/application/run-tests.ts index d5c1b3de382..4ce09d3c73a 100644 --- a/apps/sim/lib/workflow-tests/application/run-tests.ts +++ b/apps/sim/lib/workflow-tests/application/run-tests.ts @@ -55,7 +55,7 @@ async function executeTestRun(params: { const buffer = await fetchWorkspaceFileBuffer(file, { maxBytes: MAX_TEST_SOURCE_BYTES }) const source = buffer.toString('utf-8') sourceHash = testSourceHash(source) - const { report, enteredWorkflowIds } = await runWorkflowTestFile({ + const { report, entered } = await runWorkflowTestFile({ principal, workspaceId: test.workspaceId, source, @@ -69,9 +69,8 @@ async function executeTestRun(params: { ) const ranAgainst = await readRanAgainst({ executionIds, - enteredWorkflowIds, + entered, workspaceId: test.workspaceId, - version, }) await completeWorkflowTestRun(runId, report, sourceHash, ranAgainst) } catch (error) { diff --git a/apps/sim/lib/workflow-tests/repository.ts b/apps/sim/lib/workflow-tests/repository.ts index 5ea1d280954..52a62f0a276 100644 --- a/apps/sim/lib/workflow-tests/repository.ts +++ b/apps/sim/lib/workflow-tests/repository.ts @@ -142,77 +142,50 @@ export interface RanAgainstEntry { deploymentVersionId: string | null /** A `runWorkflow()` execution of it, whose snapshot is the workflow as it ran; null for a child. */ executionId: string | null - /** For a draft, the workflow's `updatedAt` when the run ended; a later edit makes the run stale. */ + /** For a draft, the workflow's `updatedAt` before the run loaded it; a later edit makes the run stale. */ draftUpdatedAt: string | null } +/** A workflow version a test run loaded, recorded as the executor entered it. */ +export type EnteredWorkflow = Omit + type ExecutedWorkflow = Omit +/** When each of the workspace's drafts last changed; a test run reads them before it loads any. */ +export async function readDraftTimestamps(workspaceId: string): Promise> { + const rows = await db + .select({ id: workflow.id, updatedAt: workflow.updatedAt }) + .from(workflow) + .where(and(eq(workflow.workspaceId, workspaceId), isNull(workflow.archivedAt))) + return new Map(rows.map((row) => [row.id, row.updatedAt.toISOString()])) +} + +const enteredKey = (entry: { workflowId: string; deploymentVersionId: string | null }) => + `${entry.workflowId}:${entry.deploymentVersionId ?? 'draft'}` + /** - * Every workflow a run executed: the ones `runWorkflow()` started, from their execution logs, and - * the children they called, which run inside their parent's execution and so log nothing of their - * own. A child ran the same version as its run: its live deployment, or its draft. + * Every workflow version a run executed, as each was loaded: the ones `runWorkflow()` started, + * with the execution whose log holds their snapshot, and the children they called, which run + * inside their parent's execution and so log nothing of their own. */ export async function readRanAgainst(params: { executionIds: string[] - enteredWorkflowIds: string[] + entered: EnteredWorkflow[] workspaceId: string - version: WorkflowTestVersion }): Promise { const started = await readExecutedDeployments(params.executionIds, params.workspaceId) - const startedIds = new Set(started.map((entry) => entry.workflowId)) - const childIds = params.enteredWorkflowIds.filter((id) => !startedIds.has(id)) - const children = await readChildVersions(childIds, params.version) - const executed = [...started, ...children] - const drafts = executed.filter((entry) => entry.deploymentVersionId === null) - const updatedAt = new Map( - drafts.length === 0 - ? [] - : ( - await db - .select({ id: workflow.id, updatedAt: workflow.updatedAt }) - .from(workflow) - .where( - inArray( - workflow.id, - drafts.map((entry) => entry.workflowId) - ) - ) - ).map((row) => [row.id, row.updatedAt.toISOString()]) - ) - return executed.map((entry) => ({ - ...entry, - draftUpdatedAt: - entry.deploymentVersionId === null ? (updatedAt.get(entry.workflowId) ?? null) : null, - })) -} - -async function readChildVersions( - workflowIds: string[], - version: WorkflowTestVersion -): Promise { - if (workflowIds.length === 0) return [] - if (version === 'draft') { - return workflowIds.map((workflowId) => ({ - workflowId, - deploymentVersionId: null, - executionId: null, - })) - } - const active = await db - .select({ workflowId: workflowDeploymentVersion.workflowId, id: workflowDeploymentVersion.id }) - .from(workflowDeploymentVersion) - .where( - and( - inArray(workflowDeploymentVersion.workflowId, workflowIds), - eq(workflowDeploymentVersion.isActive, true) - ) - ) - return active.map((row) => ({ - workflowId: row.workflowId, - deploymentVersionId: row.id, - executionId: null, - })) + const entered = new Map(params.entered.map((entry) => [enteredKey(entry), entry])) + const startedKeys = new Set(started.map(enteredKey)) + return [ + // A run that failed before its executor started logged an execution but loaded nothing. + ...started.map((entry) => ({ + ...entry, + draftUpdatedAt: entered.get(enteredKey(entry))?.draftUpdatedAt ?? null, + })), + ...params.entered + .filter((entry) => !startedKeys.has(enteredKey(entry))) + .map((entry) => ({ ...entry, executionId: null })), + ] } /** The workflows and deployments these executions ran, from their execution logs. */ diff --git a/apps/sim/lib/workflow-tests/run-test-file.ts b/apps/sim/lib/workflow-tests/run-test-file.ts index 15ac944c457..49d91d389be 100644 --- a/apps/sim/lib/workflow-tests/run-test-file.ts +++ b/apps/sim/lib/workflow-tests/run-test-file.ts @@ -6,6 +6,7 @@ import { type TestReport, testReportSchema, } from '@/lib/workflow-tests/protocol' +import type { EnteredWorkflow } from '@/lib/workflow-tests/repository' import { WorkflowTestSession } from '@/lib/workflow-tests/session' import { closeWorkflowTestSession, @@ -32,7 +33,7 @@ export interface RunWorkflowTestFileInput { */ export async function runWorkflowTestFile( input: RunWorkflowTestFileInput -): Promise<{ report: TestReport; enteredWorkflowIds: string[] }> { +): Promise<{ report: TestReport; entered: EnteredWorkflow[] }> { const requestId = generateShortId(12) const session = new WorkflowTestSession({ workspaceId: input.workspaceId, @@ -50,7 +51,7 @@ export async function runWorkflowTestFile( ) return { report: testReportSchema.parse(JSON.parse(Buffer.from(bytes).toString('utf-8'))), - enteredWorkflowIds: [...session.enteredWorkflowIds], + entered: session.enteredWorkflows(), } } finally { closeWorkflowTestSession(requestId) diff --git a/apps/sim/lib/workflow-tests/session.ts b/apps/sim/lib/workflow-tests/session.ts index 762e180a677..8654dd114d0 100644 --- a/apps/sim/lib/workflow-tests/session.ts +++ b/apps/sim/lib/workflow-tests/session.ts @@ -15,6 +15,7 @@ import { type ParkedMockCall, } from '@/lib/workflow-tests/mock-channel' import type { TestCaseStatus, TestEvent, TestTarget } from '@/lib/workflow-tests/protocol' +import { type EnteredWorkflow, readDraftTimestamps } from '@/lib/workflow-tests/repository' import { runWorkflowForTest, type WorkflowTestRunResult, @@ -250,7 +251,7 @@ class RunMatcher implements MockMatcher { constructor( targets: TestTarget[], private readonly loadChildBlocks: (workflowId: string) => Promise>, - private readonly onEnter: (workflowId: string) => void + private readonly onEnter: (workflowId: string, deploymentVersionId: string | null) => void ) { this.targets = [...targets].sort( (a, b) => Number(a.workflow === null) - Number(b.workflow === null) @@ -259,10 +260,12 @@ class RunMatcher implements MockMatcher { async enterWorkflow({ workflowId, + deploymentVersionId, blocks, resolvedSecretTraceRegistry, }: { workflowId: string + deploymentVersionId: string | null blocks: TestWorkflowBlock[] resolvedSecretTraceRegistry: ResolvedSecretTraceRegistry | undefined }): Promise { @@ -270,7 +273,7 @@ class RunMatcher implements MockMatcher { this.entered = true this.outputRegistry = resolvedSecretTraceRegistry } - this.onEnter(workflowId) + this.onEnter(workflowId, deploymentVersionId) const workflowName = await this.workflowName(workflowId) for (const block of blocks) this.blocks.set(block.id, block) await this.learnWorkflow( @@ -520,8 +523,8 @@ class TestWorkflowRun { */ export class WorkflowTestSession { private readonly runs = new Map() - /** Every workflow this file's runs executed, children included. */ - readonly enteredWorkflowIds = new Set() + /** Every workflow version this file's runs executed, children included, as each loaded it. */ + private readonly enteredVersions = new Map() /** Set when the sandbox run ends; a start still awaiting its lookups then starts nothing. */ private closed = false /** Stops `toMatchRubric` model calls still in flight when the sandbox run ends. */ @@ -540,9 +543,10 @@ export class WorkflowTestSession { const blocks = await loadWorkflowBlocks(workflowId, workspaceId, version) const triggerBlockId = chooseTrigger(args.workflow, blocks, args.trigger) const targets = await resolveToolTargets(workspaceId, args.targets) + const draftsBeforeLoad = version === 'draft' ? await readDraftTimestamps(workspaceId) : null if (this.closed) throw new Error('This workflow test run has ended') - const matcher = new RunMatcher(targets, loadChildBlockNames, (id) => - this.enteredWorkflowIds.add(id) + const matcher = new RunMatcher(targets, loadChildBlockNames, (id, deploymentVersionId) => + this.recordEntered(id, deploymentVersionId, draftsBeforeLoad) ) const channel = createMockChannel(matcher) const abort = new AbortController() @@ -586,6 +590,30 @@ export class WorkflowTestSession { } } + /** + * A draft records its timestamp from before the run loaded it, so any edit after that point, + * including one the load may have missed, leaves the run stale. The first run to enter a + * workflow keeps its entry, the earliest timestamp. + */ + private recordEntered( + workflowId: string, + deploymentVersionId: string | null, + draftsBeforeLoad: Map | null + ): void { + const key = `${workflowId}:${deploymentVersionId ?? 'draft'}` + if (this.enteredVersions.has(key)) return + this.enteredVersions.set(key, { + workflowId, + deploymentVersionId, + draftUpdatedAt: + deploymentVersionId === null ? (draftsBeforeLoad?.get(workflowId) ?? null) : null, + }) + } + + enteredWorkflows(): EnteredWorkflow[] { + return [...this.enteredVersions.values()] + } + close(): void { this.closed = true this.judgeAbort.abort(new Error('The test run ended'))