diff --git a/.claude/rules/sim-testing.md b/.claude/rules/sim-testing.md index 96553945475..8684de09d8d 100644 --- a/.claude/rules/sim-testing.md +++ b/.claude/rules/sim-testing.md @@ -169,6 +169,7 @@ The suite's wall time is bound by the single Vite server thread that serves ever 3. Mock heavy graphs a test does not need and the setup does not already mock: `@/blocks`, `@/triggers/registry`, `@/tools/generated/*`. 4. No real timers: `vi.useFakeTimers()`, `flushMicrotasks()`, or `flushMacrotask()`. + `check:test-patterns` (`real-wait`) fails on a new wall-clock wait in a unit test. 5. Absolute imports only. ### Running diff --git a/.cursor/rules/sim-testing.mdc b/.cursor/rules/sim-testing.mdc index d71779a2cf0..f8e665e0c90 100644 --- a/.cursor/rules/sim-testing.mdc +++ b/.cursor/rules/sim-testing.mdc @@ -167,6 +167,7 @@ The suite's wall time is bound by the single Vite server thread that serves ever 3. Mock heavy graphs a test does not need and the setup does not already mock: `@/blocks`, `@/triggers/registry`, `@/tools/generated/*`. 4. No real timers: `vi.useFakeTimers()`, `flushMicrotasks()`, or `flushMacrotask()`. + `check:test-patterns` (`real-wait`) fails on a new wall-clock wait in a unit test. 5. Absolute imports only. ### Running diff --git a/scripts/check-test-patterns.test.ts b/scripts/check-test-patterns.test.ts index 09ffb4106b6..76b0b9e9b05 100644 --- a/scripts/check-test-patterns.test.ts +++ b/scripts/check-test-patterns.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from 'vitest' -import { findStubAndHookViolations, parseSource } from './check-test-patterns' +import { + findRealWaitViolations, + findStubAndHookViolations, + parseSource, +} from './check-test-patterns' function rules(source: string, file = 'apps/sim/lib/example.test.ts') { return findStubAndHookViolations(file, parseSource(file, source)).map( @@ -76,3 +80,308 @@ describe('redundant-hook', () => { ]) }) }) + +describe('real-wait', () => { + const HELPERS = "import { sleep, interruptibleSleep } from '@sim/utils/helpers'" + + function waits(source: string, file = 'apps/sim/lib/example.test.ts') { + return findRealWaitViolations(file, parseSource(file, source)).map(({ detail }) => detail) + } + + it.each<[string, string[], string[]]>([ + ['an imported sleep', [HELPERS, "it('t', async () => { await sleep(10) })"], ['sleep() #1']], + [ + 'an aliased sleep', + ["import { sleep as wait } from '@sim/utils/helpers'", 'await wait(10)'], + ['sleep() #1'], + ], + [ + 'interruptibleSleep', + [HELPERS, 'await interruptibleSleep(10, signal)'], + ['interruptibleSleep() #1'], + ], + ['a sleep with a computed delay', [HELPERS, 'await sleep(delayMs)'], ['sleep() #1']], + [ + 'timers/promises setTimeout', + ["import { setTimeout as delay } from 'node:timers/promises'", 'await delay(50)'], + ['timers/promises setTimeout() #1'], + ], + [ + 'a promise resolved by setTimeout', + ['await new Promise((r) => setTimeout(r, 5))'], + ['setTimeout promise #1'], + ], + [ + 'a promise resolved from a setTimeout callback', + ['await new Promise((resolve) => { globalThis.setTimeout(() => resolve(), ms) })'], + ['setTimeout promise #1'], + ], + [ + 'fake timers that leave setTimeout real', + [HELPERS, "beforeEach(() => { vi.useFakeTimers({ toFake: ['Date'] }) })", 'await sleep(10)'], + ['sleep() #1'], + ], + [ + 'fake timers that advance with the wall clock', + [ + HELPERS, + "it('t', async () => {", + ' vi.useFakeTimers({ shouldAdvanceTime: true })', + ' await sleep(10)', + '})', + ], + ['sleep() #1'], + ], + [ + 'a sleep before the test installs fake timers', + [HELPERS, "it('t', async () => {", ' await sleep(10)', ' vi.useFakeTimers()', '})'], + ['sleep() #1'], + ], + [ + 'a sleep in a test beside one that installs fake timers', + [ + HELPERS, + "it('a', () => { vi.useFakeTimers() })", + "it('b', async () => { await sleep(10) })", + ], + ['sleep() #1'], + ], + [ + 'a sleep before the install in the hook that installs fake timers', + [ + HELPERS, + "describe('d', () => {", + ' beforeEach(async () => {', + ' await sleep(100)', + ' vi.useFakeTimers()', + ' })', + " it('t', async () => { await sleep(10) })", + '})', + ], + ['sleep() #1'], + ], + [ + 'a sleep after the test restores real timers', + [ + HELPERS, + "it('t', async () => {", + ' vi.useFakeTimers()', + ' vi.useRealTimers()', + ' await sleep(10)', + '})', + ], + ['sleep() #1'], + ], + [ + 'a sleep in a nested suite that restores real timers', + [ + HELPERS, + "describe('outer', () => {", + ' beforeEach(() => { vi.useFakeTimers() })', + " describe('inner', () => {", + ' beforeEach(() => { vi.useRealTimers() })', + " it('t', async () => { await sleep(10) })", + ' })', + '})', + ], + ['sleep() #1'], + ], + [ + 'a sleep in a beforeAll beside a beforeEach that installs fake timers', + [ + HELPERS, + 'beforeEach(() => { vi.useFakeTimers() })', + 'beforeAll(async () => { await sleep(10) })', + ], + ['sleep() #1'], + ], + [ + 'a sleep in a beforeEach declared before the one that installs fake timers', + [ + HELPERS, + 'beforeEach(async () => { await sleep(10) })', + 'beforeEach(() => { vi.useFakeTimers() })', + ], + ['sleep() #1'], + ], + [ + 'fake timers with a config the AST cannot read', + [ + HELPERS, + 'beforeEach(() => { vi.useFakeTimers(config) })', + "it('t', async () => { await sleep(10) })", + ], + ['sleep() #1'], + ], + [ + 'fake timers with a spread config', + [ + HELPERS, + "it('t', async () => {", + ' vi.useFakeTimers({ ...config })', + ' await sleep(10)', + '})', + ], + ['sleep() #1'], + ], + [ + 'namespace imports', + [ + "import * as helpers from '@sim/utils/helpers'", + "import * as timers from 'node:timers/promises'", + 'await helpers.sleep(10)', + 'await timers.setTimeout(10)', + ], + ['sleep() #1', 'timers/promises setTimeout() #1'], + ], + [ + 'a module-scope wait before a module-scope install', + [HELPERS, 'await sleep(10)', 'vi.useFakeTimers()'], + ['sleep() #1'], + ], + [ + 'a helper wait after module-scope real timers are restored', + [ + HELPERS, + 'async function settle() { await sleep(10) }', + 'vi.useFakeTimers()', + 'vi.useRealTimers()', + ], + ['sleep() #1'], + ], + [ + 'a helper called at module scope before a module-scope install', + [ + HELPERS, + 'async function settle() { await sleep(10) }', + 'await settle()', + 'vi.useFakeTimers()', + ], + ['sleep() #1'], + ], + [ + 'an arrow helper called at module scope before a module-scope install', + [ + HELPERS, + 'const settle = async () => { await sleep(10) }', + 'await settle()', + 'vi.useFakeTimers()', + ], + ['sleep() #1'], + ], + [ + 'a wrapped arrow helper called at module scope before a module-scope install', + [ + HELPERS, + 'const settle = (async () => { await sleep(10) }) satisfies (() => Promise)', + 'await settle()', + 'vi.useFakeTimers()', + ], + ['sleep() #1'], + ], + [ + 'every occurrence, numbered per kind', + [HELPERS, 'await sleep(10)', 'await new Promise((r) => setTimeout(r, 5))', 'await sleep(20)'], + ['sleep() #1', 'setTimeout promise #1', 'sleep() #2'], + ], + ])('flags %s', (_name, lines, expected) => { + expect(waits(lines.join('\n'))).toEqual(expected) + }) + + it.each<[string, string[]]>([ + [ + 'a sleep not imported from @sim/utils/helpers', + ["import { sleep } from './helpers'", 'await sleep(10)'], + ], + [ + 'a mocked @sim/utils/helpers', + [HELPERS, "vi.mock('@sim/utils/helpers', () => utilsHelpersMock)", 'await sleep(10)'], + ], + ['a zero-delay sleep', [HELPERS, 'await sleep(0)']], + [ + 'a zero-delay setTimeout promise', + ['await new Promise((r) => setTimeout(r, 0))', 'await new Promise((r) => setTimeout(r))'], + ], + [ + 'a setTimeout that only rejects', + [ + "await new Promise((resolve, reject) => setTimeout(() => reject(new Error('timeout')), 50))", + ], + ], + [ + 'a sleep after the test installs fake timers', + [ + HELPERS, + "it('t', async () => {", + ' vi.useFakeTimers()', + ' const done = sleep(10)', + ' await vi.advanceTimersByTimeAsync(10)', + ' await done', + '})', + ], + ], + [ + 'a sleep in a describe whose beforeEach installs fake timers', + [ + HELPERS, + "describe('d', () => {", + " beforeEach(() => { vi.useFakeTimers({ toFake: ['setTimeout'] }) })", + " it('t', async () => { await sleep(10) })", + '})', + ], + ], + [ + 'a sleep under a module-level beforeAll that installs fake timers', + [HELPERS, "it('t', async () => { await sleep(10) })", 'beforeAll(() => vi.useFakeTimers())'], + ], + [ + 'a sleep after the install in the hook that installs fake timers', + [HELPERS, 'beforeAll(async () => {', ' vi.useFakeTimers()', ' await sleep(100)', '})'], + ], + [ + 'a sleep after real timers are restored and fake ones installed again', + [ + HELPERS, + "it('t', async () => {", + ' vi.useRealTimers()', + ' vi.useFakeTimers()', + ' await sleep(10)', + '})', + ], + ], + [ + 'a sleep in an afterEach beside a beforeEach that installs fake timers', + [ + HELPERS, + 'beforeEach(() => { vi.useFakeTimers() })', + 'afterEach(async () => { await sleep(10) })', + ], + ], + [ + 'a helper declared before a module-scope install', + [ + HELPERS, + "vi.mock('@/lib/example', () => ({}))", + 'const mocks = vi.hoisted(() => ({ run: vi.fn() }))', + 'const DELAY_MS = 10', + 'type Options = { delay: number }', + 'async function settle() { await sleep(DELAY_MS) }', + 'const settleLater = async () => { await settle() }', + 'const run = vi.fn()', + 'const fixture = createFixture()', + 'vi.useFakeTimers()', + 'await settle()', + "it('t', async () => { await settle() })", + ], + ], + ])('ignores %s', (_name, lines) => { + expect(waits(lines.join('\n'))).toEqual([]) + }) + + it.each(['apps/sim/lib/example.integration.ts', 'apps/sim/tools/example.live.test.ts'])( + 'exempts %s, which waits on real services', + (file) => { + expect(waits([HELPERS, 'await sleep(10)'].join('\n'), file)).toEqual([]) + } + ) +}) diff --git a/scripts/check-test-patterns.ts b/scripts/check-test-patterns.ts index 97aca30ef3a..622bfba77ab 100644 --- a/scripts/check-test-patterns.ts +++ b/scripts/check-test-patterns.ts @@ -16,6 +16,10 @@ * - `module-scope-stub`: a non-integration test calls `vi.stubGlobal`, `vi.stubEnv` or `vi.spyOn` * at module scope, where the shared config undoes it before the first test. * - `test-dir`: a test lives in a `__tests__/` or `tests/` directory instead of next to its source. + * - `real-wait`: a unit test waits on the wall clock (`sleep`, `new Promise(r => setTimeout(r, ms))`) + * with no fake timers in scope. Integration and `*.live.test.ts` files wait on real Postgres, + * Redis and external services, whose time is real, so the rule skips them; e2e scripts and + * Playwright specs are not `*.test.ts` files and are never scanned. * * A central mock, or a global mock in `apps/sim/vitest.setup.ts`, that `vi.mock`s an `@/…` module * id that no longer resolves fails outright: it mocks nothing, and a dead central `@example` id @@ -199,6 +203,359 @@ export function findStubAndHookViolations( return violations } +const FUNCTION_TYPES = new Set([ + 'ArrowFunctionExpression', + 'FunctionExpression', + 'FunctionDeclaration', + 'ObjectMethod', + 'ClassMethod', +]) + +/** Like `walk`, but hands each node its ancestors, nearest last. */ +function walkWithAncestors( + node: unknown, + visit: (node: Node, ancestors: Node[]) => void, + ancestors: Node[] = [] +): void { + if (!node || typeof node !== 'object') return + if (Array.isArray(node)) { + for (const child of node) walkWithAncestors(child, visit, ancestors) + return + } + const current = node as Node + const isNode = typeof current.type === 'string' + if (isNode) visit(current, ancestors) + const next = isNode ? [...ancestors, current] : ancestors + for (const [key, value] of Object.entries(current)) { + if (key === 'loc' || key === 'start' || key === 'end' || key === 'extra') continue + walkWithAncestors(value, visit, next) + } +} + +/** Index of the nearest function among `ancestors[0..before)`, or -1. */ +function nearestFunctionIndex(ancestors: Node[], before: number): number { + for (let index = before - 1; index >= 0; index--) { + if (FUNCTION_TYPES.has(ancestors[index].type)) return index + } + return -1 +} + +function nodeRange(node: Node): [number, number] { + return [node.start as number, node.end as number] +} + +/** A `setTimeout` delay that is a real wait: anything but an omitted or literal `0` delay. */ +function isNonZeroDelay(delay: Node | undefined): boolean { + return !(delay === undefined || (delay.type === 'NumericLiteral' && delay.value === 0)) +} + +/** + * Whether a `vi.useFakeTimers(config)` call provably fakes `setTimeout` on a clock that only moves + * when told: no config, or a literal one whose `toFake`/`toNotFake` keep `setTimeout` faked and + * that does not set `shouldAdvanceTime`. A config the AST cannot read is not proof. + */ +function fakesSetTimeout(call: Node): boolean { + const args = call.arguments as Node[] + if (args.length === 0) return true + const config = args[0] + if (args.length > 1 || config.type !== 'ObjectExpression') return false + for (const property of config.properties as Node[]) { + if (property.type !== 'ObjectProperty' || property.computed) return false + const key = property.key as { name?: string; value?: string } + const name = key.name ?? key.value + const value = property.value as Node + if (name === 'shouldAdvanceTime' && !(value.type === 'BooleanLiteral' && !value.value)) { + return false + } + if (name === 'toFake' || name === 'toNotFake') { + if (value.type !== 'ArrayExpression') return false + const listed = (value.elements as Node[]).some( + (element) => element?.type === 'StringLiteral' && element.value === 'setTimeout' + ) + if (name === 'toFake' ? !listed : listed) return false + } + } + return true +} + +/** Whether `node` calls `setTimeout` (bare or as `globalThis.setTimeout`/`window.setTimeout`). */ +function isSetTimeoutCall(node: Node): boolean { + if (node.type !== 'CallExpression') return false + const callee = node.callee as Node + return ( + (callee.type === 'Identifier' && callee.name === 'setTimeout') || + (callee.type === 'MemberExpression' && + (callee.property as { name?: string }).name === 'setTimeout') + ) +} + +function referencesIdentifier(node: Node, name: string): boolean { + if (node.type === 'Identifier') return node.name === name + let found = false + walk(node, (child) => { + if (child.type === 'Identifier' && child.name === name) found = true + }) + return found +} + +/** + * `new Promise((resolve) => setTimeout(resolve, ms))` and its variants: a promise whose executor + * resolves it from a non-zero `setTimeout`. A timer that only rejects is a timeout guard, not a + * wait, and a zero delay is a macrotask flush. + */ +function isTimedPromise(node: Node): boolean { + if ( + node.type !== 'NewExpression' || + (node.callee as Node).type !== 'Identifier' || + (node.callee as { name: string }).name !== 'Promise' + ) { + return false + } + const executor = (node.arguments as Node[])[0] + if (!executor || !FUNCTION_TYPES.has(executor.type)) return false + const resolve = (executor.params as Node[])[0] + if (resolve?.type !== 'Identifier') return false + const resolveName = resolve.name as string + let timed = false + walk(executor.body, (child) => { + if (!isSetTimeoutCall(child)) return + const [callback, delay] = child.arguments as Node[] + if (callback && isNonZeroDelay(delay) && referencesIdentifier(callback, resolveName)) { + timed = true + } + }) + return timed +} + +const WAIT_EXPORTS = { + '@sim/utils/helpers': ['sleep', 'interruptibleSleep'], + 'node:timers/promises': ['setTimeout'], + 'timers/promises': ['setTimeout'], +} as const +const WAIT_EXPORTS_BY_SOURCE: Readonly> = WAIT_EXPORTS + +const HOOK_KINDS = ['beforeAll', 'beforeEach', 'afterEach', 'afterAll'] as const +type HookKind = (typeof HOOK_KINDS)[number] +const HOOK_KIND_NAMES: readonly string[] = HOOK_KINDS + +/** + * A call and where it runs: its nearest function (the module when `inModule`), the module-scope + * helper it sits in (a function declaration or a variable bound to a function), and the innermost + * hook callback around it. For a module-scope `vi.useFakeTimers()`, `calledBefore` names the + * functions module-scope code calls before it. + */ +interface Site { + node: Node + fn: [number, number] + inModule: boolean + helper?: string + calledBefore?: Set + hook?: { kind: HookKind; range: [number, number]; suite: [number, number] } +} + +function within(position: number, range: [number, number]): boolean { + return range[0] <= position && position < range[1] +} + +function siteOf(node: Node, ancestors: Node[], programRange: [number, number]): Site { + const fnIndex = nearestFunctionIndex(ancestors, ancestors.length) + const site: Site = { + node, + fn: fnIndex === -1 ? programRange : nodeRange(ancestors[fnIndex]), + inModule: fnIndex === -1, + helper: helperName(ancestors), + } + for (let index = ancestors.length - 1; index > 0; index--) { + const fn = ancestors[index] + const call = ancestors[index - 1] + if (!FUNCTION_TYPES.has(fn.type) || call.type !== 'CallExpression') continue + const callee = call.callee as Node + const kind = callee.type === 'Identifier' ? (callee.name as string) : '' + if (!HOOK_KIND_NAMES.includes(kind) || (call.arguments as Node[])[0] !== fn) continue + const owner = nearestFunctionIndex(ancestors, index - 1) + site.hook = { + kind: kind as HookKind, + range: nodeRange(fn), + suite: owner === -1 ? programRange : nodeRange(ancestors[owner]), + } + break + } + return site +} + +/** The module-scope function declaration, or variable bound to a function, that `ancestors` sit in. */ +function helperName(ancestors: Node[]): string | undefined { + let index = 1 + while (ancestors[index]?.type.startsWith('Export')) index++ + const statement = ancestors[index] + const inner = ancestors[index + 1] + const init = inner?.type === 'VariableDeclarator' ? unwrap(inner.init as Node) : undefined + if (statement?.type === 'FunctionDeclaration') { + return (statement.id as Node | null)?.name as string | undefined + } + if ( + statement?.type === 'VariableDeclaration' && + inner?.type === 'VariableDeclarator' && + (inner.id as Node).type === 'Identifier' && + init && + FUNCTION_TYPES.has(init.type) + ) { + return (inner.id as Node).name as string + } + return undefined +} + +/** Names of the functions `node` calls directly by identifier when it runs, outside nested functions. */ +function directCalls(node: unknown, names: Set): Set { + if (!node || typeof node !== 'object') return names + if (Array.isArray(node)) { + for (const child of node) directCalls(child, names) + return names + } + const current = node as Node + if (FUNCTION_TYPES.has(current.type)) return names + if (current.type === 'CallExpression' && (current.callee as Node).type === 'Identifier') { + names.add((current.callee as Node).name as string) + } + for (const [key, value] of Object.entries(current)) { + if (key !== 'loc' && key !== 'start' && key !== 'end' && key !== 'extra') { + directCalls(value, names) + } + } + return names +} + +/** + * Whether a `vi.useFakeTimers()`/`vi.useRealTimers()` call at `timer` is in effect at `target`. + * Inside its own function it applies to what follows it. At module scope it also applies to every + * function body, since functions run after the module loads, except an install skips helpers that + * module-scope code calls before it (directly only: a helper reached through another call is + * missed). From a `beforeAll` it also applies to the + * rest of its suite except `beforeAll` hooks declared before it; from a `beforeEach`, to the + * suite's tests, `afterEach` hooks, and `beforeEach` hooks declared after it or in nested suites. + */ +function inEffect(timer: Site, target: Site): boolean { + const position = target.node.start as number + if (timer.inModule && !target.inModule) { + return !(target.helper && timer.calledBefore?.has(target.helper)) + } + if (within(position, timer.fn)) return position > (timer.node.start as number) + const hook = timer.hook + if (!hook || hook.range[0] !== timer.fn[0] || !within(position, hook.suite)) return false + if (hook.kind !== 'beforeAll' && hook.kind !== 'beforeEach') return false + const other = target.hook + if (!other) return true + const earlierSibling = other.suite[0] === hook.suite[0] && other.range[0] < hook.range[0] + if (hook.kind === 'beforeAll') return !(other.kind === 'beforeAll' && earlierSibling) + if (other.kind === 'beforeAll' || other.kind === 'afterAll') return false + return !(other.kind === 'beforeEach' && earlierSibling) +} + +/** + * `real-wait` violations in one unit test file: a call that waits on the wall clock — `sleep` or + * `interruptibleSleep` from `@sim/utils/helpers`, `setTimeout` from `node:timers/promises` (named + * or through a namespace import), or a promise resolved by `setTimeout` — with a non-zero delay + * and no fake clock in effect. + * + * Vitest's fake clock fakes each of these whenever it fakes `setTimeout`, and then the wait costs + * no wall-clock time. A wait is exempt when a `vi.useFakeTimers()` that provably fakes + * `setTimeout` is in effect there (see `inEffect`) and no `vi.useRealTimers()` in effect there runs + * after it. A file that mocks `@sim/utils/helpers` gets the mock's `sleep`, so its imports are not + * real waits. Each occurrence is its own numbered entry, so the baseline caps the count per file. + */ +export function findRealWaitViolations( + file: string, + program: ReturnType +): Violation[] { + if (file.endsWith('.integration.ts') || file.endsWith('.live.test.ts')) return [] + let helpersMocked = false + walk(program, (node) => { + if (mockedId(node) === '@sim/utils/helpers') helpersMocked = true + }) + const waitImports = new Map() + const namespaces = new Map() + for (const statement of program.body as Node[]) { + if (statement.type !== 'ImportDeclaration') continue + const source = (statement.source as { value: string }).value + const exported = + source === '@sim/utils/helpers' && helpersMocked ? [] : WAIT_EXPORTS_BY_SOURCE[source] + if (!exported?.length) continue + for (const specifier of statement.specifiers as Node[]) { + const local = (specifier.local as { name: string }).name + if (specifier.type === 'ImportNamespaceSpecifier') namespaces.set(local, exported) + if (specifier.type !== 'ImportSpecifier') continue + const imported = specifier.imported as { name?: string; value?: string } + const name = imported.name ?? imported.value ?? '' + if (exported.includes(name)) waitImports.set(local, waitLabel(name)) + } + } + + function importedWait(node: Node): string | undefined { + if (node.type !== 'CallExpression' || !isNonZeroDelay((node.arguments as Node[])[0])) { + return undefined + } + const callee = node.callee as Node + if (callee.type === 'Identifier') return waitImports.get(callee.name as string) + const object = callee.object as Node | undefined + const property = callee.property as Node | undefined + if ( + callee.type !== 'MemberExpression' || + callee.computed || + object?.type !== 'Identifier' || + property?.type !== 'Identifier' + ) { + return undefined + } + const exported = namespaces.get(object.name as string) + const name = property.name as string + return exported?.includes(name) ? waitLabel(name) : undefined + } + const programRange: [number, number] = [program.start ?? 0, program.end ?? 0] + const fakeClocks: Site[] = [] + const realClocks: Site[] = [] + const waits: Array<{ site: Site; label: string }> = [] + walkWithAncestors(program, (node, ancestors) => { + if (isViCall(node, 'useFakeTimers')) { + if (fakesSetTimeout(node)) { + const site = siteOf(node, ancestors, programRange) + if (site.inModule) { + const before = (program.body as Node[]).filter( + (statement) => (statement.end as number) <= (node.start as number) + ) + site.calledBefore = directCalls(before, new Set()) + } + fakeClocks.push(site) + } + return + } + if (isViCall(node, 'useRealTimers')) { + realClocks.push(siteOf(node, ancestors, programRange)) + return + } + const label = isTimedPromise(node) ? 'setTimeout promise' : importedWait(node) + if (label) waits.push({ site: siteOf(node, ancestors, programRange), label }) + }) + + const counts = new Map() + const violations: Violation[] = [] + for (const { site, label } of waits) { + const faked = fakeClocks.some( + (fake) => + inEffect(fake, site) && + !realClocks.some((real) => inEffect(real, site) && inEffect(fake, real)) + ) + if (faked) continue + const count = (counts.get(label) ?? 0) + 1 + counts.set(label, count) + violations.push({ file, rule: 'real-wait', detail: `${label} #${count}` }) + } + return violations +} + +function waitLabel(name: string): string { + return name === 'setTimeout' ? 'timers/promises setTimeout()' : `${name}()` +} + function mockedId(node: Node): string | undefined { if (!isViCall(node, 'mock')) return undefined const first = (node.arguments as Node[])[0] @@ -348,6 +705,7 @@ function collect(globals: Set, declaredCentralIds: Set): Violati } violations.push(...findStubAndHookViolations(file, program)) + violations.push(...findRealWaitViolations(file, program)) walk(program, (node) => { const id = mockedId(node) @@ -408,6 +766,10 @@ const FIX: Record = { 'module-scope-stub': 'move the stub into a beforeEach or the test; the shared config undoes it before the first test', 'test-dir': 'move the test next to its source file', + 'real-wait': + 'replace the wall-clock wait: vi.useFakeTimers() + await vi.advanceTimersByTimeAsync(ms) for ' + + 'time-based code, await vi.waitFor(() => …) for a condition, or flushMicrotasks()/flushMacrotask() ' + + 'from @sim/testing to let pending work settle', } /** Prints each new violation, then the fix for every rule involved. */ diff --git a/scripts/test-patterns-baseline.json b/scripts/test-patterns-baseline.json index faeda51cc7b..336ba63d942 100644 --- a/scripts/test-patterns-baseline.json +++ b/scripts/test-patterns-baseline.json @@ -70,5 +70,126 @@ "local-factory\tapps/sim/serializer/index.edge-case-blocks.test.ts\t@/blocks", "local-factory\tapps/sim/serializer/private-inputs.test.ts\t@/blocks", "local-factory\tapps/sim/tools/assistant-execution.test.ts\t@/tools/utils", + "real-wait\tapps/desktop/src/main/browser-import/chromium-cookies.test.ts\tsleep() #1", + "real-wait\tapps/desktop/src/main/desktop-executor/executor.test.ts\tsleep() #1", + "real-wait\tapps/desktop/src/main/desktop-executor/executor.test.ts\tsleep() #2", + "real-wait\tapps/desktop/src/main/desktop-executor/executor.test.ts\tsleep() #3", + "real-wait\tapps/desktop/src/main/desktop-executor/executor.test.ts\tsleep() #4", + "real-wait\tapps/desktop/src/main/desktop-executor/executor.test.ts\tsleep() #5", + "real-wait\tapps/desktop/src/main/desktop-executor/service.test.ts\tsleep() #1", + "real-wait\tapps/desktop/src/main/desktop-executor/service.test.ts\tsleep() #2", + "real-wait\tapps/desktop/src/main/desktop-executor/service.test.ts\tsleep() #3", + "real-wait\tapps/desktop/src/main/desktop-executor/service.test.ts\tsleep() #4", + "real-wait\tapps/desktop/src/main/desktop-executor/service.test.ts\tsleep() #5", + "real-wait\tapps/desktop/src/main/terminal/service.test.ts\tsleep() #1", + "real-wait\tapps/desktop/src/main/terminal/service.test.ts\tsleep() #2", + "real-wait\tapps/desktop/src/main/terminal/tmux.test.ts\tsleep() #1", + "real-wait\tapps/desktop/src/main/terminal/tmux.test.ts\tsleep() #2", + "real-wait\tapps/desktop/src/main/terminal/tmux.test.ts\tsleep() #3", + "real-wait\tapps/realtime/src/handlers/file-doc-store.test.ts\tsleep() #1", + "real-wait\tapps/realtime/src/handlers/file-doc.redis-store.test.ts\tsetTimeout promise #1", + "real-wait\tapps/sim/app/(interfaces)/chat/hooks/use-chat-streaming.test.tsx\tsetTimeout promise #1", + "real-wait\tapps/sim/app/api/v2/chat/route.test.ts\tsleep() #1", + "real-wait\tapps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/dirty-on-open.test.ts\tsleep() #1", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #1", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #10", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #11", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #12", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #13", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #14", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #15", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #16", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #17", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #18", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #19", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #2", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #20", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #21", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #22", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #23", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #24", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #25", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #26", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #27", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #28", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #29", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #3", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #30", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #31", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #32", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #33", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #34", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #35", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #36", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #37", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #38", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #39", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #4", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #40", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #41", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #5", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #6", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #7", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #8", + "real-wait\tapps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx\tsleep() #9", + "real-wait\tapps/sim/ee/workspace-forking/lib/copy/copy-resources.test.ts\tsleep() #1", + "real-wait\tapps/sim/executor/execution/engine.test.ts\tsleep() #1", + "real-wait\tapps/sim/executor/execution/engine.test.ts\tsleep() #2", + "real-wait\tapps/sim/executor/execution/engine.test.ts\tsleep() #3", + "real-wait\tapps/sim/executor/execution/engine.test.ts\tsleep() #4", + "real-wait\tapps/sim/executor/execution/engine.test.ts\tsleep() #5", + "real-wait\tapps/sim/executor/execution/engine.test.ts\tsleep() #6", + "real-wait\tapps/sim/executor/execution/engine.test.ts\tsleep() #7", + "real-wait\tapps/sim/hooks/queries/mcp.test.tsx\tsleep() #1", + "real-wait\tapps/sim/hooks/queries/mothership-chat-effort.test.tsx\tsleep() #1", + "real-wait\tapps/sim/hooks/queries/mothership-chats.test.ts\tsleep() #1", + "real-wait\tapps/sim/hooks/queries/mothership-chats.test.ts\tsleep() #2", + "real-wait\tapps/sim/hooks/queries/organization-accounts.test.tsx\tsleep() #1", + "real-wait\tapps/sim/hooks/queries/workflow-mcp-servers.test.tsx\tsleep() #1", + "real-wait\tapps/sim/hooks/queries/workspace-files.test.tsx\tsleep() #1", + "real-wait\tapps/sim/hooks/queries/workspace-files.test.tsx\tsleep() #2", + "real-wait\tapps/sim/hooks/queries/workspace.test.tsx\tsleep() #1", + "real-wait\tapps/sim/lib/billing/core/reporting-usage-cache.test.ts\tsleep() #1", + "real-wait\tapps/sim/lib/billing/core/reporting-usage-cache.test.ts\tsleep() #2", + "real-wait\tapps/sim/lib/billing/core/usage-gate-cache.test.ts\tsleep() #1", + "real-wait\tapps/sim/lib/concurrency/leader-lock.test.ts\tsleep() #1", + "real-wait\tapps/sim/lib/concurrency/singleflight.test.ts\tsleep() #1", + "real-wait\tapps/sim/lib/core/async-jobs/backends/database.test.ts\tsleep() #1", + "real-wait\tapps/sim/lib/core/async-jobs/backends/database.test.ts\tsleep() #2", + "real-wait\tapps/sim/lib/core/async-jobs/backends/database.test.ts\tsleep() #3", + "real-wait\tapps/sim/lib/core/utils/background.test.ts\tsleep() #1", + "real-wait\tapps/sim/lib/core/utils/background.test.ts\tsleep() #2", + "real-wait\tapps/sim/lib/execution/isolated-vm.test.ts\tsleep() #1", + "real-wait\tapps/sim/lib/execution/remote-sandbox/e2b-session.test.ts\tsleep() #1", + "real-wait\tapps/sim/lib/execution/remote-sandbox/e2b-session.test.ts\tsleep() #2", + "real-wait\tapps/sim/lib/execution/remote-sandbox/session-sandbox.test.ts\tsleep() #1", + "real-wait\tapps/sim/lib/execution/remote-sandbox/session-sandbox.test.ts\tsleep() #2", + "real-wait\tapps/sim/lib/execution/remote-sandbox/session-sandbox.test.ts\tsleep() #3", + "real-wait\tapps/sim/lib/execution/remote-sandbox/session-sandbox.test.ts\tsleep() #4", + "real-wait\tapps/sim/lib/execution/remote-sandbox/session-sandbox.test.ts\tsleep() #5", + "real-wait\tapps/sim/lib/internal/identity-center/concurrency.test.ts\tsleep() #1", + "real-wait\tapps/sim/lib/internal/identity-center/concurrency.test.ts\tsleep() #2", + "real-wait\tapps/sim/lib/internal/identity-center/concurrency.test.ts\tsleep() #3", + "real-wait\tapps/sim/lib/knowledge/projection/enqueue-inline.test.ts\tsleep() #1", + "real-wait\tapps/sim/lib/knowledge/transfer/export-archive.test.ts\tsleep() #1", + "real-wait\tapps/sim/lib/knowledge/transfer/export-archive.test.ts\tsleep() #2", + "real-wait\tapps/sim/lib/mcp/oauth/storage.test.ts\tsetTimeout promise #1", + "real-wait\tapps/sim/lib/mcp/storage/memory-cache.test.ts\tsleep() #1", + "real-wait\tapps/sim/lib/mothership/agent-cli/engines/universal-grep.test.ts\tsleep() #1", + "real-wait\tapps/sim/lib/mothership/agent-cli/engines/universal-grep.test.ts\tsleep() #2", + "real-wait\tapps/sim/lib/mothership/agent-cli/engines/universal-grep.test.ts\tsleep() #3", + "real-wait\tapps/sim/lib/mothership/request/handlers/handlers.test.ts\tsleep() #1", + "real-wait\tapps/sim/lib/mothership/tools/client/browser-tool-execution.test.ts\tsleep() #1", + "real-wait\tapps/sim/lib/mothership/tools/client/local-filesystem.test.ts\tsleep() #1", + "real-wait\tapps/sim/lib/mothership/tools/client/local-filesystem.test.ts\tsleep() #2", + "real-wait\tapps/sim/lib/mothership/tools/client/run-tool-execution.test.ts\tsleep() #1", + "real-wait\tapps/sim/lib/mothership/tools/server/files/file-intent-store.test.ts\tsleep() #1", + "real-wait\tapps/sim/lib/table/import.test.ts\tsleep() #1", + "real-wait\tpackages/audit/src/log.test.ts\tsleep() #1", + "real-wait\tpackages/sim-cli/src/commands/protocol/files-get.test.ts\tsleep() #1", + "real-wait\tpackages/sim-cli/src/config/profile.test.ts\tsleep() #1", + "real-wait\tpackages/sim-cli/src/config/profile.test.ts\tsleep() #2", + "real-wait\tpackages/sim-cli/src/embed.test.ts\tsleep() #1", + "real-wait\tpackages/sim-cli/src/http/client.test.ts\tsleep() #1", "redundant-hook\tapps/sim/lib/mothership/tools/hosted-workbench.live.test.ts\tvi.restoreAllMocks()" ]