From 30418a87671e4bdbfcbcabb51a12a5826dc20692 Mon Sep 17 00:00:00 2001 From: Hillary Mutisya <150286414+hillary-mutisya@users.noreply.github.com> Date: Fri, 23 Oct 2026 13:27:47 -0700 Subject: [PATCH 1/2] Reduce blocking macOS CI coverage and scope flaky-test retries Keep representative macOS correctness checks blocking and run full macOS coverage nightly. Preserve independent script-test accounting, retry failed Jest files safely, canonicalize generated-file paths, and move history scaling to periodic regressions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build-ts.yml | 28 +- .github/workflows/memory-regression.yml | 75 +++++ ts/README.md | 60 ++++ ts/package.json | 4 +- ts/packages/agents/browser/jest.config.js | 2 + .../agents/browserExtension/jest.config.js | 2 + ts/packages/cli/jest.config.js | 7 + ts/packages/memory/service/README.md | 24 ++ ts/packages/memory/service/package.json | 1 + .../test/viewHistoryRecords.regression.ts | 22 ++ .../service/test/viewHistoryRecords.spec.ts | 224 +------------- .../test/viewHistoryRecordsTestCommon.ts | 214 +++++++++++++ ts/tools/scripts/lib/generatedFileHealing.mjs | 4 +- ts/tools/scripts/lib/testRetry.mjs | 89 ++++++ ts/tools/scripts/runLocalTestPackages.mjs | 129 ++++++++ ts/tools/scripts/runTestLocalWithSummary.mjs | 241 +++++++++++---- .../test/generatedFileHealing.spec.mjs | 33 ++- ts/tools/scripts/test/jestConfig.spec.mjs | 67 +++-- .../scripts/test/localTestProfile.spec.mjs | 159 ++++++++++ .../test/runTestLocalWithSummary.spec.mjs | 280 ++++++++++++++++++ ts/tools/scripts/test/testRetry.spec.mjs | 158 ++++++++++ 21 files changed, 1516 insertions(+), 307 deletions(-) create mode 100644 .github/workflows/memory-regression.yml create mode 100644 ts/packages/memory/service/test/viewHistoryRecords.regression.ts create mode 100644 ts/packages/memory/service/test/viewHistoryRecordsTestCommon.ts create mode 100644 ts/tools/scripts/runLocalTestPackages.mjs create mode 100644 ts/tools/scripts/test/localTestProfile.spec.mjs create mode 100644 ts/tools/scripts/test/runTestLocalWithSummary.spec.mjs diff --git a/.github/workflows/build-ts.yml b/.github/workflows/build-ts.yml index 3907e0d580..69846db776 100644 --- a/.github/workflows/build-ts.yml +++ b/.github/workflows/build-ts.yml @@ -7,6 +7,8 @@ name: build-ts on: workflow_dispatch: + schedule: + - cron: "17 8 * * *" push: branches: ["main"] pull_request: @@ -15,8 +17,8 @@ on: branches: ["main"] concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name != 'schedule' }} permissions: pull-requests: read @@ -31,7 +33,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-latest", "windows-latest", "macos-latest"] + os: ${{ github.event_name == 'schedule' && fromJSON('["macos-latest"]') || fromJSON('["ubuntu-latest", "windows-latest", "macos-latest"]') }} version: ["22.23.4", "24.22.1", "26.12.1"] runs-on: ${{ matrix.os }} @@ -142,17 +144,27 @@ jobs: working-directory: ts run: | pnpm run postinstall:better-sqlite3-node-restore - # Runs the unit suite once, then reruns only the packages that owned a - # failing test up to two times (serially), stopping early on success. - # Flaky tests are reported as annotations / a step summary but do not fail - # the build; the final retry package suite must otherwise pass. + # macOS gates use an explicit compatibility selection. Scheduled and + # manual runs retain full macOS coverage on all supported Node versions. - name: Test if: ${{ github.event_name != 'pull_request' || steps.filter.outputs.ts != 'false' }} working-directory: ts + shell: pwsh env: TYPEAGENT_SKIP_POWERSHELL_BROKER_TESTS: ${{ runner.os == 'Windows' && '1' || '0' }} + TYPEAGENT_TEST_PROFILE: ${{ runner.os == 'macOS' && github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' && 'macos-compat' || 'full' }} run: | - node tools/scripts/runTestLocalWithSummary.mjs + $log = Join-Path $env:RUNNER_TEMP 'typeagent-test.log' + node tools/scripts/runTestLocalWithSummary.mjs 2>&1 | Tee-Object -FilePath $log + exit $LASTEXITCODE + - name: Upload test log + if: ${{ always() && (github.event_name != 'pull_request' || steps.filter.outputs.ts != 'false') }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: test-log-${{ matrix.os }}-${{ matrix.version }} + path: ${{ runner.temp }}/typeagent-test.log + if-no-files-found: ignore + retention-days: 14 - name: UI tests (requires display) if: ${{ (github.event_name != 'pull_request' || steps.filter.outputs.ts != 'false') && runner.os == 'Linux' }} working-directory: ts diff --git a/.github/workflows/memory-regression.yml b/.github/workflows/memory-regression.yml new file mode 100644 index 0000000000..bdb593eac2 --- /dev/null +++ b/.github/workflows/memory-regression.yml @@ -0,0 +1,75 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +name: memory-regression + +on: + workflow_dispatch: + schedule: + - cron: "23 8 * * 1" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +env: + NODE_OPTIONS: --max_old_space_size=8192 + +jobs: + memory: + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest", "windows-latest", "macos-latest"] + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + defaults: + run: + shell: pwsh + working-directory: ts + steps: + - name: Setup Git LF + working-directory: . + run: git config --global core.autocrlf false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install Linux prerequisites + if: runner.os == 'Linux' + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y libsecret-1-dev + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + with: + package_json_file: ts/package.json + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "24.22.1" + cache: "pnpm" + cache-dependency-path: ts/pnpm-lock.yaml + - name: Install dependencies + run: pnpm install --frozen-lockfile --strict-peer-dependencies + - name: Build memory service + id: build + run: | + if (Test-Path $PROFILE) { . $PROFILE } + pnpm run build @typeagent/memory-service + - name: Restore better-sqlite3 for Node.js + run: pnpm run postinstall:better-sqlite3-node-restore + - name: Memory correctness tests + timeout-minutes: 30 + run: pnpm --filter @typeagent/memory-service run test:local --runInBand --json --outputFile=memory-local-results.json + - name: Memory scaling regressions + if: ${{ !cancelled() && steps.build.outcome == 'success' }} + timeout-minutes: 15 + run: pnpm --filter @typeagent/memory-service run test:regression --json --outputFile=memory-regression-results.json + - name: Upload test results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: memory-results-${{ matrix.os }} + path: ts/packages/memory/service/memory-*-results.json + if-no-files-found: warn + retention-days: 14 diff --git a/ts/README.md b/ts/README.md index 6738b3aa6a..7cbe2cc8b3 100644 --- a/ts/README.md +++ b/ts/README.md @@ -279,6 +279,66 @@ pnpm run test:live # Integration tests (*.test.ts) — requires API keys in t pnpm run test # Both local + live + shell tests ``` +CI uses `node tools/scripts/runTestLocalWithSummary.mjs`. It runs +`test:local:packages`, retries failing files or packages serially up to twice, then runs +`test:local:scripts` independently. A failed workspace phase cannot skip the +script phase or be hidden by a successful script phase; both phases must pass. +Infrastructure failures (such as a process signal or spawn error) stop the run. +Root script failures are not retried. + +For recognized single-invocation Jest package scripts, retries run the accumulated +failed files through that package's `jest-esm` wrapper. Packages with chained +phases, setup hooks, or unrecognized runners explicitly fall back to their +complete `test:local` command. This preserves setup and does not assume every +package is a standalone Jest suite. + +Shared and standalone Jest configurations use one worker when `CI=true`, and +disable their Jest cache when `TYPEAGENT_JEST_NO_CACHE=true` during package +retries. These settings do not change local worker defaults. + +#### macOS blocking and nightly coverage + +The [build-ts workflow](../.github/workflows/build-ts.yml) keeps full local +correctness coverage on Ubuntu and Windows across all supported Node versions. +For pull requests, merge queues, and pushes, macOS runs the explicit +`macos-compat` selection in +[runLocalTestPackages.mjs](./tools/scripts/runLocalTestPackages.mjs), plus **all +root script tests**, across the same Node versions: + +| Area | Selected package suites | +| ----------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| Config/file URLs | `fileLink` | +| Native SQLite | `sqliteCollection`, `sqliteDataFrame` | +| Memory locking, publication, persistence, privacy and Git history | `corpusAccess`, `filesystemPublication`, `markdownPersistence`, `revisionTextStore`, `viewHistoryRecords` | +| Dispatcher locking and session lifecycle | `instanceLock`, `sessionContext`, `codingSessionLifecycle` | +| Cache round trips | `constructionCacheRoundTrip1`, `constructionCacheRoundTripMerge1` | + +Root script tests retain generated-file healing, aliased-directory containment +and subprocess/installer coverage. The smaller package selection is deliberate; +it is not full macOS correctness coverage. Keep it a positive list with contract +tests, and update it when new platform-sensitive behavior is introduced. + +Every day at **08:17 UTC**, the same workflow runs the **full** local correctness +suite on macOS for all supported Node versions. Scheduled runs use the default +branch. Manual dispatch runs full coverage on all three operating systems. +These runs exclude the separately scheduled memory scaling regressions. +Test logs are uploaded for 14 days and phase/failure summaries appear in GitHub. +Scheduled failures do not block an already merged PR: TypeAgent maintainers +should triage them before subsequent merges and add a blocking regression to +the compatibility selection when a missing platform-sensitive case is found. + +The default developer profile remains `full`. To validate the compatibility +selection locally after building, run from `ts` in PowerShell: + +```powershell +$env:TYPEAGENT_TEST_PROFILE = 'macos-compat' +try { + pnpm run test:local +} finally { + Remove-Item Env:TYPEAGENT_TEST_PROFILE +} +``` + To run tests for a single package: ```bash diff --git a/ts/package.json b/ts/package.json index 7935949d87..87755e90c6 100644 --- a/ts/package.json +++ b/ts/package.json @@ -84,7 +84,9 @@ "test": "pnpm run test:local && pnpm run test:live && pnpm run shell:test", "test:keys": "npx tsx tools/scripts/testServiceKeys.ts", "test:live": "pnpm -r ---no-bail -no-sort --stream --workspace-concurrency=1 run test:live", - "test:local": "pnpm -r --no-bail --no-sort --stream --workspace-concurrency=3 run test:local && node --test tools/scripts/test/*.spec.mjs", + "test:local": "pnpm run test:local:packages && pnpm run test:local:scripts", + "test:local:packages": "node tools/scripts/runLocalTestPackages.mjs", + "test:local:scripts": "node --test tools/scripts/test/*.spec.mjs", "test:ui": "pnpm -r --no-bail --no-sort --stream --if-present run test:ui" }, "devDependencies": { diff --git a/ts/packages/agents/browser/jest.config.js b/ts/packages/agents/browser/jest.config.js index 5e0437aa17..82583d7172 100644 --- a/ts/packages/agents/browser/jest.config.js +++ b/ts/packages/agents/browser/jest.config.js @@ -5,6 +5,8 @@ import { fileURLToPath } from "node:url"; /** @type {import('ts-jest').JestConfigWithTsJest} */ export default { + ...(process.env.CI === "true" ? { maxWorkers: 1 } : {}), + ...(process.env.TYPEAGENT_JEST_NO_CACHE === "true" ? { cache: false } : {}), preset: "ts-jest/presets/default-esm", extensionsToTreatAsEsm: [".ts"], testEnvironment: "jsdom", diff --git a/ts/packages/agents/browserExtension/jest.config.js b/ts/packages/agents/browserExtension/jest.config.js index 05ea393085..33a1ad081a 100644 --- a/ts/packages/agents/browserExtension/jest.config.js +++ b/ts/packages/agents/browserExtension/jest.config.js @@ -5,6 +5,8 @@ import { fileURLToPath } from "node:url"; /** @type {import('ts-jest').JestConfigWithTsJest} */ export default { + ...(process.env.CI === "true" ? { maxWorkers: 1 } : {}), + ...(process.env.TYPEAGENT_JEST_NO_CACHE === "true" ? { cache: false } : {}), preset: "ts-jest/presets/default-esm", extensionsToTreatAsEsm: [".ts"], testEnvironment: "jsdom", diff --git a/ts/packages/cli/jest.config.js b/ts/packages/cli/jest.config.js index c29f38db29..c80fd14077 100644 --- a/ts/packages/cli/jest.config.js +++ b/ts/packages/cli/jest.config.js @@ -1,8 +1,15 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +import sharedConfig from "../../jest.config.js"; + /** @type {import("ts-jest").JestConfigWithTsJest} */ export default { + ...(process.env.CI === "true" ? { maxWorkers: 1 } : {}), + ...(process.env.TYPEAGENT_JEST_NO_CACHE === "true" ? { cache: false } : {}), + ...(sharedConfig.reporters === undefined + ? {} + : { reporters: sharedConfig.reporters }), preset: "ts-jest/presets/default-esm", extensionsToTreatAsEsm: [".ts"], testEnvironment: "node", diff --git a/ts/packages/memory/service/README.md b/ts/packages/memory/service/README.md index 73fe4a1834..49234c865f 100644 --- a/ts/packages/memory/service/README.md +++ b/ts/packages/memory/service/README.md @@ -23,6 +23,30 @@ snapshot; actual qualification runs still require an explicit fixture root. No generated memory stores, recordings, provider responses or run receipts are included in these fixtures. +## Test suites + +Build from the TypeScript workspace root before running compiled tests: + +```powershell +pnpm run build @typeagent/memory-service +pnpm --filter @typeagent/memory-service run test:local +pnpm --filter @typeagent/memory-service run test:regression +``` + +`test:local` runs the offline `.spec.ts` correctness suite used by PR CI. +View-history ordering, exact records, authorship and selective parsing are checked +with ten commits; privacy-purge, recovery and concurrent-read cases remain in +that suite. The same assertions run against 100 and 1,000 commits in +`test:regression`, which selects only `.regression.ts` files and runs serially. +These scaling cases are excluded from normal Jest discovery and `test:local`; +no tests are skipped. + +The [memory-regression workflow](../../../../../.github/workflows/memory-regression.yml) +runs both suites every Monday at 08:23 UTC and on manual dispatch, on Ubuntu, +Windows and macOS with Node 24.22.1. Each OS uploads Jest JSON results, including +individual test durations. No model credentials or external fixture checkout +are required. Model-backed `.test.ts` rehearsals remain separately opt-in. + ## Automatic maintenance foundation When a Wiki candidate overlaps an existing page, the service checks the page's diff --git a/ts/packages/memory/service/package.json b/ts/packages/memory/service/package.json index be6bce555c..8c63593480 100644 --- a/ts/packages/memory/service/package.json +++ b/ts/packages/memory/service/package.json @@ -28,6 +28,7 @@ "prettier:fix": "prettier --write . --ignore-path ../../../.prettierignore", "test": "npm run test:local", "test:local": "pnpm run jest-esm --testPathPattern=\".*[.]spec[.]js\"", + "test:regression": "pnpm run jest-esm --runInBand --testMatch=\"**/dist/test/**/*.regression.js\"", "tsc": "tsc -b" }, "dependencies": { diff --git a/ts/packages/memory/service/test/viewHistoryRecords.regression.ts b/ts/packages/memory/service/test/viewHistoryRecords.regression.ts new file mode 100644 index 0000000000..136c8c80fb --- /dev/null +++ b/ts/packages/memory/service/test/viewHistoryRecords.regression.ts @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { mkdtemp, rm } from "node:fs/promises"; +import path from "node:path"; +import { checkHistoryRecords } from "./viewHistoryRecordsTestCommon.js"; + +test.each([100, 1000])( + "%i commits match legacy ordering and exact records without parsing unrelated state", + async (count) => { + const root = await mkdtemp( + path.join(process.cwd(), ".b12-view-records-regression-"), + ); + try { + await checkHistoryRecords(root, count); + } finally { + import.meta.jest.restoreAllMocks(); + await rm(root, { recursive: true, force: true }); + } + }, + 180_000, +); diff --git a/ts/packages/memory/service/test/viewHistoryRecords.spec.ts b/ts/packages/memory/service/test/viewHistoryRecords.spec.ts index 1ff9252928..4577e8575b 100644 --- a/ts/packages/memory/service/test/viewHistoryRecords.spec.ts +++ b/ts/packages/memory/service/test/viewHistoryRecords.spec.ts @@ -8,131 +8,14 @@ import path from "node:path"; import { inflateSync } from "node:zlib"; import { ViewHistory } from "../src/viewHistory.js"; import { TypedViewStore } from "../src/viewStore.js"; -import type { ViewHistoryEntry, ViewVersion } from "../src/viewTypes.js"; - -interface State { - views: Record; - corpusSequence: number; - unrelatedCorpusState: string; -} - -function version(viewId: string, revision: number): ViewVersion { - const createdAt = "2026-10-19T00:00:00.000Z"; - return { - corpusId: "corpus", - viewId, - revisionId: `${viewId}-${revision}`, - version: revision, - state: revision === 3 ? "archived" : "draft", - actor: "human-owner", - createdAt, - ...(revision > 1 - ? { baseRevisionId: `${viewId}-${revision - 1}` } - : {}), - definition: { - viewId, - revisionId: "definition-revision", - kind: "troubleshootingGuide", - selector: { - kind: "sources", - sources: [ - { sourceId: `${viewId}-source`, revisionId: "source-v1" }, - ], - }, - }, - content: { - kind: "troubleshootingGuide", - title: `${viewId} title ${revision}`, - citations: [], - sections: [ - { - id: "diagnose", - role: "diagnostic", - heading: "Diagnosis", - body: `${viewId} body ${revision}`, - }, - ], - }, - provenance: "merged", - relationships: [], - protectedTargets: ["title"], - edits: [ - { - id: "editorial-edit", - target: "title", - actor: "human-owner", - createdAt, - oldHash: "original-title-hash", - oldValue: "Original", - newValue: "Protected", - operation: "set", - status: "merged", - protection: ["title"], - unlockedTargets: ["sections.diagnose.body"], - }, - ], - }; -} - -function initialState(): State { - const target = version("target", 1); - target.generation = { - candidateId: "candidate", - content: structuredClone(target.content), - fingerprint: "generation-fingerprint", - input: { - corpusId: "corpus", - actor: "generator", - definition: target.definition, - expectedVersion: 0, - bounds: {}, - inputs: [], - influenceSources: [ - { sourceId: "target-source", revisionId: "source-v1" }, - ], - pipeline: "troubleshooting-v1", - model: "offline-model", - fingerprint: "generation-fingerprint", - }, - }; - const unrelated = version("unrelated", 1); - unrelated.content.sections[0].body = "unrelated-view-body".repeat(512); - return { - views: { target: [target], unrelated: [unrelated] }, - corpusSequence: 0, - unrelatedCorpusState: "unrelated-corpus-body".repeat(512), - }; -} - -function legacyEntries( - commits: Array<{ commitId: string; state: State }>, -): ViewHistoryEntry[] { - const seen = new Set(); - return commits - .reverse() - .flatMap(({ commitId, state }) => - (state.views.target ?? []).flatMap((version) => { - if (seen.has(version.revisionId)) return []; - seen.add(version.revisionId); - return [{ commitId, version }]; - }), - ) - .reverse(); -} - -function parsingCounts(calls: unknown[][]) { - const texts = calls.map((call) => String(call[0])); - return { - corpus: texts.filter((text) => text.includes('"corpusSequence":')) - .length, - unrelated: texts.filter((text) => text.includes("unrelated-view-body")) - .length, - selected: texts.filter( - (text) => - text.startsWith("[") && text.includes('"viewId":"target"'), - ).length, - }; -} +import { + checkHistoryRecords, + initialState, + legacyEntries, + parsingCounts, + version, + type State, +} from "./viewHistoryRecordsTestCommon.js"; describe("full-fidelity per-View history records", () => { let root: string; @@ -155,94 +38,9 @@ describe("full-fidelity per-View history records", () => { ); } - test.each([100, 1000])( - "%i commits match legacy ordering and exact records without parsing unrelated state", - async (count) => { - const indexed = history(); - const state = initialState(); - let head: string | null = null; - const started = performance.now(); - for (let index = 0; index < count; index++) { - state.corpusSequence = index; - state.views.unrelated[0].content.title = `Unrelated mutation ${index}`; - if (index === Math.floor(count / 3)) - state.views.target.push(version("target", 2)); - if (index === Math.floor(count / 2)) - state.views.target[0].actor = "later-metadata-mutation"; - if (index === Math.floor((2 * count) / 3)) - state.views.target.push(version("target", 3)); - if (index === count - 1) delete state.views.target; - head = await indexed.commit( - head, - state, - {}, - "test", - `Commit ${index}`, - ); - } - const writeMs = performance.now() - started; - const parse = import.meta.jest.spyOn(JSON, "parse"); - const baselineStart = performance.now(); - const expected = legacyEntries(await indexed.history()); - const baselineMs = performance.now() - baselineStart; - const baselineCounts = parsingCounts(parse.mock.calls); - expect(baselineCounts.corpus).toBe(count + 1); - parse.mockClear(); - - const store = new TypedViewStore(root); - const queryStart = performance.now(); - const actual = await store.getViewHistory({ - corpusId: "corpus", - viewId: "target", - }); - const queryMs = performance.now() - queryStart; - expect(actual).toEqual(expected); - const counts = parsingCounts(parse.mock.calls); - expect(counts).toEqual({ corpus: 0, unrelated: 0, selected: 4 }); - expect(expected.map((entry) => entry.version.version)).toEqual([ - 3, 2, 1, - ]); - expect(expected.at(-1)?.version.actor).toBe("human-owner"); - parse.mockClear(); - expect( - await new TypedViewStore(root).getViewHistory({ - corpusId: "corpus", - viewId: "target", - revisionId: "target-2", - }), - ).toEqual( - expected.filter( - (entry) => entry.version.revisionId === "target-2", - ), - ); - expect(parsingCounts(parse.mock.calls).corpus).toBe(0); - parse.mockClear(); - expect( - await store.getViewHistory({ - corpusId: "corpus", - viewId: "missing", - }), - ).toEqual([]); - expect(parsingCounts(parse.mock.calls)).toEqual({ - corpus: 0, - unrelated: 0, - selected: 0, - }); - process.stdout.write( - JSON.stringify({ - commits: count, - writeMs, - baselineMs, - queryMs, - baselineCorpusParses: baselineCounts.corpus, - baselineUnrelatedParses: baselineCounts.unrelated, - baselineSelectedRecordParses: baselineCounts.selected, - ...counts, - }) + "\n", - ); - }, - 180_000, - ); + test("10 commits match legacy ordering and exact records without parsing unrelated state", async () => { + await checkHistoryRecords(root, 10); + }); test("legacy indexes rebuild once, survive restart, and recover missing or malformed indexes", async () => { const legacy = history(false); diff --git a/ts/packages/memory/service/test/viewHistoryRecordsTestCommon.ts b/ts/packages/memory/service/test/viewHistoryRecordsTestCommon.ts new file mode 100644 index 0000000000..cd85c0cc40 --- /dev/null +++ b/ts/packages/memory/service/test/viewHistoryRecordsTestCommon.ts @@ -0,0 +1,214 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import path from "node:path"; +import { ViewHistory } from "../src/viewHistory.js"; +import { TypedViewStore } from "../src/viewStore.js"; +import type { ViewHistoryEntry, ViewVersion } from "../src/viewTypes.js"; + +export interface State { + views: Record; + corpusSequence: number; + unrelatedCorpusState: string; +} + +export function version(viewId: string, revision: number): ViewVersion { + const createdAt = "2026-10-19T00:00:00.000Z"; + return { + corpusId: "corpus", + viewId, + revisionId: `${viewId}-${revision}`, + version: revision, + state: revision === 3 ? "archived" : "draft", + actor: "human-owner", + createdAt, + ...(revision > 1 + ? { baseRevisionId: `${viewId}-${revision - 1}` } + : {}), + definition: { + viewId, + revisionId: "definition-revision", + kind: "troubleshootingGuide", + selector: { + kind: "sources", + sources: [ + { sourceId: `${viewId}-source`, revisionId: "source-v1" }, + ], + }, + }, + content: { + kind: "troubleshootingGuide", + title: `${viewId} title ${revision}`, + citations: [], + sections: [ + { + id: "diagnose", + role: "diagnostic", + heading: "Diagnosis", + body: `${viewId} body ${revision}`, + }, + ], + }, + provenance: "merged", + relationships: [], + protectedTargets: ["title"], + edits: [ + { + id: "editorial-edit", + target: "title", + actor: "human-owner", + createdAt, + oldHash: "original-title-hash", + oldValue: "Original", + newValue: "Protected", + operation: "set", + status: "merged", + protection: ["title"], + unlockedTargets: ["sections.diagnose.body"], + }, + ], + }; +} + +export function initialState(): State { + const target = version("target", 1); + target.generation = { + candidateId: "candidate", + content: structuredClone(target.content), + fingerprint: "generation-fingerprint", + input: { + corpusId: "corpus", + actor: "generator", + definition: target.definition, + expectedVersion: 0, + bounds: {}, + inputs: [], + influenceSources: [ + { sourceId: "target-source", revisionId: "source-v1" }, + ], + pipeline: "troubleshooting-v1", + model: "offline-model", + fingerprint: "generation-fingerprint", + }, + }; + const unrelated = version("unrelated", 1); + unrelated.content.sections[0].body = "unrelated-view-body".repeat(512); + return { + views: { target: [target], unrelated: [unrelated] }, + corpusSequence: 0, + unrelatedCorpusState: "unrelated-corpus-body".repeat(512), + }; +} + +export function legacyEntries( + commits: Array<{ commitId: string; state: State }>, +): ViewHistoryEntry[] { + const seen = new Set(); + return commits + .reverse() + .flatMap(({ commitId, state }) => + (state.views.target ?? []).flatMap((version) => { + if (seen.has(version.revisionId)) return []; + seen.add(version.revisionId); + return [{ commitId, version }]; + }), + ) + .reverse(); +} + +export function parsingCounts(calls: unknown[][]) { + const texts = calls.map((call) => String(call[0])); + return { + corpus: texts.filter((text) => text.includes('"corpusSequence":')) + .length, + unrelated: texts.filter((text) => text.includes("unrelated-view-body")) + .length, + selected: texts.filter( + (text) => + text.startsWith("[") && text.includes('"viewId":"target"'), + ).length, + }; +} + +export async function checkHistoryRecords( + root: string, + count: number, +): Promise { + const indexed = new ViewHistory( + path.join(root, "corpus", "views"), + initialState, + undefined, + (state) => state.views, + ); + const state = initialState(); + let head: string | null = null; + const started = performance.now(); + for (let index = 0; index < count; index++) { + state.corpusSequence = index; + state.views.unrelated[0].content.title = `Unrelated mutation ${index}`; + if (index === Math.floor(count / 3)) + state.views.target.push(version("target", 2)); + if (index === Math.floor(count / 2)) + state.views.target[0].actor = "later-metadata-mutation"; + if (index === Math.floor((2 * count) / 3)) + state.views.target.push(version("target", 3)); + if (index === count - 1) delete state.views.target; + head = await indexed.commit(head, state, {}, "test", `Commit ${index}`); + } + const writeMs = performance.now() - started; + const parse = import.meta.jest.spyOn(JSON, "parse"); + const baselineStart = performance.now(); + const expected = legacyEntries(await indexed.history()); + const baselineMs = performance.now() - baselineStart; + const baselineCounts = parsingCounts(parse.mock.calls); + expect(baselineCounts.corpus).toBe(count + 1); + parse.mockClear(); + + const store = new TypedViewStore(root); + const queryStart = performance.now(); + const actual = await store.getViewHistory({ + corpusId: "corpus", + viewId: "target", + }); + const queryMs = performance.now() - queryStart; + expect(actual).toEqual(expected); + const counts = parsingCounts(parse.mock.calls); + expect(counts).toEqual({ corpus: 0, unrelated: 0, selected: 4 }); + expect(expected.map((entry) => entry.version.version)).toEqual([3, 2, 1]); + expect(expected.at(-1)?.version.actor).toBe("human-owner"); + parse.mockClear(); + expect( + await new TypedViewStore(root).getViewHistory({ + corpusId: "corpus", + viewId: "target", + revisionId: "target-2", + }), + ).toEqual( + expected.filter((entry) => entry.version.revisionId === "target-2"), + ); + expect(parsingCounts(parse.mock.calls).corpus).toBe(0); + parse.mockClear(); + expect( + await store.getViewHistory({ + corpusId: "corpus", + viewId: "missing", + }), + ).toEqual([]); + expect(parsingCounts(parse.mock.calls)).toEqual({ + corpus: 0, + unrelated: 0, + selected: 0, + }); + process.stdout.write( + JSON.stringify({ + commits: count, + writeMs, + baselineMs, + queryMs, + baselineCorpusParses: baselineCounts.corpus, + baselineUnrelatedParses: baselineCounts.unrelated, + baselineSelectedRecordParses: baselineCounts.selected, + ...counts, + }) + "\n", + ); +} diff --git a/ts/tools/scripts/lib/generatedFileHealing.mjs b/ts/tools/scripts/lib/generatedFileHealing.mjs index 23bbfbf097..aa69e26256 100644 --- a/ts/tools/scripts/lib/generatedFileHealing.mjs +++ b/ts/tools/scripts/lib/generatedFileHealing.mjs @@ -706,7 +706,7 @@ function assertRegularGeneratedConflict(worktree, path, environment) { function containedRegularPath(root, path) { const normalized = normalizeCandidatePath(path); const rootPath = realpathSync(root); - const destination = resolve(root, ...normalized.split("/")); + const destination = resolve(rootPath, ...normalized.split("/")); const relativeDestination = relative(rootPath, destination); if ( relativeDestination === "" || @@ -833,7 +833,7 @@ function copyManifestInputs(worktree, scratchDirectory) { ) { blockCandidate(`Manifest source escapes the worktree: ${path}`); } - const destination = resolve(scratchDirectory, ...components.slice(1)); + const destination = resolve(resolvedScratch, ...components.slice(1)); const destinationRelative = relative(resolvedScratch, destination); if ( destinationRelative === "" || diff --git a/ts/tools/scripts/lib/testRetry.mjs b/ts/tools/scripts/lib/testRetry.mjs index 3c6dd127f6..b135598cf8 100644 --- a/ts/tools/scripts/lib/testRetry.mjs +++ b/ts/tools/scripts/lib/testRetry.mjs @@ -8,6 +8,95 @@ import path from "node:path"; export const MAX_TEST_RETRIES = 2; +const JEST_ESM = + "node --no-warnings --experimental-vm-modules ./node_modules/jest/bin/jest.js"; + +// Accept a deliberately small grammar, not arbitrary shell commands. In +// particular, never bypass setup, lifecycle hooks or additional runner phases. +export function planPackageRetry(manifest, files, packageDir) { + const scripts = manifest.scripts ?? {}; + const fallback = (reason) => ({ + narrowed: false, + reason, + cwd: packageDir, + args: ["run", "test:local"], + }); + if ( + [ + "pretest:local", + "posttest:local", + "prejest-esm", + "postjest-esm", + "prejest", + "postjest", + ].some((name) => scripts[name]) + ) { + return fallback( + "package script lifecycle hooks require the whole package", + ); + } + let script = scripts["test:local"]; + if (/^(?:npm|pnpm) run jest$/.test(script ?? "")) { + script = scripts.jest; + } + const invocation = + /^pnpm run jest-esm(?: --(?:testMatch|testPathPattern)=(?:"([^"$`\\\r\n]+)"|'([^'$`\\\r\n]+)'|([^\s"'$`\\;&|<>()]+)))?$/; + if (scripts["jest-esm"] !== JEST_ESM || !invocation.test(script ?? "")) { + return fallback( + "test:local is not a recognized single jest-esm invocation", + ); + } + if (files.length === 0) { + return fallback("no failed files could be identified"); + } + const absoluteFiles = [ + ...new Set( + files.map((file) => path.resolve(packageDir, toFsPath(file))), + ), + ]; + if ( + absoluteFiles.some((file) => { + const relative = path.relative(packageDir, file); + return ( + relative === "" || + relative === ".." || + relative.startsWith(`..${path.sep}`) || + path.isAbsolute(relative) + ); + }) + ) { + return fallback("failed file is outside the owning package"); + } + return { + narrowed: true, + cwd: packageDir, + args: [ + "run", + "jest-esm", + "--runInBand", + "--no-cache", + "--passWithNoTests=false", + "--runTestsByPath", + ...absoluteFiles, + ], + }; +} + +export function windowsPnpmCommand(args) { + // cmd expands percent expressions even within quotes. Refuse these rather + // than accidentally running a different path or command. + if (args.some((arg) => /[%"\r\n\0]/.test(arg))) { + throw new Error("Unsafe argument for Windows pnpm invocation"); + } + return `"pnpm ${args.map((arg) => `"${arg}"`).join(" ")}"`; +} + +export function runLocalTestPhases(runWorkspaceTests, runScriptTests) { + const workspaceStatus = runWorkspaceTests(); + const scriptStatus = runScriptTests(); + return workspaceStatus !== 0 || scriptStatus !== 0 ? 1 : 0; +} + // Convert a possibly-file:// path into a plain filesystem path. Done manually // (rather than url.fileURLToPath) because on Windows fileURLToPath throws for a // driveless URL like file:///repo/..., and reporters run on all three OSes. diff --git a/ts/tools/scripts/runLocalTestPackages.mjs b/ts/tools/scripts/runLocalTestPackages.mjs new file mode 100644 index 0000000000..1176a56488 --- /dev/null +++ b/ts/tools/scripts/runLocalTestPackages.mjs @@ -0,0 +1,129 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import fs from "node:fs"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +export const macosCompatibilitySuites = [ + { + directory: "packages/config", + files: ["fileLink"], + }, + { + directory: "packages/memory/storage", + files: ["sqliteCollection", "sqliteDataFrame"], + }, + { + directory: "packages/memory/service", + files: [ + "corpusAccess", + "filesystemPublication", + "markdownPersistence", + "revisionTextStore", + "viewHistoryRecords", + ], + }, + { + directory: "packages/dispatcher/dispatcher", + files: ["instanceLock", "sessionContext", "codingSessionLifecycle"], + }, + { + directory: "packages/defaultAgentProvider", + files: [ + "constructionCacheRoundTrip1", + "constructionCacheRoundTripMerge1", + ], + }, +]; + +export function localTestCommands(profile, root) { + if (profile === "full") { + return [ + { + cwd: root, + args: [ + "-r", + "--no-bail", + "--no-sort", + "--stream", + "--workspace-concurrency=3", + "run", + "test:local", + ], + }, + ]; + } + if (profile !== "macos-compat") { + throw new Error(`Unknown TYPEAGENT_TEST_PROFILE: ${profile}`); + } + return macosCompatibilitySuites.map(({ directory, files }) => ({ + cwd: path.resolve(root, directory), + args: [ + "run", + "jest-esm", + "--runTestsByPath", + ...files.map((file) => + path.resolve( + root, + directory, + "dist", + "test", + `${file}.spec.js`, + ), + ), + ], + })); +} + +export function runLocalTestPackages({ + profile, + root, + pnpmPath, + run = spawnSync, +}) { + const commands = localTestCommands(profile, root); + if (!pnpmPath || !fs.existsSync(pnpmPath)) { + throw new Error("Run local package tests through pnpm run."); + } + if (profile === "macos-compat") { + for (const { args } of commands) { + for (const file of args.slice(3)) { + if (!fs.statSync(file).isFile()) { + throw new Error( + `Missing compiled compatibility suite: ${file}`, + ); + } + } + } + } + let status = 0; + console.error(`Local package test profile: ${profile}`); + for (const { cwd, args } of commands) { + console.error(`Testing ${cwd}: ${args.join(" ")}`); + const result = run(process.execPath, [pnpmPath, ...args], { + cwd, + env: process.env, + stdio: "inherit", + }); + if (result.error) throw result.error; + if (result.signal) { + process.kill(process.pid, result.signal); + return 1; + } + if (result.status !== 0) status = 1; + } + return status; +} + +if ( + process.argv[1] && + path.resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { + process.exitCode = runLocalTestPackages({ + profile: process.env.TYPEAGENT_TEST_PROFILE ?? "full", + root: process.cwd(), + pnpmPath: process.env.npm_execpath, + }); +} diff --git a/ts/tools/scripts/runTestLocalWithSummary.mjs b/ts/tools/scripts/runTestLocalWithSummary.mjs index 6f30ef08d5..451fbe0ffd 100644 --- a/ts/tools/scripts/runTestLocalWithSummary.mjs +++ b/ts/tools/scripts/runTestLocalWithSummary.mjs @@ -5,16 +5,15 @@ // // 1. Run the whole suite once, capturing individual failures via the reporter // that writes to TYPEAGENT_TEST_FAILURES_DIR. -// 2. If anything failed, rerun ONLY the packages that owned a failing test up -// to two times, serially, with a fresh Jest cache and failures directory, -// stopping as soon as the package suite passes. +// 2. Retry failed files for recognized Jest scripts, otherwise whole packages, +// up to twice serially, with caching disabled and fresh reporter directories. // 3. Classify every observed failure as flaky (absent from the final attempt) // or build-blocking (present in the final attempt), then print a summary and // emit GitHub annotations / a step summary. The build fails unless a retry // command completes successfully. +// 4. Run root script tests independently, even if workspace tests still fail. import fs from "node:fs"; -import os from "node:os"; import path from "node:path"; import { spawnSync } from "node:child_process"; @@ -28,6 +27,9 @@ import { buildStepSummaryMarkdown, toFsPath, MAX_TEST_RETRIES, + runLocalTestPhases, + planPackageRetry, + windowsPnpmCommand, } from "./lib/testRetry.mjs"; // build-ts runs this from the `ts` directory, which is the pnpm workspace root. @@ -38,16 +40,55 @@ const inGithubActions = process.env.GITHUB_ACTIONS === "true"; const firstDirectory = makeFailuresDirectory("run1"); const failureDirectories = [firstDirectory]; try { - const firstResult = runPnpm(["run", "test:local"], firstDirectory); + process.exitCode = runLocalTestPhases( + () => { + const status = runWorkspaceTests(); + reportPhase("Workspace tests (including retries)", status); + return status; + }, + () => { + console.error("\nROOT SCRIPT TEST PHASE"); + const result = runPnpm( + ["run", "test:local:scripts"], + firstDirectory, + ); + checkInfrastructureFailure(result); + const status = result.status ?? 1; + reportPhase("Root script tests", status); + return status; + }, + ); +} finally { + for (const directory of failureDirectories) { + fs.rmSync(directory, { recursive: true, force: true }); + } +} - // A spawn error or a killing signal is an infrastructure failure, not a - // test failure we can retry; surface it exactly like before. - if (firstResult.error !== undefined) { - throw firstResult.error; +function checkInfrastructureFailure(result) { + if (result.error !== undefined || result.signal !== null) { + console.error( + `Test command infrastructure failure: ${result.error?.message ?? result.signal}`, + ); + return true; } - if (firstResult.signal !== null) { - process.kill(process.pid, firstResult.signal); + return false; +} + +function reportPhase(name, status) { + console.error(`\n${name}: exit ${status}`); + if (inGithubActions) { + if (status !== 0) + console.log(`::error::${name} failed (exit ${status}).`); + appendGithubSummary( + `### ${name}\n\nResult: **${status === 0 ? "passed" : "failed"}** (exit ${status}).`, + ); } +} + +function runWorkspaceTests() { + console.error("\nWORKSPACE TEST PHASE"); + const firstResult = runPnpm(["run", "test:local:packages"], firstDirectory); + if (checkInfrastructureFailure(firstResult)) return 1; const firstFailures = dedupeFailures( readFailures(firstDirectory), @@ -60,14 +101,9 @@ try { "\nFAILED TEST SUMMARY\nNo individual test failures were captured. See the test output above.", ); } - process.exitCode = firstResult.status ?? 0; - } else { - process.exitCode = handleFailures(firstFailures); - } -} finally { - for (const directory of failureDirectories) { - fs.rmSync(directory, { recursive: true, force: true }); + return firstResult.status ?? 1; } + return handleFailures(firstFailures); } function handleFailures(firstFailures) { @@ -95,43 +131,89 @@ function handleFailures(firstFailures) { for (let attempt = 1; attempt <= MAX_TEST_RETRIES; attempt++) { console.error(`\nRetry ${attempt} of ${MAX_TEST_RETRIES}`); - const retryDirectory = makeFailuresDirectory(`retry${attempt}`); - failureDirectories.push(retryDirectory); - const retryResult = runPnpm( - [ - ...packages.flatMap((name) => ["--filter", name]), - // Serial rerun of only the affected packages isolates - // genuine failures from resource-contention flakiness. - "--workspace-concurrency=1", - "--no-sort", - "--stream", - "--no-bail", - "run", - "test:local", - ], - retryDirectory, - true, - ); - - finalFailures = dedupeFailures( - readFailures(retryDirectory), - workspaceRoot, - ); - retryFailed = - retryResult.error !== undefined || - retryResult.signal !== null || - (retryResult.status ?? 1) !== 0; - - // A failed command with no captured failures cannot safely be - // classified or cleared by another retry. - const retryCrashed = - retryResult.error !== undefined || - retryResult.signal !== null || - (retryFailed && finalFailures.length === 0); - if (retryCrashed) { - retryTrusted = false; - break; + finalFailures = []; + retryFailed = false; + const packageNames = [ + ...new Set( + priorFailures + .map((failure) => failure.package) + .filter((name) => name !== undefined), + ), + ]; + for (const name of packageNames) { + const failures = priorFailures.filter( + (failure) => failure.package === name, + ); + const packageDir = resolvePackage( + failures[0].testFilePath, + )?.directory; + if (packageDir === undefined) { + retryTrusted = false; + retryFailed = true; + break; + } + const manifest = JSON.parse( + fs.readFileSync( + path.join(packageDir, "package.json"), + "utf8", + ), + ); + let plan = planPackageRetry( + manifest, + failures.map((failure) => + toAbsolutePath(failure.testFilePath), + ), + packageDir, + ); + if ( + plan.narrowed && + plan.args + .slice(6) + .some((file) => !isRegularPackageFile(file, packageDir)) + ) { + plan = { + narrowed: false, + reason: "failed path is not a regular file contained in the owning package", + cwd: packageDir, + args: ["run", "test:local"], + }; + } + console.error( + plan.narrowed + ? `Retrying failed files in ${name}` + : `Full-package retry for ${name}: ${plan.reason}`, + ); + const retryDirectory = makeFailuresDirectory(`retry${attempt}`); + failureDirectories.push(retryDirectory); + const retryResult = runPnpm( + plan.args, + retryDirectory, + true, + plan.cwd, + ); + const observed = readFailures(retryDirectory); + finalFailures.push(...observed); + const failed = + retryResult.error !== undefined || + retryResult.signal !== null || + (retryResult.status ?? 1) !== 0; + retryFailed ||= failed; + // Check each command independently: another package's artifacts + // cannot make a crashed or unreported invocation trustworthy. + if ( + retryResult.error !== undefined || + retryResult.signal !== null || + (failed && observed.length === 0) + ) { + console.error( + `Untrusted retry for ${name}: command failed without reliable failure artifacts.`, + ); + retryTrusted = false; + break; + } } + finalFailures = dedupeFailures(finalFailures, workspaceRoot); + if (!retryTrusted) break; if (!retryFailed) { break; } @@ -158,7 +240,7 @@ function handleFailures(firstFailures) { function annotatePackages(failures) { return failures.map((failure) => ({ ...failure, - package: resolvePackageName(failure.testFilePath), + package: resolvePackage(failure.testFilePath)?.name, })); } @@ -179,19 +261,52 @@ function handleFailures(firstFailures) { function makeFailuresDirectory(label) { return fs.mkdtempSync( - path.join(os.tmpdir(), `typeagent-test-failures-${label}-`), + path.join(workspaceRoot, `.typeagent-test-failures-${label}-`), ); } -function runPnpm(pnpmArguments, failuresDirectory, disableJestCache = false) { +function isRegularPackageFile(file, packageDir) { + try { + const relative = path.relative( + fs.realpathSync(packageDir), + fs.realpathSync(file), + ); + return ( + relative !== "" && + relative !== ".." && + !relative.startsWith(`..${path.sep}`) && + !path.isAbsolute(relative) && + fs.statSync(file).isFile() + ); + } catch { + return false; + } +} + +function runPnpm( + pnpmArguments, + failuresDirectory, + disableJestCache = false, + cwd = workspaceRoot, +) { + let windowsCommand; + try { + windowsCommand = isWindows + ? windowsPnpmCommand(pnpmArguments) + : undefined; + } catch (error) { + return { error, signal: null, status: null }; + } const [command, commandArguments] = isWindows ? [ process.env.ComSpec ?? "cmd.exe", - ["/d", "/s", "/c", ["pnpm", ...pnpmArguments].join(" ")], + ["/d", "/v:off", "/s", "/c", windowsCommand], ] : ["pnpm", pnpmArguments]; return spawnSync(command, commandArguments, { + cwd, + ...(isWindows ? { windowsVerbatimArguments: true } : {}), env: { ...process.env, TYPEAGENT_TEST_FAILURES_DIR: failuresDirectory, @@ -221,7 +336,7 @@ function readFailureArtifact(directory, fileName) { } } -function resolvePackageName(testFilePath) { +function resolvePackage(testFilePath) { const packageDir = findPackageDir( toAbsolutePath(testFilePath), workspaceRoot, @@ -235,7 +350,7 @@ function resolvePackageName(testFilePath) { fs.readFileSync(path.join(packageDir, "package.json"), "utf8"), ); return typeof manifest.name === "string" && manifest.name.length > 0 - ? manifest.name + ? { name: manifest.name, directory: packageDir } : undefined; } catch { return undefined; @@ -256,8 +371,12 @@ function emitGithubReport(classification) { for (const annotation of buildGithubAnnotations(classification)) { console.log(annotation); } - const summaryFile = process.env.GITHUB_STEP_SUMMARY; const markdown = buildStepSummaryMarkdown(classification); + appendGithubSummary(markdown); +} + +function appendGithubSummary(markdown) { + const summaryFile = process.env.GITHUB_STEP_SUMMARY; if (summaryFile !== undefined && markdown.length > 0) { try { fs.appendFileSync(summaryFile, `${markdown}\n`); diff --git a/ts/tools/scripts/test/generatedFileHealing.spec.mjs b/ts/tools/scripts/test/generatedFileHealing.spec.mjs index bcdea09b74..2493c7121b 100644 --- a/ts/tools/scripts/test/generatedFileHealing.spec.mjs +++ b/ts/tools/scripts/test/generatedFileHealing.spec.mjs @@ -422,7 +422,7 @@ function prepare(fixture, options = {}) { RUNNER_TEMP: runnerTemp, TEST_SECRET: "sentinel-test-secret", }, - workDirectory: join(fixture.root, "prepare"), + workDirectory: options.workDirectory ?? join(fixture.root, "prepare"), now: () => new Date("2026-10-20T20:00:00.000Z"), }); return { summary, outputDirectory, auditPath, pnpmCommand }; @@ -627,6 +627,37 @@ test("trusted healer prepares a candidate without exposing credentials or runnin } }); +test("generated files remain contained when the work directory has a filesystem alias", () => { + const fixture = createFixture(); + try { + const canonical = join(fixture.root, "canonical-work"); + const alias = join(fixture.root, "aliased-work"); + mkdirSync(canonical); + symlinkSync( + canonical, + alias, + process.platform === "win32" ? "junction" : "dir", + ); + const prepared = prepare(fixture, { + workDirectory: join(alias, "prepare"), + }); + assert.deepEqual(prepared.summary.skipped, []); + assert.equal(prepared.summary.candidates.length, 1); + const { plan } = verify(fixture, prepared); + assert.equal(plan.candidates.length, 1); + assert.equal( + git( + plan.repositoryPath, + "show", + `${plan.candidates[0].candidateSha}:ts/packages/app/README.AUTOGEN.md`, + ), + "pull request readme", + ); + } finally { + fixture.cleanup(); + } +}); + test("prepare CLI consumes metadata without invoking GitHub CLI or APIs", () => { const fixture = createFixture(); try { diff --git a/ts/tools/scripts/test/jestConfig.spec.mjs b/ts/tools/scripts/test/jestConfig.spec.mjs index 0ea8001185..1b669b5009 100644 --- a/ts/tools/scripts/test/jestConfig.spec.mjs +++ b/ts/tools/scripts/test/jestConfig.spec.mjs @@ -6,25 +6,50 @@ import { spawnSync } from "node:child_process"; import { fileURLToPath } from "node:url"; import test from "node:test"; -const configPath = fileURLToPath( - new URL("../../../jest.config.js", import.meta.url), -); - -for (const ci of ["true", "false"]) { - test(`Jest limits workers only in CI: CI=${ci}`, () => { - const result = spawnSync( - process.execPath, - [ - "-e", - "process.stdout.write(JSON.stringify(require(process.argv[1])))", - configPath, - ], - { env: { ...process.env, CI: ci }, encoding: "utf8" }, - ); - assert.ifError(result.error); - assert.equal(result.status, 0, result.stderr); - const config = JSON.parse(result.stdout); - if (ci === "true") assert.equal(config.maxWorkers, 1); - else assert.equal(Object.hasOwn(config, "maxWorkers"), false); - }); +for (const config of [ + "jest.config.js", + "packages/agents/browser/jest.config.js", + "packages/agents/browserExtension/jest.config.js", + "packages/cli/jest.config.js", +]) { + for (const enabled of ["true", "false"]) { + test(`Jest CI, retry cache and reporter settings: ${config}, enabled=${enabled}`, () => { + const configPath = fileURLToPath( + new URL(`../../../${config}`, import.meta.url), + ); + const env = { + ...process.env, + CI: enabled, + TYPEAGENT_JEST_NO_CACHE: enabled, + }; + delete env.TYPEAGENT_TEST_FAILURES_DIR; + if (enabled === "true") { + env.TYPEAGENT_TEST_FAILURES_DIR = + "unused-config-test-directory"; + } + const result = spawnSync( + process.execPath, + [ + "--input-type=module", + "-e", + "import { pathToFileURL } from 'node:url'; const config = await import(pathToFileURL(process.argv[1]).href); process.stdout.write(JSON.stringify(config.default));", + configPath, + ], + { env, encoding: "utf8" }, + ); + assert.ifError(result.error); + assert.equal(result.status, 0, result.stderr); + const actual = JSON.parse(result.stdout); + if (enabled === "true") { + assert.equal(actual.maxWorkers, 1); + assert.equal(actual.cache, false); + assert.equal(actual.reporters[0], "default"); + assert.match(actual.reporters[1], /jestFailureReporter\.cjs$/); + } else { + assert.equal(Object.hasOwn(actual, "maxWorkers"), false); + assert.equal(Object.hasOwn(actual, "cache"), false); + assert.equal(Object.hasOwn(actual, "reporters"), false); + } + }); + } } diff --git a/ts/tools/scripts/test/localTestProfile.spec.mjs b/ts/tools/scripts/test/localTestProfile.spec.mjs new file mode 100644 index 0000000000..38fe26169b --- /dev/null +++ b/ts/tools/scripts/test/localTestProfile.spec.mjs @@ -0,0 +1,159 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; +import vm from "node:vm"; +import yaml from "js-yaml"; +import { + localTestCommands, + macosCompatibilitySuites, + runLocalTestPackages, +} from "../runLocalTestPackages.mjs"; + +const root = fileURLToPath(new URL("../../../", import.meta.url)); + +test("workflow retains blocking Linux/Windows, macOS compatibility, and full nightly/manual coverage", () => { + const workflow = yaml.load( + fs.readFileSync( + new URL( + "../../../../.github/workflows/build-ts.yml", + import.meta.url, + ), + "utf8", + ), + ); + assert.equal(workflow.on.schedule[0].cron, "17 8 * * *"); + const job = workflow.jobs.build_ts; + const step = job.steps.find((step) => step.name === "Test"); + const evaluate = (expression, event, os) => + vm.runInNewContext(expression.slice(3, -2), { + github: { event_name: event }, + runner: { os }, + fromJSON: JSON.parse, + }); + for (const event of [ + "pull_request", + "merge_group", + "push", + "workflow_dispatch", + "schedule", + ]) { + const matrix = Array.from(evaluate(job.strategy.matrix.os, event)); + assert.deepEqual( + matrix, + event === "schedule" + ? ["macos-latest"] + : ["ubuntu-latest", "windows-latest", "macos-latest"], + ); + for (const os of ["Linux", "Windows", "macOS"]) { + assert.equal( + evaluate(step.env.TYPEAGENT_TEST_PROFILE, event, os), + os === "macOS" && + !["schedule", "workflow_dispatch"].includes(event) + ? "macos-compat" + : "full", + ); + } + } + assert.deepEqual(job.strategy.matrix.version, [ + "22.23.4", + "24.22.1", + "26.12.1", + ]); + assert.equal(step.shell, "pwsh"); + assert.match(step.run, /exit \$LASTEXITCODE/); +}); + +test("full profile preserves the existing workspace selection and concurrency", () => { + assert.deepEqual(localTestCommands("full", root), [ + { + cwd: root, + args: [ + "-r", + "--no-bail", + "--no-sort", + "--stream", + "--workspace-concurrency=3", + "run", + "test:local", + ], + }, + ]); +}); + +test("compatibility selection is explicit, unique, and references existing source tests", () => { + const commands = localTestCommands("macos-compat", root); + assert.equal(commands.length, 5); + const files = commands.flatMap(({ args }) => args.slice(3)); + assert.equal(files.length, 13); + assert.equal(new Set(files).size, files.length); + for (const { directory, files } of macosCompatibilitySuites) { + const manifest = JSON.parse( + fs.readFileSync(path.join(root, directory, "package.json"), "utf8"), + ); + assert.match(manifest.scripts["jest-esm"], /jest/); + for (const file of files) { + assert.ok( + fs + .statSync( + path.join(root, directory, "test", `${file}.spec.ts`), + ) + .isFile(), + ); + } + } +}); + +test("unknown profiles fail instead of silently selecting a different suite", () => { + assert.throws(() => localTestCommands("macos-typo", root), /Unknown/); +}); + +test("compatibility validates compiled files before running any package", (t) => { + const empty = fs.mkdtempSync(path.join(os.tmpdir(), "typeagent-profile-")); + t.after(() => fs.rmSync(empty, { force: true, recursive: true })); + assert.throws( + () => + runLocalTestPackages({ + profile: "macos-compat", + root: empty, + pnpmPath: process.execPath, + run: () => assert.fail("Must not start with missing tests"), + }), + /ENOENT|Missing compiled/, + ); +}); + +test("profile runner preserves errors and invokes pnpm without shell interpolation", () => { + const calls = []; + assert.equal( + runLocalTestPackages({ + profile: "full", + root, + pnpmPath: process.execPath, + run: (command, args, options) => { + calls.push({ command, args, cwd: options.cwd }); + return { status: 1 }; + }, + }), + 1, + ); + assert.equal(calls.length, 1); + assert.equal(calls[0].command, process.execPath); + assert.equal(calls[0].cwd, root); + const error = new Error("spawn failed"); + assert.throws( + () => + runLocalTestPackages({ + profile: "full", + root, + pnpmPath: process.execPath, + run: () => ({ error }), + }), + error, + ); +}); diff --git a/ts/tools/scripts/test/runTestLocalWithSummary.spec.mjs b/ts/tools/scripts/test/runTestLocalWithSummary.spec.mjs new file mode 100644 index 0000000000..50e74d329b --- /dev/null +++ b/ts/tools/scripts/test/runTestLocalWithSummary.spec.mjs @@ -0,0 +1,280 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +const runner = fileURLToPath( + new URL("../runTestLocalWithSummary.mjs", import.meta.url), +); + +for (const scenario of [ + { + name: "passed package remains in retry union", + initial: 1, + retry: 0, + scripts: 0, + secondPackage: true, + mixedRecovery: true, + }, + { + name: "missing file falls back safely", + initial: 1, + retry: 0, + scripts: 0, + missingFile: true, + fallback: true, + }, + { name: "first pass succeeds", initial: 0, retry: 0, scripts: 0 }, + { name: "failed file retry recovers", initial: 1, retry: 0, scripts: 0 }, + { name: "script failure after recovery", initial: 1, retry: 0, scripts: 1 }, + { name: "package retry stays failing", initial: 1, retry: 1, scripts: 0 }, + { + name: "unknown runner falls back", + initial: 1, + retry: 0, + scripts: 0, + script: "node --test test/*.spec.mjs", + fallback: true, + }, + { + name: "browser chained phases fall back", + initial: 1, + retry: 0, + scripts: 0, + script: "jest && npm run test:views:gateway && npm run test:views:client", + fallback: true, + }, + { + name: "setup phases fall back", + initial: 1, + retry: 0, + scripts: 0, + script: 'pnpm run build && pnpm run jest-esm --testPathPattern=".*[.]spec[.]js"', + fallback: true, + }, + { + name: "retry introduces another failed file", + initial: 1, + retry: 0, + scripts: 0, + newFile: true, + }, + { + name: "unreported retry cannot recover", + initial: 1, + retry: 1, + scripts: 0, + unreportedRetry: true, + }, + { + name: "one unreported package cannot hide behind another package", + initial: 1, + retry: 1, + scripts: 0, + secondPackage: true, + unreportedRetry: true, + }, + { + name: "unreported package failure", + initial: 1, + retry: 0, + scripts: 0, + unreported: true, + }, +]) { + test(`local test accounting: ${scenario.name}`, (t) => { + const root = fs.mkdtempSync( + path.join( + path.dirname(fileURLToPath(import.meta.url)), + ".typeagent phases & fixtures-", + ), + ); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + for (const name of scenario.secondPackage + ? ["example", "other"] + : ["example"]) { + const packageDir = path.join(root, "packages", name); + fs.mkdirSync(packageDir, { recursive: true }); + fs.mkdirSync(path.join(packageDir, "test"), { recursive: true }); + for (const file of ["x file.spec.js", "new file.spec.js"]) { + if (!scenario.missingFile) + fs.writeFileSync(path.join(packageDir, "test", file), ""); + } + fs.writeFileSync( + path.join(packageDir, "package.json"), + JSON.stringify({ + name, + scripts: { + "test:local": + scenario.script ?? + 'pnpm run jest-esm --testPathPattern=".*[.]spec[.]js"', + "jest-esm": + "node --no-warnings --experimental-vm-modules ./node_modules/jest/bin/jest.js", + }, + }), + ); + } + const fakePnpm = path.join(root, "fakePnpm.mjs"); + fs.writeFileSync( + fakePnpm, + `#!/usr/bin/env node +import fs from "node:fs"; +import path from "node:path"; +const args = process.argv.slice(2); +const root = process.env.TEST_PHASE_ROOT; +const log = path.join(root, "calls.jsonl"); +const earlier = fs.existsSync(log) ? fs.readFileSync(log, "utf8").trim().split("\\n").map(JSON.parse) : []; +fs.appendFileSync(log, JSON.stringify({args, cwd: process.cwd(), + failures: process.env.TYPEAGENT_TEST_FAILURES_DIR, + noCache: process.env.TYPEAGENT_JEST_NO_CACHE}) + "\\n"); +const scenario = JSON.parse(process.env.TEST_PHASE_SCENARIO); +const scripts = args.includes("test:local:scripts"); +const retry = !scripts && !args.includes("test:local:packages"); +const retryNumber = earlier.filter((call) => call.args.includes("--runTestsByPath")).length + 1; +const status = scripts ? scenario.scripts : retry ? + ((scenario.newFile && retryNumber === 1) || + (scenario.mixedRecovery && retryNumber === 2) ? 1 : scenario.retry) : scenario.initial; +if (status && !scripts && !scenario.unreported && + !(retry && scenario.unreportedRetry && (!scenario.secondPackage || path.basename(process.cwd()) === "other"))) { + const names = !retry && scenario.secondPackage ? ["example", "other"] : [retry ? path.basename(process.cwd()) : "example"]; + fs.writeFileSync(path.join(process.env.TYPEAGENT_TEST_FAILURES_DIR, "failures.json"), + JSON.stringify(names.map((name) => ({ + testFilePath: path.join(root, "packages", name, "test", scenario.newFile && retry ? "new file.spec.js" : "x file.spec.js"), + fullName: "fails", failureMessages: ["expected failure"]})))); +} +process.exitCode = status; +`, + ); + if (process.platform === "win32") { + fs.writeFileSync( + path.join(root, "pnpm.cmd"), + `@"${process.execPath}" "${fakePnpm}" %*\r\n`, + ); + } else { + fs.copyFileSync(fakePnpm, path.join(root, "pnpm")); + fs.chmodSync(path.join(root, "pnpm"), 0o755); + } + const result = spawnSync(process.execPath, [runner], { + cwd: root, + env: { + ...process.env, + PATH: `${root}${path.delimiter}${process.env.PATH}`, + GITHUB_ACTIONS: "true", + GITHUB_STEP_SUMMARY: path.join(root, "summary.md"), + TEST_PHASE_SCENARIO: JSON.stringify(scenario), + TEST_PHASE_ROOT: root, + }, + encoding: "utf8", + }); + assert.ifError(result.error); + const expectedFailure = + scenario.scripts !== 0 || + (scenario.initial !== 0 && + (scenario.unreported || scenario.retry !== 0)); + assert.equal(result.status, expectedFailure ? 1 : 0, result.stderr); + const summary = fs.readFileSync(path.join(root, "summary.md"), "utf8"); + const workspaceFailed = + scenario.initial !== 0 && + (scenario.unreported || scenario.retry !== 0); + assert.ok( + summary.includes( + `### Workspace tests (including retries)\n\nResult: **${workspaceFailed ? "failed" : "passed"}**`, + ), + ); + assert.ok( + summary.includes( + `### Root script tests\n\nResult: **${scenario.scripts ? "failed" : "passed"}**`, + ), + ); + const calls = fs + .readFileSync(path.join(root, "calls.jsonl"), "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + const retryCalls = calls.filter( + ({ args }) => + !args.includes("test:local:packages") && + !args.includes("test:local:scripts"), + ); + assert.deepEqual(calls[0].args, ["run", "test:local:packages"]); + assert.deepEqual(calls.at(-1).args, ["run", "test:local:scripts"]); + assert.equal( + calls.filter(({ args }) => args.includes("test:local:scripts")) + .length, + 1, + ); + assert.equal( + retryCalls.length, + scenario.initial && !scenario.unreported + ? scenario.mixedRecovery + ? 4 + : scenario.secondPackage || scenario.newFile + ? 2 + : scenario.unreportedRetry + ? 1 + : scenario.retry + ? 2 + : 1 + : 0, + ); + assert.equal( + new Set(retryCalls.map((call) => call.failures)).size, + retryCalls.length, + ); + for (const call of retryCalls) { + assert.equal(call.noCache, "true"); + assert.ok(call.cwd.startsWith(path.join(root, "packages"))); + if (scenario.fallback) { + assert.deepEqual(call.args, ["run", "test:local"]); + assert.match( + result.stderr, + /Full-package retry.*(?:not a recognized single jest-esm invocation|not a regular file contained)/, + ); + } else { + assert.deepEqual(call.args.slice(0, 6), [ + "run", + "jest-esm", + "--runInBand", + "--no-cache", + "--passWithNoTests=false", + "--runTestsByPath", + ]); + assert.ok( + call.args.slice(6).every((file) => path.isAbsolute(file)), + ); + assert.ok( + call.args.includes( + path.join(call.cwd, "test", "x file.spec.js"), + ), + ); + } + } + if (scenario.newFile) { + assert.deepEqual(retryCalls[1].args.slice(6), [ + path.join( + root, + "packages", + "example", + "test", + "x file.spec.js", + ), + path.join( + root, + "packages", + "example", + "test", + "new file.spec.js", + ), + ]); + } + if (scenario.unreportedRetry) { + assert.match(result.stderr, /Untrusted retry/); + assert.doesNotMatch(result.stderr, /FLAKY TESTS/); + } + }); +} diff --git a/ts/tools/scripts/test/testRetry.spec.mjs b/ts/tools/scripts/test/testRetry.spec.mjs index 274c1b8329..5a72d83a68 100644 --- a/ts/tools/scripts/test/testRetry.spec.mjs +++ b/ts/tools/scripts/test/testRetry.spec.mjs @@ -17,10 +17,168 @@ import { buildGithubAnnotations, buildStepSummaryMarkdown, MAX_TEST_RETRIES, + runLocalTestPhases, + planPackageRetry, + windowsPnpmCommand, } from "../lib/testRetry.mjs"; const cwd = "/repo/ts"; +const jestEsm = + "node --no-warnings --experimental-vm-modules ./node_modules/jest/bin/jest.js"; + +for (const script of [ + "pnpm run jest-esm", + 'pnpm run jest-esm --testPathPattern=".*[.]spec[.]js"', + 'pnpm run jest-esm --testMatch="**/dist/test/**/*.spec.js"', + "pnpm run jest-esm --testMatch='**/dist/test/**/*.spec.js'", +]) { + test(`narrows single Jest invocation: ${script}`, () => { + const dir = path.resolve("packages", "a space"); + const file = path.join(dir, "test", "a file.spec.js"); + const plan = planPackageRetry( + { + scripts: { "test:local": script, "jest-esm": jestEsm }, + }, + [file, file], + dir, + ); + assert.equal(plan.narrowed, true); + assert.equal(plan.cwd, dir); + assert.deepEqual(plan.args, [ + "run", + "jest-esm", + "--runInBand", + "--no-cache", + "--passWithNoTests=false", + "--runTestsByPath", + file, + ]); + }); +} + +for (const manager of ["npm", "pnpm"]) { + test(`recognizes exact ${manager} Jest script alias`, () => { + const plan = planPackageRetry( + { + scripts: { + "test:local": `${manager} run jest`, + jest: "pnpm run jest-esm", + "jest-esm": jestEsm, + }, + }, + ["test/a.spec.js"], + path.resolve("package"), + ); + assert.equal(plan.narrowed, true); + }); +} + +for (const script of [ + undefined, + "jest", + "node --test test/*.spec.mjs", + "jest && npm run test:views:gateway && npm run test:views:client", + "pnpm run build && pnpm run jest-esm", + "pnpm run jest-esm && node check.mjs", + "pnpm run jest-esm --config custom.js", + "pnpm run jest-esm --passWithNoTests", + 'pnpm run jest-esm --testMatch="$PATTERN"', + "pnpm run jest-esm\nnode check.mjs", +]) { + test(`whole-package fallback for unsupported script: ${script}`, () => { + const plan = planPackageRetry( + { + scripts: { + "test:local": script, + "jest-esm": jestEsm, + }, + }, + ["test/a.spec.js"], + cwd, + ); + assert.equal(plan.narrowed, false); + assert.match(plan.reason, /not a recognized single/); + assert.deepEqual(plan.args, ["run", "test:local"]); + }); +} + +test("does not bypass modified wrappers or package lifecycle hooks", () => { + for (const overrides of [ + { "jest-esm": `${jestEsm} && node check.mjs` }, + { "pretest:local": "node setup.mjs" }, + { "posttest:local": "node check.mjs" }, + { "prejest-esm": "node setup.mjs" }, + { postjest: "node check.mjs" }, + ]) { + assert.equal( + planPackageRetry( + { + scripts: { + "test:local": "pnpm run jest-esm", + "jest-esm": jestEsm, + ...overrides, + }, + }, + ["test/a.spec.js"], + cwd, + ).narrowed, + false, + ); + } +}); + +test("does not narrow files outside the owning package", () => { + const packageDir = path.resolve("packages", "a"); + const plan = planPackageRetry( + { + scripts: { + "test:local": "pnpm run jest-esm", + "jest-esm": jestEsm, + }, + }, + [path.resolve("packages", "b", "test.spec.js")], + packageDir, + ); + assert.equal(plan.narrowed, false); + assert.match(plan.reason, /outside the owning package/); +}); + +test("quotes Windows argument boundaries, spaces and shell metacharacters", () => { + assert.equal( + windowsPnpmCommand([ + "run", + "jest-esm", + "--runTestsByPath", + "D:\\repo space & files\\test\\a file.spec.js", + ]), + '"pnpm "run" "jest-esm" "--runTestsByPath" "D:\\repo space & files\\test\\a file.spec.js""', + ); + for (const value of ['a"b', "%PATH%", "a\nb", "a\0b"]) { + assert.throws(() => windowsPnpmCommand([value]), /Unsafe argument/); + } +}); + +for (const workspaceStatus of [0, 1]) { + for (const scriptStatus of [0, 1]) { + test(`both phases run: workspace=${workspaceStatus}, scripts=${scriptStatus}`, () => { + const calls = []; + const status = runLocalTestPhases( + () => { + calls.push("workspace including retries"); + return workspaceStatus; + }, + () => { + calls.push("scripts"); + return scriptStatus; + }, + ); + assert.deepEqual(calls, ["workspace including retries", "scripts"]); + assert.equal(status, workspaceStatus || scriptStatus); + }); + } +} + test("allows two retries for three total attempts", () => { assert.equal(MAX_TEST_RETRIES, 2); }); From 717e3a5094169f3945b7cd95fd8c30949b2472c6 Mon Sep 17 00:00:00 2001 From: Hillary Mutisya <150286414+hillary-mutisya@users.noreply.github.com> Date: Fri, 23 Oct 2026 13:48:30 -0700 Subject: [PATCH 2/2] Reduce retry orchestration complexity Extract individual package retries without changing failure classification or test phase accounting. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ts/tools/scripts/runTestLocalWithSummary.mjs | 121 +++++++++---------- 1 file changed, 55 insertions(+), 66 deletions(-) diff --git a/ts/tools/scripts/runTestLocalWithSummary.mjs b/ts/tools/scripts/runTestLocalWithSummary.mjs index 451fbe0ffd..d692f9b955 100644 --- a/ts/tools/scripts/runTestLocalWithSummary.mjs +++ b/ts/tools/scripts/runTestLocalWithSummary.mjs @@ -141,73 +141,14 @@ function handleFailures(firstFailures) { ), ]; for (const name of packageNames) { - const failures = priorFailures.filter( - (failure) => failure.package === name, + const result = retryPackage( + name, + priorFailures.filter((failure) => failure.package === name), + attempt, ); - const packageDir = resolvePackage( - failures[0].testFilePath, - )?.directory; - if (packageDir === undefined) { - retryTrusted = false; - retryFailed = true; - break; - } - const manifest = JSON.parse( - fs.readFileSync( - path.join(packageDir, "package.json"), - "utf8", - ), - ); - let plan = planPackageRetry( - manifest, - failures.map((failure) => - toAbsolutePath(failure.testFilePath), - ), - packageDir, - ); - if ( - plan.narrowed && - plan.args - .slice(6) - .some((file) => !isRegularPackageFile(file, packageDir)) - ) { - plan = { - narrowed: false, - reason: "failed path is not a regular file contained in the owning package", - cwd: packageDir, - args: ["run", "test:local"], - }; - } - console.error( - plan.narrowed - ? `Retrying failed files in ${name}` - : `Full-package retry for ${name}: ${plan.reason}`, - ); - const retryDirectory = makeFailuresDirectory(`retry${attempt}`); - failureDirectories.push(retryDirectory); - const retryResult = runPnpm( - plan.args, - retryDirectory, - true, - plan.cwd, - ); - const observed = readFailures(retryDirectory); - finalFailures.push(...observed); - const failed = - retryResult.error !== undefined || - retryResult.signal !== null || - (retryResult.status ?? 1) !== 0; - retryFailed ||= failed; - // Check each command independently: another package's artifacts - // cannot make a crashed or unreported invocation trustworthy. - if ( - retryResult.error !== undefined || - retryResult.signal !== null || - (failed && observed.length === 0) - ) { - console.error( - `Untrusted retry for ${name}: command failed without reliable failure artifacts.`, - ); + finalFailures.push(...result.failures); + retryFailed ||= result.failed; + if (!result.trusted) { retryTrusted = false; break; } @@ -259,6 +200,54 @@ function handleFailures(firstFailures) { return hasHardFailures(classification) ? 1 : 0; } +function retryPackage(name, failures, attempt) { + const packageDir = resolvePackage(failures[0].testFilePath)?.directory; + if (packageDir === undefined) { + return { failures: [], failed: true, trusted: false }; + } + const manifest = JSON.parse( + fs.readFileSync(path.join(packageDir, "package.json"), "utf8"), + ); + let plan = planPackageRetry( + manifest, + failures.map((failure) => toAbsolutePath(failure.testFilePath)), + packageDir, + ); + if ( + plan.narrowed && + plan.args + .slice(6) + .some((file) => !isRegularPackageFile(file, packageDir)) + ) { + plan = { + narrowed: false, + reason: "failed path is not a regular file contained in the owning package", + cwd: packageDir, + args: ["run", "test:local"], + }; + } + console.error( + plan.narrowed + ? `Retrying failed files in ${name}` + : `Full-package retry for ${name}: ${plan.reason}`, + ); + const retryDirectory = makeFailuresDirectory(`retry${attempt}`); + failureDirectories.push(retryDirectory); + const result = runPnpm(plan.args, retryDirectory, true, plan.cwd); + const observed = readFailures(retryDirectory); + const infrastructureFailed = + result.error !== undefined || result.signal !== null; + const failed = infrastructureFailed || (result.status ?? 1) !== 0; + // Each command needs its own artifacts; another package cannot establish trust. + const trusted = !infrastructureFailed && !(failed && observed.length === 0); + if (!trusted) { + console.error( + `Untrusted retry for ${name}: command failed without reliable failure artifacts.`, + ); + } + return { failures: observed, failed, trusted }; +} + function makeFailuresDirectory(label) { return fs.mkdtempSync( path.join(workspaceRoot, `.typeagent-test-failures-${label}-`),