diff --git a/apps/code/snapshots.yml b/apps/code/snapshots.yml index 1a1c7bd850..31a3b4ce8a 100644 --- a/apps/code/snapshots.yml +++ b/apps/code/snapshots.yml @@ -40,62 +40,18 @@ snapshots: hash: v1.k4693efd2.58144cdaf083f0ff98b136a5bae61686a2a856a0fce7835d5d11647e106b0b4f.-G4zNrpohMozh41d_NvfZ-pD1DUHkANR7XkHBaV6CCQ archive-archivedtasksview--with-missing-task--light: hash: v1.k4693efd2.2aad97fbf529d8e0c0896105051e140af8322d9c699653b0c60d6375dd2ea4c3.mc318EfElvU4a7rtpUNV9VJbFkYNpei4y4vhOcf3GRI - autoresearch-configurable-full-dashboard--before-baseline--dark: - hash: v1.k4693efd2.34b322085d4a0cc5723c4cc8b984fd81cf89809714f11de742605666e6979db2.D9JzgYGAuE95x0QkPVy4uBUYgUIkwrCCA08KMKXkKcg - autoresearch-configurable-full-dashboard--before-baseline--light: - hash: v1.k4693efd2.b4b99cafb1e4d7694726e86243910650f8ccb9156a8e81e5497f0705ab4ac383.H1b2sQFL1TV2nIDVf2RDkTSsUoA79-JM_FYlD-DTGzE - autoresearch-configurable-full-dashboard--completed-run--dark: - hash: v1.k4693efd2.2d10fc5b66537e110c096ca6349e1b46beffc2fcc8314e20a08a2c919c392ec7.xK39m0L7j5UNMcu1hrpcg7nPSPV0zwQLIq2UFbmoeaM - autoresearch-configurable-full-dashboard--completed-run--light: - hash: v1.k4693efd2.c03f43fe95a2dd45a722daacb531779cb6ebba5808b9790a4ccabbec8e16150f.95LMNvhl0FSreegkTDvktiIn_Pu47xydgq5KqtUOsCY - autoresearch-configurable-full-dashboard--playground--dark: - hash: v1.k4693efd2.67b1deb79c2c8a600bc77090e45a1586645df4af8a862ae90566583ba74e0c8c.0ae-YcXq7CDAUW5tZWa1nCJ2bK8XzG0U-f_0onDkF_Y - autoresearch-configurable-full-dashboard--playground--light: - hash: v1.k4693efd2.3944a480bec8fc56e71d16b8c44fd6dacb36736dad00c7116b24bb2f56bc3131.V-VD__LGWRlzMVNRv8QxglnuD1i_xHRBZg2fmwfXa70 - autoresearch-observability--active-experiment--dark: - hash: v1.k4693efd2.7635193c423ee44a6f8ac40f1a2c885cec49e9131e187ff164b03b051ea71143.NSfHpkBCo_0DJaaVHP71WGkWASEwHc48GBXF4o4VUhc - autoresearch-observability--active-experiment--light: - hash: v1.k4693efd2.e5c9b0dfc9d814f5022401d9b6c8271165c28a28f74b4708e155c4800e7b690a.8TP9QEzM6yKB-_gt61fVEAsj6DRry5LFAUIsHVINo_8 - autoresearch-observability--waiting-for-plan--dark: - hash: v1.k4693efd2.36e182d9438abfdbda7251f25255a1f6db00192111eaf6dde5c4adf74aa5ed85.jw3FQJBnplmsYY-xEJXq8MulEs2bReiud2xK4eYNVDI - autoresearch-observability--waiting-for-plan--light: - hash: v1.k4693efd2.261edaaca1e0d8b9932ca1af0d79e35365d6e53345252869b3ef79ebb0530d61.ejvug_EfWycvKL5XL93pJFLyxr22HfuK7lV6nqIRAco - autoresearch-research-map--establishing-baseline--dark: - hash: v1.k4693efd2.5162214d6100856e8a233f364cb0035845ae943dcac07afa0267ddc4d07bc53d.sDh8juzfN9gbXqX8suNdCWAEUqhuyRlUy4jEsh2ETlQ - autoresearch-research-map--establishing-baseline--light: - hash: v1.k4693efd2.21a9c5d13c2ca2592e5c73312ffcc39f16868bf38427ede85cba1f2b476d9de5.hJ9ITN88WX-wzhjiDY40OLDd_DBCV4hCq3FPSy-1e8w - autoresearch-research-map--findings-by-code-area--dark: - hash: v1.k4693efd2.24241937b04bc1845e1c0d5d6561f84f32d3512d32628fedb7a8be571d46c4e0._rYrAun17fUqJJEmIuYJQC6svmW2qQmJsSa2zMcpoao - autoresearch-research-map--findings-by-code-area--light: - hash: v1.k4693efd2.db68afd28cd0e204ab0ca02602b66adff99ec157918303a02a8f228d5fa23aeb.0iIfBWYqdNGwka2L3P-qS37Y_T0AWGHwgEDixWQL9X8 - autoresearch-results-and-summary--baseline-just-recorded--dark: - hash: v1.k4693efd2.fd89d5a250d7c9ba75656208f5fa85a1aff2d27f5e7a62d4dbe19c374668b0ed.oM02rII7mDCEH5GFyZ0oJgzizclMQWMb1S7hn0F8vZA - autoresearch-results-and-summary--baseline-just-recorded--light: - hash: v1.k4693efd2.cdb1596e320e7afa01d150626530df417351bbc5562cac2be32d9df77e3ea738.YzCAe389zQW1meP8ie4HfA2qIj65qCz7FX0oWu0daPg - autoresearch-results-and-summary--completed-run--dark: - hash: v1.k4693efd2.9aed29fb95ba712a8a8bb04cbd4c4f704879525c91b57db620cf4a91238d7017.5CT3vEkBAVI1-Ekd8x2WTHoBodZcebTUAeXMa6hdtQE - autoresearch-results-and-summary--completed-run--light: - hash: v1.k4693efd2.c37fe3b4fa73de73dabc3dc8480292bde19d4d84f4820819ec48ad3be3c5d698.ElWCwCPyshrgyzvqTDxx7WuGvtGFzZmnpvusc0LBlAI - autoresearch-rundashboard--maximize-completed--dark: - hash: v1.k4693efd2.80aa97a95ac4830be972d1366b77567b85f935dd368175bcdabe40a76d6b353c.Ne1OrLmuP1SEO4m3fiAcmatww9ivj3JLrDBMr_H5O5Y - autoresearch-rundashboard--maximize-completed--light: - hash: v1.k4693efd2.618b533f0d5e83f66f157deb8d1c4d8175f130c1b21d65a09f0bc3dab1dd0400.qTssqMO2-FgGs4NBvZPVHh7s0qgzkc-YSLCyqSqm9iA - autoresearch-rundashboard--minimize-in-progress--dark: - hash: v1.k4693efd2.d13eebe8c14696a1cd48b6b29097e0d96841b36f2ba9fecc299615d01c256d7f.qjZSf4ehb8NGy72JTAb_PhsCL9EVosZ6UZb6HwQPCNU - autoresearch-rundashboard--minimize-in-progress--light: - hash: v1.k4693efd2.eadc597c9bc11eca735f15c903ae63f09a57e4f781374ca29ea2d3213243889f.ZAeEfQLOMhLC_hwkcoDYpfRVbVIdT4MTskEQscvtJQU - autoresearch-rundashboard--no-iterations-yet--dark: - hash: v1.k4693efd2.a6940dbc2b9658b27b378a9682aa0a620aa9257264dfbd4fcb50171b67ab3bbc.L5nlctLAoM5-lzijBFcR4dZPEcvRBbtE88esDyjxjXA - autoresearch-rundashboard--no-iterations-yet--light: - hash: v1.k4693efd2.1a3a3769b49045c8ff70c895c6a840c8652cd2a34dfe52ddac26a2e27d367400.Wv7VCEDJVF8GYlXQC1j10XznFekbNVR4lXPO8lmd9XA - autoresearch-runtime-stats--before-usage-update--dark: - hash: v1.k4693efd2.f31283c7e391d88a500507127c4bf327b6ba8e47be04ca975b140c07824cf032.Z07es_PMn-0tcNOj5f11fMTbzm_juPqb0PCXjEbrSjY - autoresearch-runtime-stats--before-usage-update--light: - hash: v1.k4693efd2.cdbd6673dad805fd4f5763e2d050c8bb6db7a5b84725dccd9f530d951db920cf.m8ngO8I6Jsb1OcWI_6zltKZdjsK97tF0XcjImqpJS8k - autoresearch-runtime-stats--with-context-usage--dark: - hash: v1.k4693efd2.b76d39ed8142d4449cb5befcde538bb11a6bffcbed66e81bdc3aa8e2746b7e06.GSIfBsgE-3RYuqlLUpluZ87C6-KjBUaTj2eu_a5elZ8 - autoresearch-runtime-stats--with-context-usage--light: - hash: v1.k4693efd2.18889b5eda942cd19b0f7be00a87925d632529ca4111fac180d6339f7d34d1ca.X3s9IXrx2-TdQaJEjf1td3hwvNzeIp3Xq3uQh9Gwt5A + autoresearch-dashboard--active-run--dark: + hash: v1.k4693efd2.4b0e10f2aff5473d6dda9983e889748831b6dd930d345c074dde8cff7e95f979.QMSLjN0Ey631SdSe2D-SvT62ftICU65KnsiuZR_LQPo + autoresearch-dashboard--active-run--light: + hash: v1.k4693efd2.d190408b5d3c167d64da65b15581c903ade25b43a5de0254779f337647ec93f6.hAJZFZjDp0c1LwlXDyQ-mcLVwC1xwEUVVQth-NHlGAw + autoresearch-dashboard--completed-run--dark: + hash: v1.k4693efd2.74ee05d10c67c6878ee4c19059cc66d22af2c5d18ba27ef584c04f421ab80302.OD5XadbFRLiMCl2Scp7tIS3oew-jSh02LQmiqSdO0bM + autoresearch-dashboard--completed-run--light: + hash: v1.k4693efd2.910162d6bf5ad1588b5453e4f5464e2d3c12ee77c890e58a9bc60c2ae6d936d1.SzyIVMxc-AMRfA1X3_Vi59wmQ1xIljuwAw2Hn2jKXKc + autoresearch-dashboard--researching-baseline--dark: + hash: v1.k4693efd2.8936e61d920c886326a8a252f6eb937e66caa5ae43b76daf6f8ead0e9f9a05f3.5ZC4Fj11UEOvgDP7owvjlydg61nBBF2-cMxiltr5Gk0 + autoresearch-dashboard--researching-baseline--light: + hash: v1.k4693efd2.35673a495a589ea3862abaf7ef46bdb4c3bb87c0bf79a563a8bf5782e7b6e74d.gW02sNxx7mUnKRvoexWGTRnjYgglyBrSLtz37Wic5TA billing-usagemeter--exceeded--dark: hash: v1.k4693efd2.dc026c5cace4e44c4cc713b95654e278d7bf0439a4862e9ceea39a033974815d.m5Be4dAOwwmt-UlEnhbhrKiym1rlGUiGZ4oyUonUoVg billing-usagemeter--exceeded--light: diff --git a/packages/core/src/autoresearch/autoresearch.test.ts b/packages/core/src/autoresearch/autoresearch.test.ts index d071303ea9..68790cf444 100644 --- a/packages/core/src/autoresearch/autoresearch.test.ts +++ b/packages/core/src/autoresearch/autoresearch.test.ts @@ -270,6 +270,7 @@ function makeRun( researchFindings: [], iterations: [], startedAt: 1_000, + pauseIntervals: [], endedAt: null, endReason: null, interruptedReason: null, @@ -941,6 +942,46 @@ describe("AutoresearchService", () => { }); describe("pause and resume", () => { + it("records and settles paused duration", () => { + vi.setSystemTime(1_000); + const run = service.startRun(baseConfig); + + vi.setSystemTime(11_000); + service.pauseRun(run.id); + expect(activeRun().pausedAt).toBe(11_000); + expect(activeRun().pausedDurationMs).toBe(0); + + vi.setSystemTime(31_000); + service.resumeRun(run.id); + expect(activeRun().pausedAt).toBeNull(); + expect(activeRun().pausedDurationMs).toBe(20_000); + expect(activeRun().pauseIntervals).toEqual([ + { startedAt: 11_000, endedAt: 31_000 }, + ]); + }); + + it("excludes interruption downtime and records its interval", async () => { + vi.setSystemTime(1_000); + service.startRun(baseConfig); + await flush(); + + vi.setSystemTime(11_000); + sessionStoreSetters.updateSession(TASK_RUN_ID, { status: "error" }); + expect(activeRun().status).toBe("interrupted"); + expect(activeRun().pausedAt).toBe(11_000); + + vi.setSystemTime(31_000); + sessionStoreSetters.updateSession(TASK_RUN_ID, { status: "connected" }); + await flush(); + + expect(activeRun().status).toBe("running"); + expect(activeRun().pausedAt).toBeNull(); + expect(activeRun().pausedDurationMs).toBe(20_000); + expect(activeRun().pauseIntervals).toEqual([ + { startedAt: 11_000, endedAt: 31_000 }, + ]); + }); + it("records iterations while paused but does not continue the loop", () => { const run = service.startRun(baseConfig); service.pauseRun(run.id); @@ -989,13 +1030,22 @@ describe("AutoresearchService", () => { }); it("resume waits for the agent when a turn is in flight", () => { + vi.setSystemTime(1_000); const run = service.startRun(baseConfig); + + vi.setSystemTime(11_000); service.pauseRun(run.id); beginTurn(); sentPrompts = []; + vi.setSystemTime(31_000); service.resumeRun(run.id); expect(sentPrompts).toHaveLength(0); + expect(activeRun().pausedAt).toBeNull(); + expect(activeRun().pausedDurationMs).toBe(20_000); + expect(activeRun().pauseIntervals).toEqual([ + { startedAt: 11_000, endedAt: 31_000 }, + ]); completeTurn(reportText(10)); expect(activeRun().iterations).toHaveLength(1); diff --git a/packages/core/src/autoresearch/autoresearch.ts b/packages/core/src/autoresearch/autoresearch.ts index 6b0f64bbab..7b399e8921 100644 --- a/packages/core/src/autoresearch/autoresearch.ts +++ b/packages/core/src/autoresearch/autoresearch.ts @@ -162,6 +162,9 @@ export class AutoresearchService { researchFindings: [], iterations: [], startedAt: Date.now(), + pausedAt: null, + pausedDurationMs: 0, + pauseIntervals: [], endedAt: null, endReason: null, interruptedReason: null, @@ -213,6 +216,7 @@ export class AutoresearchService { this.clearPendingReaction(runId); this.clearRecoveryTimer(runId); this.remindersSent.delete(runId); + this.pauseClock(runId); autoresearchStoreActions.setRunStatus(runId, "paused"); this.persist(runId); this.restoreOriginalStage(run); @@ -231,6 +235,7 @@ export class AutoresearchService { return; } + this.settlePausedClock(runId); this.clearRecoveryTimer(runId); const session = getSessionForTask( sessionStore.getState(), @@ -257,6 +262,7 @@ export class AutoresearchService { autoresearchStoreActions.setRunStatus(runId, "interrupted", { interruptedReason: run.interruptedReason ?? "session-error", }); + this.pauseClock(runId); this.persist(runId); void this.attemptRecovery(runId); return; @@ -765,6 +771,7 @@ export class AutoresearchService { if (run?.status === "running") { this.clearPendingReaction(runId); this.remindersSent.delete(runId); + this.pauseClock(runId); autoresearchStoreActions.setRunStatus(runId, "paused"); this.persist(runId); this.restoreOriginalStage(run); @@ -806,6 +813,7 @@ export class AutoresearchService { this.clearPendingReaction(runId); // A fresh interruption gets its own reminder budget on resume. this.remindersSent.delete(runId); + this.pauseClock(runId); autoresearchStoreActions.setRunStatus(runId, "interrupted", { interruptedReason: reason, lastError, @@ -885,6 +893,7 @@ export class AutoresearchService { return; } + this.settlePausedClock(runId); this.clearRecoveryTimer(runId); const session = getSessionForTask( sessionStore.getState(), @@ -915,6 +924,7 @@ export class AutoresearchService { status: "completed" | "stopped" | "failed", options: { endReason: AutoresearchEndReason; lastError?: string }, ): void { + this.settlePausedClock(runId); const run = autoresearchStore.getState().runs[runId]; autoresearchStoreActions.setRunStatus(runId, status, options); this.persist(runId); @@ -928,6 +938,29 @@ export class AutoresearchService { if (run) this.restoreOriginalStage(run); } + private pauseClock(runId: string): void { + const run = autoresearchStore.getState().runs[runId]; + if (!run || run.pausedAt != null) return; + autoresearchStoreActions.setPauseTiming( + runId, + Date.now(), + run.pausedDurationMs ?? 0, + run.pauseIntervals ?? [], + ); + } + + private settlePausedClock(runId: string): void { + const run = autoresearchStore.getState().runs[runId]; + if (!run || run.pausedAt == null) return; + const endedAt = Date.now(); + autoresearchStoreActions.setPauseTiming( + runId, + null, + (run.pausedDurationMs ?? 0) + Math.max(0, endedAt - run.pausedAt), + [...(run.pauseIntervals ?? []), { startedAt: run.pausedAt, endedAt }], + ); + } + private clearPendingReaction(runId: string): void { const timer = this.pendingReactions.get(runId); if (timer) clearTimeout(timer); diff --git a/packages/core/src/autoresearch/autoresearchStore.ts b/packages/core/src/autoresearch/autoresearchStore.ts index dcf82fc0d3..7a00f53a6e 100644 --- a/packages/core/src/autoresearch/autoresearchStore.ts +++ b/packages/core/src/autoresearch/autoresearchStore.ts @@ -3,6 +3,7 @@ import { type AutoresearchEndReason, type AutoresearchInterruptionReason, type AutoresearchIteration, + type AutoresearchPauseInterval, type AutoresearchPhase, type AutoresearchResearchFinding, type AutoresearchRun, @@ -75,6 +76,20 @@ export const autoresearchStoreActions = { updateRun(runId, (run) => ({ ...run, phase })); }, + setPauseTiming( + runId: string, + pausedAt: number | null, + pausedDurationMs: number, + pauseIntervals: AutoresearchPauseInterval[], + ): void { + updateRun(runId, (run) => ({ + ...run, + pausedAt, + pausedDurationMs, + pauseIntervals, + })); + }, + setRunStatus( runId: string, status: AutoresearchRunStatus, diff --git a/packages/core/src/autoresearch/schemas.test.ts b/packages/core/src/autoresearch/schemas.test.ts index 6f10d4f1b7..b84b55590d 100644 --- a/packages/core/src/autoresearch/schemas.test.ts +++ b/packages/core/src/autoresearch/schemas.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { AUTORESEARCH_MAX_ITERATIONS_LIMIT, autoresearchConfigSchema, @@ -86,10 +86,13 @@ describe("parseStoredAutoresearchRun", () => { expect(run?.interruptedReason).toBeNull(); }); - it("restores a running run as an app-restart interruption", () => { + it("restores a running run as a paused app-restart interruption", () => { + vi.setSystemTime(20_000); const run = parseStoredAutoresearchRun(storedRun("running")); expect(run?.status).toBe("interrupted"); expect(run?.interruptedReason).toBe("app-restart"); + expect(run?.pausedAt).toBe(20_000); + expect(run?.pausedDurationMs).toBe(0); }); it("defaults interruptedReason for blobs persisted before the field existed", () => { diff --git a/packages/core/src/autoresearch/schemas.ts b/packages/core/src/autoresearch/schemas.ts index 16d775ab7a..49b8f2a2cf 100644 --- a/packages/core/src/autoresearch/schemas.ts +++ b/packages/core/src/autoresearch/schemas.ts @@ -124,6 +124,14 @@ export function isTerminalRunStatus(status: AutoresearchRunStatus): boolean { export const autoresearchPhaseSchema = z.enum(["implement", "measure"]); export type AutoresearchPhase = z.infer; +export const autoresearchPauseIntervalSchema = z.object({ + startedAt: z.number(), + endedAt: z.number(), +}); +export type AutoresearchPauseInterval = z.infer< + typeof autoresearchPauseIntervalSchema +>; + export const autoresearchRunSchema = z.object({ id: z.string().min(1), config: autoresearchConfigSchema, @@ -150,6 +158,9 @@ export const autoresearchRunSchema = z.object({ researchFindings: z.array(autoresearchResearchFindingSchema).default([]), iterations: z.array(autoresearchIterationSchema), startedAt: z.number(), + pausedAt: z.number().nullable().optional(), + pausedDurationMs: z.number().min(0).optional(), + pauseIntervals: z.array(autoresearchPauseIntervalSchema).optional(), endedAt: z.number().nullable(), endReason: autoresearchEndReasonSchema.nullable(), interruptedReason: autoresearchInterruptionReasonSchema @@ -176,12 +187,17 @@ export function parseStoredAutoresearchRun( } const parsed = autoresearchRunSchema.safeParse(raw); if (!parsed.success) return null; - const run = parsed.data; + let run = parsed.data; + if (run.status === "paused" && run.pausedAt === undefined) { + run = { ...run, pausedAt: Date.now(), pausedDurationMs: 0 }; + } if (run.status !== "running") return run; return { ...run, status: "interrupted", interruptedReason: "app-restart", + pausedAt: Date.now(), + pausedDurationMs: run.pausedDurationMs ?? 0, }; } diff --git a/packages/core/src/autoresearch/stats.test.ts b/packages/core/src/autoresearch/stats.test.ts index 47489a6a54..9e1efd0813 100644 --- a/packages/core/src/autoresearch/stats.test.ts +++ b/packages/core/src/autoresearch/stats.test.ts @@ -3,6 +3,7 @@ import type { AutoresearchIteration, AutoresearchRun } from "./schemas"; import { computeBest, evaluateContinuation, + getAutoresearchElapsedMs, isImprovement, meetsTarget, summarizeRun, @@ -32,6 +33,11 @@ function makeRun(overrides: { direction?: "maximize" | "minimize"; targetValue?: number | null; maxIterations?: number; + startedAt?: number; + pausedAt?: number | null; + pausedDurationMs?: number; + pauseIntervals?: AutoresearchRun["pauseIntervals"]; + endedAt?: number | null; }): AutoresearchRun { return { id: "ar-1", @@ -54,14 +60,43 @@ function makeRun(overrides: { originalEffort: null, researchFindings: [], iterations: overrides.iterations ?? [], - startedAt: 0, - endedAt: null, + startedAt: overrides.startedAt ?? 0, + pausedAt: overrides.pausedAt, + pausedDurationMs: overrides.pausedDurationMs, + pauseIntervals: overrides.pauseIntervals ?? [], + endedAt: overrides.endedAt ?? null, endReason: null, interruptedReason: null, lastError: null, }; } +describe("getAutoresearchElapsedMs", () => { + it("freezes while paused and excludes completed pause intervals", () => { + expect( + getAutoresearchElapsedMs( + makeRun({ startedAt: 1_000, pausedAt: 11_000 }), + 31_000, + ), + ).toBe(10_000); + expect( + getAutoresearchElapsedMs( + makeRun({ startedAt: 1_000, pausedDurationMs: 20_000 }), + 36_000, + ), + ).toBe(15_000); + expect( + getAutoresearchElapsedMs( + makeRun({ + startedAt: 1_000, + pauseIntervals: [{ startedAt: 11_000, endedAt: 31_000 }], + }), + 36_000, + ), + ).toBe(15_000); + }); +}); + describe("isImprovement", () => { it.each([ ["maximize", 5, 3, true], diff --git a/packages/core/src/autoresearch/stats.ts b/packages/core/src/autoresearch/stats.ts index 5c0165d766..d9c7413aec 100644 --- a/packages/core/src/autoresearch/stats.ts +++ b/packages/core/src/autoresearch/stats.ts @@ -9,6 +9,26 @@ import type { AutoresearchRun, } from "./schemas"; +export function getAutoresearchElapsedMs( + run: AutoresearchRun, + now: number, +): number { + const effectiveEnd = run.endedAt ?? run.pausedAt ?? now; + const trackedPausedDurationMs = (run.pauseIntervals ?? []).reduce( + (total, interval) => { + const overlapStart = Math.max(run.startedAt, interval.startedAt); + const overlapEnd = Math.min(effectiveEnd, interval.endedAt); + return total + Math.max(0, overlapEnd - overlapStart); + }, + 0, + ); + const pausedDurationMs = Math.max( + trackedPausedDurationMs, + run.pausedDurationMs ?? 0, + ); + return Math.max(0, effectiveEnd - run.startedAt - pausedDurationMs); +} + export function isImprovement( candidate: number, reference: number | null, diff --git a/packages/ui/src/features/autoresearch/AutoresearchFullDashboard.stories.tsx b/packages/ui/src/features/autoresearch/AutoresearchFullDashboard.stories.tsx index 54aa83c194..313ef6d205 100644 --- a/packages/ui/src/features/autoresearch/AutoresearchFullDashboard.stories.tsx +++ b/packages/ui/src/features/autoresearch/AutoresearchFullDashboard.stories.tsx @@ -58,7 +58,7 @@ function FullDashboardStory(props: FullDashboardStoryProps) { : null; return ( -
+
{run.iterations.length === 0 ? ( @@ -149,36 +149,11 @@ function buildRun(props: FullDashboardStoryProps): AutoresearchRun { phase: null, originalModel: null, originalEffort: null, - researchFindings: props.showResearch - ? [ - { - index: 1, - area: "build", - summary: "Located the measurement path", - finding: - "The metric can be reproduced from production build output.", - nextStep: "Measure the baseline", - at: STARTED_AT + 60_000, - }, - { - index: 2, - area: "frontend", - summary: "Mapped the eager dependency graph", - finding: "Several modal modules load before the user opens them.", - nextStep: "Test a lazy loading boundary", - at: STARTED_AT + 3 * 60_000, - }, - ] - : [], + researchFindings: props.showResearch ? buildResearchFindings() : [], iterations: buildIterations(props), startedAt: STARTED_AT, endedAt: ended ? STORY_NOW : null, - endReason: - props.status === "completed" - ? "max-iterations" - : props.status === "stopped" - ? "stopped-by-user" - : null, + endReason: endReasonForStatus(props.status), interruptedReason: props.status === "interrupted" ? "session-error" : null, lastError: props.status === "failed" @@ -187,6 +162,90 @@ function buildRun(props: FullDashboardStoryProps): AutoresearchRun { }; } +function buildResearchFindings(): AutoresearchRun["researchFindings"] { + return [ + { + index: 1, + area: "build", + summary: "Located the production bundle measurement", + finding: + "The dashboard marginal bundle can be isolated from esbuild metadata.", + nextStep: "Establish the baseline bundle size", + at: STARTED_AT + 60_000, + }, + { + index: 2, + area: "build", + summary: "Found the dashboard entry chunk", + finding: + "The route entry includes editor and modal code before either feature is opened.", + nextStep: "Compare the entry chunk with lazy boundaries enabled", + at: STARTED_AT + 2 * 60_000, + }, + { + index: 3, + area: "frontend", + summary: "Mapped eager dashboard imports", + finding: "Dashboard modals load before users open them.", + nextStep: "Inspect modal boundaries", + at: STARTED_AT + 3 * 60_000, + }, + { + index: 4, + area: "frontend", + summary: "Traced editor initialization", + finding: + "The rich text editor initializes with the dashboard even when no editor is visible.", + nextStep: "Move editor setup behind the edit action", + at: STARTED_AT + 4 * 60_000, + }, + { + index: 5, + area: "data", + summary: "Identified duplicate insight requests", + finding: + "The summary and chart issue equivalent requests during the first render.", + nextStep: "Share the initial query result", + at: STARTED_AT + 5 * 60_000, + }, + { + index: 6, + area: "data", + summary: "Measured oversized response fields", + finding: + "Dashboard cards receive metadata that is only needed in the detail view.", + nextStep: "Select the minimal card response shape", + at: STARTED_AT + 6 * 60_000, + }, + { + index: 7, + area: "testing", + summary: "Located the bundle regression test", + finding: + "The current threshold covers the full application instead of the dashboard entry.", + nextStep: "Add a dashboard-specific bundle assertion", + at: STARTED_AT + 7 * 60_000, + }, + { + index: 8, + area: "testing", + summary: "Confirmed a stable measurement command", + finding: + "The production build emits deterministic metadata for the dashboard chunk.", + nextStep: "Establish the baseline bundle size", + at: STARTED_AT + 8 * 60_000, + }, + ]; +} + +function endReasonForStatus( + status: AutoresearchRunStatus, +): AutoresearchRun["endReason"] { + if (status === "completed") return "max-iterations"; + if (status === "stopped") return "stopped-by-user"; + return null; +} + function buildIterations( props: FullDashboardStoryProps, ): AutoresearchIteration[] { @@ -210,12 +269,7 @@ function buildIterations( value, bestValue: best ?? value, delta: previous === null ? null : value - previous, - summary: - index === 1 - ? "Established the baseline" - : index % 3 === 1 - ? "Measured a regression and changed direction" - : `Completed focused experiment ${index}`, + summary: iterationSummary(index), hypothesis: index === 1 ? "The baseline captures the current production behavior" @@ -232,6 +286,12 @@ function buildIterations( }); } +function iterationSummary(index: number): string { + if (index === 1) return "Established the baseline"; + if (index % 3 === 1) return "Measured a regression and changed direction"; + return `Completed focused experiment ${index}`; +} + function buildEvents(props: FullDashboardStoryProps): AcpMessage[] { if (!props.showActivity) return []; return [ @@ -244,22 +304,82 @@ function buildEvents(props: FullDashboardStoryProps): AcpMessage[] { }), sessionEvent(STARTED_AT + 13 * 60_000, { sessionUpdate: "tool_call", - title: "Search relevant modules", + toolCallId: "search", + title: "Find autoresearch dashboard components", kind: "search", status: "completed", }), - sessionEvent(STARTED_AT + 17 * 60_000, { + sessionEvent(STARTED_AT + 14 * 60_000, { + sessionUpdate: "tool_call", + toolCallId: "read", + title: "Read existing timeline implementation", + kind: "read", + status: "completed", + }), + sessionEvent(STARTED_AT + 15 * 60_000, { + sessionUpdate: "tool_call", + toolCallId: "server", + title: "Start Storybook dev server", + kind: "execute", + rawInput: { command: "pnpm --filter code storybook" }, + status: props.status === "running" ? "in_progress" : "completed", + }), + sessionEvent(STARTED_AT + 16 * 60_000, { sessionUpdate: "tool_call", - title: "Edit the selected implementation", + toolCallId: "layout", + title: "Add responsive timeline layout", + kind: "edit", + status: "completed", + }), + sessionEvent(STARTED_AT + 18 * 60_000, { + sessionUpdate: "tool_call", + toolCallId: "typecheck", + title: "Typecheck UI package", + kind: "execute", + rawInput: { command: "pnpm --filter @posthog/ui typecheck" }, + status: "completed", + }), + sessionEvent(STARTED_AT + 19 * 60_000, { + sessionUpdate: "tool_call", + toolCallId: "screenshots", + title: "Inspect Storybook screenshots", + kind: "read", + status: "completed", + }), + sessionEvent(STARTED_AT + 20 * 60_000, { + sessionUpdate: "tool_call", + toolCallId: "stories", + title: "Add dashboard story variants", kind: "edit", status: "completed", }), sessionEvent(STARTED_AT + 22 * 60_000, { sessionUpdate: "tool_call", - title: "Run the metric measurement", + toolCallId: "benchmark", + title: "Run bundle benchmark", kind: "execute", + rawInput: { command: "pnpm bench:dashboard" }, status: props.status === "running" ? "in_progress" : "completed", }), + sessionEvent(STARTED_AT + 23 * 60_000, { + sessionUpdate: "tool_call", + toolCallId: "status", + title: "Check repository status", + kind: "execute", + rawInput: { command: "git status --short" }, + status: "completed", + }), + sessionEvent(STARTED_AT + 24 * 60_000, { + sessionUpdate: "tool_call", + toolCallId: "visual-tests", + title: "Capture visual snapshots", + kind: "execute", + rawInput: { + command: + "pnpm exec test-storybook --browsers chromium AutoresearchFullDashboard.stories.tsx", + }, + status: "completed", + }), ]; } @@ -276,7 +396,7 @@ function sessionEvent(ts: number, update: Record): AcpMessage { } const meta = { - title: "Autoresearch/Configurable Full Dashboard", + title: "Autoresearch/Dashboard", component: FullDashboardStory, parameters: { layout: "fullscreen" }, argTypes: { @@ -326,12 +446,21 @@ const meta = { export default meta; type Story = StoryObj; -export const Playground: Story = {}; +export const ActiveRun: Story = { + parameters: { testOptions: { viewport: { width: 1280, height: 2200 } } }, +}; -export const BeforeBaseline: Story = { +export const ResearchingBaseline: Story = { args: { iterationCount: 0, showResearch: true, status: "running" }, + parameters: { + testOptions: { + waitForLoadersToDisappear: false, + viewport: { width: 1280, height: 2500 }, + }, + }, }; export const CompletedRun: Story = { args: { status: "completed", iterationCount: 8 }, + parameters: { testOptions: { viewport: { width: 1280, height: 2400 } } }, }; diff --git a/packages/ui/src/features/autoresearch/AutoresearchObservability.stories.tsx b/packages/ui/src/features/autoresearch/AutoresearchObservability.stories.tsx deleted file mode 100644 index 4e826d19b7..0000000000 --- a/packages/ui/src/features/autoresearch/AutoresearchObservability.stories.tsx +++ /dev/null @@ -1,93 +0,0 @@ -import type { AutoresearchRun } from "@posthog/core/autoresearch/schemas"; -import type { AcpMessage } from "@posthog/shared"; -import type { Meta, StoryObj } from "@storybook/react-vite"; -import { AutoresearchObservability } from "./AutoresearchObservability"; - -const STARTED_AT = Date.parse("2026-07-10T14:00:00Z"); - -const run: AutoresearchRun = { - id: "run-observability", - config: { - taskId: "task-1", - direction: "minimize", - targetValue: 300, - maxIterations: 10, - implementModel: null, - measureModel: null, - implementEffort: null, - measureEffort: null, - instructions: "Reduce dashboard loading time.", - }, - status: "running", - metricName: "dashboard bundle", - metricUnit: "KiB", - phase: null, - originalModel: null, - originalEffort: null, - researchFindings: [], - iterations: [], - startedAt: STARTED_AT, - endedAt: STARTED_AT + 10 * 60_000, - endReason: null, - interruptedReason: null, - lastError: null, -}; - -function event(ts: number, update: Record): AcpMessage { - return { - type: "acp_message", - ts, - message: { - jsonrpc: "2.0", - method: "session/update", - params: { update }, - }, - } as AcpMessage; -} - -const events = [ - event(STARTED_AT + 30_000, { - sessionUpdate: "agent_message_chunk", - content: { - type: "text", - text: "```autoresearch\ntype: plan\nhypothesis: eager modal imports dominate the dashboard entry bundle\nplan: lazy load dashboard modals and rerun the bundle measurement\napproach: code splitting\n```", - }, - }), - event(STARTED_AT + 60_000, { - sessionUpdate: "tool_call", - title: "Search dashboard imports", - kind: "search", - status: "completed", - }), - event(STARTED_AT + 3 * 60_000, { - sessionUpdate: "tool_call", - title: "Edit DashboardScene", - kind: "edit", - status: "completed", - }), - event(STARTED_AT + 7 * 60_000, { - sessionUpdate: "tool_call", - title: "Measure dashboard bundle", - kind: "execute", - status: "in_progress", - }), -]; - -const meta: Meta = { - title: "Autoresearch/Observability", - component: AutoresearchObservability, - decorators: [ - (Story) => ( -
- -
- ), - ], -}; - -export default meta; -type Story = StoryObj; - -export const ActiveExperiment: Story = { args: { run, events } }; - -export const WaitingForPlan: Story = { args: { run, events: [] } }; diff --git a/packages/ui/src/features/autoresearch/AutoresearchObservability.test.tsx b/packages/ui/src/features/autoresearch/AutoresearchObservability.test.tsx new file mode 100644 index 0000000000..8d7dbad7b9 --- /dev/null +++ b/packages/ui/src/features/autoresearch/AutoresearchObservability.test.tsx @@ -0,0 +1,311 @@ +import type { AutoresearchRun } from "@posthog/core/autoresearch/schemas"; +import type { AcpMessage } from "@posthog/shared"; +import { Theme } from "@radix-ui/themes"; +import { render, screen, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it } from "vitest"; +import { AutoresearchObservability } from "./AutoresearchObservability"; + +function updateEvent(ts: number, update: Record): AcpMessage { + return { + type: "acp_message", + ts, + message: { + jsonrpc: "2.0", + method: "session/update", + params: { update }, + }, + } as AcpMessage; +} + +function makeRun(overrides: Partial = {}): AutoresearchRun { + return { + id: "run-1", + config: { + taskId: "task-1", + direction: "minimize", + targetValue: null, + maxIterations: 10, + implementModel: null, + measureModel: null, + implementEffort: null, + measureEffort: null, + instructions: "Reduce memory usage.", + }, + status: "running", + metricName: "memory", + metricUnit: "MB", + phase: null, + originalModel: null, + originalEffort: null, + researchFindings: [ + { + index: 1, + summary: "Mapped the hot path", + finding: "Serialization dominates the current measurement.", + nextStep: "Inspect the serializer", + area: "workspace server", + at: 1_500, + }, + ], + iterations: [], + startedAt: 1_000, + endedAt: null, + endReason: null, + interruptedReason: null, + lastError: null, + ...overrides, + }; +} + +describe("AutoresearchObservability", () => { + it("renders timeline-shaped skeletons before the first activity", () => { + render( + + + , + ); + + const loading = screen.getByRole("status", { + name: "Loading live timeline", + }); + expect(within(loading).getAllByRole("listitem")).toHaveLength(3); + expect(loading.querySelectorAll('[data-slot="skeleton"]')).toHaveLength(10); + }); + + it("renders current findings and a reconciled newest-first timeline", () => { + const events = [ + updateEvent(2_000, { + sessionUpdate: "tool_call", + toolCallId: "search-1", + title: "Search serializers", + kind: "search", + status: "in_progress", + }), + updateEvent(3_000, { + sessionUpdate: "tool_call_update", + toolCallId: "search-1", + status: "completed", + }), + updateEvent(5_000, { + sessionUpdate: "tool_call", + toolCallId: "edit-1", + title: "Edit serializer", + kind: "edit", + status: "in_progress", + }), + ]; + + render( + + + , + ); + + expect(screen.getByText("Current findings")).toBeVisible(); + expect(screen.getByText("Mapped the hot path")).toBeVisible(); + const timeline = screen.getByText("Edit serializer").closest("ol"); + const collapsible = timeline?.closest('[data-slot="collapsible"]'); + expect(collapsible).toHaveClass( + "bg-transparent", + "hover:bg-transparent", + "data-open:bg-transparent", + ); + expect(screen.getByRole("button", { name: "Live timeline" })).toHaveClass( + "aria-expanded:bg-transparent", + ); + expect(timeline?.parentElement).toHaveClass("pt-3"); + const items = within(timeline as HTMLElement).getAllByRole("listitem"); + expect(items[0]).toHaveTextContent("Edit serializer"); + expect(items[0]).toHaveTextContent("Now"); + expect(items[0].querySelector("time")).toHaveAttribute( + "datetime", + new Date(5_000).toISOString(), + ); + expect(items[1]).toHaveTextContent("Search serializers"); + expect(items[1]).not.toHaveTextContent("Now"); + }); + + it("toggles the live timeline", async () => { + const user = userEvent.setup(); + const events = [ + updateEvent(2_000, { + sessionUpdate: "tool_call", + title: "Search serializers", + kind: "search", + status: "completed", + }), + ]; + + render( + + + , + ); + + const trigger = screen.getByRole("button", { name: "Live timeline" }); + expect(screen.getByText("Search serializers")).toBeVisible(); + + await user.click(trigger); + expect(screen.queryByText("Search serializers")).not.toBeInTheDocument(); + + await user.click(trigger); + expect(screen.getByText("Search serializers")).toBeVisible(); + }); + + it("labels only the newest live command as now", () => { + const events = [ + updateEvent(2_000, { + sessionUpdate: "tool_call", + title: "Start dev server", + kind: "execute", + status: "in_progress", + }), + updateEvent(4_000, { + sessionUpdate: "tool_call", + title: "Run benchmark", + kind: "execute", + status: "in_progress", + }), + ]; + + render( + + + , + ); + + const timeline = screen.getByText("Run benchmark").closest("ol"); + const items = within(timeline as HTMLElement).getAllByRole("listitem"); + expect(items[0]).toHaveTextContent("Run benchmark"); + expect(items[0]).toHaveTextContent("Now"); + expect(items[0].querySelector("[data-timeline-details]")).toHaveTextContent( + "Now", + ); + expect(within(items[0]).getByText("Now").parentElement).toHaveClass( + "ml-auto", + ); + expect(items[0]).not.toHaveClass("bg-blue-2/50"); + expect(within(items[0]).getByText("Now")).not.toHaveClass( + "motion-safe:animate-pulse", + ); + expect(items[1]).toHaveTextContent("Start dev server"); + expect(items[1]).toHaveTextContent("Background"); + expect(within(items[1]).getByText("Background")).toHaveClass( + "text-gray-10", + ); + expect(screen.getAllByText("Now")).toHaveLength(1); + }); + + it("renders commands as single-line code tooltip triggers", () => { + const command = + "/bin/zsh -lc 'pnpm bench:memory -- --label autoresearch-baseline'"; + const events = [ + updateEvent(2_000, { + sessionUpdate: "tool_call", + title: command, + kind: "execute", + status: "in_progress", + }), + ]; + + render( + + + , + ); + + const trigger = screen.getByText(command); + expect(trigger.tagName).toBe("CODE"); + expect(trigger.parentElement).toHaveClass( + "w-full", + "max-w-full", + "truncate", + "font-mono", + "bg-gray-3", + ); + expect(trigger.parentElement).toHaveAttribute("type", "button"); + expect( + trigger.closest("li")?.querySelector("[data-activity-icon]"), + ).toHaveClass("h-5", "w-3.5", "items-center", "justify-center"); + const commandColumn = trigger + .closest("li") + ?.querySelector("[data-timeline-command-column]"); + expect(commandColumn).toContainElement(trigger); + expect(commandColumn?.querySelector(".text-\\[11px\\]")).toBeTruthy(); + const timelineItem = trigger.closest("li"); + expect(timelineItem).toHaveClass( + "grid", + "w-full", + "min-w-0", + "grid-cols-[auto_minmax(0,1fr)]", + ); + expect(timelineItem?.parentElement).not.toHaveClass("overflow-hidden"); + }); + + it("sorts observed time from longest to shortest", () => { + const events = [ + updateEvent(2_000, { + sessionUpdate: "tool_call", + title: "Search serializers", + kind: "search", + status: "completed", + }), + updateEvent(7_000, { + sessionUpdate: "tool_call", + title: "Edit serializer", + kind: "edit", + status: "completed", + }), + ]; + + render( + + + , + ); + + const observedTime = screen.getByText("Observed time").closest("section"); + expect(observedTime).not.toBeNull(); + const rows = observedTime?.querySelectorAll("[data-observed-kind]"); + expect( + Array.from(rows ?? []).map((row) => + row.getAttribute("data-observed-kind"), + ), + ).toEqual([ + "research", + "implementation", + "reasoning", + "measurement", + "execution", + ]); + }); + + it("freezes observed time at the pause timestamp", () => { + const events = [ + updateEvent(2_000, { + sessionUpdate: "tool_call", + title: "Run benchmark", + kind: "execute", + status: "in_progress", + }), + ]; + + const { container } = render( + + + , + ); + + expect( + container.querySelector('[data-observed-kind="measurement"]'), + ).toHaveTextContent("9s"); + expect(screen.queryByText("Now")).not.toBeInTheDocument(); + }); +}); diff --git a/packages/ui/src/features/autoresearch/AutoresearchObservability.tsx b/packages/ui/src/features/autoresearch/AutoresearchObservability.tsx index f047768fff..68a49604cd 100644 --- a/packages/ui/src/features/autoresearch/AutoresearchObservability.tsx +++ b/packages/ui/src/features/autoresearch/AutoresearchObservability.tsx @@ -1,13 +1,27 @@ import { Binoculars, + Check, Code, Compass, Lightbulb, ListChecks, MagnifyingGlass, + Terminal, TestTube, } from "@phosphor-icons/react"; import type { AutoresearchRun } from "@posthog/core/autoresearch/schemas"; +import { getAutoresearchElapsedMs } from "@posthog/core/autoresearch/stats"; +import { + Collapsible, + CollapsibleContent, + CollapsibleTrigger, + Badge as QuillBadge, + Skeleton, + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, +} from "@posthog/quill"; import type { AcpMessage } from "@posthog/shared"; import { Badge, Progress, Text } from "@radix-ui/themes"; import { useEffect, useMemo, useState } from "react"; @@ -24,86 +38,288 @@ export function AutoresearchObservability({ run: AutoresearchRun; events: AcpMessage[]; }) { - const now = useLiveNow(run.endedAt === null); + const live = run.endedAt === null && run.status === "running"; + const now = useLiveNow(live); + const observationEnd = run.endedAt ?? run.pausedAt ?? now; const activity = useMemo( - () => analyzeAutoresearchActivity(events, run.startedAt, run.endedAt, now), - [events, now, run.endedAt, run.startedAt], + () => + analyzeAutoresearchActivity( + events, + run.startedAt, + run.endedAt, + observationEnd, + { + live, + pauseIntervals: run.pauseIntervals, + pausedDurationMs: run.pausedDurationMs, + }, + ), + [ + events, + live, + observationEnd, + run.endedAt, + run.pauseIntervals, + run.pausedDurationMs, + run.startedAt, + ], ); const lastIteration = run.iterations.at(-1); const hypothesis = activity.currentPlan?.hypothesis ?? lastIteration?.hypothesis; const plan = activity.currentPlan?.plan ?? lastIteration?.plan; const approach = activity.currentPlan?.approach ?? lastIteration?.approach; + const observedTime = ( + Object.keys(activity.timeByKind) as AutoresearchActivityKind[] + ).sort( + (left, right) => activity.timeByKind[right] - activity.timeByKind[left], + ); return ( -
-
- } - title="Current experiment" - /> - - - {approach && ( -
- - {approach} - -
- )} -
- -
- } title="Observed time" /> -
- {(Object.keys(activity.timeByKind) as AutoresearchActivityKind[]).map( - (kind) => ( +
+
+
+ } + title="Current experiment" + /> + + + {approach && ( +
+ + {approach} + +
+ )} +
+ +
+ } title="Observed time" /> +
+ {observedTime.map((kind) => ( - ), - )} -
-
- -
- } title="Live activity" /> - {activity.items.length === 0 ? ( - - Waiting for the first observable tool action. - - ) : ( -
    - {activity.items.map((item) => ( -
  1. - - - {item.label} - - {item.active && Active} -
  2. ))} -
- )} -
+
+
+
+ + + + + + + } + title="Live timeline" + /> + + + {activity.items.length === 0 ? ( + + ) : ( +
    + {activity.items.map((item) => ( +
  1. + + + + +
    + + + {activityLabelContent(item.command, item.label)} + + + {item.label} + + +
    + + {TIME_LABEL[item.kind]} ·{" "} + {" "} + · {formatTimelineTime(item.at - run.startedAt)} + + +
    +
    +
  2. + ))} +
+ )} +
+
+
); } +const TIMELINE_SKELETON_ROWS = [ + { id: "current", commandWidth: "w-3/5", detailWidth: "w-2/5" }, + { id: "recent", commandWidth: "w-4/5", detailWidth: "w-1/3" }, + { id: "earlier", commandWidth: "w-1/2", detailWidth: "w-2/5" }, +]; + +function TimelineLoadingState() { + return ( + +
    + {TIMELINE_SKELETON_ROWS.map((row, index) => ( +
  1. + + + + +
    + +
    + + {index === 0 && ( + + )} +
    +
    +
  2. + ))} +
+
+ ); +} + +function EmptyTimelineState({ live }: { live: boolean }) { + if (live) return ; + return ( + + No timeline activity was recorded. + + ); +} + +function CurrentFindings({ run }: { run: AutoresearchRun }) { + const findings = run.researchFindings.slice(-3).reverse(); + return ( +
+ } + title="Current findings" + /> + {findings.length === 0 ? ( + + Findings appear here as the agent records research checkpoints. + + ) : ( +
    + {findings.map((finding) => ( +
  1. +
    + + {finding.summary} + + {finding.area && } +
    + + {finding.finding} + + {finding.nextStep && ( + + Next: {finding.nextStep} + + )} +
  2. + ))} +
+ )} +
+ ); +} + +function FindingAreaBadge({ area }: { area: string }) { + return ( + + + + } + > + + {area} + + + {area} + + + ); +} + +function activityLabelTrigger(command: boolean) { + const typography = command ? "font-mono text-[11px]" : "text-xs"; + const background = command ? "bg-gray-3 px-1.5 py-0.5" : "bg-transparent"; + return ( +