diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1491d220a8..a72ef42c90 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -925,6 +925,14 @@ jobs: echo "### Timeline viewport gate" echo "- Default arm exit: ${DEFAULT_STATUS}" echo "- Explicitly disabled arm exit: ${DISABLED_STATUS}" + for arm in default disabled; do + node -e ' + const e = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); + const b = e.environment.appliedBudgets; + for (const a of e.attempts) + console.log(`- ${process.argv[2]} attempt ${a.attempt}: interaction p95 ${a.interactionP95Ms.toFixed(1)}/${b.interactionP95Ms} ms, frame p95 ${a.frameIntervalP95Ms.toFixed(1)}/${b.frameIntervalP95Ms} ms, ${a.passingRuns}/${a.runs.length} runs, ${a.passed ? "pass" : "fail"}`); + ' "/tmp/timeline-gate-${arm}.json" "$arm" || echo "- ${arm}: no evidence" + done } >> "$GITHUB_STEP_SUMMARY" if (( DEFAULT_STATUS != 0 || DISABLED_STATUS != 0 )); then diff --git a/packages/studio/src/player/lib/timelineViewportBudgets.test.ts b/packages/studio/src/player/lib/timelineViewportBudgets.test.ts index a75d630ebe..3c8a56496c 100644 --- a/packages/studio/src/player/lib/timelineViewportBudgets.test.ts +++ b/packages/studio/src/player/lib/timelineViewportBudgets.test.ts @@ -19,6 +19,8 @@ describe("timeline viewport budgets", () => { interactionP95Ms: 50, constrainedInteractionP95Ms: 75, constrainedFrameIntervalP95Ms: 75, + ciVirtualizedInteractionP95Ms: 58.3, + ciVirtualizedFrameIntervalP95Ms: 25, longTaskLimitMs: 50, constrainedLongTaskLimitMs: 300, posterCoverageRatio: 0.9, diff --git a/packages/studio/src/player/lib/timelineViewportBudgets.ts b/packages/studio/src/player/lib/timelineViewportBudgets.ts index 771a37cd32..4e4c2e3e1c 100644 --- a/packages/studio/src/player/lib/timelineViewportBudgets.ts +++ b/packages/studio/src/player/lib/timelineViewportBudgets.ts @@ -28,6 +28,8 @@ export interface TimelineViewportBudgets { frameIntervalP95Ms: number; constrainedInteractionP95Ms: number; constrainedFrameIntervalP95Ms: number; + ciVirtualizedInteractionP95Ms: number; + ciVirtualizedFrameIntervalP95Ms: number; longTaskLimitMs: number; constrainedLongTaskLimitMs: number; memoryReturnToleranceRatio: number; @@ -86,6 +88,8 @@ export const TIMELINE_VIEWPORT_BUDGETS: Readonly = Obje frameIntervalP95Ms: 33.3, constrainedInteractionP95Ms: 75, constrainedFrameIntervalP95Ms: 75, + ciVirtualizedInteractionP95Ms: 58.3, + ciVirtualizedFrameIntervalP95Ms: 25, longTaskLimitMs: 50, constrainedLongTaskLimitMs: 300, memoryReturnToleranceRatio: 0.15, diff --git a/packages/studio/tests/e2e/timeline-viewport-verdict.mjs b/packages/studio/tests/e2e/timeline-viewport-verdict.mjs index d9f0f3f639..1e7bcfdf78 100644 --- a/packages/studio/tests/e2e/timeline-viewport-verdict.mjs +++ b/packages/studio/tests/e2e/timeline-viewport-verdict.mjs @@ -9,6 +9,24 @@ export function percentile(values, ratio) { return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * ratio) - 1)]; } +/** The p95 pair a tier is held to; the CI virtualized arm has its own, tighter one. */ +export function responsivenessLimits(budgets, tier, rowVirtualization) { + if (tier === "primary") + return { + interactionLimitMs: budgets.interactionP95Ms, + frameIntervalLimitMs: budgets.frameIntervalP95Ms, + }; + if (tier === "ci" && rowVirtualization === "on") + return { + interactionLimitMs: budgets.ciVirtualizedInteractionP95Ms, + frameIntervalLimitMs: budgets.ciVirtualizedFrameIntervalP95Ms, + }; + return { + interactionLimitMs: budgets.constrainedInteractionP95Ms, + frameIntervalLimitMs: budgets.constrainedFrameIntervalP95Ms, + }; +} + function assertSampleCount(expected, interactions, frameIntervals) { if (expected > 0 && interactions.length === expected && frameIntervals.length === expected) return; @@ -38,17 +56,14 @@ export function judgeResponsiveness( }; } -export function gatePassed({ - directScrollApproved, - responsivenessPassed, - passingRuns, - requiredPassingRuns, - memoryReturned, -}) { - return ( - directScrollApproved && - responsivenessPassed && - passingRuns >= requiredPassingRuns && - memoryReturned - ); +/** A failed timing attempt is measured once more, so one bad stretch of a shared runner cannot fail the gate alone. */ +export const TIMING_ATTEMPTS = 2; + +export function attemptPassed({ responsivenessPassed, passingRuns, requiredPassingRuns }) { + return responsivenessPassed && passingRuns >= requiredPassingRuns; +} + +export function gatePassed({ directScrollApproved, attempts, memoryReturned }) { + const timingPassed = attempts.slice(0, TIMING_ATTEMPTS).some((attempt) => attempt.passed); + return directScrollApproved && timingPassed && memoryReturned; } diff --git a/packages/studio/tests/e2e/timeline-viewport-verdict.test.mjs b/packages/studio/tests/e2e/timeline-viewport-verdict.test.mjs index c5a54af4b9..91c52153ef 100644 --- a/packages/studio/tests/e2e/timeline-viewport-verdict.test.mjs +++ b/packages/studio/tests/e2e/timeline-viewport-verdict.test.mjs @@ -1,5 +1,12 @@ import { describe, expect, it } from "vitest"; -import { gatePassed, judgeResponsiveness, percentile } from "./timeline-viewport-verdict.mjs"; +import { TIMELINE_VIEWPORT_BUDGETS } from "../../src/player/lib/timelineViewportBudgets"; +import { + attemptPassed, + gatePassed, + judgeResponsiveness, + percentile, + responsivenessLimits, +} from "./timeline-viewport-verdict.mjs"; const LIMITS = { samplesPerRun: 63, interactionLimitMs: 75, frameIntervalLimitMs: 75 }; const FAST = 49; @@ -69,20 +76,73 @@ describe("judgeResponsiveness", () => { }); }); -describe("gatePassed", () => { - const passing = { - directScrollApproved: true, - responsivenessPassed: true, - passingRuns: 5, - requiredPassingRuns: 4, - memoryReturned: true, +describe("the CI virtualized arm's limits", () => { + const limits = { + samplesPerRun: 63, + ...responsivenessLimits(TIMELINE_VIEWPORT_BUDGETS, "ci", "on"), }; + // Two frames is a normal step, four a step two frames late; one frame is a normal interval, two a dropped one. + const steps = (count, normal, slow) => + Array.from({ length: 5 }, (_, run) => + Array.from({ length: 63 }, (_, step) => (run * 63 + step < count ? slow : normal)), + ); + const interactionRuns = (count) => + steps(count, 33.3, 66.7).map((interactions) => ({ + interactions, + frameIntervals: interactions.map(() => 16.7), + })); + const frameRuns = (count) => + steps(count, 16.7, 33.3).map((frameIntervals) => ({ + interactions: frameIntervals.map(() => 33.3), + frameIntervals, + })); + + it("fails 16 of 315 steps two frames late and passes 15", () => { + expect(judgeResponsiveness(interactionRuns(16), limits).passed).toBe(false); + expect(judgeResponsiveness(interactionRuns(15), limits).passed).toBe(true); + }); + + it("fails 16 of 315 frame intervals that drop a frame and passes 15", () => { + expect(judgeResponsiveness(frameRuns(16), limits).passed).toBe(false); + expect(judgeResponsiveness(frameRuns(15), limits).passed).toBe(true); + }); + + it("leaves the unvirtualized arm and the other constrained tiers at 75 ms", () => { + const loose = { interactionLimitMs: 75, frameIntervalLimitMs: 75 }; + expect(responsivenessLimits(TIMELINE_VIEWPORT_BUDGETS, "ci", "off")).toEqual(loose); + expect(responsivenessLimits(TIMELINE_VIEWPORT_BUDGETS, "low-resource", "on")).toEqual(loose); + expect(responsivenessLimits(TIMELINE_VIEWPORT_BUDGETS, "primary", "on")).toEqual({ + interactionLimitMs: 50, + frameIntervalLimitMs: 33.3, + }); + }); +}); + +describe("attemptPassed", () => { + const passing = { responsivenessPassed: true, passingRuns: 5, requiredPassingRuns: 4 }; + + it("needs pooled responsiveness and enough passing runs", () => { + expect(attemptPassed(passing)).toBe(true); + expect(attemptPassed({ ...passing, responsivenessPassed: false })).toBe(false); + expect(attemptPassed({ ...passing, passingRuns: 3 })).toBe(false); + }); +}); + +describe("gatePassed", () => { + const pass = { passed: true }; + const fail = { passed: false }; + const passing = { directScrollApproved: true, attempts: [pass], memoryReturned: true }; it("passes only when every check holds", () => { expect(gatePassed(passing)).toBe(true); - expect(gatePassed({ ...passing, responsivenessPassed: false })).toBe(false); expect(gatePassed({ ...passing, directScrollApproved: false })).toBe(false); - expect(gatePassed({ ...passing, passingRuns: 3 })).toBe(false); expect(gatePassed({ ...passing, memoryReturned: false })).toBe(false); }); + + it("fails timing only when the attempt and its one rerun both fail", () => { + expect(gatePassed({ ...passing, attempts: [fail, pass] })).toBe(true); + expect(gatePassed({ ...passing, attempts: [fail, fail] })).toBe(false); + expect(gatePassed({ ...passing, attempts: [fail, fail, pass] })).toBe(false); + expect(gatePassed({ ...passing, attempts: [] })).toBe(false); + }); }); diff --git a/packages/studio/tests/e2e/timeline-virtualization.mjs b/packages/studio/tests/e2e/timeline-virtualization.mjs index ff6435755f..b53c07065a 100644 --- a/packages/studio/tests/e2e/timeline-virtualization.mjs +++ b/packages/studio/tests/e2e/timeline-virtualization.mjs @@ -10,9 +10,9 @@ * TIMELINE_TIER selects the budget set and the emulation applied. "primary" is * a developer machine and holds the strict budgets. "low-resource" and * "high-dpr" add CPU throttling and a 2x scale factor respectively. "ci" is a - * shared runner: no emulation, but the constrained budgets, because a hosted - * runner is already slower and noisier than the machine the strict numbers were - * recorded on. Throttling it further would measure the throttle, not the build. + * shared runner with no emulation: its virtualized arm holds its own limits + * (responsivenessLimits), its unvirtualized arm the constrained ones. Throttling + * it further would measure the throttle, not the build. * CI also requires production React and reports the observed runtime so Vite's * development-only checks can never contaminate the shipped-code measurement. * @@ -23,7 +23,13 @@ */ import { platform, arch } from "node:os"; import { launchStudioChrome } from "./chrome-executable.mjs"; -import { gatePassed, judgeResponsiveness } from "./timeline-viewport-verdict.mjs"; +import { + attemptPassed, + gatePassed, + judgeResponsiveness, + responsivenessLimits, + TIMING_ATTEMPTS, +} from "./timeline-viewport-verdict.mjs"; const STUDIO_URL = process.env.STUDIO_URL; const PROFILE = process.env.TIMELINE_PROFILE || "dense-short"; @@ -290,36 +296,61 @@ try { ); } - const runs = []; - const interactionLimitMs = - TIER === "primary" ? budgets.interactionP95Ms : budgets.constrainedInteractionP95Ms; - const frameIntervalLimitMs = - TIER === "primary" ? budgets.frameIntervalP95Ms : budgets.constrainedFrameIntervalP95Ms; - for (let index = 0; index < budgets.warmupRuns + budgets.measuredRuns; index += 1) { - const run = await collectRun(page); - if (index >= budgets.warmupRuns) runs.push(run); - } + const { interactionLimitMs, frameIntervalLimitMs } = responsivenessLimits( + budgets, + TIER, + ROW_VIRTUALIZATION, + ); // Latency, long tasks and memory are product promises and hold for both // builds. The DOM-size budgets describe what windowing achieves, so they only // apply when windowing is on. They are skipped explicitly rather than relaxed, // so a skipped budget never reads as a passed one. const domBudgetsApply = ROW_VIRTUALIZATION === "on"; - for (const run of runs) { - run.longTaskPassed = run.longestTaskMs <= longTaskLimitMs; - run.timelineMounted = run.diagnostics.timelineRoots === 1; - run.domSizePassed = domBudgetsApply - ? run.diagnostics.mountedRows <= budgets.maxMountedRows && - run.diagnostics.mountedClipRoots <= budgets.maxMountedClipRoots && - run.diagnostics.maxMountedClipRootsInOneRow <= budgets.maxMountedClipRootsPerRow && - run.diagnostics.mountedTimelineDescendants <= budgets.maxMountedTimelineDescendants - : null; - run.passed = run.longTaskPassed && run.timelineMounted && run.domSizePassed !== false; + const attempts = []; + while (attempts.length < TIMING_ATTEMPTS && !attempts.some((attempt) => attempt.passed)) { + const runs = []; + for (let index = 0; index < budgets.warmupRuns + budgets.measuredRuns; index += 1) { + const run = await collectRun(page); + if (index >= budgets.warmupRuns) runs.push(run); + } + for (const run of runs) { + run.longTaskPassed = run.longestTaskMs <= longTaskLimitMs; + run.timelineMounted = run.diagnostics.timelineRoots === 1; + run.domSizePassed = domBudgetsApply + ? run.diagnostics.mountedRows <= budgets.maxMountedRows && + run.diagnostics.mountedClipRoots <= budgets.maxMountedClipRoots && + run.diagnostics.maxMountedClipRootsInOneRow <= budgets.maxMountedClipRootsPerRow && + run.diagnostics.mountedTimelineDescendants <= budgets.maxMountedTimelineDescendants + : null; + run.passed = run.longTaskPassed && run.timelineMounted && run.domSizePassed !== false; + } + const responsiveness = judgeResponsiveness(runs, { + samplesPerRun: budgets.scrollSamplesPerRun, + interactionLimitMs, + frameIntervalLimitMs, + }); + const passingRuns = runs.filter((run) => run.passed).length; + const attempt = { + attempt: attempts.length + 1, + interactionP95Ms: responsiveness.interactionP95Ms, + frameIntervalP95Ms: responsiveness.frameIntervalP95Ms, + responsivenessPassed: responsiveness.passed, + passingRuns, + passed: attemptPassed({ + responsivenessPassed: responsiveness.passed, + passingRuns, + requiredPassingRuns: budgets.requiredPassingRuns, + }), + runs, + }; + attempts.push(attempt); + console.error( + `timeline gate ${ROW_VIRTUALIZATION} attempt ${attempt.attempt}: ` + + `interaction p95 ${attempt.interactionP95Ms.toFixed(1)}/${interactionLimitMs} ms, ` + + `frame p95 ${attempt.frameIntervalP95Ms.toFixed(1)}/${frameIntervalLimitMs} ms, ` + + `${passingRuns}/${runs.length} runs passed, ${attempt.passed ? "PASS" : "FAIL"}`, + ); } - const responsiveness = judgeResponsiveness(runs, { - samplesPerRun: budgets.scrollSamplesPerRun, - interactionLimitMs, - frameIntervalLimitMs, - }); await page.evaluate(() => window.__studioTest.resetTimelinePerformanceFixture()); await page.waitForFunction( @@ -331,8 +362,10 @@ try { const returnedHeapBytes = await collectHeapBytes(client); const memoryReturned = returnedHeapBytes <= baselineHeapBytes * (1 + budgets.memoryReturnToleranceRatio); - const passingRuns = runs.filter((run) => run.passed).length; - const maxTimelineContentWidthPx = Math.max(0, ...runs.map((run) => run.scrollWidth)); + const maxTimelineContentWidthPx = Math.max( + 0, + ...attempts.flatMap((attempt) => attempt.runs.map((run) => run.scrollWidth)), + ); const directScrollGate = { safetyEnvelopePx: budgets.directScrollSafetyPx, maxTimelineContentWidthPx, @@ -369,15 +402,13 @@ try { warmups: budgets.warmupRuns, measured: budgets.measuredRuns, requiredPassing: budgets.requiredPassingRuns, + timingAttempts: TIMING_ATTEMPTS, }, }, directScrollGate, - runs, + attempts, aggregate: { - interactionP95Ms: responsiveness.interactionP95Ms, - frameIntervalP95Ms: responsiveness.frameIntervalP95Ms, - responsivenessPassed: responsiveness.passed, - passingRuns, + timingPassed: attempts.some((attempt) => attempt.passed), baselineHeapBytes, returnedHeapBytes, memoryReturned, @@ -386,9 +417,7 @@ try { console.log(JSON.stringify(evidence, null, 2)); exitCode = gatePassed({ directScrollApproved: directScrollGate.decision === "approved", - responsivenessPassed: responsiveness.passed, - passingRuns, - requiredPassingRuns: budgets.requiredPassingRuns, + attempts, memoryReturned, }) ? 0