From eeca1d1110a0ba0502378f8ed521f49418978657 Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Sun, 25 Oct 2026 02:06:48 +0300 Subject: [PATCH 1/2] feat(studio): scores grouped by eval #/evals/scores is its own screen now: one row per eval from ?aggregate=eval (pass rate, scores, mean, last seen); expanding a row (open=) loads that eval's rows and shows the pass-rate trend from the eval-level scores (one mean point per bucket), per-check pass rates and the newest failures, each linking to #/traces/?obs=. Filters eval/agent/source/range live in the query string, defaults unwritten. The daemon-down verdict + retry-while-down is a shared useDaemonDown hook, now used by the Trace, Monitoring and Scores screens. Co-Authored-By: Claude Opus 5.5 --- apps/agent-forge/e2e/a11y.spec.ts | 11 +- .../observe/__tests__/observeClient.test.ts | 11 + apps/agent-forge/src/observe/observeClient.ts | 18 ++ apps/agent-forge/src/observe/types.ts | 16 ++ apps/agent-forge/src/observe/useObserve.ts | 44 ++++ .../screens/monitoring/MonitoringScreen.tsx | 9 +- apps/agent-forge/src/screens/routes.tsx | 6 +- .../src/screens/scores/EvalDetail.tsx | 108 +++++++++ .../src/screens/scores/ScoresScreen.tsx | 154 +++++++++++++ .../scores/__tests__/ScoresScreen.test.tsx | 210 ++++++++++++++++++ .../agent-forge/src/screens/scores/scores.css | 94 ++++++++ .../src/screens/scores/scoresData.ts | 135 +++++++++++ .../src/screens/traces/TraceScreen.tsx | 15 +- 13 files changed, 810 insertions(+), 21 deletions(-) create mode 100644 apps/agent-forge/src/screens/scores/EvalDetail.tsx create mode 100644 apps/agent-forge/src/screens/scores/ScoresScreen.tsx create mode 100644 apps/agent-forge/src/screens/scores/__tests__/ScoresScreen.test.tsx create mode 100644 apps/agent-forge/src/screens/scores/scores.css create mode 100644 apps/agent-forge/src/screens/scores/scoresData.ts diff --git a/apps/agent-forge/e2e/a11y.spec.ts b/apps/agent-forge/e2e/a11y.spec.ts index 49c798ed..f784bd99 100644 --- a/apps/agent-forge/e2e/a11y.spec.ts +++ b/apps/agent-forge/e2e/a11y.spec.ts @@ -113,7 +113,7 @@ test('Ctrl+K opens the command palette; ? lists the shortcuts', async ({ page }) }); // One
, one h1, the banner and the primary nav on every screen, not just Agents. -for (const route of ['#/traces', '#/traces/e2e-trace?obs=e2e-obs', '#/monitoring', '#/settings', '#/playground']) { +for (const route of ['#/traces', '#/traces/e2e-trace?obs=e2e-obs', '#/monitoring', '#/evals/scores', '#/settings', '#/playground']) { test(`${route} keeps the landmarks and exactly one main`, async ({ page }) => { await page.goto(`/?token=e2e-studio-token${route}`); await expect(page.locator('.screen-main')).toBeVisible(); @@ -145,6 +145,15 @@ test('#/monitoring with no daemon shows the down state and keeps window/agent in expect(new URL(page.url()).hash).toBe('#/monitoring?window=7d&agent=e2e-agent'); }); +test('#/evals/scores with no daemon shows the down state and keeps the filters in the URL', async ({ page }) => { + await page.route('**/observe/**', (route) => + route.fulfill({ status: 502, contentType: 'application/json', body: '{"error":"down"}' }), + ); + await page.goto('/?token=e2e-studio-token#/evals/scores?eval=e2e-eval&source=eval&range=7d'); + await expect(page.getByRole('main').getByText('lousho observe start')).toBeVisible(); + expect(new URL(page.url()).hash).toBe('#/evals/scores?eval=e2e-eval&source=eval&range=7d'); +}); + test('#/settings shows the settings panel inside the screen', async ({ page }) => { await page.goto('/?token=e2e-studio-token#/settings'); const main = page.getByRole('main'); diff --git a/apps/agent-forge/src/observe/__tests__/observeClient.test.ts b/apps/agent-forge/src/observe/__tests__/observeClient.test.ts index f2a2f698..7c9c9908 100644 --- a/apps/agent-forge/src/observe/__tests__/observeClient.test.ts +++ b/apps/agent-forge/src/observe/__tests__/observeClient.test.ts @@ -90,4 +90,15 @@ describe('ObserveClient', () => { const down = clientWith((async () => jsonResponse(502, {})) as typeof fetch); await expect(down.postScore({ trace_id: 'x', name: 'feedback', value: 0 })).rejects.toBeInstanceOf(ObserveDaemonDownError); }); + + it('scorePassRates asks for the eval aggregate with the score filters', async () => { + const evals = [{ name: 'support', count: 2, passed: 1, pass_rate: 0.5, avg_value: 0.5, last_ts: 1 }]; + const fetchFn = vi.fn(async () => jsonResponse(200, { evals })); + const client = clientWith(fetchFn as typeof fetch); + expect(await client.scorePassRates({ agent_id: 'a1', source: 'eval', since: 10, until: 20 })).toEqual(evals); + expect(fetchFn).toHaveBeenCalledWith( + '/observe/api/v1/scores?agent_id=a1&source=eval&since=10&until=20&aggregate=eval', + expect.anything() + ); + }); }); diff --git a/apps/agent-forge/src/observe/observeClient.ts b/apps/agent-forge/src/observe/observeClient.ts index c4c5e5de..3b8c6bc8 100644 --- a/apps/agent-forge/src/observe/observeClient.ts +++ b/apps/agent-forge/src/observe/observeClient.ts @@ -14,6 +14,7 @@ import type { ObserveAgentRow, ObserveHealth, ObserveInsights, + ObserveScorePassRate, ObserveScoreRow, ObserveTraceDetail, ObserveTraceList, @@ -72,8 +73,17 @@ export interface TraceListFilter { export interface ScoreListFilter { trace_id?: string; + /** Exact score name (`` is the eval-level row of each case). */ name?: string; + /** Eval name: every score named `` or `/`. */ + eval?: string; agent_id?: string; + /** `eval` | `judge` | `human`. */ + source?: string; + /** `created_at` lower bound (epoch ms). */ + since?: number; + /** `created_at` upper bound (epoch ms). */ + until?: number; limit?: number; } @@ -200,6 +210,14 @@ export class ObserveClient { return body.scores; } + /** `GET /api/v1/scores?aggregate=eval` - pass rate, mean and last seen per eval name (same filters as {@link listScores}). */ + async scorePassRates(filter: Omit = {}): Promise { + const body = await this.request<{ evals: ObserveScorePassRate[] }>('/api/v1/scores', { + query: { ...filter, aggregate: 'eval' }, + }); + return body.evals; + } + /** `POST /api/v1/scores` - a human feedback score (append-only); the daemon's 404/400 surface as `ObserveApiError`. */ postScore(score: NewScore): Promise<{ score_id: string }> { return this.request('/api/v1/scores', { method: 'POST', body: score }); diff --git a/apps/agent-forge/src/observe/types.ts b/apps/agent-forge/src/observe/types.ts index 60501cd5..aafdf988 100644 --- a/apps/agent-forge/src/observe/types.ts +++ b/apps/agent-forge/src/observe/types.ts @@ -76,6 +76,22 @@ export interface ObserveScoreRow { created_at: number; } +/** One row of `GET /api/v1/scores?aggregate=eval`: every score whose name is `` or `/*`, folded. */ +export interface ObserveScorePassRate { + /** The eval name (the score-name prefix before the first '/'). */ + name: string; + /** Score rows in the group (the eval-level row of each case plus its per-check rows). */ + count: number; + /** Rows that passed: no failure comment, or a perfect 1. */ + passed: number; + /** `passed / count`. */ + pass_rate: number; + /** Mean value over the group's non-null values; null when every value is null. */ + avg_value: number | null; + /** Newest row's `created_at`. */ + last_ts: number; +} + export interface ObserveTraceDetail { trace_id: string; observations: ObserveObservationRow[]; diff --git a/apps/agent-forge/src/observe/useObserve.ts b/apps/agent-forge/src/observe/useObserve.ts index 254eef1e..3fbd37bd 100644 --- a/apps/agent-forge/src/observe/useObserve.ts +++ b/apps/agent-forge/src/observe/useObserve.ts @@ -11,6 +11,7 @@ import type { ObserveAgentRow, ObserveHealth, ObserveInsights, + ObserveScorePassRate, ObserveScoreRow, ObserveTraceDetail, ObserveTraceList, @@ -107,3 +108,46 @@ export function useInsights(filter: InsightsFilter, refreshKey?: unknown): Obser export function useScores(filter: ScoreListFilter, refreshKey?: unknown): ObserveQuery { return useObserveQuery(() => observeClient.listScores(filter), [JSON.stringify(filter), refreshKey]); } + +/** One eval's score rows (`eval=` plus `filter`); no request while `evalName` is empty. */ +export function useEvalScores( + evalName: string, + filter: Omit, + refreshKey?: unknown +): ObserveQuery { + return useObserveQuery( + () => (evalName ? observeClient.listScores({ ...filter, eval: evalName }) : Promise.resolve(undefined)), + [evalName, JSON.stringify(filter), refreshKey] + ); +} + +export function useScorePassRates( + filter: Omit, + refreshKey?: unknown +): ObserveQuery { + return useObserveQuery(() => observeClient.scorePassRates(filter), [JSON.stringify(filter), refreshKey]); +} + +/** + * The observe screens' shared "daemon down" verdict: the health probe says so, + * or one of `queries` failed to reach it. While down, every query re-runs each + * `pollMs`, so the screen recovers in place (URL untouched) once the daemon is + * back even if the health probe never saw it go. + */ +export function useDaemonDown( + healthDown: boolean, + queries: readonly Pick, 'error' | 'refresh'>[], + pollMs = 5000 +): boolean { + const down = healthDown || queries.some((q) => q.error instanceof ObserveDaemonDownError); + const latest = useRef(queries); + latest.current = queries; + useEffect(() => { + if (!down || pollMs <= 0) return undefined; + const timer = setInterval(() => { + for (const q of latest.current) q.refresh(); + }, pollMs); + return () => clearInterval(timer); + }, [down, pollMs]); + return down; +} diff --git a/apps/agent-forge/src/screens/monitoring/MonitoringScreen.tsx b/apps/agent-forge/src/screens/monitoring/MonitoringScreen.tsx index 69680ee8..96ad3027 100644 --- a/apps/agent-forge/src/screens/monitoring/MonitoringScreen.tsx +++ b/apps/agent-forge/src/screens/monitoring/MonitoringScreen.tsx @@ -1,8 +1,7 @@ import { useEffect, useMemo, useState } from 'react'; import { AgentSelect, ObserveDaemonDown, ObserveError } from '../../observe/chrome'; -import { ObserveDaemonDownError } from '../../observe/observeClient'; -import { useAgents, useDaemonHealth, useInsights } from '../../observe/useObserve'; +import { useAgents, useDaemonDown, useDaemonHealth, useInsights } from '../../observe/useObserve'; import { navigate } from '../../router/router'; import { useQueryState } from '../../router/useRoute'; import { EmptyState } from '../../ui/EmptyState'; @@ -68,9 +67,9 @@ function useMonitoringData(win: MonitoringWindow, agentId: string) { { since: chart.from, until: chart.to, agent_id: agentId || undefined, granularity: chart.granularity }, refreshKey, ); - const errors = [agents.error, fleetInsights.error, insights.error]; - const down = health.down || errors.some((e) => e instanceof ObserveDaemonDownError); - return { agents, fleetInsights, insights, chart, down, error: errors.find((e) => e !== undefined) }; + const queries = [agents, fleetInsights, insights]; + const down = useDaemonDown(health.down, queries); + return { agents, fleetInsights, insights, chart, down, error: queries.find((q) => q.error)?.error }; } /** diff --git a/apps/agent-forge/src/screens/routes.tsx b/apps/agent-forge/src/screens/routes.tsx index d87714db..573eb51b 100644 --- a/apps/agent-forge/src/screens/routes.tsx +++ b/apps/agent-forge/src/screens/routes.tsx @@ -7,6 +7,7 @@ import { ObserveView, type ObserveTab } from '../observe/ObserveView'; import { AgentsScreen } from './agents/AgentsScreen'; import { TraceScreen } from './traces/TraceScreen'; import { MonitoringScreen } from './monitoring/MonitoringScreen'; +import { ScoresScreen } from './scores/ScoresScreen'; import { useLeaveAgentsGuard } from './agents/useLeaveAgentsGuard'; import { Placeholder } from './Placeholder'; @@ -63,8 +64,8 @@ export function useScreen(): ScreenId { return screen ?? 'agents'; } -/** Until the Phase 3 screens land, Traces/Scores are the existing observe panels, one per route. */ -const OBSERVE_TAB: Partial> = { traces: 'traces', scores: 'scores' }; +/** Until its Phase 3 screen lands, Traces is the existing observe panel (Scores has its own screen now). */ +const OBSERVE_TAB: Partial> = { traces: 'traces' }; const TAB_PATH: Record = { traces: '/traces', scores: '/evals/scores', @@ -110,6 +111,7 @@ export function Screens() { // Same element type at the same position for all three, so the drill-down filter survives a route change. if (screen === 'trace') content = ; else if (screen === 'monitoring') content = ; + else if (screen === 'scores') content = ; else if (observeTab) content = navigate(TAB_PATH[tab])} />; else if (placeholder) content = ; else diff --git a/apps/agent-forge/src/screens/scores/EvalDetail.tsx b/apps/agent-forge/src/screens/scores/EvalDetail.tsx new file mode 100644 index 00000000..b427fd35 --- /dev/null +++ b/apps/agent-forge/src/screens/scores/EvalDetail.tsx @@ -0,0 +1,108 @@ +import { useMemo, type ReactNode } from 'react'; +import { ObserveError } from '../../observe/chrome'; +import { fmtDateTime, fmtInt, fmtPercent } from '../../observe/format'; +import type { ObserveQuery } from '../../observe/useObserve'; +import type { ObserveScoreRow } from '../../observe/types'; +import { DataTable, type Column } from '../../ui/DataTable'; +import { EmptyState } from '../../ui/EmptyState'; +import { TimeSeriesChart } from '../../ui/TimeSeriesChart'; +import { groupEvalScores, passRateSeries, scoreStats, traceHref, type CheckGroup, type TrendWindow } from './scoresData'; + +/** Rows fetched for one eval; a full page means older rows were left out. */ +export const EVAL_ROW_LIMIT = 1000; + +const fmtMean = (v: number | null) => (v === null ? '-' : v.toFixed(2)); + +const CHECK_COLUMNS: Column[] = [ + { key: 'check', header: 'Check', render: (c) => {c.check} }, + { key: 'pass', header: 'Pass rate', align: 'right', render: (c) => fmtPercent(c.passRate) }, + { key: 'count', header: 'Scores', align: 'right', render: (c) => fmtInt(c.count) }, + { key: 'mean', header: 'Mean', align: 'right', hideBelow: 480, render: (c) => fmtMean(c.mean) }, +]; + +function TraceLink({ row }: { row: ObserveScoreRow }) { + return ( + + {row.trace_id.slice(0, 10)} + + ); +} + +function Failures({ evalName, rows }: { evalName: string; rows: ObserveScoreRow[] }) { + if (rows.length === 0) return

No failing scores in this window.

; + return ( +
    + {rows.map((row) => ( +
  • + {fmtDateTime(row.created_at)} + {row.name === evalName ? 'eval' : row.name.slice(evalName.length + 1)} + {row.value === null ? '-' : row.value} + + {row.comment} +
    • + ))} +
+ ); +} + +/** + * One expanded eval: its pass-rate trend (from the eval-level row of each + * case), the per-check pass rates (worst first) and the newest failing rows, + * each linking to its trace. + */ +export function EvalDetail({ + evalName, + query, + window: w, +}: { + evalName: string; + query: ObserveQuery; + window: TrendWindow; +}) { + const rows = query.data; + const groups = useMemo(() => groupEvalScores(evalName, rows ?? []), [evalName, rows]); + const runs = scoreStats(groups.evalRows); + const series = useMemo(() => passRateSeries(groups.evalRows, w), [groups.evalRows, w]); + + let body: ReactNode; + if (query.error) body = ; + else if (rows === undefined) body =
Loading {evalName}...
; + else + body = ( + <> +

+ {fmtInt(runs.count)} runs, {fmtPercent(runs.passRate)} passed + {rows.length >= EVAL_ROW_LIMIT && ` (newest ${fmtInt(EVAL_ROW_LIMIT)} scores only)`} +

+
+

Pass-rate trend

+ {series.points.length ? ( + + ) : ( +

No eval-level scores in this window.

+ )} +
+
+

Checks

+ c.name} + density="compact" + empty={} + /> +
+
+

Latest failures

+ +
+ + ); + return ( +
+

{evalName}

+ {body} +
+ ); +} diff --git a/apps/agent-forge/src/screens/scores/ScoresScreen.tsx b/apps/agent-forge/src/screens/scores/ScoresScreen.tsx new file mode 100644 index 00000000..0118d368 --- /dev/null +++ b/apps/agent-forge/src/screens/scores/ScoresScreen.tsx @@ -0,0 +1,154 @@ +import { useMemo } from 'react'; + +import { ObserveDaemonDown, ObserveError } from '../../observe/chrome'; +import { fmtDateTime, fmtInt, fmtPercent } from '../../observe/format'; +import type { ObserveAgentRow, ObserveScorePassRate } from '../../observe/types'; +import { useAgents, useDaemonDown, useDaemonHealth, useEvalScores, useScorePassRates } from '../../observe/useObserve'; +import { useQueryState } from '../../router/useRoute'; +import { DataTable, type Column } from '../../ui/DataTable'; +import { EmptyState } from '../../ui/EmptyState'; +import { FilterBar, type FacetDef } from '../../ui/FilterBar'; +import { formatTimeRange, toWindow, type TimeRange } from '../../ui/TimeRangePicker'; +import { EVAL_ROW_LIMIT, EvalDetail } from './EvalDetail'; +import { + formatEvals, + formatRange, + formatText, + parseEvals, + parseRange, + parseSource, + parseText, + SCORE_SOURCES, + trendWindow, +} from './scoresData'; +// The shared daemon-down / error presentation is styled there. +import '../../observe/observe.css'; +import './scores.css'; + +/** The query-string filters: `eval` (comma list), `agent`, `source`, `range`, and the expanded eval `open`. */ +function useScoreFilters() { + const [evals, setEvals] = useQueryState('eval', parseEvals, formatEvals); + const [agent, setAgent] = useQueryState('agent', parseText, formatText); + const [source, setSource] = useQueryState('source', parseSource, formatText); + const [range, setRange] = useQueryState('range', parseRange, formatRange); + const [open, setOpen] = useQueryState('open', parseText, formatText); + return { evals, setEvals, agent, setAgent, source, setSource, range, setRange, open, setOpen }; +} + +/** The daemon's time bounds for a range; a preset's clock is re-read only when the range (or refresh) changes. */ +function useBounds(range: TimeRange, refreshKey: unknown) { + const key = formatTimeRange(range); + return useMemo(() => { + const now = Date.now(); + const w = toWindow(range, now); + return { since: w.from, until: w.to, trend: trendWindow(range, now) }; + // `range` is identified by its serialised key. + }, [key, refreshKey]); +} + +/** Single-select facet over a multi-select FilterBar facet: the newest tick wins. */ +function singleFacet(key: string, label: string, options: FacetDef['options'], value: string, set: (v: string) => void): FacetDef { + return { key, label, options, value: value ? [value] : [], onChange: (next) => set(next.at(-1) ?? '') }; +} + +function agentOptions(agents: ObserveAgentRow[] | undefined, selected: string): FacetDef['options'] { + const opts = (agents ?? []).map((a) => ({ value: a.agent_id, label: a.name ?? a.agent_id.slice(0, 8) })); + return selected && !opts.some((o) => o.value === selected) ? [...opts, { value: selected, label: selected }] : opts; +} + +function evalOptions(rates: ObserveScorePassRate[] | undefined, selected: string[]): FacetDef['options'] { + const names = new Set([...(rates ?? []).map((r) => r.name), ...selected]); + const count = new Map((rates ?? []).map((r) => [r.name, r.count])); + return [...names].sort().map((name) => ({ value: name, label: name, count: count.get(name) })); +} + +function evalColumns(open: string, toggle: (name: string) => void): Column[] { + return [ + { + key: 'name', + header: 'Eval', + render: (r) => ( + + ), + }, + { key: 'pass', header: 'Pass rate', align: 'right', render: (r) => fmtPercent(r.pass_rate) }, + { key: 'count', header: 'Scores', align: 'right', render: (r) => fmtInt(r.count) }, + { key: 'mean', header: 'Mean', align: 'right', hideBelow: 480, render: (r) => (r.avg_value === null ? '-' : r.avg_value.toFixed(2)) }, + { key: 'last', header: 'Last seen', hideBelow: 560, render: (r) => fmtDateTime(r.last_ts) }, + ]; +} + +/** Pass rates per eval, the expanded eval's rows and the agent list; refetched when the daemon comes back. */ +function useScoresData(f: ReturnType) { + const health = useDaemonHealth(); + const bounds = useBounds(f.range, health.down); + const filter = { agent_id: f.agent || undefined, source: f.source || undefined, since: bounds.since, until: bounds.until }; + const rates = useScorePassRates(filter, health.down); + const detail = useEvalScores(f.open, { ...filter, limit: EVAL_ROW_LIMIT }, health.down); + const agents = useAgents(undefined, health.down); + const down = useDaemonDown(health.down, [rates, detail, agents]); + return { rates, detail, agents, down, trend: bounds.trend }; +} + +/** + * `#/evals/scores?eval=&agent=&source=&range=&open=` - one row per eval + * (pass rate, scores, mean, last seen from `?aggregate=eval`); expanding a row + * (`open=`) shows that eval's pass-rate trend, per-check pass rates and newest + * failures, each linking to its trace. Every filter lives in the query string + * (defaults unwritten), so a reload keeps them. A dead daemon shows the shared + * down state with the URL kept and the screen recovers in place. + */ +export function ScoresScreen() { + const f = useScoreFilters(); + const { rates, detail, agents, down, trend } = useScoresData(f); + const rows = useMemo( + () => (rates.data ?? []).filter((r) => f.evals.length === 0 || f.evals.includes(r.name)), + [rates.data, f.evals], + ); + const toggle = (name: string) => f.setOpen(name === f.open ? '' : name); + const facets: FacetDef[] = [ + { key: 'eval', label: 'Eval', options: evalOptions(rates.data, f.evals), value: f.evals, onChange: f.setEvals }, + singleFacet('agent', 'Agent', agentOptions(agents.data, f.agent), f.agent, f.setAgent), + singleFacet('source', 'Source', SCORE_SOURCES.map((s) => ({ value: s, label: s })), f.source, f.setSource), + ]; + + if (down) + return ( +
+ +
+ ); + return ( +
+

Scores

+ + + r.name} + onRowOpen={(r) => toggle(r.name)} + loading={rates.loading && rates.data === undefined} + empty={ + + } + /> + {f.open && } +
+ ); +} diff --git a/apps/agent-forge/src/screens/scores/__tests__/ScoresScreen.test.tsx b/apps/agent-forge/src/screens/scores/__tests__/ScoresScreen.test.tsx new file mode 100644 index 00000000..0e8e0696 --- /dev/null +++ b/apps/agent-forge/src/screens/scores/__tests__/ScoresScreen.test.tsx @@ -0,0 +1,210 @@ +import { act } from 'react'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { ObserveScorePassRate, ObserveScoreRow } from '../../../observe/types'; +import { render } from '../../../ui/__tests__/render'; +import { ScoresScreen } from '../ScoresScreen'; +import { groupEvalScores, passRateSeries, traceHref, trendWindow } from '../scoresData'; + +const NOW = Date.UTC(2026, 9, 25, 12, 30); +const DAY = 86_400_000; + +let seq = 0; +function score(over: Partial & { name: string }): ObserveScoreRow { + seq += 1; + return { + score_id: `s${seq}`, + trace_id: `t${seq}`, + obs_id: null, + value: 1, + source: 'eval', + comment: null, + agent_id: 'a1', + created_at: NOW - DAY, + ...over, + }; +} + +const RATES: ObserveScorePassRate[] = [ + { name: 'support', count: 6, passed: 4, pass_rate: 4 / 6, avg_value: 0.67, last_ts: NOW - 1000 }, + { name: 'billing', count: 2, passed: 2, pass_rate: 1, avg_value: 1, last_ts: NOW - DAY }, +]; + +const SUPPORT_ROWS: ObserveScoreRow[] = [ + score({ name: 'support', value: 1, created_at: NOW - 3 * DAY }), + score({ name: 'support/tone', value: 1, created_at: NOW - 3 * DAY }), + score({ name: 'support/refund', value: 1, created_at: NOW - 3 * DAY }), + score({ name: 'support', value: 0, comment: 'refund', created_at: NOW - 1000, trace_id: 'trace-fail', obs_id: 'obs-9' }), + score({ name: 'support/tone', value: 1, created_at: NOW - 1000 }), + score({ name: 'support/refund', value: 0, comment: 'expected a refund link', created_at: NOW - 1000, trace_id: 'trace-fail' }), +]; + +function json(status: number, body: unknown): Response { + return new Response(JSON.stringify(body), { status, headers: { 'Content-Type': 'application/json' } }); +} + +/** A stand-in daemon behind `/observe`; `down` fails every request like a refused connection. */ +function daemon() { + const state = { down: false }; + const fetchFn = vi.fn(async (input: RequestInfo | URL) => { + if (state.down) throw new TypeError('fetch failed'); + const url = String(input); + const q = new URL(url, 'http://x').searchParams; + if (url.startsWith('/observe/health')) return json(200, { service: 'lousho-observe', ok: true }); + if (url.startsWith('/observe/__target')) return json(200, { url: 'http://127.0.0.1:7777' }); + if (url.startsWith('/observe/api/v1/agents')) return json(200, { agents: [{ agent_id: 'a1', name: 'Support bot' }] }); + if (url.startsWith('/observe/api/v1/scores') && q.get('aggregate') === 'eval') return json(200, { evals: RATES }); + if (url.startsWith('/observe/api/v1/scores')) return json(200, { scores: q.get('eval') === 'support' ? SUPPORT_ROWS : [] }); + return json(404, { error: `unexpected ${url}` }); + }); + vi.stubGlobal('fetch', fetchFn); + const calls = (match: string) => fetchFn.mock.calls.map(([u]) => String(u)).filter((u) => u.includes(match)); + return { state, calls }; +} + +async function waitFor(check: () => boolean, turns = 50) { + for (let i = 0; !check(); i++) { + if (i > turns) throw new Error('waitFor gave up'); + await act(async () => { + await new Promise((r) => setTimeout(r, 0)); + }); + } +} + +const evalRow = (c: HTMLElement, name: string) => + [...c.querySelectorAll('.data-table tbody tr, table tbody tr')].find((tr) => + tr.querySelector('.scores-toggle')?.textContent?.includes(name), + ) as HTMLElement | undefined; +const detail = (c: HTMLElement) => c.querySelector('[data-eval]') as HTMLElement | null; + +beforeEach(() => { + vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval', 'Date'] }); + vi.setSystemTime(NOW); + window.history.replaceState(null, '', '/#/evals/scores'); +}); +afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + window.history.replaceState(null, '', '/'); +}); + +describe('scoresData', () => { + it('splits eval-level rows from per-check groups, worst check first', () => { + const g = groupEvalScores('support', [...SUPPORT_ROWS, score({ name: 'supportive/x' })]); + expect(g.evalRows.map((r) => r.value)).toEqual([1, 0]); + expect(g.checks.map((c) => [c.check, c.count, c.passRate])).toEqual([ + ['refund', 2, 0.5], + ['tone', 2, 1], + ]); + expect(g.failures.map((r) => r.name)).toEqual(['support', 'support/refund']); + }); + + it('pre-aggregates the trend to one mean point per bucket', () => { + const w = trendWindow({ preset: '7d' }, NOW); + const t = NOW - 2 * DAY; + const s = passRateSeries( + [score({ name: 'e', value: 1, created_at: t }), score({ name: 'e', value: 0, created_at: t + 60_000 }), score({ name: 'e', value: null, created_at: t })], + w, + ); + expect(s.points).toEqual([{ t: t - (t % DAY), v: 0.5 }]); + }); + + it('links a score to its trace, and to the step when it has one', () => { + expect(traceHref({ trace_id: 'tr 1', obs_id: null })).toBe('#/traces/tr%201'); + expect(traceHref({ trace_id: 'tr', obs_id: 'o1' })).toBe('#/traces/tr?obs=o1'); + }); +}); + +describe('ScoresScreen', () => { + it('lists one row per eval from the aggregate', async () => { + const { calls } = daemon(); + const c = render(); + await waitFor(() => evalRow(c, 'billing') !== undefined); + const support = evalRow(c, 'support')!; + expect(support.textContent).toContain('66.7%'); + expect(support.textContent).toContain('6'); + expect(calls('aggregate=eval')[0]).toContain(`since=${NOW - 30 * DAY}`); + // Nothing is expanded, so no per-eval rows are fetched. + expect(calls('/api/v1/scores').filter((u) => !u.includes('aggregate'))).toEqual([]); + }); + + it('expands an eval into its runs, per-check pass rates and failures linking to the trace', async () => { + const { calls } = daemon(); + const c = render(); + await waitFor(() => evalRow(c, 'support') !== undefined); + act(() => (evalRow(c, 'support')!.querySelector('.scores-toggle') as HTMLButtonElement).click()); + expect(window.location.hash).toBe('#/evals/scores?open=support'); + await waitFor(() => detail(c)?.querySelector('[data-eval-runs]') != null); + expect(calls('eval=support')[0]).toContain('limit=1000'); + const d = detail(c)!; + expect(d.querySelector('[data-eval-runs]')!.getAttribute('data-eval-runs')).toBe('2'); + expect(d.textContent).toContain('2 runs, 50.0% passed'); + const checks = [...d.querySelectorAll('[aria-label="support checks"] tbody tr')].map((tr) => tr.textContent); + expect(checks[0]).toContain('refund'); + expect(checks[0]).toContain('50.0%'); + expect(checks[1]).toContain('tone'); + const failures = [...d.querySelectorAll('.scores-failure')]; + expect(failures).toHaveLength(2); + expect(failures[0]!.querySelector('a')!.getAttribute('href')).toBe('#/traces/trace-fail?obs=obs-9'); + expect(failures[1]!.querySelector('a')!.getAttribute('href')).toBe('#/traces/trace-fail'); + expect(failures[1]!.textContent).toContain('expected a refund link'); + expect(evalRow(c, 'support')!.querySelector('.scores-toggle')!.getAttribute('aria-expanded')).toBe('true'); + }); + + it('draws the trend with one point per bucket', async () => { + window.history.replaceState(null, '', '/#/evals/scores?open=support'); + daemon(); + const c = render(); + await waitFor(() => detail(c)?.querySelector('.tsc-point') != null); + // Two eval-level rows in two non-adjacent day buckets: two isolated points, no line. + expect(detail(c)!.querySelectorAll('circle.tsc-point[data-series="pass-rate"]')).toHaveLength(2); + expect(detail(c)!.querySelector('.tsc-line')).toBeNull(); + }); + + it('round-trips the filters through the URL and queries with them', async () => { + window.history.replaceState(null, '', '/#/evals/scores?eval=billing&agent=a1&source=human&range=7d'); + const { calls } = daemon(); + const c = render(); + await waitFor(() => evalRow(c, 'billing') !== undefined); + expect(evalRow(c, 'support')).toBeUndefined(); + const call = calls('aggregate=eval').at(-1)!; + expect(call).toContain('agent_id=a1'); + expect(call).toContain('source=human'); + expect(call).toContain(`since=${NOW - 7 * DAY}`); + expect(c.querySelector('.time-range [aria-pressed="true"]')?.textContent).toBe('7d'); + + const preset = (label: string) => + [...c.querySelectorAll('.time-range button')].find((b) => b.textContent === label) as HTMLButtonElement; + act(() => preset('30d').click()); + expect(window.location.hash).toBe('#/evals/scores?eval=billing&agent=a1&source=human'); + act(() => preset('24h').click()); + expect(window.location.hash).toBe('#/evals/scores?eval=billing&agent=a1&source=human&range=24h'); + await waitFor(() => calls(`since=${NOW - DAY}`).length > 0); + }); + + it('ignores an unknown source= value', async () => { + window.history.replaceState(null, '', '/#/evals/scores?source=bogus'); + const { calls } = daemon(); + const c = render(); + await waitFor(() => evalRow(c, 'billing') !== undefined); + expect(calls('aggregate=eval')[0]).not.toContain('source='); + }); + + it('shows the daemon-down state with the URL kept and recovers in place', async () => { + window.history.replaceState(null, '', '/#/evals/scores?range=7d&open=support'); + const { state } = daemon(); + state.down = true; + const c = render(); + await waitFor(() => c.textContent!.includes('lousho observe start')); + expect(window.location.hash).toBe('#/evals/scores?range=7d&open=support'); + + state.down = false; + await act(async () => { + vi.advanceTimersByTime(5000); + }); + await waitFor(() => detail(c)?.querySelector('[data-eval-runs]') != null); + expect(c.textContent).not.toContain('lousho observe start'); + expect(window.location.hash).toBe('#/evals/scores?range=7d&open=support'); + }); +}); diff --git a/apps/agent-forge/src/screens/scores/scores.css b/apps/agent-forge/src/screens/scores/scores.css new file mode 100644 index 00000000..042e02ee --- /dev/null +++ b/apps/agent-forge/src/screens/scores/scores.css @@ -0,0 +1,94 @@ +/* + * Scores screen (#/evals/scores): filters over the per-eval table, then the + * expanded eval's trend, checks and latest failures. Colours from + * src/theme.css tokens only. + */ + +.scores-screen { + display: flex; + flex-direction: column; + gap: 14px; + min-height: 0; + padding: 16px; + font-family: var(--font-ui); + color: var(--text); +} + +.scores-toggle { + font-weight: 600; + text-align: left; +} +.scores-name { + font-family: var(--font-mono); +} +.scores-muted { + margin: 0; + font-size: 12px; + color: var(--text-muted); +} +.scores-link { + font-family: var(--font-mono); + color: var(--text); + text-decoration: underline; +} + +.scores-detail { + display: flex; + flex-direction: column; + gap: 12px; + padding: 12px 16px; + background: var(--surface); + border: 1px solid var(--border); + border-radius: 8px; +} +.scores-detail-title { + margin: 0; + font-family: var(--font-mono); + font-size: 14px; +} +.scores-subtitle { + margin: 0 0 6px; + font-size: 12px; + font-weight: 600; + color: var(--text-muted); + text-transform: uppercase; + letter-spacing: 0.4px; +} + +.scores-failures { + display: flex; + flex-direction: column; + gap: 4px; + margin: 0; + padding: 0; + list-style: none; +} +.scores-failure { + display: flex; + flex-wrap: wrap; + align-items: baseline; + gap: 4px 12px; + padding: 6px 8px; + border-left: 3px solid var(--danger); + background: var(--surface-2); + border-radius: 4px; + font-size: 12px; +} +.scores-when { + color: var(--text-faint); +} +.scores-value { + color: var(--danger); + font-weight: 600; +} +.scores-comment { + flex-basis: 100%; + color: var(--text-muted); + overflow-wrap: anywhere; +} + +@media (max-width: 480px) { + .scores-screen { + padding: 12px; + } +} diff --git a/apps/agent-forge/src/screens/scores/scoresData.ts b/apps/agent-forge/src/screens/scores/scoresData.ts new file mode 100644 index 00000000..6b237fd7 --- /dev/null +++ b/apps/agent-forge/src/screens/scores/scoresData.ts @@ -0,0 +1,135 @@ +/** + * Pure helpers for the Scores screen: the query-string filters, eval-level vs + * per-check grouping of one eval's score rows, the pass-rate trend series and + * the score -> trace deep link. + * + * Score names are `` (the eval-level row each case writes: 1 iff no + * gate failed, else 0 with the failed gates as comment) and + * `/` (one per check). A row passes like the daemon's + * aggregate counts it: no failure comment, or a perfect 1. + */ +import type { ObserveScoreRow } from '../../observe/types'; +import type { TimeSeries } from '../../ui/TimeSeriesChart'; +import { formatTimeRange, parseTimeRange, toWindow, type TimeRange } from '../../ui/TimeRangePicker'; + +/** The score sources the daemon accepts for `source=`. */ +export const SCORE_SOURCES = ['eval', 'judge', 'human'] as const; + +const DEFAULT_RANGE: TimeRange = { preset: '30d' }; +const HOUR = 3_600_000; +const DAY = 86_400_000; +/** Windows up to this long trend by the hour, longer ones by the day. */ +const HOURLY_UP_TO = 2 * DAY; +/** How many failing rows the detail lists. */ +const LATEST_FAILURES = 10; + +/** `?range=`: a TimeRangePicker value; missing or garbage is the 30d default. */ +export const parseRange = (s: string | null): TimeRange => parseTimeRange(s) ?? DEFAULT_RANGE; +/** The default range is not written to the URL. */ +export const formatRange = (r: TimeRange): string | null => + formatTimeRange(r) === formatTimeRange(DEFAULT_RANGE) ? null : formatTimeRange(r); + +/** `?eval=a,b`: the selected eval names. */ +export const parseEvals = (s: string | null): string[] => (s ? s.split(',').filter(Boolean) : []); +export const formatEvals = (v: string[]): string | null => (v.length ? v.join(',') : null); + +/** A free-text param (`agent`, `open`): absent is ''. */ +export const parseText = (s: string | null): string => s ?? ''; +export const formatText = (v: string): string | null => v || null; + +/** `?source=`: one of {@link SCORE_SOURCES}, anything else is "all". */ +export const parseSource = (s: string | null): string => ((SCORE_SOURCES as readonly string[]).includes(s ?? '') ? s! : ''); + +/** Passed the way `?aggregate=eval` counts it. */ +function isPass(row: Pick): boolean { + return !row.comment || row.value === 1; +} + +/** The trace screen deep link for a score (`?obs=` when it scores one step). */ +export function traceHref(row: Pick): string { + const base = `#/traces/${encodeURIComponent(row.trace_id)}`; + return row.obs_id ? `${base}?obs=${encodeURIComponent(row.obs_id)}` : base; +} + +/** Pass rate, count and mean over a set of rows. */ +export interface ScoreStats { + count: number; + passRate: number | null; + mean: number | null; + lastTs: number | null; +} + +export function scoreStats(rows: readonly ObserveScoreRow[]): ScoreStats { + const values = rows.flatMap((r) => (r.value === null ? [] : [r.value])); + return { + count: rows.length, + passRate: rows.length ? rows.filter(isPass).length / rows.length : null, + mean: values.length ? values.reduce((a, b) => a + b, 0) / values.length : null, + lastTs: rows.length ? Math.max(...rows.map((r) => r.created_at)) : null, + }; +} + +/** One check of an eval: its short name (after the '/') and its rows' stats. */ +export interface CheckGroup extends ScoreStats { + name: string; + check: string; +} + +/** One eval's rows split into the eval-level rows (one per case) and per-check groups (worst pass rate first). */ +export interface EvalGroups { + evalRows: ObserveScoreRow[]; + checks: CheckGroup[]; + /** Newest failing rows (eval-level or check), at most {@link LATEST_FAILURES}. */ + failures: ObserveScoreRow[]; +} + +export function groupEvalScores(evalName: string, rows: readonly ObserveScoreRow[]): EvalGroups { + const evalRows: ObserveScoreRow[] = []; + const byCheck = new Map(); + for (const row of rows) { + if (row.name === evalName) evalRows.push(row); + else if (row.name.startsWith(`${evalName}/`)) byCheck.set(row.name, [...(byCheck.get(row.name) ?? []), row]); + } + const checks = [...byCheck.entries()] + .map(([name, list]) => ({ name, check: name.slice(evalName.length + 1), ...scoreStats(list) })) + .sort((a, b) => (a.passRate ?? 1) - (b.passRate ?? 1) || a.check.localeCompare(b.check)); + const failures = rows + .filter((r) => !isPass(r) && (r.name === evalName || r.name.startsWith(`${evalName}/`))) + .sort((a, b) => b.created_at - a.created_at) + .slice(0, LATEST_FAILURES); + return { evalRows, checks, failures }; +} + +/** The trend chart's window: the range's bounds aligned out to whole UTC buckets (hourly up to 2 days, else daily). */ +export interface TrendWindow { + from: number; + to: number; + bucketMs: number; +} + +export function trendWindow(range: TimeRange, now: number): TrendWindow { + const w = toWindow(range, now); + const bucketMs = w.to - w.from <= HOURLY_UP_TO ? HOUR : DAY; + const from = w.from - (w.from % bucketMs); + const to = w.to % bucketMs === 0 ? w.to : w.to - (w.to % bucketMs) + bucketMs; + return { from, to, bucketMs }; +} + +/** + * The eval's pass-rate trend from its eval-level rows: ONE point per bucket, + * the mean value of the rows in it (TimeSeriesChart sums points that share a + * bucket, so this pre-aggregates). Null values are skipped. + */ +export function passRateSeries(evalRows: readonly ObserveScoreRow[], w: TrendWindow): TimeSeries { + const buckets = new Map(); + for (const row of evalRows) { + if (row.value === null || row.created_at < w.from || row.created_at >= w.to) continue; + const start = w.from + Math.floor((row.created_at - w.from) / w.bucketMs) * w.bucketMs; + const b = buckets.get(start) ?? { sum: 0, n: 0 }; + b.sum += row.value; + b.n += 1; + buckets.set(start, b); + } + const points = [...buckets.entries()].sort(([a], [b]) => a - b).map(([t, b]) => ({ t, v: b.sum / b.n })); + return { id: 'pass-rate', label: 'Pass rate', color: 'success', points }; +} diff --git a/apps/agent-forge/src/screens/traces/TraceScreen.tsx b/apps/agent-forge/src/screens/traces/TraceScreen.tsx index 675cb400..5f4e8490 100644 --- a/apps/agent-forge/src/screens/traces/TraceScreen.tsx +++ b/apps/agent-forge/src/screens/traces/TraceScreen.tsx @@ -1,10 +1,9 @@ import { useCallback, useEffect, useMemo, useState } from 'react'; import { ObserveDaemonDown, ObserveError } from '../../observe/chrome'; -import { ObserveDaemonDownError } from '../../observe/observeClient'; import { nestObservations } from '../../observe/tree'; import type { ObserveObservationRow } from '../../observe/types'; -import { useAgents, useDaemonHealth, useTraceDetail } from '../../observe/useObserve'; +import { useAgents, useDaemonDown, useDaemonHealth, useTraceDetail } from '../../observe/useObserve'; import { navigate } from '../../router/router'; import { useQueryState } from '../../router/useRoute'; import { EmptyState } from '../../ui/EmptyState'; @@ -32,15 +31,6 @@ function useNow(running: boolean): number { return now; } -/** Re-runs `retry` every `pollMs` while `active` (the daemon-down state), so the screen recovers in place. */ -function usePollWhile(active: boolean, retry: () => void, pollMs: number): void { - useEffect(() => { - if (!active || pollMs <= 0) return undefined; - const timer = setInterval(retry, pollMs); - return () => clearInterval(timer); - }, [active, retry, pollMs]); -} - function rootOf(rows: ObserveObservationRow[]): ObserveObservationRow | undefined { return nestObservations(rows)[0]?.row; } @@ -65,8 +55,7 @@ function useTrace(traceId: string, pollMs: number) { // A health flip back to "up" re-fetches the trace at once. const detail = useTraceDetail(traceId, health.down); const agents = useAgents(undefined, health.down); - const down = health.down || detail.error instanceof ObserveDaemonDownError; - usePollWhile(down, detail.refresh, pollMs); + const down = useDaemonDown(health.down, [detail], pollMs); const observations = detail.data?.observations; const root = useMemo(() => (observations ? rootOf(observations) : undefined), [observations]); From 1bf5f88b521f8d2b8f04120655d9e2c72b61dfbf Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Sun, 25 Oct 2026 02:10:50 +0300 Subject: [PATCH 2/2] fix(studio): label eval aggregate figures per score; load a newly opened eval The aggregate=eval columns span every check score of an eval, so they read "Score pass rate" / "Mean score" with a tooltip saying so; the expanded panel's per-run figure is labelled "Run pass rate". Switching the expanded eval shows Loading until its own rows land instead of the previous eval's. Co-Authored-By: Claude Opus 5.5 --- apps/agent-forge/src/observe/useObserve.ts | 11 ++++++-- .../src/screens/scores/EvalDetail.tsx | 2 +- .../src/screens/scores/ScoresScreen.tsx | 20 +++++++++++-- .../scores/__tests__/ScoresScreen.test.tsx | 28 +++++++++++++++++-- 4 files changed, 53 insertions(+), 8 deletions(-) diff --git a/apps/agent-forge/src/observe/useObserve.ts b/apps/agent-forge/src/observe/useObserve.ts index 3fbd37bd..89c9111d 100644 --- a/apps/agent-forge/src/observe/useObserve.ts +++ b/apps/agent-forge/src/observe/useObserve.ts @@ -109,16 +109,21 @@ export function useScores(filter: ScoreListFilter, refreshKey?: unknown): Observ return useObserveQuery(() => observeClient.listScores(filter), [JSON.stringify(filter), refreshKey]); } -/** One eval's score rows (`eval=` plus `filter`); no request while `evalName` is empty. */ +/** + * One eval's score rows (`eval=` plus `filter`); no request while + * `evalName` is empty. `data` is only ever rows of the current `evalName`: + * after a switch it stays undefined (loading) until the new eval's rows land. + */ export function useEvalScores( evalName: string, filter: Omit, refreshKey?: unknown ): ObserveQuery { - return useObserveQuery( - () => (evalName ? observeClient.listScores({ ...filter, eval: evalName }) : Promise.resolve(undefined)), + const query = useObserveQuery( + async () => (evalName ? { evalName, rows: await observeClient.listScores({ ...filter, eval: evalName }) } : undefined), [evalName, JSON.stringify(filter), refreshKey] ); + return { ...query, data: query.data?.evalName === evalName ? query.data.rows : undefined }; } export function useScorePassRates( diff --git a/apps/agent-forge/src/screens/scores/EvalDetail.tsx b/apps/agent-forge/src/screens/scores/EvalDetail.tsx index b427fd35..f5300ef1 100644 --- a/apps/agent-forge/src/screens/scores/EvalDetail.tsx +++ b/apps/agent-forge/src/screens/scores/EvalDetail.tsx @@ -71,7 +71,7 @@ export function EvalDetail({ body = ( <>

- {fmtInt(runs.count)} runs, {fmtPercent(runs.passRate)} passed + {fmtInt(runs.count)} runs · Run pass rate {fmtPercent(runs.passRate)} {rows.length >= EVAL_ROW_LIMIT && ` (newest ${fmtInt(EVAL_ROW_LIMIT)} scores only)`}

diff --git a/apps/agent-forge/src/screens/scores/ScoresScreen.tsx b/apps/agent-forge/src/screens/scores/ScoresScreen.tsx index 0118d368..90b865e8 100644 --- a/apps/agent-forge/src/screens/scores/ScoresScreen.tsx +++ b/apps/agent-forge/src/screens/scores/ScoresScreen.tsx @@ -62,6 +62,9 @@ function evalOptions(rates: ObserveScorePassRate[] | undefined, selected: string return [...names].sort().map((name) => ({ value: name, label: name, count: count.get(name) })); } +/** The aggregate folds every score of an eval, so its figures are per score, not per run. */ +const AGGREGATE_NOTE = "Across every score of the eval: each run's eval-level score plus all its check scores. Expand the eval for the run pass rate."; + function evalColumns(open: string, toggle: (name: string) => void): Column[] { return [ { @@ -82,9 +85,22 @@ function evalColumns(open: string, toggle: (name: string) => void): Column ), }, - { key: 'pass', header: 'Pass rate', align: 'right', render: (r) => fmtPercent(r.pass_rate) }, + { + key: 'pass', + header: Score pass rate, + label: 'Score pass rate', + align: 'right', + render: (r) => fmtPercent(r.pass_rate), + }, { key: 'count', header: 'Scores', align: 'right', render: (r) => fmtInt(r.count) }, - { key: 'mean', header: 'Mean', align: 'right', hideBelow: 480, render: (r) => (r.avg_value === null ? '-' : r.avg_value.toFixed(2)) }, + { + key: 'mean', + header: Mean score, + label: 'Mean score', + align: 'right', + hideBelow: 480, + render: (r) => (r.avg_value === null ? '-' : r.avg_value.toFixed(2)), + }, { key: 'last', header: 'Last seen', hideBelow: 560, render: (r) => fmtDateTime(r.last_ts) }, ]; } diff --git a/apps/agent-forge/src/screens/scores/__tests__/ScoresScreen.test.tsx b/apps/agent-forge/src/screens/scores/__tests__/ScoresScreen.test.tsx index 0e8e0696..ff267beb 100644 --- a/apps/agent-forge/src/screens/scores/__tests__/ScoresScreen.test.tsx +++ b/apps/agent-forge/src/screens/scores/__tests__/ScoresScreen.test.tsx @@ -45,7 +45,7 @@ function json(status: number, body: unknown): Response { /** A stand-in daemon behind `/observe`; `down` fails every request like a refused connection. */ function daemon() { - const state = { down: false }; + const state: { down: boolean; hold?: Promise } = { down: false }; const fetchFn = vi.fn(async (input: RequestInfo | URL) => { if (state.down) throw new TypeError('fetch failed'); const url = String(input); @@ -54,6 +54,7 @@ function daemon() { if (url.startsWith('/observe/__target')) return json(200, { url: 'http://127.0.0.1:7777' }); if (url.startsWith('/observe/api/v1/agents')) return json(200, { agents: [{ agent_id: 'a1', name: 'Support bot' }] }); if (url.startsWith('/observe/api/v1/scores') && q.get('aggregate') === 'eval') return json(200, { evals: RATES }); + if (url.startsWith('/observe/api/v1/scores') && state.hold) await state.hold; if (url.startsWith('/observe/api/v1/scores')) return json(200, { scores: q.get('eval') === 'support' ? SUPPORT_ROWS : [] }); return json(404, { error: `unexpected ${url}` }); }); @@ -122,8 +123,14 @@ describe('ScoresScreen', () => { const c = render(); await waitFor(() => evalRow(c, 'billing') !== undefined); const support = evalRow(c, 'support')!; + // The aggregate spans every check score, so its columns say "score", not "run". expect(support.textContent).toContain('66.7%'); expect(support.textContent).toContain('6'); + const headers = [...c.querySelectorAll('thead th')].map((th) => th.textContent); + expect(headers).toEqual(expect.arrayContaining(['Score pass rate', 'Scores', 'Mean score'])); + expect(headers).not.toContain('Pass rate'); + const passHeader = [...c.querySelectorAll('thead th span[title]')].find((el) => el.textContent === 'Score pass rate')!; + expect(passHeader.getAttribute('title')).toContain('every score of the eval'); expect(calls('aggregate=eval')[0]).toContain(`since=${NOW - 30 * DAY}`); // Nothing is expanded, so no per-eval rows are fetched. expect(calls('/api/v1/scores').filter((u) => !u.includes('aggregate'))).toEqual([]); @@ -139,7 +146,7 @@ describe('ScoresScreen', () => { expect(calls('eval=support')[0]).toContain('limit=1000'); const d = detail(c)!; expect(d.querySelector('[data-eval-runs]')!.getAttribute('data-eval-runs')).toBe('2'); - expect(d.textContent).toContain('2 runs, 50.0% passed'); + expect(d.textContent).toContain('2 runs · Run pass rate 50.0%'); const checks = [...d.querySelectorAll('[aria-label="support checks"] tbody tr')].map((tr) => tr.textContent); expect(checks[0]).toContain('refund'); expect(checks[0]).toContain('50.0%'); @@ -152,6 +159,23 @@ describe('ScoresScreen', () => { expect(evalRow(c, 'support')!.querySelector('.scores-toggle')!.getAttribute('aria-expanded')).toBe('true'); }); + it('shows the loading state, not the previous eval, while a newly opened eval loads', async () => { + window.history.replaceState(null, '', '/#/evals/scores?open=support'); + const { state } = daemon(); + const c = render(); + await waitFor(() => detail(c)?.querySelector('[data-eval-runs]') != null); + let release = () => {}; + state.hold = new Promise((r) => (release = r)); + act(() => (evalRow(c, 'billing')!.querySelector('.scores-toggle') as HTMLButtonElement).click()); + await waitFor(() => detail(c)?.getAttribute('data-eval') === 'billing', 5); + expect(detail(c)!.textContent).toContain('Loading billing...'); + expect(detail(c)!.querySelector('[data-eval-runs]')).toBeNull(); + state.hold = undefined; + release(); + await waitFor(() => detail(c)?.querySelector('[data-eval-runs]') != null); + expect(detail(c)!.textContent).toContain('0 runs'); + }); + it('draws the trend with one point per bucket', async () => { window.history.replaceState(null, '', '/#/evals/scores?open=support'); daemon();