diff --git a/apps/agent-forge/e2e/a11y.spec.ts b/apps/agent-forge/e2e/a11y.spec.ts index 3b8918af0..e03dfbb82 100644 --- a/apps/agent-forge/e2e/a11y.spec.ts +++ b/apps/agent-forge/e2e/a11y.spec.ts @@ -113,7 +113,7 @@ test('Ctrl+K opens the command palette; ? lists the shortcuts', async ({ page }) }); // One
, one h1, the banner and the primary nav on every screen, not just Agents. -for (const route of ['#/traces', '#/monitoring', '#/settings', '#/playground']) { +for (const route of ['#/traces', '#/traces/e2e-trace?obs=e2e-obs', '#/monitoring', '#/settings', '#/playground']) { test(`${route} keeps the landmarks and exactly one main`, async ({ page }) => { await page.goto(`/?token=e2e-studio-token${route}`); await expect(page.locator('.screen-main')).toBeVisible(); @@ -126,6 +126,16 @@ for (const route of ['#/traces', '#/monitoring', '#/settings', '#/playground']) }); } +test('a #/traces/:id deep link with no daemon shows the down state and keeps the URL', async ({ page }) => { + // Answer like the studio proxy does for a dead daemon, whatever runs on the machine. + await page.route('**/observe/**', (route) => + route.fulfill({ status: 502, contentType: 'application/json', body: '{"error":"down"}' }), + ); + await page.goto('/?token=e2e-studio-token#/traces/e2e-trace?obs=e2e-obs'); + await expect(page.getByRole('main').getByText('lousho observe start')).toBeVisible(); + expect(new URL(page.url()).hash).toBe('#/traces/e2e-trace?obs=e2e-obs'); +}); + test('#/settings shows the settings panel inside the screen', async ({ page }) => { await page.goto('/?token=e2e-studio-token#/settings'); const main = page.getByRole('main'); diff --git a/apps/agent-forge/server/__tests__/observeProxy.test.ts b/apps/agent-forge/server/__tests__/observeProxy.test.ts index aa026b467..69d855773 100644 --- a/apps/agent-forge/server/__tests__/observeProxy.test.ts +++ b/apps/agent-forge/server/__tests__/observeProxy.test.ts @@ -1,8 +1,9 @@ /** * `/observe/**` -> the observe daemon: the studio server forwards the * panels' GETs same-origin, because the daemon itself serves no CORS - * headers. A dead daemon answers 502 with the `lousho observe start` hint, - * and non-GET methods stay reporter-only (405). + * headers. A dead daemon answers 502 with the `lousho observe start` hint; + * writes are forwarded only for the allowlist (feedback scores), every other + * non-GET stays reporter-only (405). */ import { describe, it, expect, beforeEach, afterEach } from 'vitest'; import * as http from 'node:http'; @@ -38,8 +39,19 @@ describe('observe proxy', () => { // A stand-in for `lousho observe`: echoes the path it was asked for. daemon = http.createServer((req, res) => { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ echoed: req.url })); + let body = ''; + req.on('data', (chunk: Buffer) => (body += chunk.toString())); + req.on('end', () => { + res.writeHead(req.method === 'POST' ? 201 : 200, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ + echoed: req.url, + method: req.method, + contentType: req.headers['content-type'] ?? null, + body: body ? JSON.parse(body) : null, + }), + ); + }); }); daemon.listen(0, '127.0.0.1'); await once(daemon, 'listening'); @@ -74,8 +86,17 @@ describe('observe proxy', () => { expect(res.body.error).toContain('lousho observe start'); }); - it('does not forward writes - Studio is read-only over the daemon API', async () => { - const res = await request(app).post('/observe/api/v1/ingest').send({ batch: [] }); - expect(res.status).toBe(405); + it('forwards POST /api/v1/scores with its JSON body (the write allowlist)', async () => { + const score = { trace_id: 't1', obs_id: 'o1', name: 'feedback', value: 1, comment: 'good' }; + const res = await request(app).post('/observe/api/v1/scores').send(score); + expect(res.status).toBe(201); + expect(res.body).toEqual({ echoed: '/api/v1/scores', method: 'POST', contentType: 'application/json', body: score }); + }); + + it('does not forward other writes - ingest and unlisted routes stay 405', async () => { + expect((await request(app).post('/observe/api/v1/ingest').send({ batch: [] })).status).toBe(405); + expect((await request(app).post('/observe/api/v1/traces').send({})).status).toBe(405); + expect((await request(app).put('/observe/api/v1/scores').send({})).status).toBe(405); + expect((await request(app).delete('/observe/api/v1/scores')).status).toBe(405); }); }); diff --git a/apps/agent-forge/server/app.ts b/apps/agent-forge/server/app.ts index 6499f183c..487904945 100644 --- a/apps/agent-forge/server/app.ts +++ b/apps/agent-forge/server/app.ts @@ -18,6 +18,7 @@ * POST /runs/:id/fork -> body: ForkRunRequest; starts the fork, 202 ForkRunResponse * GET /runs/compare?a=&b= -> RunComparisonPayload (compareTrajectories) * GET /observe/** -> proxied to the `lousho observe` daemon's read API (observeProxy.ts) + * POST /observe/api/v1/scores -> proxied too (the proxy's write allowlist: human feedback scores) * * `WS /agents/:id/stream` is wired separately in wsServer.ts (attached to * the same underlying http.Server, since `ws` needs the raw HTTP upgrade @@ -667,8 +668,9 @@ export function createApp({ registerProfileRoutes(app, settings); registerDeployRoutes(app, agentStore, baseDir); // The observe panels' read path: `/observe/**` -> the `lousho observe` - // daemon (see observeProxy.ts) - before the SPA fallback so a proxied - // GET never gets index.html. + // daemon (see observeProxy.ts), plus its small write allowlist (feedback + // scores; bodies come from the express.json above) - before the SPA + // fallback so a proxied GET never gets index.html. registerObserveProxy(app); registerStaticClient(app, staticDir); diff --git a/apps/agent-forge/server/observeProxy.ts b/apps/agent-forge/server/observeProxy.ts index 309234ff2..cdb03e040 100644 --- a/apps/agent-forge/server/observeProxy.ts +++ b/apps/agent-forge/server/observeProxy.ts @@ -8,7 +8,10 @@ * the file. `LOUSHO_OBSERVE_URL` repoints the upstream (another port, or a * hosted collector implementing the same ingest/read API). * - * Only GET is forwarded: Studio is a read surface; the ingest endpoint stays + * GET is forwarded for every path; writes only for the explicit + * {@link WRITE_ROUTES} allowlist (today: human feedback scores), with the + * JSON body app.ts's `express.json` (2 MB cap) already parsed. Everything + * else non-GET - the ingest endpoint above all - answers 405 and stays * reporter-only. `/observe` sits in app.ts's API_PREFIXES, so it carries the * same per-launch token requirement as the rest of the studio API. */ @@ -18,14 +21,22 @@ const DEFAULT_OBSERVE_URL = 'http://127.0.0.1:7777'; const PROXY_PREFIX = '/observe'; const UPSTREAM_TIMEOUT_MS = 8000; +/** The daemon writes Studio may forward (method + path below `/observe`). Add one entry per new write surface. */ +const WRITE_ROUTES: { method: 'POST' | 'PUT'; path: RegExp }[] = [{ method: 'POST', path: /^\/api\/v1\/scores$/ }]; + +function isAllowedWrite(method: string, path: string): boolean { + return WRITE_ROUTES.some((route) => route.method === method && route.path.test(path)); +} + function observeTarget(): string { return (process.env.LOUSHO_OBSERVE_URL ?? DEFAULT_OBSERVE_URL).replace(/\/+$/, ''); } export function registerObserveProxy(app: Express): void { app.use(PROXY_PREFIX, async (req, res) => { - if (req.method !== 'GET') { - res.status(405).json({ error: 'The observe proxy only forwards GET reads.' }); + const write = req.method !== 'GET'; + if (write && !isAllowedWrite(req.method, req.path)) { + res.status(405).json({ error: 'The observe proxy only forwards GET reads and allowlisted writes.' }); return; } // Which daemon this proxy points at, so the down state can name the right LOUSHO_OBSERVE value. @@ -37,8 +48,9 @@ export function registerObserveProxy(app: Express): void { let response: Response; try { response = await fetch(upstream, { - method: 'GET', - headers: { accept: 'application/json' }, + method: req.method, + headers: write ? { accept: 'application/json', 'content-type': 'application/json' } : { accept: 'application/json' }, + body: write ? JSON.stringify(req.body ?? {}) : undefined, signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS), }); } catch { diff --git a/apps/agent-forge/src/observe/__tests__/observeClient.test.ts b/apps/agent-forge/src/observe/__tests__/observeClient.test.ts index 2a2a0d4d7..f2a2f6982 100644 --- a/apps/agent-forge/src/observe/__tests__/observeClient.test.ts +++ b/apps/agent-forge/src/observe/__tests__/observeClient.test.ts @@ -68,4 +68,26 @@ describe('ObserveClient', () => { window.sessionStorage.removeItem('lousho-studio-token'); } }); + + it('postScore POSTs the JSON body to /api/v1/scores and returns the score id', async () => { + const fetchFn = vi.fn(async (_url: unknown, init?: RequestInit) => { + void init; + return jsonResponse(201, { score_id: 'sc1' }); + }); + const client = clientWith(fetchFn as typeof fetch); + const body = { trace_id: 't1', obs_id: 'o1', name: 'feedback', value: 1, comment: 'nice' }; + expect(await client.postScore(body)).toEqual({ score_id: 'sc1' }); + const [url, init] = fetchFn.mock.calls[0] as [string, RequestInit]; + expect(url).toBe('/observe/api/v1/scores'); + expect(init.method).toBe('POST'); + expect(new Headers(init.headers).get('content-type')).toBe('application/json'); + expect(JSON.parse(init.body as string)).toEqual(body); + }); + + it('postScore keeps the error semantics: 404 is an ObserveApiError, 502 is daemon down', async () => { + const notFound = clientWith((async () => jsonResponse(404, { error: "No trace 'x'." })) as typeof fetch); + await expect(notFound.postScore({ trace_id: 'x', name: 'feedback', value: 0 })).rejects.toBeInstanceOf(ObserveApiError); + const down = clientWith((async () => jsonResponse(502, {})) as typeof fetch); + await expect(down.postScore({ trace_id: 'x', name: 'feedback', value: 0 })).rejects.toBeInstanceOf(ObserveDaemonDownError); + }); }); diff --git a/apps/agent-forge/src/observe/observeClient.ts b/apps/agent-forge/src/observe/observeClient.ts index fd13207b6..c4c5e5dee 100644 --- a/apps/agent-forge/src/observe/observeClient.ts +++ b/apps/agent-forge/src/observe/observeClient.ts @@ -77,6 +77,15 @@ export interface ScoreListFilter { limit?: number; } +/** The body of {@link ObserveClient.postScore}: `obs_id` omitted scores the whole trace. */ +export interface NewScore { + trace_id: string; + obs_id?: string; + name: string; + value: number | null; + comment?: string; +} + export interface InsightsFilter { days?: number; since?: number; @@ -106,7 +115,10 @@ export class ObserveClient { return `${this.baseUrl}${path}${qs ? `?${qs}` : ''}`; } - private async request(path: string, query?: Record): Promise { + private async request( + path: string, + opts: { query?: Record; method?: 'GET' | 'POST'; body?: unknown } = {} + ): Promise { const headers = new Headers({ accept: 'application/json' }); // The studio token only travels same-origin (to the studio server's // proxy, which requires it); a configured remote collector gets no header. @@ -114,9 +126,15 @@ export class ObserveClient { const token = loadStudioToken(); if (token) headers.set(STUDIO_TOKEN_HEADER, token); } + const init: RequestInit = { headers }; + if (opts.method && opts.method !== 'GET') init.method = opts.method; + if (opts.body !== undefined) { + headers.set('content-type', 'application/json'); + init.body = JSON.stringify(opts.body); + } let res: Response; try { - res = await this.fetchFn(this.url(path, query), { headers }); + res = await this.fetchFn(this.url(path, opts.query), init); } catch { throw new ObserveDaemonDownError(); } @@ -152,7 +170,7 @@ export class ObserveClient { /** `GET /api/v1/traces` - the paginated, filterable trace list. */ listTraces(filter: TraceListFilter = {}): Promise { - return this.request('/api/v1/traces', { ...filter }); + return this.request('/api/v1/traces', { query: { ...filter } }); } /** `GET /api/v1/traces/:id` - flat observation rows + scores; `undefined` on the daemon's 404. */ @@ -167,20 +185,25 @@ export class ObserveClient { /** `GET /api/v1/agents` - the fleet registry with read-time freshness. */ async listAgents(project?: string): Promise { - const body = await this.request<{ agents: ObserveAgentRow[] }>('/api/v1/agents', { project }); + const body = await this.request<{ agents: ObserveAgentRow[] }>('/api/v1/agents', { query: { project } }); return body.agents; } /** `GET /api/v1/insights` - per-agent cards + rollup series. */ insights(filter: InsightsFilter = {}): Promise { - return this.request('/api/v1/insights', { ...filter }); + return this.request('/api/v1/insights', { query: { ...filter } }); } /** `GET /api/v1/scores` - eval/judge/human score rows. */ async listScores(filter: ScoreListFilter = {}): Promise { - const body = await this.request<{ scores: ObserveScoreRow[] }>('/api/v1/scores', { ...filter }); + const body = await this.request<{ scores: ObserveScoreRow[] }>('/api/v1/scores', { query: { ...filter } }); return body.scores; } + + /** `POST /api/v1/scores` - a human feedback score (append-only); the daemon's 404/400 surface as `ObserveApiError`. */ + postScore(score: NewScore): Promise<{ score_id: string }> { + return this.request('/api/v1/scores', { method: 'POST', body: score }); + } } export const observeClient = new ObserveClient(); diff --git a/apps/agent-forge/src/screens/__tests__/routes.test.tsx b/apps/agent-forge/src/screens/__tests__/routes.test.tsx index 716100f04..660b09c65 100644 --- a/apps/agent-forge/src/screens/__tests__/routes.test.tsx +++ b/apps/agent-forge/src/screens/__tests__/routes.test.tsx @@ -59,7 +59,7 @@ describe('screenFor', () => { expect(screenFor('#/agents')).toBe('agents'); expect(screenFor('#/agents/support-bot')).toBe('agents'); expect(screenFor('#/traces')).toBe('traces'); - expect(screenFor('#/traces/abc?obs=1')).toBe('traces'); + expect(screenFor('#/traces/abc?obs=1')).toBe('trace'); expect(screenFor('#/monitoring?window=24h')).toBe('monitoring'); expect(screenFor('#/evals/scores')).toBe('scores'); expect(screenFor('#/evals/datasets')).toBe('evals'); diff --git a/apps/agent-forge/src/screens/routes.tsx b/apps/agent-forge/src/screens/routes.tsx index 3ffe22768..8b6a1cf49 100644 --- a/apps/agent-forge/src/screens/routes.tsx +++ b/apps/agent-forge/src/screens/routes.tsx @@ -5,6 +5,7 @@ import { Topbar } from '../components/Topbar'; import { SettingsPanel } from '../components/SettingsPanel'; import { ObserveView, type ObserveTab } from '../observe/ObserveView'; import { AgentsScreen } from './agents/AgentsScreen'; +import { TraceScreen } from './traces/TraceScreen'; import { useLeaveAgentsGuard } from './agents/useLeaveAgentsGuard'; import { Placeholder } from './Placeholder'; @@ -12,6 +13,7 @@ export type ScreenId = | 'agents' | 'playground' | 'traces' + | 'trace' | 'threads' | 'monitoring' | 'scores' @@ -26,7 +28,7 @@ const ROUTES: { pattern: string; screen: ScreenId }[] = [ { pattern: '/agents/:id', screen: 'agents' }, { pattern: '/playground', screen: 'playground' }, { pattern: '/traces', screen: 'traces' }, - { pattern: '/traces/:traceId', screen: 'traces' }, + { pattern: '/traces/:traceId', screen: 'trace' }, { pattern: '/threads', screen: 'threads' }, { pattern: '/threads/:sessionId', screen: 'threads' }, { pattern: '/monitoring', screen: 'monitoring' }, @@ -80,6 +82,13 @@ const PLACEHOLDERS: Partial> hub: { title: 'Context Hub', blurb: 'The shared context namespaces your agents read from, browsable by namespace.' }, }; +/** `#/traces/:traceId`: the trace id comes from the hash; a new id remounts the screen. */ +function TraceRoute() { + const { path } = useRoute(); + const traceId = matchRoute(`#${path}`, ['/traces/:traceId'])?.params.traceId ?? ''; + return ; +} + /** Every screen but Agents: the shell top bar over the screen's content. */ function ScreenFrame({ children }: { children: ReactNode }) { return ( @@ -100,7 +109,8 @@ export function Screens() { const placeholder = PLACEHOLDERS[screen]; let content: ReactNode; // Same element type at the same position for all three, so the drill-down filter survives a route change. - if (observeTab) content = navigate(TAB_PATH[tab])} />; + if (screen === 'trace') content = ; + else if (observeTab) content = navigate(TAB_PATH[tab])} />; else if (placeholder) content = ; else content = ( diff --git a/apps/agent-forge/src/screens/traces/TraceFeedback.tsx b/apps/agent-forge/src/screens/traces/TraceFeedback.tsx new file mode 100644 index 000000000..4861db16d --- /dev/null +++ b/apps/agent-forge/src/screens/traces/TraceFeedback.tsx @@ -0,0 +1,85 @@ +import { useState, type FormEvent } from 'react'; + +import { observeClient } from '../../observe/observeClient'; +import type { ObserveObservationRow } from '../../observe/types'; +import { useToast } from '../../ui/Toast'; + +/** The score name human thumbs up/down feedback is stored under. */ +const FEEDBACK_SCORE_NAME = 'feedback'; + +const CHOICES: { value: 1 | 0; label: string; glyph: string }[] = [ + { value: 1, label: 'Thumbs up', glyph: '\u{1F44D}' }, + { value: 0, label: 'Thumbs down', glyph: '\u{1F44E}' }, +]; + +/** + * Human feedback on one observation: thumbs up (1) / down (0) plus an + * optional comment, posted as a `feedback` score (`POST /api/v1/scores`). + * A root observation's feedback scores the whole trace (no `obs_id`). + * `onSaved` runs after a successful write so the caller can refetch. + */ +export function TraceFeedback({ observation, onSaved }: { observation: ObserveObservationRow; onSaved: () => void }) { + const toast = useToast(); + const [value, setValue] = useState<1 | 0 | undefined>(undefined); + const [comment, setComment] = useState(''); + const [saving, setSaving] = useState(false); + + const submit = (e: FormEvent) => { + e.preventDefault(); + if (value === undefined || saving) return; + const text = comment.trim(); + setSaving(true); + observeClient + .postScore({ + trace_id: observation.trace_id, + ...(observation.parent_obs_id === null ? {} : { obs_id: observation.obs_id }), + name: FEEDBACK_SCORE_NAME, + value, + ...(text ? { comment: text } : {}), + }) + .then( + () => { + toast.show({ message: 'Feedback saved', tone: 'success' }); + setValue(undefined); + setComment(''); + onSaved(); + }, + (error: unknown) => { + const reason = error instanceof Error ? error.message : String(error); + toast.show({ message: `Could not save feedback: ${reason}`, tone: 'error' }); + }, + ) + .finally(() => setSaving(false)); + }; + + return ( +
+
+ {CHOICES.map((choice) => ( + + ))} +
+