Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion apps/agent-forge/e2e/a11y.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ test('Ctrl+K opens the command palette; ? lists the shortcuts', async ({ page })
});

// One <main>, one h1, the banner and the primary nav on every screen, not just Agents.
for (const route of ['#/traces', '#/monitoring', '#/settings', '#/playground']) {
for (const route of ['#/traces', '#/traces/e2e-trace?obs=e2e-obs', '#/monitoring', '#/settings', '#/playground']) {
test(`${route} keeps the landmarks and exactly one main`, async ({ page }) => {
await page.goto(`/?token=e2e-studio-token${route}`);
await expect(page.locator('.screen-main')).toBeVisible();
Expand All @@ -126,6 +126,16 @@ for (const route of ['#/traces', '#/monitoring', '#/settings', '#/playground'])
});
}

test('a #/traces/:id deep link with no daemon shows the down state and keeps the URL', async ({ page }) => {
// Answer like the studio proxy does for a dead daemon, whatever runs on the machine.
await page.route('**/observe/**', (route) =>
route.fulfill({ status: 502, contentType: 'application/json', body: '{"error":"down"}' }),
);
await page.goto('/?token=e2e-studio-token#/traces/e2e-trace?obs=e2e-obs');
await expect(page.getByRole('main').getByText('lousho observe start')).toBeVisible();
expect(new URL(page.url()).hash).toBe('#/traces/e2e-trace?obs=e2e-obs');
});

test('#/settings shows the settings panel inside the screen', async ({ page }) => {
await page.goto('/?token=e2e-studio-token#/settings');
const main = page.getByRole('main');
Expand Down
35 changes: 28 additions & 7 deletions apps/agent-forge/server/__tests__/observeProxy.test.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
/**
* `/observe/**` -> the observe daemon: the studio server forwards the
* panels' GETs same-origin, because the daemon itself serves no CORS
* headers. A dead daemon answers 502 with the `lousho observe start` hint,
* and non-GET methods stay reporter-only (405).
* headers. A dead daemon answers 502 with the `lousho observe start` hint;
* writes are forwarded only for the allowlist (feedback scores), every other
* non-GET stays reporter-only (405).
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import * as http from 'node:http';
Expand Down Expand Up @@ -38,8 +39,19 @@ describe('observe proxy', () => {

// A stand-in for `lousho observe`: echoes the path it was asked for.
daemon = http.createServer((req, res) => {
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ echoed: req.url }));
let body = '';
req.on('data', (chunk: Buffer) => (body += chunk.toString()));
req.on('end', () => {
res.writeHead(req.method === 'POST' ? 201 : 200, { 'Content-Type': 'application/json' });
res.end(
JSON.stringify({
echoed: req.url,
method: req.method,
contentType: req.headers['content-type'] ?? null,
body: body ? JSON.parse(body) : null,
}),
);
});
});
daemon.listen(0, '127.0.0.1');
await once(daemon, 'listening');
Expand Down Expand Up @@ -74,8 +86,17 @@ describe('observe proxy', () => {
expect(res.body.error).toContain('lousho observe start');
});

it('does not forward writes - Studio is read-only over the daemon API', async () => {
const res = await request(app).post('/observe/api/v1/ingest').send({ batch: [] });
expect(res.status).toBe(405);
it('forwards POST /api/v1/scores with its JSON body (the write allowlist)', async () => {
const score = { trace_id: 't1', obs_id: 'o1', name: 'feedback', value: 1, comment: 'good' };
const res = await request(app).post('/observe/api/v1/scores').send(score);
expect(res.status).toBe(201);
expect(res.body).toEqual({ echoed: '/api/v1/scores', method: 'POST', contentType: 'application/json', body: score });
});

it('does not forward other writes - ingest and unlisted routes stay 405', async () => {
expect((await request(app).post('/observe/api/v1/ingest').send({ batch: [] })).status).toBe(405);
expect((await request(app).post('/observe/api/v1/traces').send({})).status).toBe(405);
expect((await request(app).put('/observe/api/v1/scores').send({})).status).toBe(405);
expect((await request(app).delete('/observe/api/v1/scores')).status).toBe(405);
});
});
6 changes: 4 additions & 2 deletions apps/agent-forge/server/app.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
* POST /runs/:id/fork -> body: ForkRunRequest; starts the fork, 202 ForkRunResponse
* GET /runs/compare?a=&b= -> RunComparisonPayload (compareTrajectories)
* GET /observe/** -> proxied to the `lousho observe` daemon's read API (observeProxy.ts)
* POST /observe/api/v1/scores -> proxied too (the proxy's write allowlist: human feedback scores)
*
* `WS /agents/:id/stream` is wired separately in wsServer.ts (attached to
* the same underlying http.Server, since `ws` needs the raw HTTP upgrade
Expand Down Expand Up @@ -667,8 +668,9 @@ export function createApp({
registerProfileRoutes(app, settings);
registerDeployRoutes(app, agentStore, baseDir);
// The observe panels' read path: `/observe/**` -> the `lousho observe`
// daemon (see observeProxy.ts) - before the SPA fallback so a proxied
// GET never gets index.html.
// daemon (see observeProxy.ts), plus its small write allowlist (feedback
// scores; bodies come from the express.json above) - before the SPA
// fallback so a proxied GET never gets index.html.
registerObserveProxy(app);
registerStaticClient(app, staticDir);

Expand Down
22 changes: 17 additions & 5 deletions apps/agent-forge/server/observeProxy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,10 @@
* the file. `LOUSHO_OBSERVE_URL` repoints the upstream (another port, or a
* hosted collector implementing the same ingest/read API).
*
* Only GET is forwarded: Studio is a read surface; the ingest endpoint stays
* GET is forwarded for every path; writes only for the explicit
* {@link WRITE_ROUTES} allowlist (today: human feedback scores), with the
* JSON body app.ts's `express.json` (2 MB cap) already parsed. Everything
* else non-GET - the ingest endpoint above all - answers 405 and stays
* reporter-only. `/observe` sits in app.ts's API_PREFIXES, so it carries the
* same per-launch token requirement as the rest of the studio API.
*/
Expand All @@ -18,14 +21,22 @@ const DEFAULT_OBSERVE_URL = 'http://127.0.0.1:7777';
const PROXY_PREFIX = '/observe';
const UPSTREAM_TIMEOUT_MS = 8000;

/** The daemon writes Studio may forward (method + path below `/observe`). Add one entry per new write surface. */
const WRITE_ROUTES: { method: 'POST' | 'PUT'; path: RegExp }[] = [{ method: 'POST', path: /^\/api\/v1\/scores$/ }];

function isAllowedWrite(method: string, path: string): boolean {
return WRITE_ROUTES.some((route) => route.method === method && route.path.test(path));
}

function observeTarget(): string {
return (process.env.LOUSHO_OBSERVE_URL ?? DEFAULT_OBSERVE_URL).replace(/\/+$/, '');
}

export function registerObserveProxy(app: Express): void {
app.use(PROXY_PREFIX, async (req, res) => {
if (req.method !== 'GET') {
res.status(405).json({ error: 'The observe proxy only forwards GET reads.' });
const write = req.method !== 'GET';
if (write && !isAllowedWrite(req.method, req.path)) {
res.status(405).json({ error: 'The observe proxy only forwards GET reads and allowlisted writes.' });
return;
}
// Which daemon this proxy points at, so the down state can name the right LOUSHO_OBSERVE value.
Expand All @@ -37,8 +48,9 @@ export function registerObserveProxy(app: Express): void {
let response: Response;
try {
response = await fetch(upstream, {
method: 'GET',
headers: { accept: 'application/json' },
method: req.method,
headers: write ? { accept: 'application/json', 'content-type': 'application/json' } : { accept: 'application/json' },
body: write ? JSON.stringify(req.body ?? {}) : undefined,
signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS),
});
} catch {
Expand Down
22 changes: 22 additions & 0 deletions apps/agent-forge/src/observe/__tests__/observeClient.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -68,4 +68,26 @@ describe('ObserveClient', () => {
window.sessionStorage.removeItem('lousho-studio-token');
}
});

it('postScore POSTs the JSON body to /api/v1/scores and returns the score id', async () => {
const fetchFn = vi.fn(async (_url: unknown, init?: RequestInit) => {
void init;
return jsonResponse(201, { score_id: 'sc1' });
});
const client = clientWith(fetchFn as typeof fetch);
const body = { trace_id: 't1', obs_id: 'o1', name: 'feedback', value: 1, comment: 'nice' };
expect(await client.postScore(body)).toEqual({ score_id: 'sc1' });
const [url, init] = fetchFn.mock.calls[0] as [string, RequestInit];
expect(url).toBe('/observe/api/v1/scores');
expect(init.method).toBe('POST');
expect(new Headers(init.headers).get('content-type')).toBe('application/json');
expect(JSON.parse(init.body as string)).toEqual(body);
});

it('postScore keeps the error semantics: 404 is an ObserveApiError, 502 is daemon down', async () => {
const notFound = clientWith((async () => jsonResponse(404, { error: "No trace 'x'." })) as typeof fetch);
await expect(notFound.postScore({ trace_id: 'x', name: 'feedback', value: 0 })).rejects.toBeInstanceOf(ObserveApiError);
const down = clientWith((async () => jsonResponse(502, {})) as typeof fetch);
await expect(down.postScore({ trace_id: 'x', name: 'feedback', value: 0 })).rejects.toBeInstanceOf(ObserveDaemonDownError);
});
});
35 changes: 29 additions & 6 deletions apps/agent-forge/src/observe/observeClient.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,15 @@ export interface ScoreListFilter {
limit?: number;
}

/** The body of {@link ObserveClient.postScore}: `obs_id` omitted scores the whole trace. */
export interface NewScore {
trace_id: string;
obs_id?: string;
name: string;
value: number | null;
comment?: string;
}

export interface InsightsFilter {
days?: number;
since?: number;
Expand Down Expand Up @@ -106,17 +115,26 @@ export class ObserveClient {
return `${this.baseUrl}${path}${qs ? `?${qs}` : ''}`;
}

private async request<T>(path: string, query?: Record<string, string | number | undefined>): Promise<T> {
private async request<T>(
path: string,
opts: { query?: Record<string, string | number | undefined>; method?: 'GET' | 'POST'; body?: unknown } = {}
): Promise<T> {
const headers = new Headers({ accept: 'application/json' });
// The studio token only travels same-origin (to the studio server's
// proxy, which requires it); a configured remote collector gets no header.
if (this.baseUrl.startsWith('/')) {
const token = loadStudioToken();
if (token) headers.set(STUDIO_TOKEN_HEADER, token);
}
const init: RequestInit = { headers };
if (opts.method && opts.method !== 'GET') init.method = opts.method;
if (opts.body !== undefined) {
headers.set('content-type', 'application/json');
init.body = JSON.stringify(opts.body);
}
let res: Response;
try {
res = await this.fetchFn(this.url(path, query), { headers });
res = await this.fetchFn(this.url(path, opts.query), init);
} catch {
throw new ObserveDaemonDownError();
}
Expand Down Expand Up @@ -152,7 +170,7 @@ export class ObserveClient {

/** `GET /api/v1/traces` - the paginated, filterable trace list. */
listTraces(filter: TraceListFilter = {}): Promise<ObserveTraceList> {
return this.request('/api/v1/traces', { ...filter });
return this.request('/api/v1/traces', { query: { ...filter } });
}

/** `GET /api/v1/traces/:id` - flat observation rows + scores; `undefined` on the daemon's 404. */
Expand All @@ -167,20 +185,25 @@ export class ObserveClient {

/** `GET /api/v1/agents` - the fleet registry with read-time freshness. */
async listAgents(project?: string): Promise<ObserveAgentRow[]> {
const body = await this.request<{ agents: ObserveAgentRow[] }>('/api/v1/agents', { project });
const body = await this.request<{ agents: ObserveAgentRow[] }>('/api/v1/agents', { query: { project } });
return body.agents;
}

/** `GET /api/v1/insights` - per-agent cards + rollup series. */
insights(filter: InsightsFilter = {}): Promise<ObserveInsights> {
return this.request('/api/v1/insights', { ...filter });
return this.request('/api/v1/insights', { query: { ...filter } });
}

/** `GET /api/v1/scores` - eval/judge/human score rows. */
async listScores(filter: ScoreListFilter = {}): Promise<ObserveScoreRow[]> {
const body = await this.request<{ scores: ObserveScoreRow[] }>('/api/v1/scores', { ...filter });
const body = await this.request<{ scores: ObserveScoreRow[] }>('/api/v1/scores', { query: { ...filter } });
return body.scores;
}

/** `POST /api/v1/scores` - a human feedback score (append-only); the daemon's 404/400 surface as `ObserveApiError`. */
postScore(score: NewScore): Promise<{ score_id: string }> {
return this.request('/api/v1/scores', { method: 'POST', body: score });
}
}

export const observeClient = new ObserveClient();
2 changes: 1 addition & 1 deletion apps/agent-forge/src/screens/__tests__/routes.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ describe('screenFor', () => {
expect(screenFor('#/agents')).toBe('agents');
expect(screenFor('#/agents/support-bot')).toBe('agents');
expect(screenFor('#/traces')).toBe('traces');
expect(screenFor('#/traces/abc?obs=1')).toBe('traces');
expect(screenFor('#/traces/abc?obs=1')).toBe('trace');
expect(screenFor('#/monitoring?window=24h')).toBe('monitoring');
expect(screenFor('#/evals/scores')).toBe('scores');
expect(screenFor('#/evals/datasets')).toBe('evals');
Expand Down
14 changes: 12 additions & 2 deletions apps/agent-forge/src/screens/routes.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,15 @@ import { Topbar } from '../components/Topbar';
import { SettingsPanel } from '../components/SettingsPanel';
import { ObserveView, type ObserveTab } from '../observe/ObserveView';
import { AgentsScreen } from './agents/AgentsScreen';
import { TraceScreen } from './traces/TraceScreen';
import { useLeaveAgentsGuard } from './agents/useLeaveAgentsGuard';
import { Placeholder } from './Placeholder';

export type ScreenId =
| 'agents'
| 'playground'
| 'traces'
| 'trace'
| 'threads'
| 'monitoring'
| 'scores'
Expand All @@ -26,7 +28,7 @@ const ROUTES: { pattern: string; screen: ScreenId }[] = [
{ pattern: '/agents/:id', screen: 'agents' },
{ pattern: '/playground', screen: 'playground' },
{ pattern: '/traces', screen: 'traces' },
{ pattern: '/traces/:traceId', screen: 'traces' },
{ pattern: '/traces/:traceId', screen: 'trace' },
{ pattern: '/threads', screen: 'threads' },
{ pattern: '/threads/:sessionId', screen: 'threads' },
{ pattern: '/monitoring', screen: 'monitoring' },
Expand Down Expand Up @@ -80,6 +82,13 @@ const PLACEHOLDERS: Partial<Record<ScreenId, { title: string; blurb: string }>>
hub: { title: 'Context Hub', blurb: 'The shared context namespaces your agents read from, browsable by namespace.' },
};

/** `#/traces/:traceId`: the trace id comes from the hash; a new id remounts the screen. */
function TraceRoute() {
const { path } = useRoute();
const traceId = matchRoute(`#${path}`, ['/traces/:traceId'])?.params.traceId ?? '';
return <TraceScreen key={traceId} traceId={traceId} />;
}

/** Every screen but Agents: the shell top bar over the screen's content. */
function ScreenFrame({ children }: { children: ReactNode }) {
return (
Expand All @@ -100,7 +109,8 @@ export function Screens() {
const placeholder = PLACEHOLDERS[screen];
let content: ReactNode;
// Same element type at the same position for all three, so the drill-down filter survives a route change.
if (observeTab) content = <ObserveView tab={observeTab} onTabChange={(tab) => navigate(TAB_PATH[tab])} />;
if (screen === 'trace') content = <TraceRoute />;
else if (observeTab) content = <ObserveView tab={observeTab} onTabChange={(tab) => navigate(TAB_PATH[tab])} />;
else if (placeholder) content = <Placeholder {...placeholder} />;
else
content = (
Expand Down
Loading
Loading