From 5ee027dd0644c7b16d0ed864366d02808f315f73 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Mon, 28 Sep 2026 00:16:29 +0530 Subject: [PATCH] feat: remember recent sources and clarify inspection copy --- docs/api-reference/openapi.json | 32 +++++ docs/dashboard/sources.mdx | 6 + docs/internals/endpoints.mdx | 3 + platform/src/durable-objects/user-account.ts | 46 +++++++ platform/src/lib/source-history-storage.ts | 88 +++++++++++++ platform/src/lib/source-history.ts | 66 ++++++++++ platform/src/openapi-audience.ts | 6 + platform/src/openapi.ts | 29 +++++ platform/src/routes/session/session.index.ts | 32 +++++ platform/test/e2e/dashboard-api.spec.ts | 93 ++++++++++++-- .../test/fixtures/agent-dashboard-server.mjs | 8 ++ .../test/user-account-do.integration.test.ts | 121 +++++++++++++++++- platform/test/user-account.wrangler.jsonc | 1 + reference/agents/platform-internals.md | 2 + reference/engineering/SOURCE_HISTORY.md | 32 +++++ web/app/dashboard/SourcesClient.tsx | 99 ++++++++++++-- web/app/dashboard/research-types.ts | 3 + web/app/globals.css | 4 + 18 files changed, 652 insertions(+), 19 deletions(-) create mode 100644 platform/src/lib/source-history-storage.ts create mode 100644 platform/src/lib/source-history.ts create mode 100644 reference/engineering/SOURCE_HISTORY.md diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json index 9a04b71..e1cad06 100644 --- a/docs/api-reference/openapi.json +++ b/docs/api-reference/openapi.json @@ -8461,6 +8461,38 @@ ], "additionalProperties": false }, + "RecentSource": { + "type": "object", + "required": [ + "id", + "input", + "title", + "kind", + "updatedAt" + ], + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "input": { + "type": "string" + }, + "title": { + "type": "string" + }, + "kind": { + "type": "string", + "enum": [ + "search", + "inspection" + ] + }, + "updatedAt": { + "type": "integer" + } + } + }, "CompactAgentRun": { "type": "object", "properties": { diff --git a/docs/dashboard/sources.mdx b/docs/dashboard/sources.mdx index dc5f35f..63f4ad8 100644 --- a/docs/dashboard/sources.mdx +++ b/docs/dashboard/sources.mdx @@ -18,3 +18,9 @@ Opening a playlist does not automatically fetch transcripts or comments for ever When an upstream source is unavailable, existing results remain visible and the dashboard identifies which requested dataset could not be loaded. + +## Recent sources + +Sources remembers your thirty most recent successful searches and inspected links. Select an entry to reopen its saved results or datasets after reloading the page or signing in on another device. Opening recent sources reads saved data without a new YouTube request or credit charge. Use Refresh data when you want current video details or comments. + +Search terms and links belong to your account. Video data stays in the shared catalog, and account deletion removes your history without deleting public assets used by other accounts. diff --git a/docs/internals/endpoints.mdx b/docs/internals/endpoints.mdx index b0394ab..7529636 100644 --- a/docs/internals/endpoints.mdx +++ b/docs/internals/endpoints.mdx @@ -33,6 +33,9 @@ Called by the video2ctx web application or an explicit signed-in account action. | `GET` | `/v1/oauth/youtube/connect` | `createYouTubeConnectUrl` | Create a YouTube OAuth URL | `sessionCookie` or `demoUser` | Returns a state-bound OAuth URL; start it only from a user-initiated connection flow. | | `POST` | `/v1/resolve` | `resolveInput` | Route universal UI input | `sessionCookie` or `cliSession` or `bearerApiKey` or `apiKey` or `demoUser` | First-party input router; its dispatch behavior is not a stable public API contract. | | `POST` | `/v1/scale-inquiries` | `submitScaleInquiry` | Submit a Scale plan inquiry | Public or protocol-signed | Public lead form; validate Turnstile and rate limits, and never let the caller choose the notification recipient. | +| `GET` | `/v1/sources/recent` | `listRecentSources` | List recent Sources inputs | `sessionCookie` or `demoUser` | Browser-session only. Lists inputs belonging to the authenticated user without exposing shared asset references. | +| `POST` | `/v1/sources/recent` | `saveRecentSource` | Remember a Sources search or inspection | `sessionCookie` or `demoUser` | Browser-session only. Resolves existing provider assets server-side; never accepts client-supplied R2 keys or video payloads. | +| `GET` | `/v1/sources/recent/{id}` | `getRecentSource` | Restore saved Sources data | `sessionCookie` or `demoUser` | Browser-session only. Verifies user ownership before reading the saved shared references. | ## Callbacks and signed links diff --git a/platform/src/durable-objects/user-account.ts b/platform/src/durable-objects/user-account.ts index 5252a4f..30fa20f 100644 --- a/platform/src/durable-objects/user-account.ts +++ b/platform/src/durable-objects/user-account.ts @@ -2,6 +2,7 @@ import { AgentAdmissionQueue } from '../agents/runtime/admission-queue'; import type { AgentRequest, AgentAdmission } from '../agents/contracts'; import { DurableObject } from 'cloudflare:workers'; import { z } from 'zod'; +import { RECENT_SOURCE_LIMIT, saveReferencedSourceSchema, sourceReferenceSchema, sourceIdentity, type RecentSource, type SaveReferencedSource, type SourceReference } from '../lib/source-history'; const MAX_SEARCH_TEXT_LENGTH = 32_000; const MAX_TITLE_LENGTH = 80; @@ -100,6 +101,7 @@ export class UserAccountDO extends DurableObject { this.ctx.storage.sql.exec('DELETE FROM user_session_runs'); this.ctx.storage.sql.exec('DELETE FROM user_sessions'); this.ctx.storage.sql.exec('DELETE FROM agent_conversations'); + this.ctx.storage.sql.exec('DELETE FROM recent_sources'); // Keep only a tombstone so already-authenticated requests cannot recreate data. } @@ -109,6 +111,46 @@ export class UserAccountDO extends DurableObject { } } + saveSource(value: SaveReferencedSource): RecentSource { + this.assertActive(); + const input = saveReferencedSourceSchema.parse(value); + const snapshot = JSON.stringify(input.snapshot); + const key = sourceIdentity(input); + const existing = this.ctx.storage.sql.exec<{ id: string }>('SELECT id FROM recent_sources WHERE source_key = ?', key).toArray()[0]; + const entry: RecentSource = { id: existing?.id ?? crypto.randomUUID(), input: input.input, + title: input.title, + kind: input.snapshot.kind, updatedAt: this.nextSourceUpdate() }; + this.ctx.storage.transactionSync(() => { + this.ctx.storage.sql.exec(`INSERT INTO recent_sources (id, source_key, input, title, kind, updated_at, snapshot) + VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(source_key) DO UPDATE SET + input=excluded.input, title=excluded.title, kind=excluded.kind, updated_at=excluded.updated_at, snapshot=excluded.snapshot`, + entry.id, key, entry.input, entry.title, entry.kind, entry.updatedAt, snapshot); + this.ctx.storage.sql.exec(`DELETE FROM recent_sources WHERE id NOT IN + (SELECT id FROM recent_sources ORDER BY updated_at DESC, rowid DESC LIMIT ?)`, RECENT_SOURCE_LIMIT); + }); + return entry; + } + + listSources(): RecentSource[] { + this.assertActive(); + return this.ctx.storage.sql.exec<{ id: string; input: string; title: string; kind: RecentSource['kind']; updated_at: number }>( + 'SELECT id, input, title, kind, updated_at FROM recent_sources ORDER BY updated_at DESC, rowid DESC LIMIT ?', RECENT_SOURCE_LIMIT, + ).toArray().map(({ updated_at, ...row }) => ({ ...row, updatedAt: updated_at })); + } + + getSource(id: string): { source: RecentSource; snapshot: SourceReference } | null { + this.assertActive(); + const row = this.ctx.storage.sql.exec<{ snapshot: string }>('SELECT snapshot FROM recent_sources WHERE id = ?', z.string().uuid().parse(id)).toArray()[0]; + if (!row) return null; + this.ctx.storage.sql.exec('UPDATE recent_sources SET updated_at = ? WHERE id = ?', this.nextSourceUpdate(), id); + return { source: this.listSources().find(source => source.id === id)!, snapshot: sourceReferenceSchema.parse(JSON.parse(row.snapshot)) }; + } + + private nextSourceUpdate(): number { + const latest = this.ctx.storage.sql.exec<{ latest: number | null }>('SELECT MAX(updated_at) AS latest FROM recent_sources').one().latest; + return Math.max(Date.now(), (latest ?? 0) + 1); + } + recordSession(value: RecordSessionInput): UserSessionSummary { this.assertActive(); const input = recordSessionInputSchema.parse(value); @@ -283,6 +325,10 @@ export class UserAccountDO extends DurableObject { } private ensureSchema(): void { + this.ctx.storage.sql.exec(`CREATE TABLE IF NOT EXISTS recent_sources ( + id TEXT PRIMARY KEY, source_key TEXT NOT NULL UNIQUE, input TEXT NOT NULL, title TEXT NOT NULL, + kind TEXT NOT NULL, updated_at INTEGER NOT NULL, snapshot TEXT NOT NULL + )`); this.ctx.storage.sql.exec('CREATE TABLE IF NOT EXISTS account_deletion (id INTEGER PRIMARY KEY)'); this.ctx.storage.sql.exec('CREATE TABLE IF NOT EXISTS agent_conversations (conversation_id TEXT PRIMARY KEY)'); this.ctx.storage.sql.exec(` diff --git a/platform/src/lib/source-history-storage.ts b/platform/src/lib/source-history-storage.ts new file mode 100644 index 0000000..9f67102 --- /dev/null +++ b/platform/src/lib/source-history-storage.ts @@ -0,0 +1,88 @@ +import { ApiError, sha256 } from './http'; +import { videoCatalog, type VideoAssetReference } from './video-catalog'; +import { videoResourceKey } from './video-resources'; +import { readYouTubeCacheEntry } from './youtube-cache-coordinator'; +import { routeInput, withYouTubeMetadata } from './youtube'; +import { sourceSnapshotSchema, type SaveSourceInput, type SaveReferencedSource, type SourceReference, type SourceSnapshot } from './source-history'; + +// Only public provider payloads enter this bucket. Inputs and user selections stay in the user DO. +async function saveShared(env: Env, value: unknown): Promise { + const payload = JSON.stringify(value); + const key = `youtube/source-history/${await sha256(payload)}.json`; + await env.VIDEO_ASSETS.put(key, payload, { httpMetadata: { contentType: 'application/json' } }); + return key; +} + +export async function referenceSource(env: Env, value: SaveSourceInput): Promise { + if (!env.VIDEO_ASSETS || !videoCatalog(env)) throw new ApiError(503, 'SOURCE_STORAGE_UNAVAILABLE', 'Recent sources storage is unavailable.'); + const { input, snapshot } = value; + const cachedPublicData = async (type: string, id: string): Promise> => { + const key = `youtube:v1:${await sha256(JSON.stringify([type, id]))}`; + const cached = await readYouTubeCacheEntry>(env, key, type); + if (!cached) throw new ApiError(409, 'SOURCE_ASSET_NOT_SAVED', 'Source data has not finished saving. Inspect the source again.'); + return withYouTubeMetadata(cached.value); + }; + if (snapshot.kind === 'search') { + const resolved = routeInput(input); + if (resolved.kind !== 'search') throw new ApiError(422, 'INVALID_SOURCE', 'Expected search terms.'); + const key = await sha256(JSON.stringify({ query: resolved.query, filters: { type: 'video' } })); + const search = await cachedPublicData('search-v3', key); + const items = (search.results as Array<{ type: string }>).filter(item => item.type === 'video'); + return { input, title: input, snapshot: { + kind: 'search', selectedData: snapshot.selectedData, results: await saveShared(env, items), + } }; + } + const source = snapshot.inspector; + const assets: Partial> = {}; + let data: Record = {}; + if (source.type === 'video') { + const operations = [ + { field: 'metadata' as const, op: { kind: 'video' as const, id: source.id } }, + ...(source.loadedData.includes('transcript') ? [{ field: 'transcript' as const, op: { kind: 'transcript' as const, id: source.id, granularity: 'word' as const } }] : []), + ...(source.loadedData.includes('comments') ? [{ field: 'comments' as const, op: { kind: 'comments' as const, id: source.id } }] : []), + ]; + await Promise.all(operations.map(async ({ field, op }) => { + if (source.dataErrors[field]) return; + const saved = await videoCatalog(env)!.readSaved(videoResourceKey(op)!); + const reference = saved?.catalogVersions?.[0]; + if (reference) { assets[field] = reference; if (field === 'metadata') data = saved!.value as Record; } + else throw new ApiError(409, 'SOURCE_ASSET_NOT_SAVED', 'Source data has not finished saving. Retry the source.'); + })); + } else data = await cachedPublicData(source.type === 'playlist' ? 'playlist-v2' : 'channel-v5', source.id); + const channelId = (data.channel as { id?: string } | undefined)?.id; + const channel = source.loadedData.includes('channel') && !source.dataErrors.channel && channelId + ? await cachedPublicData('channel-v5', channelId) : undefined; + return { input, title: String(data.title ?? data.name ?? input).slice(0, 300), snapshot: { + kind: 'inspection', inspector: { provider: source.provider, type: source.type, id: source.id, + requestedData: source.requestedData, dataErrors: source.dataErrors, assets, + entity: source.type !== 'video' ? await saveShared(env, data) : undefined, + channel: channel ? await saveShared(env, channel) : undefined, + }, + } }; +} + +export async function restoreSource(env: Env, reference: SourceReference): Promise { + const readShared = async (key: string) => { + const object = await env.VIDEO_ASSETS.get(key); + if (!object) throw new ApiError(404, 'SOURCE_ASSET_MISSING', 'Saved source data is unavailable. Inspect this source again.'); + const payload = await object.text(); + if (`youtube/source-history/${await sha256(payload)}.json` !== key) throw new ApiError(500, 'SOURCE_ASSET_INVALID', 'Saved source data could not be verified.'); + return JSON.parse(payload); + }; + if (reference.kind === 'search') return sourceSnapshotSchema.parse({ + kind: 'search', selectedData: reference.selectedData, items: await readShared(reference.results), + }); + const { assets, entity, channel, ...source } = reference.inspector; + const restored: Record = { ...source, + data: entity ? await readShared(entity) : { id: source.id, url: `https://youtube.com/watch?v=${encodeURIComponent(source.id)}` }, + channel: channel ? await readShared(channel) : undefined, + }; + await Promise.all(Object.entries(assets).map(async ([field, asset]) => { + if (!asset) return; + const stored = await videoCatalog(env)?.readVersion(asset); + if (!stored) throw new ApiError(404, 'SOURCE_ASSET_MISSING', 'Saved source data is unavailable. Inspect this source again.'); + const value = { ...withYouTubeMetadata(stored.value as Record), freshness: { state: 'stored', fetchedAt: new Date(stored.fetchedAt).toISOString() } }; + restored[field === 'metadata' ? 'data' : field] = value; + })); + return sourceSnapshotSchema.parse({ kind: 'inspection', inspector: restored }); +} diff --git a/platform/src/lib/source-history.ts b/platform/src/lib/source-history.ts new file mode 100644 index 0000000..9a65701 --- /dev/null +++ b/platform/src/lib/source-history.ts @@ -0,0 +1,66 @@ +import { z } from 'zod'; + +export const RECENT_SOURCE_LIMIT = 30; +export const MAX_SOURCE_SNAPSHOT_BYTES = 16_000; +export const sourceIdSchema = z.string().uuid(); +const dataset = z.enum(['transcript', 'comments', 'channel']); +const metadata = z.object({ source: z.string(), fetchedAt: z.string(), partial: z.boolean(), warnings: z.array(z.string()) }).passthrough(); +const thumbnails = z.array(z.object({ url: z.string(), width: z.number().optional(), height: z.number().optional() })); +const record = z.record(z.string(), z.unknown()); +const inspector = z.object({ + provider: z.literal('youtube'), type: z.enum(['video', 'playlist', 'channel']), id: z.string().min(1).max(200), + data: record, requestedData: z.array(dataset), + dataErrors: z.partialRecord(z.enum(['metadata', 'transcript', 'comments', 'channel']), z.string()), + refreshData: z.array(z.enum(['metadata', 'transcript', 'comments', 'channel'])).optional(), + transcript: z.object({ + videoId: z.string(), text: z.string(), granularity: z.enum(['segment', 'word']).optional(), meta: metadata, + track: z.object({ name: z.string(), kind: z.string(), languageCode: z.string() }).passthrough(), + segments: z.array(z.object({ text: z.string(), startMs: z.number(), endMs: z.number(), durationMs: z.number() }).passthrough()), + }).passthrough().optional(), + comments: z.object({ videoId: z.string(), comments: z.array(record), meta: metadata, + totalCount: z.number().optional(), continuation: z.string().optional() }).passthrough().optional(), + channel: z.object({ id: z.string(), name: z.string(), thumbnails, url: z.string(), meta: metadata, + about: z.object({ description: z.string().optional(), links: z.array(z.object({ title: z.string(), displayUrl: z.string(), url: z.string() })), + moreInfo: record }) }).passthrough().optional(), +}); + +export const sourceSnapshotSchema = z.discriminatedUnion('kind', [ + z.object({ kind: z.literal('search'), selectedData: z.array(dataset).min(1), items: z.array(z.object({ + provider: z.literal('youtube').optional(), type: z.literal('video'), id: z.string(), thumbnails, + title: z.string().optional(), name: z.string().optional(), description: z.string().optional(), + channel: z.object({ id: z.string(), name: z.string() }).optional(), + durationText: z.string().optional(), viewCountText: z.string().optional(), publishedTimeText: z.string().optional(), + isLive: z.boolean().optional(), videoCountText: z.string().optional(), + })) }), + z.object({ kind: z.literal('inspection'), inspector }), +]); +export const saveSourceSchema = z.object({ input: z.string().trim().min(1).max(500), snapshot: z.discriminatedUnion('kind', [ + z.object({ kind: z.literal('search'), selectedData: z.array(dataset).min(1) }), + z.object({ kind: z.literal('inspection'), inspector: inspector.pick({ provider: true, type: true, id: true, requestedData: true, dataErrors: true }) + .extend({ loadedData: z.array(z.enum(['metadata', 'transcript', 'comments', 'channel'])) }) }), +]) }); +export type SourceSnapshot = z.infer; +export type SaveSourceInput = z.infer; +const assetReference = z.object({ videoId: z.string(), kind: z.string(), variant: z.string(), contentHash: z.string() }); +const sharedReference = z.string().regex(/^youtube\/source-history\/[a-f0-9]{64}\.json$/); +export const sourceReferenceSchema = z.discriminatedUnion('kind', [ + z.object({ kind: z.literal('search'), selectedData: z.array(dataset), results: sharedReference }), + z.object({ kind: z.literal('inspection'), inspector: z.object({ + provider: z.literal('youtube'), type: z.enum(['video', 'playlist', 'channel']), id: z.string(), + requestedData: z.array(dataset), dataErrors: inspector.shape.dataErrors, + assets: z.partialRecord(z.enum(['metadata', 'transcript', 'comments']), assetReference), + entity: sharedReference.optional(), channel: sharedReference.optional(), + }) }), +]); +export const saveReferencedSourceSchema = z.object({ input: z.string().trim().min(1).max(500), title: z.string().max(300), snapshot: sourceReferenceSchema }); +export type SourceReference = z.infer; +export type SaveReferencedSource = z.infer; +export interface RecentSource { + id: string; input: string; title: string; kind: SourceSnapshot['kind']; updatedAt: number; +} + +export function sourceIdentity(input: SaveReferencedSource): string { + return input.snapshot.kind === 'search' + ? `search:${input.input.replace(/\s+/g, ' ').toLowerCase()}` + : `${input.snapshot.inspector.provider}:${input.snapshot.inspector.type}:${input.snapshot.inspector.id}`; +} diff --git a/platform/src/openapi-audience.ts b/platform/src/openapi-audience.ts index 2953647..6f8c957 100644 --- a/platform/src/openapi-audience.ts +++ b/platform/src/openapi-audience.ts @@ -29,6 +29,9 @@ export const OPENAPI_OPERATION_AUDIENCE: Readonly> = { + listRecentSources: 'Browser-session only. Lists inputs belonging to the authenticated user without exposing shared asset references.', + saveRecentSource: 'Browser-session only. Resolves existing provider assets server-side; never accepts client-supplied R2 keys or video payloads.', + getRecentSource: 'Browser-session only. Verifies user ownership before reading the saved shared references.', inspectLandingYouTubeVideo: 'Public, rate-limited demo route; do not use it as a credentialed bulk-data API.', submitScaleInquiry: 'Public lead form; validate Turnstile and rate limits, and never let the caller choose the notification recipient.', signInWithMagicLink: 'Sends account email; rate-limit callers and never disclose whether an address is registered.', diff --git a/platform/src/openapi.ts b/platform/src/openapi.ts index 37f98e1..b6c3da2 100644 --- a/platform/src/openapi.ts +++ b/platform/src/openapi.ts @@ -1,6 +1,7 @@ import { storedExtractionDiagnosticSchema } from './lib/extraction-diagnostics'; import { transcriptDiagnosticSchema } from './agents/runtime/transcript-diagnostics'; import { z } from 'zod'; +import { saveSourceSchema, sourceSnapshotSchema } from './lib/source-history'; import { compactAgentRunSchema } from './agents/response'; import { agentRunProgressSchema } from './agents/runtime/run-progress'; import { @@ -1086,6 +1087,30 @@ export const openApiDocument = { }, }, }, + '/v1/sources/recent': { + get: { + tags: ['Projects'], operationId: 'listRecentSources', summary: 'List recent Sources inputs', security: privateSecurity, + description: 'First-party dashboard history. Returns the thirty most recent successful searches and inspections for the signed-in user.', + responses: { '200': jsonResponse('Recent source summaries.', { type: 'object', properties: { sources: { type: 'array', items: schemaRef('RecentSource') } } }), ...standardErrors }, + }, + post: { + tags: ['Projects'], operationId: 'saveRecentSource', summary: 'Remember a Sources search or inspection', security: privateSecurity, + description: 'Stores the input and references in the user Durable Object. Provider data is read from existing shared storage. This does not fetch YouTube data.', + requestBody: jsonBody(z.toJSONSchema(saveSourceSchema, { target: 'openapi-3.0' })), + responses: { '201': jsonResponse('Source remembered.', { type: 'object', properties: { source: schemaRef('RecentSource') } }), + '409': jsonResponse('Provider data is not yet present in shared storage.', schemaRef('Error')), ...standardErrors }, + }, + }, + '/v1/sources/recent/{id}': { + get: { + tags: ['Projects'], operationId: 'getRecentSource', summary: 'Restore saved Sources data', security: privateSecurity, + description: 'Loads the user-owned references, hydrates shared immutable assets, and moves the entry to the top of history. No provider request or credit charge is made.', + parameters: [pathParameter('id', 'Recent source UUID.')], + responses: { '200': jsonResponse('Saved source and displayed data.', { type: 'object', properties: { + source: schemaRef('RecentSource'), snapshot: z.toJSONSchema(sourceSnapshotSchema, { target: 'openapi-3.0' }), + } }), '404': responseRef('NotFound'), ...standardErrors }, + }, + }, '/v1/projects': { get: { tags: ['Projects'], @@ -2152,6 +2177,10 @@ export const openApiDocument = { }, }, AgentRunProgress: z.toJSONSchema(agentRunProgressSchema, { target: 'openapi-3.0' }), + RecentSource: { type: 'object', required: ['id', 'input', 'title', 'kind', 'updatedAt'], properties: { + id: { type: 'string', format: 'uuid' }, input: { type: 'string' }, title: { type: 'string' }, + kind: { type: 'string', enum: ['search', 'inspection'] }, updatedAt: { type: 'integer' }, + } }, CompactAgentRun: z.toJSONSchema(compactAgentRunSchema, { target: 'openapi-3.0' }), AgentRunReceipt: { type: 'object', diff --git a/platform/src/routes/session/session.index.ts b/platform/src/routes/session/session.index.ts index a73ced3..c446299 100644 --- a/platform/src/routes/session/session.index.ts +++ b/platform/src/routes/session/session.index.ts @@ -1,4 +1,7 @@ import { framePreviewPrefix } from '../../agents/runtime/frame-previews'; +import { userAccountInstanceName } from '../../agents/runtime/identity'; +import { MAX_SOURCE_SNAPSHOT_BYTES, saveSourceSchema, sourceIdSchema } from '../../lib/source-history'; +import { referenceSource, restoreSource } from '../../lib/source-history-storage'; import { deleteAgentAccountData } from '../../agents/runtime/account-deletion'; import { Hono } from 'hono'; import type { App, ImportPayload } from '../../types'; @@ -42,6 +45,8 @@ export const ACCOUNT_ROUTE_PATTERNS = [ ] as const; export const SESSION_ONLY_ROUTE_PATTERNS = [ + '/sources/recent', + '/sources/recent/*', '/oauth/youtube/connect', '/oauth/youtube', '/billing', @@ -50,6 +55,33 @@ export const SESSION_ONLY_ROUTE_PATTERNS = [ for (const path of ACCOUNT_ROUTE_PATTERNS) sessionRoutes.use(path, requireAccountPrincipal); for (const path of SESSION_ONLY_ROUTE_PATTERNS) sessionRoutes.use(path, requireSessionPrincipal); +sessionRoutes.get('/sources/recent', async (c) => { + const account = c.env.USER_ACCOUNT.getByName(await userAccountInstanceName(requireUser(c).id)); + return c.json({ sources: await account.listSources() }); +}); + +sessionRoutes.post('/sources/recent', async (c) => { + if (!c.req.header('content-type')?.includes('application/json')) throw new ApiError(422, 'INVALID_CONTENT_TYPE', 'Expected application/json.'); + const payload = await c.req.text(); + if (new TextEncoder().encode(payload).byteLength > MAX_SOURCE_SNAPSHOT_BYTES) throw new ApiError(422, 'SOURCE_TOO_LARGE', 'This source is too large to add to recent sources.'); + let json: unknown; + try { json = JSON.parse(payload); } catch { throw new ApiError(422, 'INVALID_JSON', 'The request body is not valid JSON.'); } + const parsed = saveSourceSchema.safeParse(json); + if (!parsed.success) throw new ApiError(422, 'INVALID_SOURCE', 'The recent source data is invalid.'); + const account = c.env.USER_ACCOUNT.getByName(await userAccountInstanceName(requireUser(c).id)); + const referenced = await referenceSource(c.env, parsed.data); + return c.json({ source: await account.saveSource(referenced) }, 201); +}); + +sessionRoutes.get('/sources/recent/:id', async (c) => { + const id = c.req.param('id'); + if (!sourceIdSchema.safeParse(id).success) throw new ApiError(422, 'INVALID_ID', 'Invalid recent source ID.'); + const account = c.env.USER_ACCOUNT.getByName(await userAccountInstanceName(requireUser(c).id)); + const saved = await account.getSource(id); + if (!saved) throw new ApiError(404, 'SOURCE_NOT_FOUND', 'This recent source was not found.'); + return c.json({ source: saved.source, snapshot: await restoreSource(c.env, saved.snapshot) }); +}); + sessionRoutes.get('/account', (c) => { const principal = requirePrincipal(c); return c.json({ diff --git a/platform/test/e2e/dashboard-api.spec.ts b/platform/test/e2e/dashboard-api.spec.ts index 6e9fb9b..a18304e 100644 --- a/platform/test/e2e/dashboard-api.spec.ts +++ b/platform/test/e2e/dashboard-api.spec.ts @@ -17,6 +17,83 @@ test.beforeEach(async ({ page, context }) => { await page.route(`**/api/platform/v1/providers/youtube/videos/${videoId}`, route => route.fulfill({ json: { id: videoId, title: 'Transcript deadline regression', thumbnails: [], channel: { id: 'channel', name: 'Creator' } } })); }); +test('recent video sources survive reload and restore datasets without provider requests', async ({ page }) => { + const source = { id: 'e98e29c2-2d42-4408-b055-5b63d5907084', input: `https://youtube.com/watch?v=${videoId}`, title: 'Saved video', kind: 'inspection', updatedAt: Date.now() }; + const snapshot = { kind: 'inspection', inspector: { provider: 'youtube', type: 'video', id: videoId, + data: { id: videoId, title: source.title, thumbnails: [] }, transcript, requestedData: ['transcript'], dataErrors: {} } }; + let remembered = false, providerReads = 0; + page.on('request', request => { if (request.url().includes('/v1/providers/')) providerReads++; }); + await page.route(`**/videos/${videoId}/transcript`, route => route.fulfill({ json: transcript })); + await page.route('**/api/platform/v1/sources/recent**', async route => { + if (route.request().method() === 'POST') { + const request = route.request().postDataJSON(); + expect(request.input).toBe(source.input); + expect(request.snapshot.inspector.id).toBe(videoId); + expect(request.snapshot.inspector.loadedData).toContain('transcript'); + expect(request.snapshot.inspector).not.toHaveProperty('transcript'); + expect(request.snapshot.inspector).not.toHaveProperty('data'); + remembered = true; return route.fulfill({ status: 201, json: { source } }); + } + return route.fulfill({ json: route.request().url().endsWith(source.id) ? { source, snapshot } : { sources: remembered ? [source] : [] } }); + }); + await page.goto('/dashboard/sources'); + await expect(page.getByRole('heading', { name: 'Recent sources', exact: true })).toBeVisible(); + await expect(page.getByRole('heading', { name: 'No recent sources yet' })).toBeVisible(); + await page.getByRole('textbox', { name: 'Video search or YouTube URL' }).fill(source.input); + await page.getByRole('button', { name: /Inspect/ }).click(); + await expect(page.getByText(transcript.text, { exact: true })).toBeVisible(); + await expect.poll(() => remembered).toBe(true); + const readsBeforeRestore = providerReads; + await page.reload(); + await page.getByRole('button', { name: /Saved video/ }).click(); + await expect(page.getByRole('heading', { name: 'Saved video' })).toBeVisible(); + await expect(page.getByText(transcript.text, { exact: true })).toBeVisible(); + expect(providerReads).toBe(readsBeforeRestore); + await page.getByRole('button', { name: 'Recent sources', exact: true }).click(); + await expect(page.getByRole('button', { name: /Saved video/ })).toBeVisible(); +}); + +test('recent searches restore their saved result list and dataset choices', async ({ page }) => { + const source = { id: 'cd2b8fe1-c3e6-4b67-bd51-0ee1a10ccbbf', input: 'Opus vs Astra', title: 'Opus vs Astra', kind: 'search', updatedAt: Date.now() }; + const items = [{ provider: 'youtube', type: 'video', id: videoId, title: 'Saved comparison result', thumbnails: [] }]; + let remembered = false, providerReads = 0; + await page.unroute('**/api/platform/v1/resolve'); + await page.route('**/api/platform/v1/resolve', route => route.fulfill({ json: { kind: 'search', query: source.input } })); + await page.route('**/api/platform/v1/providers/youtube/search?**', route => { providerReads++; return route.fulfill({ json: { results: items } }); }); + await page.route('**/api/platform/v1/sources/recent**', async route => { + if (route.request().method() === 'POST') { + expect(route.request().postDataJSON()).toEqual({ input: source.input, snapshot: { kind: 'search', selectedData: ['transcript'] } }); + remembered = true; return route.fulfill({ status: 201, json: { source } }); + } + return route.fulfill({ json: route.request().url().endsWith(source.id) + ? { source, snapshot: { kind: 'search', items, selectedData: ['comments'] } } : { sources: remembered ? [source] : [] } }); + }); + await page.goto('/dashboard/sources'); + await page.getByRole('textbox', { name: 'Video search or YouTube URL' }).fill(source.input); + await page.getByRole('button', { name: /Inspect/ }).click(); + await expect(page.getByText('Saved comparison result')).toBeVisible(); + await expect.poll(() => remembered).toBe(true); + await page.reload(); + await page.getByRole('button', { name: /Opus vs Astra/ }).click(); + await expect(page.getByText('Saved comparison result')).toBeVisible(); + await expect(page.getByRole('textbox', { name: 'Video search or YouTube URL' })).toHaveValue(source.input); + await expect(page.getByRole('checkbox', { name: 'Comments' })).toBeChecked(); + expect(providerReads).toBe(1); +}); + +test('recent source load failures expose a retry and preserve the form', async ({ page }) => { + let attempts = 0; + await page.route('**/api/platform/v1/sources/recent', route => ++attempts === 1 + ? route.fulfill({ status: 503, json: { error: { code: 'TEMPORARY', message: 'History is temporarily unavailable.' } } }) + : route.fulfill({ json: { sources: [] } })); + await page.goto('/dashboard/sources'); + await expect(page.getByRole('alert').filter({ hasText: 'History is temporarily unavailable.' })).toBeVisible(); + await expect(page.getByText('No recent sources yet')).toHaveCount(0); + await page.getByRole('button', { name: 'Retry recent sources' }).click(); + await expect(page.getByText('No recent sources yet')).toBeVisible(); + await expect(page.getByRole('textbox', { name: 'Video search or YouTube URL' })).toBeVisible(); +}); + test('slow transcript finishes after the old browser deadline', async ({ page }) => { let release!: () => void; const gate = new Promise(resolve => { release = resolve; }); @@ -26,7 +103,7 @@ test('slow transcript finishes after the old browser deadline', async ({ page }) await page.goto('/dashboard?section=discover'); await page.clock.install(); await page.getByRole('textbox', { name: 'Video search or YouTube URL' }).fill(`https://youtube.com/watch?v=${videoId}`); - await page.getByRole('button', { name: /Open video|Search videos/ }).click(); + await page.getByRole('button', { name: /Inspect/ }).click(); await requested; await page.clock.fastForward(180_000); await expect(page.getByText('A transcript is not available for this video.')).toHaveCount(0); @@ -42,7 +119,7 @@ test('source errors mirror the API and retry only the failed dataset', async ({ : route.fulfill({ json: transcript })); await page.goto('/dashboard?section=discover'); await page.getByRole('textbox', { name: 'Video search or YouTube URL' }).fill(`https://youtube.com/watch?v=${videoId}`); - await page.getByRole('button', { name: /Open video|Search videos/ }).click(); + await page.getByRole('button', { name: /Inspect/ }).click(); await expect(page.getByRole('alert').filter({ hasText: 'The API transcript deadline expired.' })).toBeVisible(); await page.getByRole('button', { name: 'Retry failed requests' }).click(); await expect(page.getByText('Transcript arrived successfully.', { exact: true })).toBeVisible(); @@ -105,7 +182,7 @@ test('transcript renders while metadata is pending, then survives its failure', await page.route(`**/videos/${videoId}/transcript`, route => { transcriptReads++; return route.fulfill({ json: transcript }); }); await page.goto('/dashboard?section=discover'); await page.getByRole('textbox', { name: 'Video search or YouTube URL' }).fill(`https://youtube.com/watch?v=${videoId}`); - await page.getByRole('button', { name: /Open video|Search videos/ }).click(); + await page.getByRole('button', { name: /Inspect/ }).click(); try { await expect(page.getByText('Transcript arrived successfully.', { exact: true })).toBeVisible(); } finally { release(); } @@ -128,7 +205,7 @@ test('metadata renders before a pending transcript and cancel preserves it', asy }); await page.goto('/dashboard?section=discover'); await page.getByRole('textbox', { name: 'Video search or YouTube URL' }).fill(`https://youtube.com/watch?v=${videoId}`); - await page.getByRole('button', { name: /Open video|Search videos/ }).click(); + await page.getByRole('button', { name: /Inspect/ }).click(); await expect(page.getByRole('heading', { name: 'Transcript deadline regression', exact: true })).toBeVisible(); await expect(page.getByRole('status', { name: 'Loading transcript' })).toBeVisible(); await expect(page.getByRole('button', { name: 'Save to project' })).toBeDisabled(); @@ -350,7 +427,7 @@ test('an active transcript finishes while settings is open and is reused on retu await page.route(`**/videos/${videoId}/transcript`,async route=>{reads++;await gate;await route.fulfill({json:transcript});}); await page.goto('/dashboard/sources'); await page.getByRole('textbox',{name:'Video search or YouTube URL'}).fill(`https://youtube.com/watch?v=${videoId}`); - await page.getByRole('button',{name:/Search videos/}).click(); + await page.getByRole('button',{name:/Inspect/}).click(); await expect.poll(()=>reads).toBe(1); await page.getByRole('link',{name:'Settings',exact:true}).click(); await expect(page.getByRole('heading',{name:'Workspace settings'})).toBeVisible(); @@ -429,7 +506,7 @@ for (const mobile of [false, true]) { }); await page.goto('/dashboard?section=discover'); await page.getByRole('textbox', { name: 'Video search or YouTube URL' }).fill(`https://youtube.com/watch?v=${videoId}`); - await page.getByRole('button', { name: /Open video|Search videos/ }).click(); + await page.getByRole('button', { name: /Inspect/ }).click(); const refresh = page.getByRole('button', { name: 'Refresh data', exact: true }); await expect(refresh).toBeEnabled(); await expect(page.getByText('Transcript arrived successfully.', { exact: true })).toBeVisible(); @@ -456,7 +533,7 @@ test('fresh video data does not offer a saved-data refresh button', async ({ pag await page.route(`**/videos/${videoId}/transcript`, route => route.fulfill({ json: { ...transcript, freshness: { state: 'fresh', fetchedAt: Date.now() } } })); await page.goto('/dashboard?section=discover'); await page.getByRole('textbox', { name: 'Video search or YouTube URL' }).fill(`https://youtube.com/watch?v=${videoId}`); - await page.getByRole('button', { name: /Open video|Search videos/ }).click(); + await page.getByRole('button', { name: /Inspect/ }).click(); await expect(page.getByText('Transcript arrived successfully.', { exact: true })).toBeVisible(); await expect(page.getByRole('button', { name: 'Refresh data', exact: true })).toHaveCount(0); }); @@ -474,7 +551,7 @@ test('a known video opens without waiting for the remote URL resolver', async ({ await page.goto('/dashboard/sources'); await page.getByRole('textbox', { name: 'Video search or YouTube URL' }).fill(`https://youtu.be/${videoId}`); try { - await page.getByRole('button', { name: /Open video|Search videos/ }).click(); + await page.getByRole('button', { name: /Inspect/ }).click(); await expect(page.getByText('Transcript arrived successfully.', { exact: true })).toBeVisible({ timeout: 1500 }); expect(resolveReads).toBe(0); } finally { release(); } diff --git a/platform/test/fixtures/agent-dashboard-server.mjs b/platform/test/fixtures/agent-dashboard-server.mjs index adef797..14928c0 100644 --- a/platform/test/fixtures/agent-dashboard-server.mjs +++ b/platform/test/fixtures/agent-dashboard-server.mjs @@ -135,6 +135,14 @@ createServer(async (req, res) => { if (runId) return reply(200, success(runId)); } if (url.pathname === '/v1/projects') return reply(200, { projects: [] }); + if (url.pathname === '/v1/sources/recent') { + if (req.method === 'POST') { + let raw = ''; for await (const chunk of req) raw += chunk; + const payload = JSON.parse(raw); + return reply(201, { source: { id: crypto.randomUUID(), input: payload.input, title: payload.input, kind: payload.snapshot.kind, updatedAt: Date.now() } }); + } + return reply(200, { sources: [] }); + } if (url.pathname === '/v1/monitors') return reply(200, { monitors: [] }); if (url.pathname === '/v1/notifications') return reply(200, { notifications: [] }); if (url.pathname === '/v1/billing') return reply(200, { plan: 'starter', status: 'active', creditBalance: 679, includedCredits: 1000, cancelAtPeriodEnd: false, currentPeriodStart: null, currentPeriodEnd: null, canManageBilling: false }); diff --git a/platform/test/user-account-do.integration.test.ts b/platform/test/user-account-do.integration.test.ts index 186fa1f..93cbfaf 100644 --- a/platform/test/user-account-do.integration.test.ts +++ b/platform/test/user-account-do.integration.test.ts @@ -1,11 +1,130 @@ -import { env, runInDurableObject } from 'cloudflare:test'; +import { env as workerEnv, runInDurableObject } from 'cloudflare:test'; import { describe, expect, test } from 'vitest'; +import type { SaveReferencedSource } from '../src/lib/source-history'; +import { referenceSource, restoreSource } from '../src/lib/source-history-storage'; +import { saveVideoResource } from '../src/lib/video-resources'; +import { sha256 } from '../src/lib/http'; +import { jsonError } from '../src/lib/http'; +import { Hono } from 'hono'; +import { sessionRoutes } from '../src/routes/session/session.index'; +import type { App, AuthPrincipal } from '../src/types'; +const env = workerEnv as Env; + +function sourceApp(userId: string, method: AuthPrincipal['method'] = 'session') { + const app = new Hono(); + app.use('*', async (c, next) => { + const user = { id: userId, email: `${userId}@example.test`, name: userId }; + c.set('principal', { user, method, permissions: {} }); c.set('user', user); await next(); + }); + app.route('/', sessionRoutes); app.onError((error, c) => jsonError(c, error)); + return app; +} const CONVERSATION_A = 'a08cff6c-326e-47f7-b771-59ff58c48846'; const CONVERSATION_B = 'e665d2a1-f9f9-4b7f-8b7c-0bf0a1393c3b'; const CONVERSATION_C = 'cfb5309a-954f-4e4a-9b0e-2d673c708f20'; describe('UserAccountDO', () => { + test('recent source routes enforce ownership and browser authentication and restore playlist data', async () => { + const playlist = { id: 'PLhistory', title: 'Saved playlist', videos: [] }; + const key = `youtube:v1:${await sha256(JSON.stringify(['playlist-v2', playlist.id]))}`; + await env.YOUTUBE_CACHE.put(key, JSON.stringify({ version: 1, fetchedAt: Date.now(), freshUntil: Date.now() + 60_000, value: playlist })); + const first = sourceApp('route-first'); + const request = () => ({ method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ + input: 'https://youtube.com/playlist?list=PLhistory', snapshot: { kind: 'inspection', inspector: { + provider: 'youtube', type: 'playlist', id: playlist.id, requestedData: [], dataErrors: {}, loadedData: ['metadata'], + } }, + }) }); + const saved = await first.request('/sources/recent', request(), env); + expect(saved.status).toBe(201); + const { source } = await saved.json() as { source: { id: string } }; + const list = await first.request('/sources/recent', {}, env); + expect(await list.json()).toMatchObject({ sources: [{ id: source.id }] }); + const restored = await first.request(`/sources/recent/${source.id}`, {}, env); + expect(restored.status).toBe(200); + expect(await restored.json()).toMatchObject({ snapshot: { kind: 'inspection', inspector: { data: playlist } } }); + expect((await sourceApp('route-second').request(`/sources/recent/${source.id}`, {}, env)).status).toBe(404); + expect((await sourceApp('route-first', 'cli-session').request('/sources/recent', {}, env)).status).toBe(403); + expect((await sourceApp('route-first', 'api-key').request('/sources/recent', request(), env)).status).toBe(403); + expect((await first.request('/sources/recent/invalid', {}, env)).status).toBe(422); + const malformed = await first.request('/sources/recent', { method: 'POST', headers: { 'content-type': 'application/json' }, body: '{}' }, env); + expect(malformed.status).toBe(422); + }); + test('recent videos store catalog references, reuse shared assets and restore the original version', async () => { + const id = 'abcdefghijk'; + const metadata = { id, title: 'Shared video', thumbnails: [] }; + const transcript = { videoId: id, text: 'Original transcript', segments: [{ text: 'Original transcript', startMs: 0, endMs: 1000, durationMs: 1000 }], + track: { name: 'English', kind: 'asr', languageCode: 'en' }, meta: { source: 'youtube', fetchedAt: new Date().toISOString(), partial: false, warnings: [] } }; + await saveVideoResource(env, { kind: 'video', id }, metadata, Date.now(), 60_000); + await saveVideoResource(env, { kind: 'transcript', id, granularity: 'word' }, transcript, Date.now(), 60_000); + const references = await referenceSource(env, { input: `https://youtu.be/${id}`, snapshot: { kind: 'inspection', inspector: { + provider: 'youtube', type: 'video', id, requestedData: ['transcript'], dataErrors: {}, loadedData: ['metadata', 'transcript'], + } } }); + const first = env.USER_ACCOUNT.getByName('source-first'), second = env.USER_ACCOUNT.getByName('source-second'); + const saved = await first.saveSource(references); + expect(await second.getSource(saved.id)).toBeNull(); + await second.saveSource(references); + await runInDurableObject(first, (_instance, state) => { + const row = state.storage.sql.exec<{ snapshot: string }>('SELECT snapshot FROM recent_sources').one(); + expect(row.snapshot).toContain('contentHash'); + expect(row.snapshot).not.toContain('Original transcript'); + expect(row.snapshot).not.toContain('segments'); + }); + const storedObjects = (await env.VIDEO_ASSETS.list()).objects.length; + await first.saveSource(references); + expect((await first.listSources())).toHaveLength(1); + expect((await env.VIDEO_ASSETS.list()).objects).toHaveLength(storedObjects); + await saveVideoResource(env, { kind: 'transcript', id, granularity: 'word' }, { ...transcript, text: 'Refreshed transcript', segments: [{ ...transcript.segments[0], text: 'Refreshed transcript' }] }, Date.now() + 1000, 60_000); + const stored = await first.getSource(saved.id); + const restored = await restoreSource(env, stored!.snapshot); + expect(restored.kind).toBe('inspection'); + if (restored.kind === 'inspection') { + expect(restored.inspector.transcript?.text).toBe('Original transcript'); + expect(restored.inspector.data.title).toBe('Shared video'); + } + await first.beginDeletion(); await first.finishDeletion(); + await expect(second.getSource((await second.listSources())[0]!.id)).resolves.not.toBeNull(); + expect(await restoreSource(env, references.snapshot)).toEqual(restored); + await runInDurableObject(first, (instance, state) => { + expect(() => instance.listSources()).toThrow('deletion'); + expect(() => instance.saveSource(references)).toThrow('deletion'); + expect(state.storage.sql.exec<{ count: number }>('SELECT COUNT(*) AS count FROM recent_sources').one().count).toBe(0); + }); + }); + + test('search history persists public results outside the DO and deduplicates normalized terms', async () => { + const input = 'Opus vs Astra'; + const results = [{ id: 'abcdefghijk', type: 'video', title: 'Model comparison', thumbnails: [] }]; + const searchKey = await sha256(JSON.stringify({ query: input, filters: { type: 'video' } })); + const cacheKey = `youtube:v1:${await sha256(JSON.stringify(['search-v3', searchKey]))}`; + await env.YOUTUBE_CACHE.put(cacheKey, JSON.stringify({ version: 1, fetchedAt: Date.now(), freshUntil: Date.now() + 60_000, value: { query: input, results } })); + const referenced = await referenceSource(env, { input, snapshot: { kind: 'search', selectedData: ['transcript'] } }); + const account = env.USER_ACCOUNT.getByName('source-search'); + const first = await account.saveSource(referenced); + const repeat = await account.saveSource({ ...referenced, input: ' OPUS vs Astra ' }); + expect(repeat.id).toBe(first.id); + expect(await account.listSources()).toHaveLength(1); + const restored = await restoreSource(env, referenced.snapshot); + expect(restored).toEqual({ kind: 'search', selectedData: ['transcript'], items: results }); + if (referenced.snapshot.kind === 'search') { + const payload = await (await env.VIDEO_ASSETS.get(referenced.snapshot.results))!.text(); + expect(payload).not.toContain(input); + await env.VIDEO_ASSETS.delete(referenced.snapshot.results); + await expect(restoreSource(env, referenced.snapshot)).rejects.toThrow('Saved source data is unavailable'); + } + }); + + test('recent sources keep the newest thirty entries', async () => { + const account = env.USER_ACCOUNT.getByName('source-limit'); + const snapshot: SaveReferencedSource['snapshot'] = { kind: 'search', selectedData: ['transcript'], results: `youtube/source-history/${'a'.repeat(64)}.json` }; + for (let i = 0; i < 32; i++) await account.saveSource({ input: `query ${i}`, title: `query ${i}`, snapshot }); + const sources = await account.listSources(); + expect(sources).toHaveLength(30); + expect(sources[0]?.input).toBe('query 31'); + expect(sources.some(source => source.input === 'query 0')).toBe(false); + await account.getSource(sources.at(-1)!.id); + expect((await account.listSources())[0]?.id).toBe(sources.at(-1)!.id); + }); test('deletion includes admissions without catalog entries and blocks late writes', async () => { const account = env.USER_ACCOUNT.getByName('user:deletion'); await account.registerConversation(CONVERSATION_A); diff --git a/platform/test/user-account.wrangler.jsonc b/platform/test/user-account.wrangler.jsonc index ae40f19..a35d757 100644 --- a/platform/test/user-account.wrangler.jsonc +++ b/platform/test/user-account.wrangler.jsonc @@ -36,6 +36,7 @@ } ], "r2_buckets": [{"binding":"RESEARCH","bucket_name":"agent-test-evidence"},{"binding":"VIDEO_ASSETS","bucket_name":"session-catalog-assets-test"}], + "kv_namespaces": [{ "binding": "YOUTUBE_CACHE", "id": "00000000000000000000000000000001" }], "vars": { "STARTER_ONBOARDING_CREDITS": "1000", "BUILDER_MONTHLY_CREDITS": "20000" diff --git a/reference/agents/platform-internals.md b/reference/agents/platform-internals.md index eea2cce..0c778ae 100644 --- a/reference/agents/platform-internals.md +++ b/reference/agents/platform-internals.md @@ -18,6 +18,8 @@ Read the root `README.md`, `docs/open-source/local-development.mdx`, and `refere For the DB-first public video catalog, R2 object layout, progressive retrieval, recovery and provisioning, read `reference/engineering/VIDEO_CATALOG.md`. +For Sources history, user DO reference ownership, and restoration from shared assets, read `reference/engineering/SOURCE_HISTORY.md`. + ## Agent session evidence For session evidence reuse, memory, citation versions and deletion invariants, read `reference/engineering/SESSION_EVIDENCE.md`. diff --git a/reference/engineering/SOURCE_HISTORY.md b/reference/engineering/SOURCE_HISTORY.md new file mode 100644 index 0000000..ab060d5 --- /dev/null +++ b/reference/engineering/SOURCE_HISTORY.md @@ -0,0 +1,32 @@ +# Recent sources + +Sources keeps the thirty most recent successful searches and inspections for each user. The user account Durable Object stores the original input, display title, dataset choices, errors, and shared asset references. It never stores video metadata, transcript segments, comments, or search result payloads in SQLite. + +The platform resolves references from its existing provider storage. Browser requests submit only source identity and dataset status. They cannot supply asset keys or write video payloads into the catalog. + +Video datasets reference immutable catalog versions through `(videoId, kind, variant, contentHash)`. Refreshing another user's copy cannot change the dataset restored by a recent entry. A user's explicit refresh updates that user's entry to the new version. + +Search, playlist, and channel caches expire. When remembering them, the platform copies the existing public response into a content-addressed JSON object under `youtube/source-history/.json` in `VIDEO_ASSETS`. Search objects contain only public results, without the user's query. Original inputs, user IDs, selections, and errors stay in the user DO. This extends persistence for history without changing the provider cache policy. + +```mermaid +%%{init: {'theme':'base','themeVariables':{'background':'#ffffff','actorBkg':'#e2e8f0','actorTextColor':'#0f172a','actorBorder':'#64748b','signalColor':'#334155','signalTextColor':'#334155','sequenceNumberColor':'#ffffff','noteBkgColor':'#f1f5f9','noteTextColor':'#0f172a'}}}%% +sequenceDiagram + autonumber + participant UI as Sources page + participant App as Platform + participant User as User DO + participant Catalog as Shared catalog and R2 + UI->>App: Remember completed input and dataset status + App->>Catalog: Resolve saved public asset references + App->>User: Save input, choices and references + UI->>App: Open a recent entry + App->>User: Read references owned by this user + App->>Catalog: Hydrate saved versions and public response objects + App-->>UI: Restore displayed data without a YouTube request +``` + +The browser-session routes are `GET /v1/sources/recent`, `POST /v1/sources/recent`, and `GET /v1/sources/recent/:id`. Opening an entry moves it to the top of history and does not charge credits. Inputs deduplicate by normalized search terms or provider entity identity. Zero-result searches and inspections with failed datasets retain their displayed state; failures remain retryable. + +Account deletion removes the user's references along with the other user DO data. Shared assets remain available to other accounts. Pruning the oldest entries removes only user references. Shared response objects follow the catalog's existing policy of retaining public assets, with no blanket bucket expiration or garbage collector. + +No new Cloudflare binding or class migration is required. `UserAccountDO` creates the history table when initialized. The platform and web changes must both be deployed to enable the feature. diff --git a/web/app/dashboard/SourcesClient.tsx b/web/app/dashboard/SourcesClient.tsx index d8d89bc..980f8f0 100644 --- a/web/app/dashboard/SourcesClient.tsx +++ b/web/app/dashboard/SourcesClient.tsx @@ -14,7 +14,7 @@ import pageStyles from './DashboardPages.module.css'; import { Icon } from './DashboardSidebar'; import { useDashboardSession } from './DashboardSessionProvider'; -import type { ProviderId, EntityType, SourceDataOption, Thumbnail, SearchItem, Segment, Transcript, CommentPage, ChannelInfo, Project, Inspector } from './research-types'; +import type { ProviderId, EntityType, SourceDataOption, Thumbnail, SearchItem, Segment, Transcript, CommentPage, ChannelInfo, Project, Inspector, RecentSource, SourceSnapshot } from './research-types'; import { DashboardSkeleton as SourceSkeleton } from './DashboardSkeleton'; const YOUTUBE_API = '/v1/providers/youtube'; const SOURCE_DATA_OPTIONS: Record = { @@ -54,11 +54,64 @@ export default function SourcesClient({ active }: {active:boolean}) { const [error, setError] = useState(''); const [notice, setNotice] = useState(''); const [operationLabel, setOperationLabel] = useState(''); + const [recentSources, setRecentSources] = useState([]); + const [historyLoading, setHistoryLoading] = useState(true); + const [historyError, setHistoryError] = useState(''); + const historyInput = useRef(''); const operationController = useRef(null); const searchInput = useRef(null); const authenticated = Boolean(user) || demoEnabled; const playlistInput = isPlaylistUrl(query); + const loadHistory = useCallback(async (signal?: AbortSignal) => { + setHistoryLoading(true); setHistoryError(''); + try { + const result = await api<{ sources: RecentSource[] }>('/v1/sources/recent', { signal }); + setRecentSources(result.sources); + } catch (cause) { if (!isAbortError(cause)) setHistoryError(cause instanceof Error ? cause.message : 'Could not load recent sources.'); } + finally { if (!signal?.aborted) setHistoryLoading(false); } + }, []); + + useEffect(() => { + if (!active || !authenticated) return; + const controller = new AbortController(); + void loadHistory(controller.signal); + return () => controller.abort(); + }, [active, authenticated, loadHistory]); + + const rememberSource = async (input: string, snapshot: SourceSnapshot, signal: AbortSignal) => { + try { + const request = snapshot.kind === 'search' ? { kind: snapshot.kind, selectedData: snapshot.selectedData } + : { kind: snapshot.kind, inspector: { + provider: snapshot.inspector.provider, type: snapshot.inspector.type, id: snapshot.inspector.id, + requestedData: snapshot.inspector.requestedData, dataErrors: snapshot.inspector.dataErrors, + loadedData: ['metadata', ...(['transcript', 'comments', 'channel'] as const).filter(field => snapshot.inspector[field])], + } }; + const { source } = await api<{ source: RecentSource }>('/v1/sources/recent', { + method: 'POST', body: JSON.stringify({ input, snapshot: request }), signal, + }); + setRecentSources(current => [source, ...current.filter(item => item.id !== source.id)].slice(0, 30)); + setHistoryError(''); + } catch (cause) { if (!isAbortError(cause)) setHistoryError(cause instanceof Error ? `Could not save recent source: ${cause.message}` : 'Could not save recent source.'); } + }; + + const openRecentSource = async (entry: RecentSource) => { + const controller = beginOperation('Loading saved source data…'); + try { + const { source, snapshot } = await api<{ source: RecentSource; snapshot: SourceSnapshot }>(`/v1/sources/recent/${entry.id}`, { signal: controller.signal }); + if (controller.signal.aborted) return; + setQuery(source.input); historyInput.current = source.input; setTranscriptQuery(''); + setRecentSources(current => [source, ...current.filter(item => item.id !== source.id)]); + if (snapshot.kind === 'search') { + setSelectedData(snapshot.selectedData); setItems(snapshot.items); setInspector(null); setHasSearched(true); + } else { + setItems([]); setHasSearched(false); setSelectedData(snapshot.inspector.requestedData.length ? snapshot.inspector.requestedData : ['transcript']); + setInspector(snapshot.inspector); + } + } catch (cause) { if (!isAbortError(cause)) setError(cause instanceof Error ? cause.message : 'Could not open recent source.'); } + finally { finishOperation(controller); } + }; + const beginOperation = useCallback((label: string) => { operationController.current?.abort(); const controller = new AbortController(); @@ -94,6 +147,7 @@ export default function SourcesClient({ active }: {active:boolean}) { const runSearch = async (event?: FormEvent) => { event?.preventDefault(); if (!query.trim()) return; + const input = query.trim(); const controller = beginOperation('Resolving your query…'); setHasSearched(true); setInspector(null); @@ -104,13 +158,15 @@ export default function SourcesClient({ active }: {active:boolean}) { method: 'POST', body: JSON.stringify({ input: query }), signal: controller.signal, }); if (resolved.kind === 'video' && resolved.id) { + setHasSearched(false); setItems([]); setOperationLabel('Opening the video and fetching your selected data…'); - await inspect('video', resolved.id, controller, resolved.provider ?? 'youtube', selectedData); + await inspect('video', resolved.id, controller, resolved.provider ?? 'youtube', selectedData, input); return; } if (resolved.kind === 'playlist' && resolved.id) { + setHasSearched(false); setItems([]); setOperationLabel('Opening the playlist and loading its videos…'); - await inspect('playlist', resolved.id, controller, resolved.provider ?? 'youtube', selectedData); + await inspect('playlist', resolved.id, controller, resolved.provider ?? 'youtube', selectedData, input); return; } if (resolved.kind !== 'search') { @@ -119,7 +175,9 @@ export default function SourcesClient({ active }: {active:boolean}) { setOperationLabel('Searching YouTube videos…'); const params = new URLSearchParams({ q: resolved.query ?? query, type: 'video' }); const data = await api<{ results: SearchItem[] }>(`${YOUTUBE_API}/search?${params}`, { signal: controller.signal }); - setItems(data.results.filter((item) => item.type === 'video').map((item) => ({ ...item, provider: 'youtube' }))); + const results = data.results.filter((item) => item.type === 'video').map((item) => ({ ...item, provider: 'youtube' as const })); + setItems(results); + await rememberSource(input, { kind: 'search', selectedData: [...selectedData], items: results }, controller.signal); } catch (cause) { if (!isAbortError(cause)) setError(cause instanceof Error ? cause.message : 'Search failed.'); } finally { finishOperation(controller); } @@ -127,6 +185,7 @@ export default function SourcesClient({ active }: {active:boolean}) { // Publish each independent result immediately. Only channel info needs metadata. const loadVideoData = async (next: Inspector, datasets: Array, controller: AbortController, refresh = false) => { + const input = historyInput.current; if (refresh) next.refreshData = [...new Set([...(next.refreshData ?? []), ...datasets])]; next.loadingData = [...datasets]; const publish = () => { @@ -158,13 +217,17 @@ export default function SourcesClient({ active }: {active:boolean}) { } await fetchSourceData(next, option, controller.signal, next.refreshData?.includes(option)); }))]); + if (!controller.signal.aborted && operationController.current === controller) { + await rememberSource(input, { kind: 'inspection', inspector: { ...next, loadingData: [] } }, controller.signal); + } }; const inspect = async ( type: EntityType, id: string, activeController?: AbortController, - provider: ProviderId = 'youtube', requestedData: SourceDataOption[] = selectedData, + provider: ProviderId = 'youtube', requestedData: SourceDataOption[] = selectedData, input?: string, ) => { const controller = activeController ?? beginOperation('Fetching your selected data…'); + historyInput.current = input ?? `https://www.youtube.com/${type === 'video' ? `watch?v=${id}` : type === 'playlist' ? `playlist?list=${id}` : `channel/${id}`}`; setError(''); try { if (type === 'video') { @@ -173,7 +236,11 @@ export default function SourcesClient({ active }: {active:boolean}) { } else { const plural = type === 'channel' ? 'channels' : 'playlists'; const data = await api>(`/v1/providers/${provider}/${plural}/${encodeURIComponent(id)}`, { signal: controller.signal }); - if (!controller.signal.aborted) setInspector({ provider, type, id, data, requestedData: [], dataErrors: {} }); + if (!controller.signal.aborted) { + const next: Inspector = { provider, type, id, data, requestedData: [], dataErrors: {} }; + setInspector(next); + await rememberSource(historyInput.current, { kind: 'inspection', inspector: next }, controller.signal); + } } } catch (cause) { if (!isAbortError(cause)) setError(cause instanceof Error ? cause.message : 'Could not open this source.'); } finally { finishOperation(controller); } @@ -288,12 +355,12 @@ export default function SourcesClient({ active }: {active:boolean}) {
<>
-

Find a video or playlist

Search YouTube or paste a link to inspect its data.

+

Search or paste a YouTube link

{(inspector || hasSearched) && }
-
setQuery(event.target.value)} placeholder='Search videos, or paste a video or playlist URL' autoComplete='off' />{playlistInput ? 'PLAYLIST' : '⌘ K'}
- +
setQuery(event.target.value)} placeholder='e.g. Opus 5.5 vs GPT 6 Astra, or a YouTube URL' autoComplete='off' />{playlistInput ? 'PLAYLIST' : '⌘ K'}
+
Include with each video @@ -323,10 +390,22 @@ export default function SourcesClient({ active }: {active:boolean}) { {inspector ? ( void retrySourceData()} onOpenComments={() => void refreshComments()} onRefresh={() => void refreshVideoData()} segments={filteredSegments} transcriptQuery={transcriptQuery} setTranscriptQuery={setTranscriptQuery} onClose={() => { cancelOperation(); setInspector(null); }} onSave={() => void saveInspector()} onMonitor={() => void addMonitor()} onOpenVideo={(id) => void inspect('video', id, undefined, inspector.provider, selectedData)} /> ) : ( - void inspect('video', id, undefined, provider, selectedData)} onStart={() => searchInput.current?.focus()} loading={loading} hasSearched={hasSearched} failed={Boolean(error)} /> + hasSearched || loading || items.length ? void inspect('video', id, undefined, provider, selectedData)} onStart={() => searchInput.current?.focus()} loading={loading} hasSearched={hasSearched} failed={Boolean(error)} /> + :
+

Recent sources

+ {historyLoading ?

Loading recent sources…

: null} + {historyError ?
{historyError}
: null} + {!historyLoading && !historyError && !recentSources.length ?

No recent sources yet

Search YouTube or inspect a link to start your history.

: null} +
{recentSources.map(source => )}
+
)}
+ {historyError && (inspector || hasSearched || loading) ?
{historyError}
: null} ); } diff --git a/web/app/dashboard/research-types.ts b/web/app/dashboard/research-types.ts index a2e7e5d..28fd045 100644 --- a/web/app/dashboard/research-types.ts +++ b/web/app/dashboard/research-types.ts @@ -74,6 +74,9 @@ export type Inspector = { comments?: CommentPage; channel?: ChannelInfo; }; +export type SourceSnapshot = { kind: 'search'; selectedData: SourceDataOption[]; items: SearchItem[] } + | { kind: 'inspection'; inspector: Inspector }; +export type RecentSource = { id: string; input: string; title: string; kind: SourceSnapshot['kind']; updatedAt: number }; export type TrendVideo = { id: string; title: string; channel: { id: string; name: string }; thumbnails: Thumbnail[]; durationSeconds?: number; publishedTimeText?: string; publishDate?: string; ageHours?: number; diff --git a/web/app/globals.css b/web/app/globals.css index 3c68b46..dde1436 100644 --- a/web/app/globals.css +++ b/web/app/globals.css @@ -1605,6 +1605,10 @@ html:has(.developer-page) body { position: relative; } +.source-result-list.recent-source-list > button { + grid-template-columns: 2.5rem minmax(0, 1fr) auto; +} + .source-result-thumb img, .source-result-thumb > i { width: 100%;