From b779a4c5a21f67aa6334a7a23702ba34285be4f3 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Mon, 28 Sep 2026 21:16:56 +0530 Subject: [PATCH 1/7] feat: reorder dashboard navigation --- platform/test/e2e/dashboard-api.spec.ts | 5 ++++- web/app/dashboard/DashboardSidebar.tsx | 2 +- web/app/dashboard/ResearchHost.tsx | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/platform/test/e2e/dashboard-api.spec.ts b/platform/test/e2e/dashboard-api.spec.ts index 4fbe2b2..c84d7fc 100644 --- a/platform/test/e2e/dashboard-api.spec.ts +++ b/platform/test/e2e/dashboard-api.spec.ts @@ -564,9 +564,12 @@ test('settings renders while navigation access checks are pending', async ({page await page.goto('/dashboard/settings',{waitUntil:'commit'}); await expect(page.getByRole('switch',{name:/In-app alerts/})).toBeEnabled(); await expect(page.getByRole('button',{name:'Upgrade to Builder'})).toBeEnabled(); + const navigation = page.getByRole('navigation', { name: 'Dashboard navigation' }); await expect(page.getByRole('link',{name:'Agent',exact:true})).toHaveCount(0); + await expect.poll(async () => (await navigation.locator('a').allTextContents()).slice(0, 2).map(label => label.trim())).toEqual(['Sources', 'Trends']); await scenario.release(); await expect(page.getByRole('link',{name:'Agent',exact:true})).toBeVisible(); + await expect.poll(async () => (await navigation.locator('a').allTextContents()).slice(0, 3).map(label => label.trim())).toEqual(['Sources', 'Agent', 'Trends']); }finally{await scenario.clear();} }); @@ -599,7 +602,7 @@ test('an active trend request survives projects navigation without restarting',a await page.getByRole('link',{name:'Projects',exact:true}).click(); await expect(page.getByRole('heading',{name:'Your projects'})).toBeVisible(); release(); - await page.getByRole('link',{name:'Trend Lab',exact:true}).click(); + await page.getByRole('link',{name:'Trends',exact:true}).click(); await expect(page.getByRole('alert').filter({hasText:'Retained scan completed'})).toBeVisible(); expect(reads).toBe(1); }); diff --git a/web/app/dashboard/DashboardSidebar.tsx b/web/app/dashboard/DashboardSidebar.tsx index b770ded..ae8321c 100644 --- a/web/app/dashboard/DashboardSidebar.tsx +++ b/web/app/dashboard/DashboardSidebar.tsx @@ -87,9 +87,9 @@ export function DashboardSidebar({ activeSection
Recent projects
- {projects.slice(0, 5).map(project => )} - {!projectsResource.ready && !projectsResource.error &&
{[0, 1, 2].map(index => )}
} + {projects.slice(0, 5).map(project => { + const expanded = expandedProjectId === project.id; + return
+ + {expanded &&
+ {!projectDetail && !projectError &&
} + {projectError && } + {projectDetail?.id === project.id && (projectDetail.items.length + ? projectDetail.items.map(item => ) + : No saved sources)} +
} +
; + })} + {!projectsResource.ready && !projectsResource.error &&
{[0, 1, 2].map(index => )}
} {projectsResource.ready && !projects.length &&

Save a source to start a project.

}
diff --git a/web/app/dashboard/NewProjectDialog.tsx b/web/app/dashboard/NewProjectDialog.tsx index b992b79..08b54f8 100644 --- a/web/app/dashboard/NewProjectDialog.tsx +++ b/web/app/dashboard/NewProjectDialog.tsx @@ -1,9 +1,54 @@ 'use client'; -import { useRef, useEffect, useState } from 'react'; -export function NewProjectDialog({ onClose,onCreate }: { onClose:()=>void;onCreate:(name:string)=>void }) { const dialogRef=useDialogFocus(onClose);const [name,setName]=useState('');return
event.stopPropagation()} onSubmit={(event)=>{event.preventDefault();if(name.trim())onCreate(name.trim());}}>

New project

Name this line of inquiry

setName(event.target.value)} placeholder='e.g. AI video research'/>
; } + +import { useEffect, useRef, useState } from 'react'; + +export function NewProjectDialog({ onClose, onCreate, error }: { + onClose: () => void; + onCreate: (name: string) => Promise; + error: string; +}) { + const [name, setName] = useState(''); + const [creating, setCreating] = useState(false); + const close = () => { if (!creating) onClose(); }; + const dialogRef = useDialogFocus(close); + + const create = async () => { + const trimmed = name.trim(); + if (!trimmed || creating) return; + setCreating(true); + const succeeded = await onCreate(trimmed); + if (!succeeded) setCreating(false); + }; + + return
+
event.stopPropagation()} + onSubmit={event => { event.preventDefault(); void create(); }} + > + +

New project

+

Keep related sources in one place.

+ + setName(event.target.value)} placeholder='e.g. AI video research' /> + {error &&

{error}

} + +
+
; +} function useDialogFocus(onClose: () => void) { const dialogRef = useRef(null); + const closeRef = useRef(onClose); + closeRef.current = onClose; useEffect(() => { const previouslyFocused = document.activeElement as HTMLElement | null; const dialog = dialogRef.current; @@ -12,9 +57,9 @@ function useDialogFocus(onClose: () => void) { preferred?.focus(); }); const onKeyDown = (event: KeyboardEvent) => { - if (event.key === 'Escape') { event.preventDefault(); onClose(); return; } + if (event.key === 'Escape') { event.preventDefault(); closeRef.current(); return; } if (event.key !== 'Tab' || !dialog) return; - const focusable = Array.from(dialog.querySelectorAll('button:not([disabled]), input:not([disabled]), select:not([disabled]), textarea:not([disabled]), [href], [tabindex]:not([tabindex="-1"])')).filter((element) => !element.hidden); + const focusable = Array.from(dialog.querySelectorAll('button:not([disabled]), input:not([disabled]), select:not([disabled]), textarea:not([disabled]), [href], [tabindex]:not([tabindex="-1"])')).filter(element => !element.hidden); if (!focusable.length) return; const first = focusable[0]; const last = focusable[focusable.length - 1]; if (event.shiftKey && document.activeElement === first) { event.preventDefault(); last.focus(); } @@ -22,6 +67,6 @@ function useDialogFocus(onClose: () => void) { }; document.addEventListener('keydown', onKeyDown); return () => { document.removeEventListener('keydown', onKeyDown); previouslyFocused?.focus(); }; - }, [onClose]); + }, []); return dialogRef; } diff --git a/web/app/dashboard/WorkspaceShell.tsx b/web/app/dashboard/WorkspaceShell.tsx index 5e6c061..5c1d8f2 100644 --- a/web/app/dashboard/WorkspaceShell.tsx +++ b/web/app/dashboard/WorkspaceShell.tsx @@ -30,6 +30,7 @@ export function WorkspaceShell({ onNavigate={(s) => router.push(dashboardPath(s))} onNewProject={() => router.push('/dashboard/projects?newProject=1')} onOpenProject={(p) => router.push(`/dashboard/projects?project=${encodeURIComponent(p.id)}`)} + onOpenProjectItem={(item) => router.push(`/dashboard/sources?type=${item.entity_type}&id=${encodeURIComponent(item.entity_id)}`)} onSignIn={() => router.push('/login')} accountName={user?.name ?? user?.email ?? (demoEnabled ? 'Local demo' : undefined)} credits={usage?.creditBalance} diff --git a/web/app/dashboard/admin/AdminAccessClient.tsx b/web/app/dashboard/admin/AdminAccessClient.tsx index 3055e8d..4855ff3 100644 --- a/web/app/dashboard/admin/AdminAccessClient.tsx +++ b/web/app/dashboard/admin/AdminAccessClient.tsx @@ -68,6 +68,7 @@ export default function AdminAccessClient() { onNavigate={section => router.push(`/dashboard/${section === 'discover' ? 'sources' : section}`)} onNewProject={() => router.push('/dashboard?section=projects')} onOpenProject={project => router.push(`/dashboard/projects?project=${encodeURIComponent(project.id)}`)} + onOpenProjectItem={item => router.push(`/dashboard/sources?type=${item.entity_type}&id=${encodeURIComponent(item.entity_id)}`)} onSignIn={() => router.push('/dashboard')} accountName={user?.name ?? user?.email} credits={credits} onSignOut={() => void signOut()} />
diff --git a/web/app/dashboard/developer/DeveloperSettingsClient.tsx b/web/app/dashboard/developer/DeveloperSettingsClient.tsx index 516c815..edeb536 100644 --- a/web/app/dashboard/developer/DeveloperSettingsClient.tsx +++ b/web/app/dashboard/developer/DeveloperSettingsClient.tsx @@ -87,6 +87,7 @@ export default function DeveloperSettingsClient({promise}:{promise:Promise router.push('/dashboard/projects?newProject=1')} onOpenProject={project => router.push(`/dashboard/projects?project=${encodeURIComponent(project.id)}`)} + onOpenProjectItem={item => router.push(`/dashboard/sources?type=${item.entity_type}&id=${encodeURIComponent(item.entity_id)}`)} onSignIn={() => router.push('/login?returnTo=%2Fdashboard%2Fdeveloper')} accountName={displayUser.name ?? displayUser.email} credits={credits} diff --git a/web/app/dashboard/projects/ProjectsClient.tsx b/web/app/dashboard/projects/ProjectsClient.tsx index 7c345b5..261958e 100644 --- a/web/app/dashboard/projects/ProjectsClient.tsx +++ b/web/app/dashboard/projects/ProjectsClient.tsx @@ -14,6 +14,7 @@ export function ProjectsClient({ promise }: { promise: Promise('selected-project', null); const [loading, setLoading] = useState(false), [error, setError] = useState(''), + [createError, setCreateError] = useState(''), [create, setCreate] = useState(false); const controller = useRef(null); const open = useCallback( @@ -41,16 +42,19 @@ export function ProjectsClient({ promise }: { promise: Promise { + const add = async (name: string): Promise => { + setCreateError(''); try { - await api('/v1/projects', { method: 'POST', body: JSON.stringify({ name }) }); - await resource.refresh(); + const project = await api('/v1/projects', { method: 'POST', body: JSON.stringify({ name }) }); + resource.setData(current => [project, ...current]); setCreate(false); + return true; } catch (cause) { - setError(cause instanceof Error ? cause.message : 'Could not create project.'); + setCreateError(cause instanceof Error ? cause.message : 'Could not create project.'); + return false; } }; return ( @@ -66,7 +70,7 @@ export function ProjectsClient({ promise }: { promise: Promise setCreate(true)} + onCreate={() => { setCreateError(''); setCreate(true); }} onOpen={(p) => void open(p)} onBack={() => { setSelected(null); @@ -78,12 +82,7 @@ export function ProjectsClient({ promise }: { promise: Promise )} - {create && ( - <> - setCreate(false)} onCreate={(name) => void add(name)} /> - {error &&

{error}

} - - )} + {create && { setCreate(false); setCreateError(''); }} onCreate={add} error={createError} />} ); } diff --git a/web/app/dashboard/sessions/SessionsClient.tsx b/web/app/dashboard/sessions/SessionsClient.tsx index e372b19..9c9f29b 100644 --- a/web/app/dashboard/sessions/SessionsClient.tsx +++ b/web/app/dashboard/sessions/SessionsClient.tsx @@ -36,8 +36,9 @@ export function AgentShell({ children }: { children: ReactNode }) { return
router.push(`/dashboard/${section === 'discover' ? 'sources' : section}`)} - onNewProject={() => router.push('/dashboard?section=projects')} - onOpenProject={() => router.push('/dashboard?section=projects')} + onNewProject={() => router.push('/dashboard/projects?newProject=1')} + onOpenProject={project => router.push(`/dashboard/projects?project=${encodeURIComponent(project.id)}`)} + onOpenProjectItem={item => router.push(`/dashboard/sources?type=${item.entity_type}&id=${encodeURIComponent(item.entity_id)}`)} onSignIn={() => router.push('/login?returnTo=%2Fdashboard%2Fsessions')} accountName={user?.name ?? user?.email} onSignOut={() => void signOut()} />
diff --git a/web/app/dashboard/settings/SettingsShell.tsx b/web/app/dashboard/settings/SettingsShell.tsx index 16a85c6..c6dcc06 100644 --- a/web/app/dashboard/settings/SettingsShell.tsx +++ b/web/app/dashboard/settings/SettingsShell.tsx @@ -18,6 +18,7 @@ export function SettingsShell({ children }: { children: ReactNode }) { router.push('/dashboard/projects?newProject=1')} onOpenProject={project => router.push(`/dashboard/projects?project=${encodeURIComponent(project.id)}`)} + onOpenProjectItem={item => router.push(`/dashboard/sources?type=${item.entity_type}&id=${encodeURIComponent(item.entity_id)}`)} onSignIn={() => router.push('/login')} accountName={user?.name ?? user?.email ?? (demoEnabled ? 'Local demo' : undefined)} credits={usage?.creditBalance} onSignOut={() => void signOut()} />
diff --git a/web/app/globals.css b/web/app/globals.css index 8758ecd..b0381f3 100644 --- a/web/app/globals.css +++ b/web/app/globals.css @@ -1039,6 +1039,13 @@ html:has(.developer-page) body { backdrop-filter: blur(.45rem); } +.new-project-dialog h2 { margin: 0 0 .375rem; } +.new-project-dialog .new-project-description { margin: 0 0 1.5rem; color: var(--color-dashboard-muted); font-size: .8125rem; } +.new-project-dialog .new-project-error { margin: .25rem 0 0; color: var(--color-dashboard-danger); font-size: .75rem; } +.new-project-dialog .button.primary { display: inline-flex; align-items: center; justify-content: center; gap: .5rem; min-width: 6.5rem; margin-top: .75rem; } +.new-project-dialog .button.primary:disabled { cursor: default; opacity: .65; } +.new-project-dialog .status-spinner { width: .875rem; height: .875rem; border-width: 2px; border-color: #ffffff66; border-top-color: #fff; } + .dialog-close, .back, .project-grid footer button, From ec6234e63fc7edb559eb87a87f4fb5a4c5e8fff9 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Mon, 28 Sep 2026 22:30:31 +0530 Subject: [PATCH 6/7] fix: link project searches and videos to saved sources --- docs/api-reference/openapi.json | 15 +++ docs/internals/endpoints.mdx | 1 + platform/src/durable-objects/user-account.ts | 106 +++++++++++++++++- platform/src/openapi-audience.ts | 2 + platform/src/openapi.ts | 22 ++++ platform/src/routes/session/session.index.ts | 25 ++++- platform/test/e2e/dashboard-api.spec.ts | 71 ++++++++++++ .../test/user-account-do.integration.test.ts | 47 ++++++++ web/app/dashboard/SourcesClient.tsx | 53 +++++++-- web/app/dashboard/projects/ProjectsClient.tsx | 13 ++- web/app/dashboard/projects/ProjectsView.tsx | 2 +- web/app/dashboard/research-types.ts | 2 +- web/app/globals.css | 11 ++ 13 files changed, 346 insertions(+), 24 deletions(-) diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json index 86f7bf4..1b92696 100644 --- a/docs/api-reference/openapi.json +++ b/docs/api-reference/openapi.json @@ -6864,6 +6864,21 @@ "description": "A persisted D1 record. Database timestamps are Unix milliseconds.", "additionalProperties": true }, + "ProjectSourceLink": { + "type": "object", + "required": [ + "item", + "added" + ], + "properties": { + "item": { + "$ref": "#/components/schemas/ProjectItem" + }, + "added": { + "type": "boolean" + } + } + }, "CreateProjectRequest": { "type": "object", "required": [ diff --git a/docs/internals/endpoints.mdx b/docs/internals/endpoints.mdx index 7529636..75db0bb 100644 --- a/docs/internals/endpoints.mdx +++ b/docs/internals/endpoints.mdx @@ -31,6 +31,7 @@ Called by the video2ctx web application or an explicit signed-in account action. | `POST` | `/v1/notification-preferences/confirm-email` | `confirmNotificationEmail` | Confirm monitor email alerts from the signed-in dashboard | `sessionCookie` or `cliSession` or `bearerApiKey` or `apiKey` or `demoUser` | Enables email delivery only for the signed-in account after validating the confirmation token. | | `DELETE` | `/v1/oauth/youtube` | `disconnectYouTube` | Disconnect the YouTube account | `sessionCookie` or `demoUser` | Mutates the account connection state; require an explicit user action. | | `GET` | `/v1/oauth/youtube/connect` | `createYouTubeConnectUrl` | Create a YouTube OAuth URL | `sessionCookie` or `demoUser` | Returns a state-bound OAuth URL; start it only from a user-initiated connection flow. | +| `POST` | `/v1/projects/{id}/sources` | `linkProjectSource` | Add a saved Sources search or inspection to a project | `sessionCookie` or `demoUser` | Browser-session only. Verify that the project and saved source belong to the same signed-in user before linking shared asset references. | | `POST` | `/v1/resolve` | `resolveInput` | Route universal UI input | `sessionCookie` or `cliSession` or `bearerApiKey` or `apiKey` or `demoUser` | First-party input router; its dispatch behavior is not a stable public API contract. | | `POST` | `/v1/scale-inquiries` | `submitScaleInquiry` | Submit a Scale plan inquiry | Public or protocol-signed | Public lead form; validate Turnstile and rate limits, and never let the caller choose the notification recipient. | | `GET` | `/v1/sources/recent` | `listRecentSources` | List recent Sources inputs | `sessionCookie` or `demoUser` | Browser-session only. Lists inputs belonging to the authenticated user without exposing shared asset references. | diff --git a/platform/src/durable-objects/user-account.ts b/platform/src/durable-objects/user-account.ts index af01f56..fd41108 100644 --- a/platform/src/durable-objects/user-account.ts +++ b/platform/src/durable-objects/user-account.ts @@ -2,7 +2,7 @@ import { AgentAdmissionQueue } from '../agents/runtime/admission-queue'; import type { AgentRequest, AgentAdmission } from '../agents/contracts'; import { DurableObject } from 'cloudflare:workers'; import { z } from 'zod'; -import { RECENT_SOURCE_LIMIT, saveReferencedSourceSchema, sourceReferenceSchema, sourceIdentity, type RecentSource, type SaveReferencedSource, type SourceReference } from '../lib/source-history'; +import { RECENT_SOURCE_LIMIT, saveReferencedSourceSchema, sourceReferenceSchema, sourceIdentity, sourceIdSchema, type RecentSource, type SaveReferencedSource, type SourceReference } from '../lib/source-history'; const MAX_SEARCH_TEXT_LENGTH = 32_000; const MAX_TITLE_LENGTH = 80; @@ -46,6 +46,26 @@ export interface UserSessionPage { nextCursor: UserSessionCursor | null; } +export interface ProjectSourceItem { + id: string; + source_id: string; + provider: 'youtube'; + entity_type: 'search' | 'video' | 'playlist' | 'channel'; + entity_id: string; + title: string; + created_at: number; +} + +interface ProjectSourceRow extends Record { + id: string; + source_id: string; + input: string; + title: string; + kind: RecentSource['kind']; + snapshot: string; + created_at: number; +} + interface SessionRow extends Record { conversation_id: string; title: string; @@ -102,6 +122,7 @@ export class UserAccountDO extends DurableObject { this.ctx.storage.sql.exec('DELETE FROM user_sessions'); this.ctx.storage.sql.exec('DELETE FROM agent_conversations'); this.ctx.storage.sql.exec('DELETE FROM recent_sources'); + this.ctx.storage.sql.exec('DELETE FROM project_sources'); // Keep only a tombstone so already-authenticated requests cannot recreate data. } @@ -153,10 +174,78 @@ export class UserAccountDO extends DurableObject { getSource(id: string): { source: RecentSource; snapshot: SourceReference } | null { this.assertActive(); - const row = this.ctx.storage.sql.exec<{ snapshot: string }>('SELECT snapshot FROM recent_sources WHERE id = ?', z.string().uuid().parse(id)).toArray()[0]; - if (!row) return null; - this.ctx.storage.sql.exec('UPDATE recent_sources SET updated_at = ? WHERE id = ?', this.nextSourceUpdate(), id); - return { source: this.listSources().find(source => source.id === id)!, snapshot: sourceReferenceSchema.parse(JSON.parse(row.snapshot)) }; + const sourceId = sourceIdSchema.parse(id); + const row = this.ctx.storage.sql.exec<{ id: string; input: string; title: string; kind: RecentSource['kind']; updated_at: number; snapshot: string }>( + 'SELECT id, input, title, kind, updated_at, snapshot FROM recent_sources WHERE id = ?', sourceId, + ).toArray()[0]; + if (row) { + const updatedAt = this.nextSourceUpdate(); + this.ctx.storage.sql.exec('UPDATE recent_sources SET updated_at = ? WHERE id = ?', updatedAt, id); + return { source: { id: row.id, input: row.input, title: row.title, kind: row.kind, updatedAt }, + snapshot: sourceReferenceSchema.parse(JSON.parse(row.snapshot)) }; + } + // A project keeps its own reference after the 30-entry recent list rotates. + const saved = this.ctx.storage.sql.exec( + 'SELECT id, source_id, input, title, kind, snapshot, created_at FROM project_sources WHERE source_id = ? LIMIT 1', sourceId, + ).toArray()[0]; + if (!saved) return null; + return { source: { id: saved.source_id, input: saved.input, title: saved.title, kind: saved.kind, updatedAt: saved.created_at }, + snapshot: sourceReferenceSchema.parse(JSON.parse(saved.snapshot)) }; + } + + linkSourceToProject(projectId: string, sourceId: string): { item: ProjectSourceItem; added: boolean } | null { + this.assertActive(); + const project = z.string().uuid().parse(projectId); + const source = sourceIdSchema.parse(sourceId); + const recent = this.ctx.storage.sql.exec<{ source_key: string; input: string; title: string; kind: RecentSource['kind']; snapshot: string }>( + 'SELECT source_key, input, title, kind, snapshot FROM recent_sources WHERE id = ?', source, + ).toArray()[0]; + if (!recent) return null; + const existing = this.ctx.storage.sql.exec<{ id: string }>( + 'SELECT id FROM project_sources WHERE project_id = ? AND source_key = ?', project, recent.source_key, + ).toArray()[0]; + const id = existing?.id ?? crypto.randomUUID(); + const createdAt = Date.now(); + this.ctx.storage.sql.exec(`INSERT INTO project_sources + (id, project_id, source_key, source_id, input, title, kind, snapshot, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(project_id, source_key) DO UPDATE SET + source_id=excluded.source_id, input=excluded.input, title=excluded.title, + kind=excluded.kind, snapshot=excluded.snapshot`, + id, project, recent.source_key, source, recent.input, recent.title, recent.kind, recent.snapshot, createdAt); + const saved = this.ctx.storage.sql.exec( + 'SELECT id, source_id, input, title, kind, snapshot, created_at FROM project_sources WHERE id = ?', id, + ).one(); + return { item: this.toProjectSourceItem(saved), added: !existing }; + } + + listProjectSources(projectId: string): ProjectSourceItem[] { + this.assertActive(); + const rows = this.ctx.storage.sql.exec( + 'SELECT id, source_id, input, title, kind, snapshot, created_at FROM project_sources WHERE project_id = ? ORDER BY created_at DESC', + z.string().uuid().parse(projectId), + ).toArray(); + return rows.map(row => this.toProjectSourceItem(row)); + } + + projectSourceCounts(): Array<{ projectId: string; count: number }> { + this.assertActive(); + return this.ctx.storage.sql.exec<{ project_id: string; count: number }>( + 'SELECT project_id, COUNT(*) AS count FROM project_sources GROUP BY project_id', + ).toArray().map(row => ({ projectId: row.project_id, count: row.count })); + } + + removeProjectSources(projectId: string): void { + this.assertActive(); + this.ctx.storage.sql.exec('DELETE FROM project_sources WHERE project_id = ?', z.string().uuid().parse(projectId)); + } + + private toProjectSourceItem(row: ProjectSourceRow): ProjectSourceItem { + const snapshot = sourceReferenceSchema.parse(JSON.parse(row.snapshot)); + return { id: row.id, source_id: row.source_id, provider: 'youtube', + entity_type: snapshot.kind === 'search' ? 'search' : snapshot.inspector.type, + entity_id: snapshot.kind === 'search' ? row.source_id : snapshot.inspector.id, + title: row.title, created_at: row.created_at }; } private nextSourceUpdate(): number { @@ -342,6 +431,13 @@ export class UserAccountDO extends DurableObject { id TEXT PRIMARY KEY, source_key TEXT NOT NULL UNIQUE, input TEXT NOT NULL, title TEXT NOT NULL, kind TEXT NOT NULL, updated_at INTEGER NOT NULL, snapshot TEXT NOT NULL )`); + this.ctx.storage.sql.exec(`CREATE TABLE IF NOT EXISTS project_sources ( + id TEXT PRIMARY KEY, project_id TEXT NOT NULL, source_key TEXT NOT NULL, + source_id TEXT NOT NULL, input TEXT NOT NULL, title TEXT NOT NULL, + kind TEXT NOT NULL, snapshot TEXT NOT NULL, created_at INTEGER NOT NULL, + UNIQUE(project_id, source_key) + )`); + this.ctx.storage.sql.exec('CREATE INDEX IF NOT EXISTS project_sources_project_idx ON project_sources (project_id, created_at DESC)'); this.ctx.storage.sql.exec('CREATE TABLE IF NOT EXISTS account_deletion (id INTEGER PRIMARY KEY)'); this.ctx.storage.sql.exec('CREATE TABLE IF NOT EXISTS agent_conversations (conversation_id TEXT PRIMARY KEY)'); this.ctx.storage.sql.exec(` diff --git a/platform/src/openapi-audience.ts b/platform/src/openapi-audience.ts index 6f8c957..702d6d1 100644 --- a/platform/src/openapi-audience.ts +++ b/platform/src/openapi-audience.ts @@ -32,6 +32,7 @@ export const OPENAPI_OPERATION_AUDIENCE: Readonly> = { listRecentSources: 'Browser-session only. Lists inputs belonging to the authenticated user without exposing shared asset references.', saveRecentSource: 'Browser-session only. Resolves existing provider assets server-side; never accepts client-supplied R2 keys or video payloads.', getRecentSource: 'Browser-session only. Verifies user ownership before reading the saved shared references.', + linkProjectSource: 'Browser-session only. Verify that the project and saved source belong to the same signed-in user before linking shared asset references.', inspectLandingYouTubeVideo: 'Public, rate-limited demo route; do not use it as a credentialed bulk-data API.', submitScaleInquiry: 'Public lead form; validate Turnstile and rate limits, and never let the caller choose the notification recipient.', signInWithMagicLink: 'Sends account email; rate-limit callers and never disclose whether an address is registered.', diff --git a/platform/src/openapi.ts b/platform/src/openapi.ts index e5200e0..a0ede21 100644 --- a/platform/src/openapi.ts +++ b/platform/src/openapi.ts @@ -1181,6 +1181,25 @@ export const openApiDocument = { }, }, }, + '/v1/projects/{id}/sources': { + post: { + tags: ['Projects'], + operationId: 'linkProjectSource', + summary: 'Add a saved Sources search or inspection to a project', + description: 'Keeps a user-owned reference to the shared source assets even after the recent Sources list rotates.', + security: privateSecurity, + parameters: [idParameter], + requestBody: jsonBody({ type: 'object', required: ['sourceId'], properties: { + sourceId: { type: 'string', format: 'uuid' }, + } }), + responses: { + '201': jsonResponse('Source linked to the project.', schemaRef('ProjectSourceLink')), + '200': jsonResponse('Existing project source refreshed.', schemaRef('ProjectSourceLink')), + ...standardErrors, + '404': responseRef('NotFound'), + }, + }, + }, '/v1/imports': { post: { tags: ['Research'], @@ -2142,6 +2161,9 @@ export const openApiDocument = { Project: { allOf: [storedRecord, { properties: { id: { type: 'string' }, name: { type: 'string' }, description: { type: 'string' }, item_count: { type: 'integer' } } }] }, ProjectDetail: { allOf: [schemaRef('Project'), { type: 'object', required: ['items'], properties: { items: { type: 'array', items: schemaRef('ProjectItem') } } }] }, ProjectItem: storedRecord, + ProjectSourceLink: { type: 'object', required: ['item', 'added'], properties: { + item: schemaRef('ProjectItem'), added: { type: 'boolean' }, + } }, CreateProjectRequest: { type: 'object', required: ['name'], properties: { name: { type: 'string', minLength: 1, maxLength: 120, example: 'Research inbox' }, diff --git a/platform/src/routes/session/session.index.ts b/platform/src/routes/session/session.index.ts index 9b6b1bd..c9fe83a 100644 --- a/platform/src/routes/session/session.index.ts +++ b/platform/src/routes/session/session.index.ts @@ -47,6 +47,7 @@ export const ACCOUNT_ROUTE_PATTERNS = [ export const SESSION_ONLY_ROUTE_PATTERNS = [ '/sources/recent', '/sources/recent/*', + '/projects/:id/sources', '/oauth/youtube/connect', '/oauth/youtube', '/billing', @@ -105,7 +106,11 @@ sessionRoutes.get('/projects', async (c) => { `SELECT p.*,COUNT(i.id) AS item_count FROM projects p LEFT JOIN project_items i ON i.project_id=p.id WHERE p.user_id=? GROUP BY p.id ORDER BY p.updated_at DESC` ).bind(user.id).all(); - return c.json({ projects: result.results }); + const account = c.env.USER_ACCOUNT.getByName(await userAccountInstanceName(user.id)); + const counts = new Map((await account.projectSourceCounts()).map(({ projectId, count }) => [projectId, count])); + return c.json({ projects: result.results.map(project => ({ + ...project, item_count: Number(project.item_count ?? 0) + (counts.get(String(project.id)) ?? 0), + })) }); }); sessionRoutes.post('/projects', async (c) => { @@ -129,7 +134,21 @@ sessionRoutes.get('/projects/:id', async (c) => { if (!project) throw new ApiError(404, 'PROJECT_NOT_FOUND', 'Project not found.'); const items = await c.env.DB.prepare('SELECT * FROM project_items WHERE project_id=? AND user_id=? ORDER BY created_at DESC') .bind(id, user.id).all(); - return c.json({ ...project, items: items.results }); + const account = c.env.USER_ACCOUNT.getByName(await userAccountInstanceName(user.id)); + const sources = await account.listProjectSources(id); + return c.json({ ...project, items: [...items.results, ...sources].sort((a, b) => Number(b.created_at) - Number(a.created_at)) }); +}); + +sessionRoutes.post('/projects/:id/sources', async (c) => { + const user = requireUser(c); + const projectId = asId(c.req.param('id')); + await ownProject(c.env, user.id, projectId); + const input = await body<{ sourceId?: string }>(c.req.raw); + if (!sourceIdSchema.safeParse(input.sourceId).success) throw new ApiError(422, 'INVALID_SOURCE_ID', 'A saved source ID is required.'); + const account = c.env.USER_ACCOUNT.getByName(await userAccountInstanceName(user.id)); + const linked = await account.linkSourceToProject(projectId, input.sourceId!); + if (!linked) throw new ApiError(404, 'SOURCE_NOT_FOUND', 'This recent source was not found.'); + return c.json(linked, linked.added ? 201 : 200); }); sessionRoutes.post('/projects/:id/items', async (c) => { @@ -176,6 +195,8 @@ sessionRoutes.delete('/projects/:id', async (c) => { await deleteProjectAssets(c.env, user.id, id); const result = await c.env.DB.prepare('DELETE FROM projects WHERE id=? AND user_id=?').bind(id, user.id).run(); if (!result.meta.changes) throw new ApiError(404, 'PROJECT_NOT_FOUND', 'Project not found.'); + const account = c.env.USER_ACCOUNT.getByName(await userAccountInstanceName(user.id)); + await account.removeProjectSources(id); return c.body(null, 204); }); diff --git a/platform/test/e2e/dashboard-api.spec.ts b/platform/test/e2e/dashboard-api.spec.ts index 4fbe2b2..6ba103a 100644 --- a/platform/test/e2e/dashboard-api.spec.ts +++ b/platform/test/e2e/dashboard-api.spec.ts @@ -558,6 +558,77 @@ test('settings sidebar opens the selected project and the new-project dialog', a } finally { await scenario.clear(); } }); +test('adding sources from a project saves the search and opened video in that project', async ({ page }) => { + const projectId = '1e498a23-56a6-4834-a1c3-57cc019b14a5'; + const query = 'codex gpt 6 astra tips'; + const projectItems: Array> = []; + const sources = new Map(); + let releaseSearchSave = () => {}; + const searchSaveGate = new Promise(resolve => { releaseSearchSave = resolve; }); + const scenario = await accountScenario(page, { responses: { + '/v1/projects': { body: { projects: [{ id: projectId, name: 'codex', item_count: 0 }] } }, + } }); + await page.route(`**/api/platform/v1/projects/${projectId}`, route => route.fulfill({ json: { + id: projectId, name: 'codex', items: projectItems, + } })); + await page.route(`**/api/platform/v1/projects/${projectId}/sources`, async route => { + const { sourceId } = route.request().postDataJSON(); + const source = sources.get(sourceId)!; + const existing = projectItems.find(item => item.source_id === sourceId); + if (existing) return route.fulfill({ status: 200, json: { item: existing, added: false } }); + const item = { id: `saved-${projectItems.length}`, source_id: sourceId, provider: 'youtube', + entity_type: source.kind === 'search' ? 'search' : 'video', entity_id: sourceId, title: source.title }; + projectItems.push(item); + return route.fulfill({ status: 201, json: { item, added: true } }); + }); + await page.route('**/api/platform/v1/sources/recent', async route => { + if (route.request().method() !== 'POST') return route.fulfill({ json: { sources: [] } }); + const input = route.request().postDataJSON(); + if (input.snapshot.kind === 'search') await searchSaveGate; + const id = crypto.randomUUID(); + const title = input.snapshot.kind === 'search' ? query : 'Codex tips video'; + sources.set(id, { title, kind: input.snapshot.kind, input: input.input }); + return route.fulfill({ status: 201, json: { source: { id, input: input.input, title, kind: input.snapshot.kind, updatedAt: Date.now() } } }); + }); + await page.route('**/api/platform/v1/sources/recent/*', route => { + const id = route.request().url().split('/').at(-1)!; + const source = sources.get(id)!; + const snapshot = source.kind === 'search' + ? { kind: 'search', selectedData: ['transcript'], items: [{ provider: 'youtube', type: 'video', id: videoId, title: 'Codex tips video', thumbnails: [] }] } + : { kind: 'inspection', inspector: { provider: 'youtube', type: 'video', id: videoId, + data: { id: videoId, title: 'Codex tips video', thumbnails: [] }, requestedData: ['transcript'], dataErrors: {} } }; + return route.fulfill({ json: { source: { id, ...source, updatedAt: Date.now() }, snapshot } }); + }); + await page.route('**/api/platform/v1/resolve', route => route.fulfill({ json: { kind: 'search', query } })); + await page.route('**/api/platform/v1/providers/youtube/search?**', route => route.fulfill({ json: { results: [{ + provider: 'youtube', type: 'video', id: videoId, title: 'Codex tips video', thumbnails: [], + }] } })); + await page.route(`**/api/platform/v1/providers/youtube/videos/${videoId}`, route => route.fulfill({ json: { + id: videoId, title: 'Codex tips video', thumbnails: [], channel: { id: 'channel', name: 'Creator' }, + } })); + await page.route(`**/api/platform/v1/providers/youtube/videos/${videoId}/transcript`, route => route.fulfill({ json: transcript })); + try { + await page.goto('/dashboard/projects'); + await page.getByRole('button', { name: 'codex 0 sources' }).click(); + await page.getByRole('button', { name: 'Add sources' }).click(); + await expect(page.getByText('Adding sources to')).toBeVisible(); + await page.getByRole('textbox', { name: 'Video search or YouTube URL' }).fill(query); + await page.getByRole('button', { name: /Inspect/ }).click(); + await page.getByRole('button', { name: /Codex tips video/ }).click(); + releaseSearchSave(); + await expect.poll(() => projectItems.length).toBe(2); + await page.getByRole('link', { name: 'Projects', exact: true }).click(); + await expect(page.getByRole('heading', { name: 'Saved sources 2' })).toBeVisible(); + await expect(page.getByRole('button', { name: /codex gpt 6 astra tips/ })).toBeVisible(); + await expect(page.getByRole('button', { name: /Codex tips video/ })).toBeVisible(); + await page.getByRole('button', { name: /codex gpt 6 astra tips/ }).click(); + await expect(page.getByRole('heading', { name: 'Results' })).toBeVisible(); + await page.getByRole('link', { name: 'Projects', exact: true }).click(); + await expect(page.getByRole('heading', { name: 'Saved sources 2' })).toBeVisible(); + expect(projectItems).toHaveLength(2); + } finally { releaseSearchSave(); await scenario.clear(); } +}); + test('settings renders while navigation access checks are pending', async ({page})=>{ const scenario=await accountScenario(page,{delays:['/v1/agent/access','/v1/admin/access']}); try{ diff --git a/platform/test/user-account-do.integration.test.ts b/platform/test/user-account-do.integration.test.ts index 6828ada..d34b6c0 100644 --- a/platform/test/user-account-do.integration.test.ts +++ b/platform/test/user-account-do.integration.test.ts @@ -138,6 +138,53 @@ describe('UserAccountDO', () => { await account.getSource(sources.at(-1)!.id); expect((await account.listSources())[0]?.id).toBe(sources.at(-1)!.id); }); + test('project source references survive recent-source eviction and remain user-owned', async () => { + const account = env.USER_ACCOUNT.getByName('source-project-owner'); + const other = env.USER_ACCOUNT.getByName('source-project-other'); + const projectId = 'c4433ab2-a989-48f8-8d84-97ab518ff429'; + const snapshot: SaveReferencedSource['snapshot'] = { kind: 'search', selectedData: ['transcript'], results: `youtube/source-history/${'a'.repeat(64)}.json` }; + const saved = await account.saveSource({ input: 'codex gpt 6 astra tips', title: 'codex gpt 6 astra tips', snapshot }); + const first = await account.linkSourceToProject(projectId, saved.id); + expect(first).toMatchObject({ added: true, item: { source_id: saved.id, entity_type: 'search', title: 'codex gpt 6 astra tips' } }); + expect((await account.linkSourceToProject(projectId, saved.id))?.added).toBe(false); + expect(await account.listProjectSources(projectId)).toHaveLength(1); + expect(await account.projectSourceCounts()).toEqual([{ projectId, count: 1 }]); + expect(await other.linkSourceToProject(projectId, saved.id)).toBeNull(); + for (let i = 0; i < 31; i++) await account.saveSource({ input: `new query ${i}`, title: `new query ${i}`, snapshot }); + expect((await account.listSources()).some(source => source.id === saved.id)).toBe(false); + expect((await account.getSource(saved.id))?.source.title).toBe('codex gpt 6 astra tips'); + expect(await other.getSource(saved.id)).toBeNull(); + await account.removeProjectSources(projectId); + expect(await account.getSource(saved.id)).toBeNull(); + expect(await account.projectSourceCounts()).toEqual([]); + }); + test('project routes count linked sources and reject links to another user’s project', async () => { + const userId = 'project-link-route-owner'; + const projectId = 'd0171de7-b960-4245-b137-973757910b43'; + const stamp = Date.now(); + await env.DB.prepare('INSERT INTO user (id, name, email, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)') + .bind(userId, 'Owner', 'project-link-route-owner@example.test', stamp, stamp).run(); + await env.DB.prepare('INSERT INTO projects (id, user_id, name, created_at, updated_at) VALUES (?, ?, ?, ?, ?)') + .bind(projectId, userId, 'codex', stamp, stamp).run(); + const account = env.USER_ACCOUNT.getByName(await userAccountInstanceName(userId)); + const snapshot: SaveReferencedSource['snapshot'] = { kind: 'search', selectedData: ['transcript'], results: `youtube/source-history/${'b'.repeat(64)}.json` }; + const source = await account.saveSource({ input: 'codex gpt 6 astra tips', title: 'codex gpt 6 astra tips', snapshot }); + const app = sourceApp(userId); + const link = () => app.request(`/projects/${projectId}/sources`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ sourceId: source.id }) }, env); + expect((await link()).status).toBe(201); + expect((await link()).status).toBe(200); + expect(await (await app.request('/projects', {}, env)).json()).toMatchObject({ projects: [{ id: projectId, item_count: 1 }] }); + expect(await (await app.request(`/projects/${projectId}`, {}, env)).json()).toMatchObject({ items: [{ + source_id: source.id, entity_type: 'search', title: 'codex gpt 6 astra tips', + }] }); + const unauthorized = await sourceApp('project-link-route-other').request(`/projects/${projectId}/sources`, { + method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ sourceId: source.id }), + }, env); + expect(unauthorized.status).toBe(404); + expect((await sourceApp(userId, 'api-key').request(`/projects/${projectId}/sources`, { + method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ sourceId: source.id }), + }, env)).status).toBe(403); + }); test('deletion includes admissions without catalog entries and blocks late writes', async () => { const account = env.USER_ACCOUNT.getByName('user:deletion'); await account.registerConversation(CONVERSATION_A); diff --git a/web/app/dashboard/SourcesClient.tsx b/web/app/dashboard/SourcesClient.tsx index d3aef94..6ec953f 100644 --- a/web/app/dashboard/SourcesClient.tsx +++ b/web/app/dashboard/SourcesClient.tsx @@ -51,6 +51,8 @@ export default function SourcesClient({ active }: {active:boolean}) { const [hasSearched, setHasSearched] = useDashboardDraft('source-searched', false); const projectsResource = useAccountResource('projects', []); const { data: projects } = projectsResource; + const projectId = params.get('project'); + const projectName = projects.find(project => project.id === projectId)?.name ?? 'project'; const [inspector, setInspector] = useDashboardDraft('source-inspector', null); const [transcriptQuery, setTranscriptQuery] = useDashboardDraft('transcript-query', ''); const [loading, setLoading] = useState(false); @@ -60,6 +62,8 @@ export default function SourcesClient({ active }: {active:boolean}) { const [recentSources, setRecentSources] = useState([]); const [historyLoading, setHistoryLoading] = useState(true); const [historyError, setHistoryError] = useState(''); + const [projectSaveError, setProjectSaveError] = useState(''); + const [pendingProjectSource, setPendingProjectSource] = useState(null); const historyInput = useRef(''); const operationController = useRef(null); const searchInput = useRef(null); @@ -82,7 +86,25 @@ export default function SourcesClient({ active }: {active:boolean}) { return () => controller.abort(); }, [active, authenticated, loadHistory]); - const rememberSource = async (input: string, snapshot: SourceSnapshot, signal: AbortSignal) => { + const linkProjectSource = async (sourceId: string) => { + if (!projectId) return; + try { + const { added } = await api<{ added: boolean }>(`/v1/projects/${encodeURIComponent(projectId)}/sources`, { + method: 'POST', body: JSON.stringify({ sourceId }), + }); + if (added) projectsResource.setData(current => current.map(project => project.id === projectId + ? { ...project, item_count: (project.item_count ?? 0) + 1 } : project)); + setPendingProjectSource(null); + setProjectSaveError(''); + setNotice(`Added to ${projectName}`); + } catch (cause) { + setPendingProjectSource(sourceId); + setProjectSaveError(cause instanceof Error ? cause.message : 'Could not add this source to the project.'); + } + }; + + const rememberSource = async (input: string, snapshot: SourceSnapshot) => { + let source: RecentSource; try { const request = snapshot.kind === 'search' ? { kind: snapshot.kind, selectedData: snapshot.selectedData } : { kind: snapshot.kind, inspector: { @@ -90,21 +112,23 @@ export default function SourcesClient({ active }: {active:boolean}) { requestedData: snapshot.inspector.requestedData, dataErrors: snapshot.inspector.dataErrors, loadedData: ['metadata', ...(['transcript', 'comments', 'channel'] as const).filter(field => snapshot.inspector[field])], } }; - const { source } = await api<{ source: RecentSource }>('/v1/sources/recent', { - method: 'POST', body: JSON.stringify({ input, snapshot: request }), signal, - }); + ({ source } = await api<{ source: RecentSource }>('/v1/sources/recent', { + method: 'POST', body: JSON.stringify({ input, snapshot: request }), + })); setRecentSources(current => [source, ...current.filter(item => item.id !== source.id)].slice(0, 30)); setHistoryError(''); - } catch (cause) { if (!isAbortError(cause)) setHistoryError(cause instanceof Error ? `Could not save recent source: ${cause.message}` : 'Could not save recent source.'); } + } catch (cause) { setHistoryError(cause instanceof Error ? `Could not save recent source: ${cause.message}` : 'Could not save recent source.'); return; } + await linkProjectSource(source.id); }; - const openRecentSource = async (entry: RecentSource) => { + const openRecentSource = async (entry: Pick) => { const controller = beginOperation('Loading saved source data…'); try { const { source, snapshot } = await api<{ source: RecentSource; snapshot: SourceSnapshot }>(`/v1/sources/recent/${entry.id}`, { signal: controller.signal }); if (controller.signal.aborted) return; setQuery(source.input); historyInput.current = source.input; setTranscriptQuery(''); setRecentSources(current => [source, ...current.filter(item => item.id !== source.id)]); + void linkProjectSource(source.id); if (snapshot.kind === 'search') { setSelectedData(snapshot.selectedData); setItems(snapshot.items); setInspector(null); setHasSearched(true); } else { @@ -190,7 +214,7 @@ export default function SourcesClient({ active }: {active:boolean}) { const data = await api<{ results: SearchItem[] }>(`${YOUTUBE_API}/search?${params}`, { signal: controller.signal }); const results = data.results.filter((item) => item.type === 'video').map((item) => ({ ...item, provider: 'youtube' as const })); setItems(results); - await rememberSource(input, { kind: 'search', selectedData: [...selectedData], items: results }, controller.signal); + await rememberSource(input, { kind: 'search', selectedData: [...selectedData], items: results }); } catch (cause) { if (!isAbortError(cause)) setError(cause instanceof Error ? cause.message : 'Search failed.'); } finally { finishOperation(controller); } @@ -231,7 +255,7 @@ export default function SourcesClient({ active }: {active:boolean}) { await fetchSourceData(next, option, controller.signal, next.refreshData?.includes(option)); }))]); if (!controller.signal.aborted && operationController.current === controller) { - await rememberSource(input, { kind: 'inspection', inspector: { ...next, loadingData: [] } }, controller.signal); + await rememberSource(input, { kind: 'inspection', inspector: { ...next, loadingData: [] } }); } }; @@ -252,7 +276,7 @@ export default function SourcesClient({ active }: {active:boolean}) { if (!controller.signal.aborted) { const next: Inspector = { provider, type, id, data, requestedData: [], dataErrors: {} }; setInspector(next); - await rememberSource(historyInput.current, { kind: 'inspection', inspector: next }, controller.signal); + await rememberSource(historyInput.current, { kind: 'inspection', inspector: next }); } } } catch (cause) { if (!isAbortError(cause)) setError(cause instanceof Error ? cause.message : 'Could not open this source.'); } @@ -297,10 +321,11 @@ export default function SourcesClient({ active }: {active:boolean}) { useEffect(() => { if (!active) return; - const q=params.get('q'), id=params.get('id'), type=params.get('type'); + const q=params.get('q'), id=params.get('id'), type=params.get('type'), saved=params.get('saved'); if (q) { setQuery(q); searchInput.current?.focus(); } if (id && (type==='video'||type==='channel'||type==='playlist')) void inspect(type,id); - if (q||id) { const next=new URLSearchParams(params); next.delete('q');next.delete('id');next.delete('type');router.replace(`/dashboard/sources${next.size?`?${next}`:''}`,{scroll:false}); } + if (saved) void openRecentSource({ id: saved }); + if (q||id||saved) { const next=new URLSearchParams(params); next.delete('q');next.delete('id');next.delete('type');next.delete('saved');router.replace(`/dashboard/sources${next.size?`?${next}`:''}`,{scroll:false}); } }, [active, params, router]); const createProject = async (name:string) => { const project = await api('/v1/projects',{method:'POST',body:JSON.stringify({name})}); @@ -309,6 +334,10 @@ export default function SourcesClient({ active }: {active:boolean}) { const saveInspector = async () => { if (!inspector) return; + if (projectId) { + await rememberSource(historyInput.current, { kind: 'inspection', inspector }); + return; + } try { await cache.load('projects'); if (cache.read('projects').error) throw new Error(cache.read('projects').error); @@ -367,6 +396,7 @@ export default function SourcesClient({ active }: {active:boolean}) { {projectsResource.error &&
{projectsResource.error}
}
<> + {projectId &&
Adding sources to {projectName}View project
}

Search or paste a YouTube link

{(inspector || hasSearched) && }
@@ -400,6 +430,7 @@ export default function SourcesClient({ active }: {active:boolean}) { {error &&
{error}{query.trim() && }
} {notice &&
{notice}
}
} + {projectSaveError && projectId &&
Could not add to {projectName}: {projectSaveError}{pendingProjectSource && }
} {inspector ? ( void retrySourceData()} onOpenComments={() => void refreshComments()} onRefresh={() => void refreshVideoData()} segments={filteredSegments} transcriptQuery={transcriptQuery} setTranscriptQuery={setTranscriptQuery} onClose={() => { cancelOperation(); setInspector(null); }} onSave={() => void saveInspector()} onMonitor={() => void addMonitor()} onOpenVideo={(id) => void inspect('video', id, undefined, inspector.provider, selectedData)} /> ) : ( diff --git a/web/app/dashboard/projects/ProjectsClient.tsx b/web/app/dashboard/projects/ProjectsClient.tsx index 7c345b5..2aeeb5a 100644 --- a/web/app/dashboard/projects/ProjectsClient.tsx +++ b/web/app/dashboard/projects/ProjectsClient.tsx @@ -12,6 +12,7 @@ export function ProjectsClient({ promise }: { promise: Promise('selected-project', null); + const resumeProject = useRef(selected?.id ?? null); const [loading, setLoading] = useState(false), [error, setError] = useState(''), [create, setCreate] = useState(false); @@ -38,11 +39,13 @@ export function ProjectsClient({ promise }: { promise: Promise () => controller.current?.abort(), []); useEffect(() => { - const id = params.get('project'), + const requestedId = params.get('project'), newProject = params.get('newProject'); + const id = requestedId ?? resumeProject.current; + resumeProject.current = null; if (id) void open({ id, name: '' }); if (newProject) setCreate(true); - if (id || newProject) router.replace('/dashboard/projects', { scroll: false }); + if (requestedId || newProject) router.replace('/dashboard/projects', { scroll: false }); }, [params, open, router]); const add = async (name: string) => { try { @@ -72,9 +75,11 @@ export function ProjectsClient({ promise }: { promise: Promise router.push('/dashboard/sources')} + onFindSources={() => selected && router.push(`/dashboard/sources?project=${encodeURIComponent(selected.id)}`)} onOpenItem={(item) => - router.push(`/dashboard/sources?type=${item.entity_type}&id=${encodeURIComponent(item.entity_id)}`) + router.push(item.source_id + ? `/dashboard/sources?project=${encodeURIComponent(selected?.id ?? '')}&saved=${encodeURIComponent(item.source_id)}` + : `/dashboard/sources?type=${item.entity_type}&id=${encodeURIComponent(item.entity_id)}`) } /> )} diff --git a/web/app/dashboard/projects/ProjectsView.tsx b/web/app/dashboard/projects/ProjectsView.tsx index 6785adb..63066ee 100644 --- a/web/app/dashboard/projects/ProjectsView.tsx +++ b/web/app/dashboard/projects/ProjectsView.tsx @@ -12,7 +12,7 @@ export function ProjectsView({ projects, selectedProject, loading, error, onCrea
{selectedProject.items.map(item => )}
- {!selectedProject.items.length &&

No sources yet

Add videos, channels, or playlists to this project.

} + {!selectedProject.items.length &&

No sources yet

Add searches, videos, channels, or playlists to this project.

} ; return

Your projects

Keep related sources and saved moments together.

diff --git a/web/app/dashboard/research-types.ts b/web/app/dashboard/research-types.ts index 75b4186..dfad48b 100644 --- a/web/app/dashboard/research-types.ts +++ b/web/app/dashboard/research-types.ts @@ -55,7 +55,7 @@ export type ChannelInfo = { meta: SourceMetadata; }; export type Project = { id: string; name: string; description?: string; item_count?: number }; -export type ProjectItem = { id: string; provider: ProviderId; entity_type: EntityType; entity_id: string; title?: string; note?: string; start_ms?: number | null; created_at?: number }; +export type ProjectItem = { id: string; provider: ProviderId; entity_type: EntityType | 'search'; entity_id: string; source_id?: string; title?: string; note?: string; start_ms?: number | null; created_at?: number }; export type ProjectDetail = Project & { items: ProjectItem[] }; export type Monitor = { id: string; provider: ProviderId; kind: string; target: string; query_json?: string; cadence?: string; diff --git a/web/app/globals.css b/web/app/globals.css index 7809499..74f5b36 100644 --- a/web/app/globals.css +++ b/web/app/globals.css @@ -1222,6 +1222,17 @@ html:has(.developer-page) body { } /* Sources studio — one input, a small data contract, then results. */ +.source-project-context { + display: flex; align-items: center; gap: .35rem; width: min(100%, 88rem); + margin: var(--space-lg) auto 0; + padding-inline: clamp(var(--space-md), 4vw, var(--space-2xl)); + color: var(--color-dashboard-muted); font-size: .75rem; +} +.source-project-context strong { color: var(--color-dashboard-ink); font-weight: 650; } +.source-project-context a { margin-left: auto; color: var(--color-dashboard-accent); text-decoration: none; font-weight: 650; } +.source-project-context a:hover { text-decoration: underline; } +.source-project-context + .source-studio { padding-top: var(--space-lg); } +.source-project-save-error { width: fit-content; max-width: calc(100% - 2rem); margin: var(--space-md) auto 0; } .source-studio, .source-results, .source-inspector { From a658e02d4e08de907552c72b179293e8c3a16efe Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Mon, 28 Sep 2026 22:54:40 +0530 Subject: [PATCH 7/7] fix: save project sources atomically and restore isolated snapshots --- docs/internals/endpoints.mdx | 1 + platform/src/durable-objects/user-account.ts | 27 ++++++- platform/src/lib/exports.ts | 11 +-- platform/src/lib/project-items.ts | 18 +++++ platform/src/lib/source-history.ts | 2 +- platform/src/openapi-audience.ts | 2 + platform/src/openapi.ts | 16 +++- platform/src/routes/session/session.index.ts | 21 +++-- platform/test/e2e/dashboard-api.spec.ts | 64 ++++++++++++--- .../test/user-account-do.integration.test.ts | 81 ++++++++++++++++++- reference/engineering/SOURCE_HISTORY.md | 10 +++ web/app/dashboard/SourcesClient.tsx | 71 ++++++++-------- web/app/dashboard/dashboard-routes.ts | 2 +- 13 files changed, 261 insertions(+), 65 deletions(-) create mode 100644 platform/src/lib/project-items.ts diff --git a/docs/internals/endpoints.mdx b/docs/internals/endpoints.mdx index 75db0bb..1a8935f 100644 --- a/docs/internals/endpoints.mdx +++ b/docs/internals/endpoints.mdx @@ -32,6 +32,7 @@ Called by the video2ctx web application or an explicit signed-in account action. | `DELETE` | `/v1/oauth/youtube` | `disconnectYouTube` | Disconnect the YouTube account | `sessionCookie` or `demoUser` | Mutates the account connection state; require an explicit user action. | | `GET` | `/v1/oauth/youtube/connect` | `createYouTubeConnectUrl` | Create a YouTube OAuth URL | `sessionCookie` or `demoUser` | Returns a state-bound OAuth URL; start it only from a user-initiated connection flow. | | `POST` | `/v1/projects/{id}/sources` | `linkProjectSource` | Add a saved Sources search or inspection to a project | `sessionCookie` or `demoUser` | Browser-session only. Verify that the project and saved source belong to the same signed-in user before linking shared asset references. | +| `GET` | `/v1/projects/{id}/sources/{itemId}` | `getProjectSource` | Restore a project source snapshot | `sessionCookie` or `demoUser` | Browser-session only. Verify project ownership and restore the item from that project without modifying saved references. | | `POST` | `/v1/resolve` | `resolveInput` | Route universal UI input | `sessionCookie` or `cliSession` or `bearerApiKey` or `apiKey` or `demoUser` | First-party input router; its dispatch behavior is not a stable public API contract. | | `POST` | `/v1/scale-inquiries` | `submitScaleInquiry` | Submit a Scale plan inquiry | Public or protocol-signed | Public lead form; validate Turnstile and rate limits, and never let the caller choose the notification recipient. | | `GET` | `/v1/sources/recent` | `listRecentSources` | List recent Sources inputs | `sessionCookie` or `demoUser` | Browser-session only. Lists inputs belonging to the authenticated user without exposing shared asset references. | diff --git a/platform/src/durable-objects/user-account.ts b/platform/src/durable-objects/user-account.ts index fd41108..b85290d 100644 --- a/platform/src/durable-objects/user-account.ts +++ b/platform/src/durable-objects/user-account.ts @@ -184,9 +184,25 @@ export class UserAccountDO extends DurableObject { return { source: { id: row.id, input: row.input, title: row.title, kind: row.kind, updatedAt }, snapshot: sourceReferenceSchema.parse(JSON.parse(row.snapshot)) }; } - // A project keeps its own reference after the 30-entry recent list rotates. + return null; + } + + saveSourceWithProject(value: SaveReferencedSource, projectId?: string) { + this.assertActive(); + const project = projectId === undefined ? undefined : z.string().uuid().parse(projectId); + return this.ctx.storage.transactionSync(() => { + const source = this.saveSource(value); + const linked = project ? this.linkSourceToProject(project, source.id) : null; + if (project && !linked) throw new Error('Saved source could not be linked.'); + return { source, linked }; + }); + } + + getProjectSource(projectId: string, itemId: string): { source: RecentSource; snapshot: SourceReference } | null { + this.assertActive(); const saved = this.ctx.storage.sql.exec( - 'SELECT id, source_id, input, title, kind, snapshot, created_at FROM project_sources WHERE source_id = ? LIMIT 1', sourceId, + 'SELECT id, source_id, input, title, kind, snapshot, created_at FROM project_sources WHERE project_id = ? AND id = ?', + z.string().uuid().parse(projectId), sourceIdSchema.parse(itemId), ).toArray()[0]; if (!saved) return null; return { source: { id: saved.source_id, input: saved.input, title: saved.title, kind: saved.kind, updatedAt: saved.created_at }, @@ -200,7 +216,12 @@ export class UserAccountDO extends DurableObject { const recent = this.ctx.storage.sql.exec<{ source_key: string; input: string; title: string; kind: RecentSource['kind']; snapshot: string }>( 'SELECT source_key, input, title, kind, snapshot FROM recent_sources WHERE id = ?', source, ).toArray()[0]; - if (!recent) return null; + if (!recent) { + const saved = this.ctx.storage.sql.exec( + 'SELECT id, source_id, input, title, kind, snapshot, created_at FROM project_sources WHERE project_id = ? AND source_id = ?', project, source, + ).toArray()[0]; + return saved ? { item: this.toProjectSourceItem(saved), added: false } : null; + } const existing = this.ctx.storage.sql.exec<{ id: string }>( 'SELECT id FROM project_sources WHERE project_id = ? AND source_key = ?', project, recent.source_key, ).toArray()[0]; diff --git a/platform/src/lib/exports.ts b/platform/src/lib/exports.ts index f0e36e4..12dfb78 100644 --- a/platform/src/lib/exports.ts +++ b/platform/src/lib/exports.ts @@ -1,4 +1,5 @@ import { ApiError, now } from './http'; +import { listProjectItems } from './project-items'; type Format = 'txt' | 'md' | 'json' | 'csv' | 'srt' | 'vtt'; @@ -17,11 +18,11 @@ export async function createProjectExport(env: Env, userId: string, projectId: s const project = await env.DB.prepare('SELECT name,description FROM projects WHERE id=? AND user_id=?') .bind(projectId, userId).first<{ name: string; description: string }>(); if (!project) throw new ApiError(404, 'PROJECT_NOT_FOUND', 'Project not found.'); - const rows = await env.DB.prepare( - `SELECT provider,title,entity_type,entity_id,start_ms,end_ms,note,tags_json - FROM project_items WHERE project_id=? AND user_id=? ORDER BY created_at` - ).bind(projectId, userId).all(); - const content = serialize(format, project, rows.results); + const items = await listProjectItems(env, userId, projectId); + const rows: ItemRow[] = items.reverse().map(({ provider, title, entity_type, entity_id, start_ms, end_ms, note, tags_json }) => ({ + provider, title, entity_type, entity_id, start_ms, end_ms, note, tags_json, + })); + const content = serialize(format, project, rows); const id = crypto.randomUUID(); const key = `private/${userId}/exports/${id}.${format}`; await env.RESEARCH.put(key, content, { httpMetadata: { contentType: contentType(format) } }); diff --git a/platform/src/lib/project-items.ts b/platform/src/lib/project-items.ts new file mode 100644 index 0000000..e38f2c8 --- /dev/null +++ b/platform/src/lib/project-items.ts @@ -0,0 +1,18 @@ +import { userAccountInstanceName } from '../agents/runtime/identity'; + +export interface ProjectItemRecord { + id: string; provider: string; entity_type: string; entity_id: string; title: string; + start_ms: number | null; end_ms: number | null; note: string; tags_json: string; created_at: number; + source_id?: string; +} + +// Both dashboard detail and exports include legacy items and saved source references. +export async function listProjectItems(env: Env, userId: string, projectId: string): Promise { + const legacy = await env.DB.prepare('SELECT * FROM project_items WHERE project_id=? AND user_id=? ORDER BY created_at DESC') + .bind(projectId, userId).all(); + const account = env.USER_ACCOUNT.getByName(await userAccountInstanceName(userId)); + const sources = (await account.listProjectSources(projectId)).map(item => ({ + ...item, start_ms: null, end_ms: null, note: '', tags_json: '[]', + })); + return [...legacy.results, ...sources].sort((a, b) => b.created_at - a.created_at); +} diff --git a/platform/src/lib/source-history.ts b/platform/src/lib/source-history.ts index e3b040c..bfb05a1 100644 --- a/platform/src/lib/source-history.ts +++ b/platform/src/lib/source-history.ts @@ -34,7 +34,7 @@ export const sourceSnapshotSchema = z.discriminatedUnion('kind', [ })) }), z.object({ kind: z.literal('inspection'), inspector }), ]); -export const saveSourceSchema = z.object({ input: z.string().trim().min(1).max(500), snapshot: z.discriminatedUnion('kind', [ +export const saveSourceSchema = z.object({ projectId: z.string().uuid().optional(), input: z.string().trim().min(1).max(500), snapshot: z.discriminatedUnion('kind', [ z.object({ kind: z.literal('search'), selectedData: z.array(dataset).min(1) }), z.object({ kind: z.literal('inspection'), inspector: inspector.pick({ provider: true, type: true, id: true, requestedData: true, dataErrors: true }) .extend({ loadedData: z.array(z.enum(['metadata', 'transcript', 'comments', 'channel'])) }) }), diff --git a/platform/src/openapi-audience.ts b/platform/src/openapi-audience.ts index 702d6d1..7acf1e4 100644 --- a/platform/src/openapi-audience.ts +++ b/platform/src/openapi-audience.ts @@ -33,6 +33,7 @@ export const OPENAPI_OPERATION_AUDIENCE: Readonly> = { listRecentSources: 'Browser-session only. Lists inputs belonging to the authenticated user without exposing shared asset references.', saveRecentSource: 'Browser-session only. Resolves existing provider assets server-side; never accepts client-supplied R2 keys or video payloads.', getRecentSource: 'Browser-session only. Verifies user ownership before reading the saved shared references.', + getProjectSource: 'Browser-session only. Verify project ownership and restore the item from that project without modifying saved references.', linkProjectSource: 'Browser-session only. Verify that the project and saved source belong to the same signed-in user before linking shared asset references.', inspectLandingYouTubeVideo: 'Public, rate-limited demo route; do not use it as a credentialed bulk-data API.', submitScaleInquiry: 'Public lead form; validate Turnstile and rate limits, and never let the caller choose the notification recipient.', diff --git a/platform/src/openapi.ts b/platform/src/openapi.ts index a0ede21..4d37d6f 100644 --- a/platform/src/openapi.ts +++ b/platform/src/openapi.ts @@ -1095,10 +1095,10 @@ export const openApiDocument = { }, post: { tags: ['Projects'], operationId: 'saveRecentSource', summary: 'Remember a Sources search or inspection', security: privateSecurity, - description: 'Stores the input and references in the user Durable Object. Provider data is read from existing shared storage. This does not fetch YouTube data.', + description: 'Stores the input and references in the user Durable Object. Provider data is read from existing shared storage. This does not fetch YouTube data. An optional projectId saves the project reference in the same user-storage transaction.', requestBody: jsonBody(z.toJSONSchema(saveSourceSchema, { target: 'openapi-3.0' })), - responses: { '201': jsonResponse('Source remembered.', { type: 'object', properties: { source: schemaRef('RecentSource') } }), - '409': jsonResponse('Provider data is not yet present in shared storage.', schemaRef('Error')), ...standardErrors }, + responses: { '201': jsonResponse('Source remembered.', { type: 'object', properties: { source: schemaRef('RecentSource'), linked: { anyOf: [schemaRef('ProjectSourceLink'), { type: 'null' }] } } }), + '409': jsonResponse('Provider data is not yet present in shared storage.', schemaRef('Error')), '404': responseRef('NotFound'), ...standardErrors }, }, }, '/v1/sources/recent/{id}': { @@ -1181,6 +1181,16 @@ export const openApiDocument = { }, }, }, + '/v1/projects/{id}/sources/{itemId}': { + get: { + tags: ['Projects'], operationId: 'getProjectSource', summary: 'Restore a project source snapshot', security: privateSecurity, + description: 'Restores this project item without updating history or project references. Survives recent-source eviction.', + parameters: [idParameter, pathParameter('itemId', 'Project item UUID, not the recent source UUID.')], + responses: { '200': jsonResponse('Saved project source and displayed data.', { type: 'object', properties: { + source: schemaRef('RecentSource'), snapshot: z.toJSONSchema(sourceSnapshotSchema, { target: 'openapi-3.0' }), + } }), '404': responseRef('NotFound'), ...standardErrors }, + }, + }, '/v1/projects/{id}/sources': { post: { tags: ['Projects'], diff --git a/platform/src/routes/session/session.index.ts b/platform/src/routes/session/session.index.ts index c9fe83a..d978a6e 100644 --- a/platform/src/routes/session/session.index.ts +++ b/platform/src/routes/session/session.index.ts @@ -1,3 +1,4 @@ +import { listProjectItems } from '../../lib/project-items'; import { framePreviewPrefix } from '../../agents/runtime/frame-previews'; import { userAccountInstanceName } from '../../agents/runtime/identity'; import { MAX_SOURCE_SNAPSHOT_BYTES, saveSourceSchema, sourceIdSchema } from '../../lib/source-history'; @@ -48,6 +49,7 @@ export const SESSION_ONLY_ROUTE_PATTERNS = [ '/sources/recent', '/sources/recent/*', '/projects/:id/sources', + '/projects/:id/sources/*', '/oauth/youtube/connect', '/oauth/youtube', '/billing', @@ -79,8 +81,9 @@ sessionRoutes.post('/sources/recent', async (c) => { const parsed = saveSourceSchema.safeParse(json); if (!parsed.success) throw new ApiError(422, 'INVALID_SOURCE', 'The recent source data is invalid.'); const account = c.env.USER_ACCOUNT.getByName(await userAccountInstanceName(requireUser(c).id)); + if (parsed.data.projectId) await ownProject(c.env, requireUser(c).id, parsed.data.projectId); const referenced = await referenceSource(c.env, parsed.data); - return c.json({ source: await account.saveSource(referenced) }, 201); + return c.json(await account.saveSourceWithProject(referenced, parsed.data.projectId), 201); }); sessionRoutes.get('/sources/recent/:id', async (c) => { @@ -132,11 +135,19 @@ sessionRoutes.get('/projects/:id', async (c) => { const id = asId(c.req.param('id')); const project = await c.env.DB.prepare('SELECT * FROM projects WHERE id=? AND user_id=?').bind(id, user.id).first(); if (!project) throw new ApiError(404, 'PROJECT_NOT_FOUND', 'Project not found.'); - const items = await c.env.DB.prepare('SELECT * FROM project_items WHERE project_id=? AND user_id=? ORDER BY created_at DESC') - .bind(id, user.id).all(); + return c.json({ ...project, items: await listProjectItems(c.env, user.id, id) }); +}); + +sessionRoutes.get('/projects/:id/sources/:itemId', async (c) => { + const user = requireUser(c); + const projectId = asId(c.req.param('id')); + await ownProject(c.env, user.id, projectId); + const itemId = c.req.param('itemId'); + if (!sourceIdSchema.safeParse(itemId).success) throw new ApiError(422, 'INVALID_ID', 'Invalid project source ID.'); const account = c.env.USER_ACCOUNT.getByName(await userAccountInstanceName(user.id)); - const sources = await account.listProjectSources(id); - return c.json({ ...project, items: [...items.results, ...sources].sort((a, b) => Number(b.created_at) - Number(a.created_at)) }); + const saved = await account.getProjectSource(projectId, itemId); + if (!saved) throw new ApiError(404, 'SOURCE_NOT_FOUND', 'This project source was not found.'); + return c.json({ source: saved.source, snapshot: await restoreSource(c.env, saved.snapshot) }); }); sessionRoutes.post('/projects/:id/sources', async (c) => { diff --git a/platform/test/e2e/dashboard-api.spec.ts b/platform/test/e2e/dashboard-api.spec.ts index ca7b4ad..b4ca9f7 100644 --- a/platform/test/e2e/dashboard-api.spec.ts +++ b/platform/test/e2e/dashboard-api.spec.ts @@ -608,26 +608,18 @@ test('adding sources from a project saves the search and opened video in that pr await page.route(`**/api/platform/v1/projects/${projectId}`, route => route.fulfill({ json: { id: projectId, name: 'codex', items: projectItems, } })); - await page.route(`**/api/platform/v1/projects/${projectId}/sources`, async route => { - const { sourceId } = route.request().postDataJSON(); - const source = sources.get(sourceId)!; - const existing = projectItems.find(item => item.source_id === sourceId); - if (existing) return route.fulfill({ status: 200, json: { item: existing, added: false } }); - const item = { id: `saved-${projectItems.length}`, source_id: sourceId, provider: 'youtube', - entity_type: source.kind === 'search' ? 'search' : 'video', entity_id: sourceId, title: source.title }; - projectItems.push(item); - return route.fulfill({ status: 201, json: { item, added: true } }); - }); await page.route('**/api/platform/v1/sources/recent', async route => { if (route.request().method() !== 'POST') return route.fulfill({ json: { sources: [] } }); const input = route.request().postDataJSON(); + expect(input.projectId).toBe(projectId); if (input.snapshot.kind === 'search') await searchSaveGate; const id = crypto.randomUUID(); const title = input.snapshot.kind === 'search' ? query : 'Codex tips video'; sources.set(id, { title, kind: input.snapshot.kind, input: input.input }); + projectItems.push({ id, source_id: id, provider: 'youtube', entity_type: input.snapshot.kind === 'search' ? 'search' : 'video', entity_id: id, title }); return route.fulfill({ status: 201, json: { source: { id, input: input.input, title, kind: input.snapshot.kind, updatedAt: Date.now() } } }); }); - await page.route('**/api/platform/v1/sources/recent/*', route => { + await page.route(`**/api/platform/v1/projects/${projectId}/sources/*`, route => { const id = route.request().url().split('/').at(-1)!; const source = sources.get(id)!; const snapshot = source.kind === 'search' @@ -662,14 +654,64 @@ test('adding sources from a project saves the search and opened video in that pr const savedInSidebar = page.getByRole('group', { name: 'Sources in codex' }).first(); await expect(savedInSidebar.getByRole('button', { name: query })).toBeVisible(); await expect(savedInSidebar.getByRole('button', { name: 'Codex tips video' })).toBeVisible(); + let restoreWrites = 0; + page.on('request', request => { if (request.method() === 'POST' && /\/v1\/(sources|projects)\//.test(request.url())) restoreWrites++; }); await savedInSidebar.getByRole('button', { name: query }).click(); await expect(page.getByRole('heading', { name: 'Results' })).toBeVisible(); await page.getByRole('link', { name: 'Projects', exact: true }).click(); await expect(page.getByRole('heading', { name: 'Saved sources 2' })).toBeVisible(); expect(projectItems).toHaveLength(2); + expect(restoreWrites).toBe(0); } finally { releaseSearchSave(); await scenario.clear(); } }); +test('failed search saves survive video success and retry into their original project', async ({ page }) => { + const first = 'cae4ebba-7a0d-402f-bf5c-852a34ef2815', second = 'b681c6e2-bdf8-4df5-b7ee-4fa218a3e9bc'; + const query = 'retry this search'; + const saved: Array<{ projectId: string; kind: string }> = []; + let searchAttempts = 0, releaseVideo = () => {}; + const videoGate = new Promise(resolve => { releaseVideo = resolve; }); + const projects = [{ id: first, name: 'First project', item_count: 0 }, { id: second, name: 'Second project', item_count: 0 }]; + const scenario = await accountScenario(page, { responses: { '/v1/projects': { body: { projects } } } }); + for (const project of projects) await page.route(`**/api/platform/v1/projects/${project.id}`, route => route.fulfill({ json: { ...project, items: [] } })); + await page.route('**/api/platform/v1/sources/recent', async route => { + if (route.request().method() !== 'POST') return route.fulfill({ json: { sources: [] } }); + const body = route.request().postDataJSON(); + if (body.snapshot.kind === 'search' && ++searchAttempts === 1) { + return route.fulfill({ status: 503, json: { error: { code: 'SAVE_FAILED', message: 'Temporary save failure' } } }); + } + if (body.snapshot.kind === 'inspection') await videoGate; + saved.push({ projectId: body.projectId, kind: body.snapshot.kind }); + return route.fulfill({ status: 201, json: { source: { id: crypto.randomUUID(), input: body.input, title: body.input, kind: body.snapshot.kind, updatedAt: Date.now() } } }); + }); + await page.route('**/api/platform/v1/resolve', route => route.fulfill({ json: { kind: 'search', query } })); + await page.route('**/api/platform/v1/providers/youtube/search?**', route => route.fulfill({ json: { results: [{ + provider: 'youtube', type: 'video', id: videoId, title: 'Retry test video', thumbnails: [], + }] } })); + await page.route(`**/api/platform/v1/providers/youtube/videos/${videoId}`, route => route.fulfill({ json: { + id: videoId, title: 'Retry test video', thumbnails: [], channel: { id: 'channel', name: 'Creator' }, + } })); + await page.route(`**/api/platform/v1/providers/youtube/videos/${videoId}/transcript`, route => route.fulfill({ json: transcript })); + try { + await page.goto(`/dashboard/sources?project=${first}`); + await page.getByRole('textbox', { name: 'Video search or YouTube URL' }).fill(query); + await page.getByRole('button', { name: /Inspect/ }).click(); + await page.getByRole('button', { name: /Retry test video/ }).click(); + const failure = page.getByRole('alert').filter({ hasText: 'Could not save retry this search to First project' }); + await expect(failure).toBeVisible(); + releaseVideo(); + await expect.poll(() => saved.length).toBe(1); + await expect(failure).toBeVisible(); + await page.getByRole('link', { name: 'Projects', exact: true }).click(); + await page.getByRole('button', { name: 'Second project 0 sources' }).click(); + await page.getByRole('button', { name: 'Add sources' }).click(); + await expect(page.getByText('Adding sources to')).toContainText('Second project'); + await failure.getByRole('button', { name: 'Retry saving' }).click(); + await expect(failure).toHaveCount(0); + expect(saved).toEqual([{ projectId: first, kind: 'inspection' }, { projectId: first, kind: 'search' }]); + } finally { releaseVideo(); await scenario.clear(); } +}); + test('project creation shows progress and adds the project without another list request', async ({ page }) => { const scenario = await accountScenario(page, { responses: { '/v1/projects': { body: { projects: [] } } } }); let release!: () => void; diff --git a/platform/test/user-account-do.integration.test.ts b/platform/test/user-account-do.integration.test.ts index d34b6c0..3dba9dc 100644 --- a/platform/test/user-account-do.integration.test.ts +++ b/platform/test/user-account-do.integration.test.ts @@ -3,6 +3,7 @@ import { describe, expect, test } from 'vitest'; import type { SaveReferencedSource } from '../src/lib/source-history'; import { referenceSource, restoreSource } from '../src/lib/source-history-storage'; import { saveVideoResource } from '../src/lib/video-resources'; +import { createProjectExport } from '../src/lib/exports'; import { sha256 } from '../src/lib/http'; import { jsonError } from '../src/lib/http'; import { Hono } from 'hono'; @@ -152,7 +153,9 @@ describe('UserAccountDO', () => { expect(await other.linkSourceToProject(projectId, saved.id)).toBeNull(); for (let i = 0; i < 31; i++) await account.saveSource({ input: `new query ${i}`, title: `new query ${i}`, snapshot }); expect((await account.listSources()).some(source => source.id === saved.id)).toBe(false); - expect((await account.getSource(saved.id))?.source.title).toBe('codex gpt 6 astra tips'); + expect(await account.getSource(saved.id)).toBeNull(); + expect((await account.getProjectSource(projectId, first!.item.id))?.source.title).toBe('codex gpt 6 astra tips'); + expect((await account.linkSourceToProject(projectId, saved.id))?.added).toBe(false); expect(await other.getSource(saved.id)).toBeNull(); await account.removeProjectSources(projectId); expect(await account.getSource(saved.id)).toBeNull(); @@ -185,6 +188,82 @@ describe('UserAccountDO', () => { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ sourceId: source.id }), }, env)).status).toBe(403); }); + test('project snapshots stay isolated across refreshes, other projects and history eviction', async () => { + const account = env.USER_ACCOUNT.getByName('project-snapshot-isolation'); + const projectA = crypto.randomUUID(), projectB = crypto.randomUUID(); + const value = (hash: string): SaveReferencedSource => ({ input: 'same query', title: 'Same query', + snapshot: { kind: 'search', selectedData: ['transcript'], results: `youtube/source-history/${hash.repeat(64)}.json` } }); + const first = await account.saveSourceWithProject(value('a'), projectA); + const second = await account.saveSourceWithProject(value('b'), projectB); + expect(first.source.id).toBe(second.source.id); + await account.saveSource(value('c')); + expect((await account.getProjectSource(projectA, first.linked!.item.id))?.snapshot).toEqual(value('a').snapshot); + expect((await account.getProjectSource(projectB, second.linked!.item.id))?.snapshot).toEqual(value('b').snapshot); + expect(await account.getProjectSource(projectB, first.linked!.item.id)).toBeNull(); + for (let i = 0; i < 31; i++) await account.saveSource({ ...value('c'), input: `eviction ${i}` }); + expect((await account.getProjectSource(projectA, first.linked!.item.id))?.snapshot).toEqual(value('a').snapshot); + expect((await account.getProjectSource(projectB, second.linked!.item.id))?.snapshot).toEqual(value('b').snapshot); + const refreshed = await account.saveSourceWithProject(value('d'), projectA); + expect(refreshed.linked).toMatchObject({ added: false, item: { id: first.linked!.item.id } }); + expect((await account.getProjectSource(projectA, first.linked!.item.id))?.snapshot).toEqual(value('d').snapshot); + expect((await account.getProjectSource(projectB, second.linked!.item.id))?.snapshot).toEqual(value('b').snapshot); + }); + + test('a project write failure rolls back history and its eviction', async () => { + const account = env.USER_ACCOUNT.getByName('project-atomic-rollback'); + const value: SaveReferencedSource = { input: 'original', title: 'Original', snapshot: { + kind: 'search', selectedData: ['transcript'], results: `youtube/source-history/${'a'.repeat(64)}.json`, + } }; + for (let i = 0; i < 30; i++) await account.saveSource({ ...value, input: `original ${i}` }); + const before = await account.listSources(); + await runInDurableObject(account, (instance, state) => { + state.storage.sql.exec(`CREATE TRIGGER fail_project_write BEFORE INSERT ON project_sources BEGIN SELECT RAISE(ABORT, 'injected failure'); END`); + expect(() => instance.saveSourceWithProject(value, crypto.randomUUID())).toThrow('injected failure'); + state.storage.sql.exec('DROP TRIGGER fail_project_write'); + }); + expect(await account.listSources()).toEqual(before); + expect(await account.projectSourceCounts()).toEqual([]); + }); + + test('atomic save and project restore routes enforce ownership and include sources in exports', async () => { + const userId = 'atomic-project-routes'; + const projectId = crypto.randomUUID(), stamp = Date.now(); + await env.DB.prepare('INSERT INTO user (id, name, email, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)') + .bind(userId, 'Owner', `${userId}@example.test`, stamp, stamp).run(); + await env.DB.prepare('INSERT INTO projects (id, user_id, name, description, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)') + .bind(projectId, userId, 'Research', '', stamp, stamp).run(); + const playlist = { id: 'PLatomic', title: 'Atomic playlist', videos: [] }; + const key = `youtube:v1:${await sha256(JSON.stringify(['playlist-v2', playlist.id]))}`; + await env.YOUTUBE_CACHE.put(key, JSON.stringify({ version: 1, fetchedAt: stamp, freshUntil: stamp + 60_000, value: playlist })); + const request = { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ projectId, + input: 'https://youtube.com/playlist?list=PLatomic', snapshot: { kind: 'inspection', inspector: { + provider: 'youtube', type: 'playlist', id: playlist.id, requestedData: [], dataErrors: {}, loadedData: ['metadata'], + } }, + }) }; + const app = sourceApp(userId); + const response = await app.request('/sources/recent', request, env); + expect(response.status).toBe(201); + const saved = await response.json() as { source: { id: string }; linked: { item: { id: string }; added: boolean } }; + expect(saved.linked.added).toBe(true); + expect(await (await app.request('/sources/recent', request, env)).json()).toMatchObject({ linked: { added: false } }); + const path = `/projects/${projectId}/sources/${saved.linked.item.id}`; + expect(await (await app.request(path, {}, env)).json()).toMatchObject({ snapshot: { inspector: { data: playlist } } }); + expect((await sourceApp('other-atomic-user').request(path, {}, env)).status).toBe(404); + expect((await sourceApp(userId, 'api-key').request(path, {}, env)).status).toBe(403); + expect((await sourceApp('other-atomic-user').request('/sources/recent', request, env)).status).toBe(404); + const other = env.USER_ACCOUNT.getByName(await userAccountInstanceName('other-atomic-user')); + expect(await other.listSources()).toEqual([]); + // Legacy saved moments remain in the same export alongside the new source. + await env.DB.prepare('INSERT INTO project_items (id, project_id, user_id, provider, entity_type, entity_id, title, start_ms, end_ms, note, tags_json, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)') + .bind(crypto.randomUUID(), projectId, userId, 'youtube', 'video', 'abcdefghijk', 'Saved moment', 1000, 2000, 'A note', '[]', stamp).run(); + const exported = await createProjectExport(env, userId, projectId, 'json'); + const content = await (await env.RESEARCH.get(exported.key))!.json() as { items: Array<{ title: string }> }; + expect(content.items.map(item => item.title).sort()).toEqual(['Atomic playlist', 'Saved moment']); + const subtitles = await createProjectExport(env, userId, projectId, 'srt'); + expect(await (await env.RESEARCH.get(subtitles.key))!.text()).toContain('A note'); + expect(await (await env.RESEARCH.get(subtitles.key))!.text()).not.toContain('Atomic playlist'); + }); + test('deletion includes admissions without catalog entries and blocks late writes', async () => { const account = env.USER_ACCOUNT.getByName('user:deletion'); await account.registerConversation(CONVERSATION_A); diff --git a/reference/engineering/SOURCE_HISTORY.md b/reference/engineering/SOURCE_HISTORY.md index fead40c..062a2da 100644 --- a/reference/engineering/SOURCE_HISTORY.md +++ b/reference/engineering/SOURCE_HISTORY.md @@ -34,3 +34,13 @@ No new Cloudflare binding or class migration is required. `UserAccountDO` create Recent inspection summaries carry a thumbnail URL reference from saved metadata. Older entries resolve that reference from their existing catalog version on the first list read. The user DO caches the URL without changing history order or copying image bytes. Missing thumbnails do not block the history list. Selecting Sources in the sidebar opens the input and recent list, clears the displayed query or inspector, and cancels its outstanding browser request. Ordinary navigation away from Sources still allows requests to finish in the background. The history loader uses shared skeleton bars within rows that match the loaded layout. + +## Project sources + +`POST /v1/sources/recent` accepts an optional `projectId`. The platform verifies project ownership before resolving shared assets. The user DO saves history and the project reference in one synchronous transaction, including history pruning. A failed project write rolls back the history write and pruning. Both tables contain references, never copied video payloads. + +Project references retain their own snapshots by `(project_id, source_key)`. Refreshing Recent sources or saving the same input to another project does not change an existing project's snapshot. `GET /v1/projects/:id/sources/:itemId` restores the selected project item without writing history or relinking it. Project URLs use the project item ID, not the Recent source ID. Saved items survive the thirty-entry history limit. + +Project detail and exports share the same reader for legacy D1 items and user-DO source references. Legacy moments retain their timestamps; full source references do not become subtitle cues. Failed browser saves retain their original input and project destination independently, so another save cannot clear their retry state. + +Deploy the platform before the web application. No new binding or class migration is required. diff --git a/web/app/dashboard/SourcesClient.tsx b/web/app/dashboard/SourcesClient.tsx index 6ec953f..a5d53c7 100644 --- a/web/app/dashboard/SourcesClient.tsx +++ b/web/app/dashboard/SourcesClient.tsx @@ -40,6 +40,8 @@ async function fetchSourceData(inspector: Inspector, option: SourceDataOption, s else delete inspector.dataErrors[option]; } +type SourceSave = { id: string; input: string; projectId: string | null; projectName: string; path: string; body: string }; + export default function SourcesClient({ active }: {active:boolean}) { const params = useSearchParams(); const cache = useDashboardCache(); @@ -62,8 +64,7 @@ export default function SourcesClient({ active }: {active:boolean}) { const [recentSources, setRecentSources] = useState([]); const [historyLoading, setHistoryLoading] = useState(true); const [historyError, setHistoryError] = useState(''); - const [projectSaveError, setProjectSaveError] = useState(''); - const [pendingProjectSource, setPendingProjectSource] = useState(null); + const [failedSaves, setFailedSaves] = useState>([]); const historyInput = useRef(''); const operationController = useRef(null); const searchInput = useRef(null); @@ -86,49 +87,46 @@ export default function SourcesClient({ active }: {active:boolean}) { return () => controller.abort(); }, [active, authenticated, loadHistory]); - const linkProjectSource = async (sourceId: string) => { - if (!projectId) return; + const persistSource = async (save: SourceSave) => { try { - const { added } = await api<{ added: boolean }>(`/v1/projects/${encodeURIComponent(projectId)}/sources`, { - method: 'POST', body: JSON.stringify({ sourceId }), - }); - if (added) projectsResource.setData(current => current.map(project => project.id === projectId - ? { ...project, item_count: (project.item_count ?? 0) + 1 } : project)); - setPendingProjectSource(null); - setProjectSaveError(''); - setNotice(`Added to ${projectName}`); + const { source } = await api<{ source?: RecentSource }>(save.path, { method: 'POST', body: save.body }); + if (source) setRecentSources(current => [source, ...current.filter(item => item.id !== source.id)].slice(0, 30)); + setFailedSaves(current => current.filter(item => item.id !== save.id)); + if (save.projectId) { + // Reconcile counts even when a retry follows a lost successful response. + void projectsResource.refresh(); + setNotice(`Added to ${save.projectName}`); + } } catch (cause) { - setPendingProjectSource(sourceId); - setProjectSaveError(cause instanceof Error ? cause.message : 'Could not add this source to the project.'); + const failed = { ...save, error: cause instanceof Error ? cause.message : 'Could not save this source.' }; + setFailedSaves(current => [...current.filter(item => item.id !== save.id), failed]); } }; const rememberSource = async (input: string, snapshot: SourceSnapshot) => { - let source: RecentSource; - try { - const request = snapshot.kind === 'search' ? { kind: snapshot.kind, selectedData: snapshot.selectedData } - : { kind: snapshot.kind, inspector: { - provider: snapshot.inspector.provider, type: snapshot.inspector.type, id: snapshot.inspector.id, - requestedData: snapshot.inspector.requestedData, dataErrors: snapshot.inspector.dataErrors, - loadedData: ['metadata', ...(['transcript', 'comments', 'channel'] as const).filter(field => snapshot.inspector[field])], - } }; - ({ source } = await api<{ source: RecentSource }>('/v1/sources/recent', { - method: 'POST', body: JSON.stringify({ input, snapshot: request }), - })); - setRecentSources(current => [source, ...current.filter(item => item.id !== source.id)].slice(0, 30)); - setHistoryError(''); - } catch (cause) { setHistoryError(cause instanceof Error ? `Could not save recent source: ${cause.message}` : 'Could not save recent source.'); return; } - await linkProjectSource(source.id); + const request = snapshot.kind === 'search' ? { kind: snapshot.kind, selectedData: snapshot.selectedData } + : { kind: snapshot.kind, inspector: { + provider: snapshot.inspector.provider, type: snapshot.inspector.type, id: snapshot.inspector.id, + requestedData: snapshot.inspector.requestedData, dataErrors: snapshot.inspector.dataErrors, + loadedData: ['metadata', ...(['transcript', 'comments', 'channel'] as const).filter(field => snapshot.inspector[field])], + } }; + await persistSource({ id: crypto.randomUUID(), input, projectId, projectName, path: '/v1/sources/recent', + body: JSON.stringify({ input, snapshot: request, ...(projectId ? { projectId } : {}) }), + }); }; - const openRecentSource = async (entry: Pick) => { + const openRecentSource = async (entry: Pick, savedProjectId?: string) => { const controller = beginOperation('Loading saved source data…'); try { - const { source, snapshot } = await api<{ source: RecentSource; snapshot: SourceSnapshot }>(`/v1/sources/recent/${entry.id}`, { signal: controller.signal }); + const { source, snapshot } = await api<{ source: RecentSource; snapshot: SourceSnapshot }>(savedProjectId ? `/v1/projects/${encodeURIComponent(savedProjectId)}/sources/${encodeURIComponent(entry.id)}` : `/v1/sources/recent/${entry.id}`, { signal: controller.signal }); if (controller.signal.aborted) return; setQuery(source.input); historyInput.current = source.input; setTranscriptQuery(''); - setRecentSources(current => [source, ...current.filter(item => item.id !== source.id)]); - void linkProjectSource(source.id); + if (!savedProjectId) { + setRecentSources(current => [source, ...current.filter(item => item.id !== source.id)].slice(0, 30)); + if (projectId) void persistSource({ id: crypto.randomUUID(), input: source.input, projectId, projectName, + path: `/v1/projects/${encodeURIComponent(projectId)}/sources`, body: JSON.stringify({ sourceId: source.id }), + }); + } if (snapshot.kind === 'search') { setSelectedData(snapshot.selectedData); setItems(snapshot.items); setInspector(null); setHasSearched(true); } else { @@ -324,7 +322,7 @@ export default function SourcesClient({ active }: {active:boolean}) { const q=params.get('q'), id=params.get('id'), type=params.get('type'), saved=params.get('saved'); if (q) { setQuery(q); searchInput.current?.focus(); } if (id && (type==='video'||type==='channel'||type==='playlist')) void inspect(type,id); - if (saved) void openRecentSource({ id: saved }); + if (saved && projectId) void openRecentSource({ id: saved }, projectId); if (q||id||saved) { const next=new URLSearchParams(params); next.delete('q');next.delete('id');next.delete('type');next.delete('saved');router.replace(`/dashboard/sources${next.size?`?${next}`:''}`,{scroll:false}); } }, [active, params, router]); const createProject = async (name:string) => { @@ -430,7 +428,10 @@ export default function SourcesClient({ active }: {active:boolean}) { {error &&
{error}{query.trim() && }
} {notice &&
{notice}
}
} - {projectSaveError && projectId &&
Could not add to {projectName}: {projectSaveError}{pendingProjectSource && }
} + {failedSaves.map(save =>
+ Could not save {save.input}{save.projectId ? ` to ${save.projectName}` : ''}: {save.error} + +
)} {inspector ? ( void retrySourceData()} onOpenComments={() => void refreshComments()} onRefresh={() => void refreshVideoData()} segments={filteredSegments} transcriptQuery={transcriptQuery} setTranscriptQuery={setTranscriptQuery} onClose={() => { cancelOperation(); setInspector(null); }} onSave={() => void saveInspector()} onMonitor={() => void addMonitor()} onOpenVideo={(id) => void inspect('video', id, undefined, inspector.provider, selectedData)} /> ) : ( diff --git a/web/app/dashboard/dashboard-routes.ts b/web/app/dashboard/dashboard-routes.ts index 2b69108..44b4114 100644 --- a/web/app/dashboard/dashboard-routes.ts +++ b/web/app/dashboard/dashboard-routes.ts @@ -4,6 +4,6 @@ export const SOURCES_HOME_EVENT = 'video2ctx:sources-home'; export function dashboardPath(section: DashboardSection) { return `/dashboard/${section === 'discover' ? 'sources' : section}`; } export function projectItemPath(projectId: string, item: ProjectItem) { return item.source_id - ? `/dashboard/sources?project=${encodeURIComponent(projectId)}&saved=${encodeURIComponent(item.source_id)}` + ? `/dashboard/sources?project=${encodeURIComponent(projectId)}&saved=${encodeURIComponent(item.id)}` : `/dashboard/sources?type=${item.entity_type}&id=${encodeURIComponent(item.entity_id)}`; }