From 01694e0e835f21cd5ed42939b640856f5f07a63a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 04:31:07 +0000 Subject: [PATCH] M1: GAM as the library Find footage for a scene in GAM, including where something is said in a video, and place that moment in the scene. Either as muted b-roll under the narration, or as a clip that plays its own sound. The moment becomes the visual's trim: - padded so a cut does not clip a word; - sized by a search placeholder's `moment` when it asks for one; - kept inside the asset. Search placeholders resolve the same way. Every scene showing one gets the trim. GVC's server calls GAM as the signed-in user by forwarding their own Notes token, so GAM's ownership checks hold and there is no service account. A token is only forwarded after this app has verified it. GAM down is a 502 gam_unavailable. GAM refusing the session is gam_rejected_session, which names the JWT secret mismatch. Thumbnails and names are looked up per page load, because GAM's signed URLs expire. They load straight from GAM, so the CSP's img-src and media-src carry GAM_BASE_URL, templated like the Notes origin. A note pushed from Notes can now finish the hand-off: "Copy into GAM" moves each file the spec still points at inside the note into the user's library. Each ref is replaced under the same key, so nothing that used it changes. It saves one version. Failures stay to retry. Only /media/ paths on the configured Notes are fetched, since a pasted spec is untrusted input, and the size cap is enforced while the file streams. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016fiRJAt34ag62iPiuMkNhm --- .env.example | 13 +- backend/app/auth.py | 24 ++ backend/app/config.py | 10 +- backend/app/gam/__init__.py | 1 + backend/app/gam/client.py | 156 ++++++++++ backend/app/main.py | 12 + backend/app/outbound.py | 24 ++ backend/app/routers/gam.py | 52 ++++ backend/app/routers/projects.py | 28 +- backend/app/schemas_gam.py | 56 ++++ backend/app/schemas_projects.py | 12 + backend/app/services/note_media.py | 119 ++++++++ backend/tests/conftest.py | 16 + backend/tests/fake_siblings.py | 101 +++++++ backend/tests/test_gam_api.py | 92 ++++++ backend/tests/test_note_media.py | 111 +++++++ docker-compose.yml | 4 + docs/plan-of-attack.md | 14 +- frontend/Dockerfile | 6 +- frontend/csp.test.ts | 9 + frontend/nginx.conf | 8 +- frontend/src/api/gam.ts | 68 +++++ frontend/src/api/projects.ts | 50 ++++ .../src/components/FootageSearch.test.tsx | 148 ++++++++++ frontend/src/components/FootageSearch.tsx | 274 ++++++++++++++++++ frontend/src/components/SceneList.tsx | 32 +- frontend/src/components/VisualStrip.test.tsx | 77 +++++ frontend/src/components/VisualStrip.tsx | 120 ++++++++ frontend/src/spec/footage.test.ts | 217 ++++++++++++++ frontend/src/spec/footage.ts | 217 ++++++++++++++ frontend/src/stores/projects.ts | 55 ++++ frontend/src/views/ProjectView.tsx | 139 ++++++++- 32 files changed, 2236 insertions(+), 29 deletions(-) create mode 100644 backend/app/gam/__init__.py create mode 100644 backend/app/gam/client.py create mode 100644 backend/app/outbound.py create mode 100644 backend/app/routers/gam.py create mode 100644 backend/app/schemas_gam.py create mode 100644 backend/app/services/note_media.py create mode 100644 backend/tests/fake_siblings.py create mode 100644 backend/tests/test_gam_api.py create mode 100644 backend/tests/test_note_media.py create mode 100644 frontend/src/api/gam.ts create mode 100644 frontend/src/components/FootageSearch.test.tsx create mode 100644 frontend/src/components/FootageSearch.tsx create mode 100644 frontend/src/components/VisualStrip.test.tsx create mode 100644 frontend/src/components/VisualStrip.tsx create mode 100644 frontend/src/spec/footage.test.ts create mode 100644 frontend/src/spec/footage.ts diff --git a/.env.example b/.env.example index bb49229..e15e2cb 100644 --- a/.env.example +++ b/.env.example @@ -17,9 +17,20 @@ CORS_ORIGIN= # over /api/config, and the frontend container's CSP connect-src, so one setting. NOTES_BASE_URL=https://notes.geckopico.com -# The media library. Read from M1 on. +# The media library. GVC's server searches it and uploads to it as the signed-in user, +# and the browser loads thumbnails and previews from it (the CSP's img-src/media-src). +# GAM must share JWT_SECRET_KEY with Notes and GVC, or it refuses the forwarded session. +# +# In development: GAM_BASE_URL=http://localhost:8001 and NOTES_BASE_URL=http://localhost:8000. +# The dev user (DEV_AUTH_USER) sends no token, so GAM must run with a dev user too. GAM_BASE_URL=https://gam.geckopico.com +# Per-request timeout for GAM, in seconds. +GAM_TIMEOUT_SECONDS=30 + +# The largest file "Copy into GAM" will move out of a note pushed from Notes, in MB. +NOTE_MEDIA_MAX_MB=4096 + # ─── Administrators ────────────────────────────────────────────────────────── # Gecko Notes user ids or usernames, comma-separated, matched exactly. Nothing in GVC diff --git a/backend/app/auth.py b/backend/app/auth.py index b163942..a2f498f 100644 --- a/backend/app/auth.py +++ b/backend/app/auth.py @@ -212,6 +212,30 @@ def current_user( CurrentUser = Annotated[UserCtx, Depends(current_user)] +def caller_token( + authorization: Annotated[Optional[str], Header()] = None, + gecko_session: Annotated[Optional[str], Cookie()] = None, +) -> Optional[str]: + """The raw token the caller signed in with, to call GAM as them. + + GAM verifies the same Notes-issued JWT, so forwarding it lets GVC search and upload + as the user, with GAM's own per-user ownership intact and no service account to + hold. Same transport order as `current_user` (header, then cookie). Only meaningful + on a route that also depends on `current_user`, which has already verified it and + applied the cookie path's origin check. + + None under the dev user: GAM then has to be running with a dev user of its own. + """ + if authorization and authorization.startswith("Bearer "): + return authorization[7:].strip() or None + if gecko_session: + return gecko_session.strip() or None + return None + + +CallerToken = Annotated[Optional[str], Depends(caller_token)] + + # ─── administrators ────────────────────────────────────────────────────────── # # The token carries no admin claim and `User.is_admin` has no writer, so an admin is diff --git a/backend/app/config.py b/backend/app/config.py index 5bff0ca..ec3b6aa 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -56,9 +56,15 @@ class Settings(BaseSettings): database_url: str = "" # ─── siblings ──────────────────────────────────────────────────────────── - # The media library. Read from M1 on; set now so a deployment's .env is complete - # before anything depends on it. + # The media library: footage, pictures and music are searched for and kept there. gam_base_url: str = "https://gam.geckopico.com" + # Per request to GAM. Generous, because a search that runs its semantic half calls + # an embedding provider before it answers. + gam_timeout_seconds: float = 30 + # The largest file "Copy into GAM" will move out of a pushed note. Streamed to a + # temp file and checked as it arrives, not after: a size checked afterwards has + # already filled the disk. + note_media_max_mb: int = 4096 # ─── http ──────────────────────────────────────────────────────────────── cors_origin: str = "" diff --git a/backend/app/gam/__init__.py b/backend/app/gam/__init__.py new file mode 100644 index 0000000..570a5ff --- /dev/null +++ b/backend/app/gam/__init__.py @@ -0,0 +1 @@ +"""GAM, the suite's media library, as GVC sees it.""" diff --git a/backend/app/gam/client.py b/backend/app/gam/client.py new file mode 100644 index 0000000..f15635b --- /dev/null +++ b/backend/app/gam/client.py @@ -0,0 +1,156 @@ +"""Calling GAM as the signed-in user. + +GVC forwards the caller's own Notes-issued token, so GAM sees the same user it would +see from its own frontend: same library, same ownership checks. No service account, +and nothing GVC could reach that the user could not. + +Every failure becomes a `GamError` with a code, which `main.py` turns into a 502 in the +suite's error shape. A 502 rather than passing GAM's status through: from the browser's +point of view GVC is the server, and GVC's upstream failed. +""" + +from __future__ import annotations + +import logging +from typing import IO, Any, Dict, Optional + +import httpx + +from app import outbound +from app.config import settings +from app.schemas_gam import GamAssetRead, GamHit, GamSearch + +logger = logging.getLogger(__name__) + + +class GamError(Exception): + def __init__(self, code: str, message: str, *, status_code: int = 502): + super().__init__(message) + self.code = code + self.message = message + self.status_code = status_code + + +def _absolute(url: Optional[str]) -> Optional[str]: + """GAM's media URLs are relative to GAM (`/media/?exp=…&sig=…`).""" + if not url: + return None + if url.startswith("http://") or url.startswith("https://"): + return url + return settings.gam_base_url.rstrip("/") + url + + +def to_asset(raw: Dict[str, Any]) -> GamAssetRead: + return GamAssetRead( + id=raw["id"], + name=raw.get("name") or "", + asset_type=raw.get("asset_type") or "", + duration_seconds=raw.get("duration_seconds"), + width=raw.get("width"), + height=raw.get("height"), + file_url=_absolute(raw.get("file_url")), + thumb_url=_absolute(raw.get("thumb_url")), + credit=raw.get("credit") or "", + parent_asset_id=raw.get("parent_asset_id"), + in_point=raw.get("in_point"), + out_point=raw.get("out_point"), + missing=bool(raw.get("missing")), + ) + + +class GamClient: + def __init__(self, token: Optional[str]): + headers = {"Authorization": f"Bearer {token}"} if token else {} + self._http = outbound.client( + settings.gam_base_url, timeout=settings.gam_timeout_seconds, headers=headers + ) + + def close(self) -> None: + self._http.close() + + def _request(self, method: str, path: str, **kwargs) -> httpx.Response: + try: + response = self._http.request(method, path, **kwargs) + except httpx.TimeoutException as exc: + raise GamError("gam_unavailable", "The asset library took too long to answer") from exc + except httpx.HTTPError as exc: + logger.warning("GAM unreachable at %s: %s", settings.gam_base_url, exc) + raise GamError("gam_unavailable", "Could not reach the asset library") from exc + + if response.status_code == 401: + # The user is signed in here, so GAM refusing the same token means the two + # apps disagree about JWT_SECRET_KEY: an operator's problem, said as one. + raise GamError( + "gam_rejected_session", + "The asset library did not accept your session. GVC and GAM probably " + "have different JWT_SECRET_KEY values.", + ) + if response.status_code >= 500: + raise GamError("gam_unavailable", f"The asset library failed ({response.status_code})") + return response + + @staticmethod + def _detail(response: httpx.Response) -> tuple[str, str]: + try: + detail = response.json().get("detail") or {} + except ValueError: + detail = {} + if isinstance(detail, dict): + return detail.get("code") or "gam_error", detail.get("message") or response.text[:200] + return "gam_error", str(detail) + + def search(self, query: str, *, asset_type: Optional[str] = None, limit: int = 30) -> GamSearch: + params: Dict[str, Any] = {"q": query, "limit": limit} + if asset_type: + params["asset_type"] = asset_type + response = self._request("GET", "/api/search", params=params) + if response.status_code != 200: + code, message = self._detail(response) + raise GamError(code, message, status_code=400 if response.status_code < 500 else 502) + body = response.json() + return GamSearch( + data=[ + GamHit( + asset=to_asset(hit["asset"]), + score=hit.get("score") or 0.0, + snippet=hit.get("snippet") or "", + start_time=hit.get("start_time"), + # Absent from a GAM older than its `end_time` change (gam#36); the + # editor then sizes the moment itself. + end_time=hit.get("end_time"), + ) + for hit in body.get("data", []) + ], + total=body.get("total", 0), + query=body.get("query", query), + semantic=bool(body.get("semantic")), + semantic_error=body.get("semantic_error"), + ) + + def get_asset(self, asset_id: str) -> Optional[GamAssetRead]: + response = self._request("GET", f"/api/assets/{asset_id}") + if response.status_code == 404: + return None + if response.status_code != 200: + code, message = self._detail(response) + raise GamError(code, message) + return to_asset(response.json()["data"]) + + def upload(self, filename: str, stream: IO[bytes], content_type: Optional[str]) -> GamAssetRead: + """Store one file in the user's library and return it.""" + response = self._request( + "POST", + "/api/assets", + files={"files": (filename, stream, content_type or "application/octet-stream")}, + ) + if response.status_code not in (200, 201): + code, message = self._detail(response) + raise GamError(code, message, status_code=502) + body = response.json() + created = body.get("created") or [] + if not created: + # GAM's partial success: a 201 that stored nothing names why per file. + rejected = (body.get("rejected") or [{}])[0] + raise GamError(rejected.get("code") or "gam_error", + rejected.get("message") or "The asset library did not store the file") + return to_asset(created[0]) diff --git a/backend/app/main.py b/backend/app/main.py index c8062e9..23cb5c5 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -19,6 +19,8 @@ from app.auth import CurrentUser, is_admin from app.config import settings from app.database import engine, get_session +from app.gam.client import GamError +from app.routers import gam as gam_router from app.routers import projects as projects_router from app.routers import spec as spec_router from app.schemas import ClientConfig, DataResponse, HealthResponse @@ -76,6 +78,15 @@ async def _http_exception_handler(request, exc: StarletteHTTPException): return JSONResponse(status_code=exc.status_code, content={"detail": detail}) +@app.exception_handler(GamError) +async def _gam_error_handler(request, exc: GamError): + """GAM failing is GVC's upstream failing: in the suite's error shape, with GAM's + own code where it gave one, so the editor can say which app is at fault.""" + return JSONResponse( + status_code=exc.status_code, content={"detail": {"code": exc.code, "message": exc.message}} + ) + + def _default_code(status_code: int) -> str: return { 400: "bad_request", @@ -126,3 +137,4 @@ def me(user: CurrentUser, session: Session = Depends(get_session)) -> DataRespon app.include_router(projects_router.router, prefix="/api/projects", tags=["projects"]) app.include_router(spec_router.router, prefix="/api/spec", tags=["spec"]) +app.include_router(gam_router.router, prefix="/api/gam", tags=["gam"]) diff --git a/backend/app/outbound.py b/backend/app/outbound.py new file mode 100644 index 0000000..44e9695 --- /dev/null +++ b/backend/app/outbound.py @@ -0,0 +1,24 @@ +"""HTTP clients for calls GVC makes to its siblings (GAM, Notes). + +One place builds them, so tests can swap the transport for an `httpx.MockTransport` +and fake both siblings without a network, the way GAM fakes fal.ai. +""" + +from typing import Dict, Optional + +import httpx + +# Tests set this; production leaves it None, which is httpx's real network transport. +TRANSPORT: Optional[httpx.BaseTransport] = None + + +def client(base_url: str, *, timeout: float, headers: Optional[Dict[str, str]] = None) -> httpx.Client: + return httpx.Client( + base_url=base_url.rstrip("/"), + timeout=timeout, + headers=headers or {}, + transport=TRANSPORT, + # Never follow a redirect to somewhere else: the base URLs are configured, and a + # sibling that answers with a redirect is misconfigured, not something to chase. + follow_redirects=False, + ) diff --git a/backend/app/routers/gam.py b/backend/app/routers/gam.py new file mode 100644 index 0000000..9e6162b --- /dev/null +++ b/backend/app/routers/gam.py @@ -0,0 +1,52 @@ +"""Searching and reading GAM, the media library, as the signed-in user.""" + +from typing import Iterator, Literal, Optional + +from fastapi import APIRouter, Depends, Query + +from app.auth import CallerToken, CurrentUser +from app.gam.client import GamClient +from app.schemas import DataResponse +from app.schemas_gam import AssetLookup, AssetLookupResult, GamSearch + +router = APIRouter() + + +def get_gam(_user: CurrentUser, token: CallerToken) -> Iterator[GamClient]: + """A GAM client acting as the caller. Depends on `current_user` first, so a token + is only ever forwarded after this app has verified it.""" + client = GamClient(token) + try: + yield client + finally: + client.close() + + +@router.get("/search", response_model=GamSearch) +def search( + q: str = Query(default="", max_length=500), + # What can go on screen or under it. GAM also holds documents, which a video has + # no use for, so they are not offered. + type: Optional[Literal["image", "video", "audio"]] = Query(default=None), + limit: int = Query(default=30, ge=1, le=100), + gam: GamClient = Depends(get_gam), +) -> GamSearch: + if not q.strip(): + return GamSearch(data=[], total=0, query="") + result = gam.search(q, asset_type=type, limit=limit) + if type is None: + # Without a type filter GAM returns documents too; keep the three media kinds. + result.data = [h for h in result.data if h.asset.asset_type in ("image", "video", "audio")] + result.total = len(result.data) + return result + + +@router.post("/assets/lookup", response_model=DataResponse[AssetLookupResult]) +def lookup(body: AssetLookup, gam: GamClient = Depends(get_gam)) -> DataResponse[AssetLookupResult]: + """Names, thumbnails and fresh signed URLs for the GAM assets a spec uses. + + One GET per id, in sequence: a project uses a handful, and GAM has no batch read. + Asked for on each page load, because the signed URLs expire. + """ + assets = {asset_id: gam.get_asset(asset_id) for asset_id in dict.fromkeys(body.ids)} + return DataResponse(data=AssetLookupResult(assets=assets)) diff --git a/backend/app/routers/projects.py b/backend/app/routers/projects.py index f05b41b..bc307a8 100644 --- a/backend/app/routers/projects.py +++ b/backend/app/routers/projects.py @@ -7,10 +7,13 @@ from app.database import get_session from app.models.project import Project, ProjectVersion from app.schemas import DataResponse, ListResponse +from app.gam.client import GamClient +from app.routers.gam import get_gam from app.schemas_projects import ( - ProjectCreate, ProjectImport, ProjectRead, ProjectSummary, SpecSave, VersionDetail, - VersionRead, + CopyFailureRead, NoteMediaCopy, ProjectCreate, ProjectImport, ProjectRead, + ProjectSummary, SpecSave, VersionDetail, VersionRead, ) +from app.services import note_media from app.services import projects as service from app.spec.starter import starter_spec @@ -146,3 +149,24 @@ def revert_project( project = service.get_owned(session, user, project_id) service.revert(session, project, service.get_version(session, project, version_id)) return DataResponse(data=_read(project)) + + +@router.post("/{project_id}/copy-note-media", response_model=DataResponse[NoteMediaCopy]) +def copy_note_media( + project_id: str, + user: CurrentUser, + session: Session = Depends(get_session), + gam: GamClient = Depends(get_gam), +) -> DataResponse[NoteMediaCopy]: + """Move the files a pushed note still points at into GAM, as one new version. + + Synchronous: a pushed note carries a handful of pictures and a music file. If that + proves slow it moves onto the job queue rendering brings (M2). + """ + project = service.get_owned(session, user, project_id) + result = note_media.copy_note_media(session, project, gam) + return DataResponse(data=NoteMediaCopy( + project=_read(project), + copied=result.copied, + failed=[CopyFailureRead(key=f.key, message=f.message) for f in result.failed], + )) diff --git a/backend/app/schemas_gam.py b/backend/app/schemas_gam.py new file mode 100644 index 0000000..f943c83 --- /dev/null +++ b/backend/app/schemas_gam.py @@ -0,0 +1,56 @@ +"""GAM's assets and search hits, in GVC's own shape. + +Normalised rather than passed through, so a field GAM renames breaks this file and its +tests instead of every screen that shows a thumbnail. Only what GVC uses is kept. +""" + +from typing import Dict, List, Optional + +from pydantic import BaseModel, Field + + +class GamAssetRead(BaseModel): + id: str + name: str + asset_type: str + duration_seconds: Optional[float] = None + width: Optional[int] = None + height: Optional[int] = None + # Absolute and signed by GAM: usable directly in /