diff --git a/backend/app/routers/video.py b/backend/app/routers/video.py index 485d8c7..148885f 100644 --- a/backend/app/routers/video.py +++ b/backend/app/routers/video.py @@ -21,9 +21,11 @@ from app.routers.media import MEDIA_DIR from app.schemas import ( DataResponse, ListResponse, VideoEstimateRead, VideoRenderJobRead, VideoRenderRequest, + VideoSpecRead, ) from app.video import ffmpeg as F from app.video import worker +from app.video.importer import import_note from app.video.options import RenderOptions from app.video.renderer import estimate @@ -211,3 +213,31 @@ def estimate_render(payload: VideoRenderRequest, request: Request, session: Sess shots=shots, narration_chars=chars, estimated_seconds=round(seconds, 1), warnings=warnings, )) + + +@router.post("/spec", response_model=DataResponse[VideoSpecRead]) +def video_spec(payload: VideoRenderRequest, request: Request, session: Session = Depends(get_session)): + """The VideoSpec a render of this note would be built from, as a document. + + Takes the same body as /estimate, so a selection and the dialog's options + shape it exactly as they would the render. Nothing is rendered or stored. + """ + user_id = _get_user_id(request) + note = _owned_note(session, payload.note_id, user_id) + options = _parse_options(payload.options) + user = session.get(User, user_id) + + imported = import_note( + payload.selected_content or note.content or "[]", + user_id=user_id, + media_dir=MEDIA_DIR, + options=options, + note_title=note.title or "Untitled", + author=(user.username if user else "") or "", + has_audio=F.probe_has_audio, + note_id=note.id, + ) + return DataResponse(data=VideoSpecRead( + spec=imported.spec.model_dump(mode="json", by_alias=True, exclude_none=True), + warnings=imported.warnings, + )) diff --git a/backend/app/schemas.py b/backend/app/schemas.py index 9df37bf..db8bb08 100644 --- a/backend/app/schemas.py +++ b/backend/app/schemas.py @@ -738,6 +738,13 @@ class VideoEstimateRead(BaseModel): warnings: List[str] = [] +class VideoSpecRead(BaseModel): + # The VideoSpec as JSON, under its own field names ("from", "in") — see + # app/video/spec.py and docs/gecko-video-creator.md §5. + spec: Dict[str, Any] + warnings: List[str] = [] + + class ActivityJobRead(BaseModel): """One running (or recently finished) background job, whatever kind it is. diff --git a/backend/app/video/compiler.py b/backend/app/video/compiler.py index a5932cb..4b0db21 100644 --- a/backend/app/video/compiler.py +++ b/backend/app/video/compiler.py @@ -150,7 +150,7 @@ def _segment_shots(segment: Segment, spec: VideoSpec, resolve, heading_ms: int) label=label, ) if index == 0 and segment.chapter: - # Set after construction, as segment() does for every section, so + # Set after construction, as the walk always has for a section, so # the mark is exactly what it was — pause markers included, which # nothing strips from a section's chapter today. shot.chapter = segment.chapter diff --git a/backend/app/video/importer.py b/backend/app/video/importer.py index 1bafd45..4e99875 100644 --- a/backend/app/video/importer.py +++ b/backend/app/video/importer.py @@ -1,16 +1,18 @@ """A note (BlockNote JSON) -> a VideoSpec. -The walk is `segmenter.segment()`'s, step for step, so that compiling what this -produces gives back exactly the shots `segment()` would have made — that -equivalence is what tests/test_video_spec.py holds it to, and what lets -`segment()` later become `compile_spec(import_note(...))`. What it records is -richer: the text under a picture is kept as one beat per block rather than one -joined string, a quote or code block is an overlay on its own beat rather than a -shot of its own, and every segment remembers which blocks it came from. - -Until the switch, the two walks must change together. Anything the import -decides — whether headings become chapter screens, whether quotes are drawn — -is settled here, from `RenderOptions`, and the spec only records the outcome. +This is the walk every render starts from: `segmenter.segment()` is +`compile_spec(import_note(...))`. It reads a note the way the segmenter always +has — every image or video a boundary, the text below it that section's +narration — but records more than shots: the text under a picture is kept as one +beat per block rather than one joined string, a quote or code block is an +overlay on its own beat rather than a shot of its own, and every segment and +beat remembers which blocks it came from. + +The walk is the original segmenter's, step for step, quirks included, and +tests/test_video_spec.py holds it to a verbatim copy of that original +(tests/segment_reference.py). Anything the import decides — whether headings +become chapter screens, whether quotes are drawn — is settled here, from +`RenderOptions`, and the spec only records the outcome. """ import json @@ -35,8 +37,7 @@ @dataclass class ImportResult: spec: VideoSpec - # Media the note references that could not be used, in the order and - # wording segment() reports them. + # Media the note references that could not be used, in document order. warnings: List[str] = field(default_factory=list) @@ -47,7 +48,7 @@ class _Part: kind: ShotKind media: Optional[Tuple[str, str]] = None # (url, "image" | "video") media_label: Optional[str] = None # a readable name, when the file has one - path: Optional[str] = None # resolved, for segment()'s comparisons + path: Optional[str] = None # resolved, for the cleanup passes label: str = "" beats: List[Beat] = field(default_factory=list) chapter: Optional[str] = None @@ -61,7 +62,7 @@ def narration(self, heading_pause_ms: int) -> str: heading_pause_ms=heading_pause_ms) def panel(self) -> bool: - """Whether segment()'s shot would carry quote_text or code_text.""" + """Whether the compiled shot will carry quote_text or code_text.""" if isinstance(self.overlay, CodeOverlay): return True if isinstance(self.overlay, QuoteOverlay): @@ -120,7 +121,8 @@ def segment_via_spec( author: str = "", has_audio: Optional[Callable[[str], Any]] = None, ) -> Segmentation: - """`segment()` by way of a VideoSpec: same arguments, same shots and warnings.""" + """What `segment()` returns: the note imported as a spec and compiled back to shots, + with the import's warnings ahead of the compile's.""" imported = import_note(content, user_id=user_id, media_dir=media_dir, options=options, note_title=note_title, author=author, has_audio=has_audio) compiled = compile_spec(imported.spec, user_id=user_id, media_dir=media_dir) @@ -128,7 +130,7 @@ def segment_via_spec( warnings=imported.warnings + compiled.segmentation.warnings) -# ── the walk (segmenter.segment(), recording beats) ────────────────────────── +# ── the walk ────────────────────────── def _walk(content, *, user_id, media_dir, options, note_title, author, has_audio, warnings) -> List[_Part]: diff --git a/backend/app/video/segmenter.py b/backend/app/video/segmenter.py index a07542e..48be7c6 100644 --- a/backend/app/video/segmenter.py +++ b/backend/app/video/segmenter.py @@ -3,7 +3,10 @@ A "shot" is one continuous piece of screen time: a background (still image, looping video, or a Pillow-composed card) plus the narration heard over it. The document is walked in order and every image/video acts as a boundary — the text -between one media block and the next becomes that media's narration. +between one media block and the next becomes that media's narration. The walk +itself is in `importer`, which records it as a VideoSpec; `segment()` compiles +that spec back to shots. This module keeps the shot model and the reading of +blocks — what counts as media, what text a block speaks — that both share. Flattening to shots rather than nesting segments is what keeps the renderer simple: a clip that carries its own audio is not a special case inside a segment, @@ -12,7 +15,6 @@ ffmpeg invocation per shot, one progress tick per shot. """ -import json import os from dataclasses import dataclass, field from typing import Any, Dict, List, Literal, Optional, Tuple @@ -56,9 +58,9 @@ class Shot: quote_attribution: Optional[str] = None # A code block drawn over this shot's background. Always drawn when set; # `narration` above only carries its text when narrate_code is on — see - # segment()'s codeBlock branch. + # the importer's codeBlock branch. code_text: Optional[str] = None - # "intro" or "outro" for a clip bracketing the article (see _bumper_shot). + # "intro" or "outro" for a clip bracketing the article (see importer._bumper). # The renderer draws nothing of its own over one of these. bumper: Optional[str] = None # Set on the second half of a sounded-clip pair, purely for readable logs. @@ -318,282 +320,19 @@ def segment( author: str = "", has_audio: Optional[Any] = None, ) -> Segmentation: - """Turn note content into shots. - - `has_audio` is a callable(path) -> bool used to decide whether a video clip - carries its own audio track; the renderer passes an ffprobe-backed one, and - tests pass a stub. When it is None every clip is treated as silent. + """Turn note content into shots, by way of a VideoSpec. + + The note is imported as a spec (app/video/importer.py) and compiled back to + the shots the renderer works from (app/video/compiler.py), so what a render + does is always something a spec can say. `has_audio` is a callable(path) -> + bool used to decide whether a video clip carries its own audio track; the + renderer passes an ffprobe-backed one, and tests pass a stub. When it is + None every clip is treated as silent. """ - result = Segmentation() - - try: - blocks = json.loads(content or "[]") - except (ValueError, TypeError): - blocks = [] - if not isinstance(blocks, list): - blocks = [] - - if options.title_card and (note_title or author): - result.shots.append(Shot( - kind="card", - card_title=note_title or "Untitled", - card_subtitle=author, - card_kind="title", - chapter=note_title or None, - label="title card", - )) - - # Text accumulated since the last media boundary, plus the shot it belongs to. - pending_text: List[str] = [] - # The shot that pending_text will be attached to when the next boundary lands. - open_shot: Optional[Shot] = None - # A heading seen since the last boundary, to hang a chapter mark on. - pending_chapter: Optional[str] = None - pending_card: Optional[Shot] = None - - def flush(next_shot: Optional[Shot]) -> None: - """Close the open shot with whatever narration has accumulated.""" - nonlocal pending_text, open_shot, pending_chapter, pending_card - text = "\n".join(t for t in pending_text if t).strip() - - if open_shot is None and text: - # Text before the first media block: give it the fallback background. - open_shot = Shot(kind="still", background=None, label="opening") - if open_shot is not None: - open_shot.narration = text - if pending_chapter and not open_shot.chapter: - open_shot.chapter = pending_chapter - if pending_card is not None: - result.shots.append(pending_card) - pending_card = None - result.shots.append(open_shot) - elif pending_card is not None: - result.shots.append(pending_card) - pending_card = None - - # A heading with nothing spoken under it (read_chapters off) has no shot - # to hang its mark on; hand it to the shot that starts here instead. - carried_chapter = None - if (pending_chapter and open_shot is None - and next_shot is not None and next_shot.kind != "card"): - carried_chapter = pending_chapter - pending_text = [] - pending_chapter = None - open_shot = next_shot - if carried_chapter and open_shot is not None and not open_shot.chapter: - open_shot.chapter = carried_chapter - - for block in _flatten(blocks): - kind = _classify(block, options) - - if kind is None: - btype = block.get("type") - if btype == "heading": - heading = _inline_text(block.get("content")).strip() - if heading: - if options.chapter_screens: - # A chapter screen interrupts: close the current shot so - # the card lands between sections rather than mid-thought. - # Nothing from this heading is recorded before the flush, - # or the section *above* it would be labelled with the - # chapter this heading is opening. - # - # The section after the card resumes on the same picture - # — cutting to the plain fallback for one screen and back - # again would read as a mistake, the same reasoning the - # quote handler below uses. A card interrupting a sounded - # clip resumes muted, matching how the clip is carried - # elsewhere; only a genuinely new image or video changes - # the background from here. - carry_kind: ShotKind = ( - "video_muted" - if open_shot is not None and open_shot.kind.startswith("video") - else "still" - ) - carry_background = open_shot.background if open_shot is not None else None - flush(Shot(kind=carry_kind, background=carry_background, - label="after chapter card")) - # The card reads its own heading. Letting the heading fall - # through to the next section instead would show it in - # silence and then speak it over the following shot, once - # the words were no longer on screen. - pending_card = Shot( - kind="card", card_title=heading, chapter=heading, - card_kind="chapter", - narration=_as_sentence(heading) if options.read_chapters else "", - label="chapter card", - ) - # The card carries the chapter mark, so the section after - # it must not claim the same one and duplicate the entry. - pending_chapter = None - continue - # Without a chapter screen the heading is read inside the - # section it introduces, so that shot carries the mark. - if pending_chapter is None: - pending_chapter = heading - - if btype == "quote" and options.quotes.enabled: - quoted = _split_attribution(_inline_text(block.get("content"))) - if quoted[0]: - # A quote gets its own screen time so the words can be on - # screen while they are read, but it keeps the background of - # the section it interrupts — cutting to a different picture - # for one sentence would read as a mistake. A sounded clip is - # carried muted, or it would replay its audio underneath. - carry_kind: ShotKind = ( - "video_muted" - if open_shot is not None and open_shot.kind.startswith("video") - else "still" - ) - carry_background = open_shot.background if open_shot is not None else None - quote_shot = Shot( - kind=carry_kind, background=carry_background, - quote_text=quoted[0], quote_attribution=quoted[1] or None, - label="quote", - ) - if (any(t.strip() for t in pending_text) - or pending_chapter is not None or pending_card is not None): - flush(quote_shot) - else: - # The open shot has nothing of its own to say, so the - # quote takes it over rather than being preceded by a - # blank copy of itself held for min_shot_seconds. - open_shot = quote_shot - # Closing the quote's own shot immediately is what keeps the - # quotation as its narration rather than whatever follows it. - pending_text.append(_as_sentence(quoted[0])) - flush(Shot(kind=carry_kind, background=carry_background, - label="after quote")) - continue - - if btype == "codeBlock": - code = _inline_text(block.get("content")) - if code.strip(): - # A code block gets its own screen time the same way a - # quote does — its own shot, over whatever background the - # section it interrupts was already showing — but unlike a - # quote it is drawn unconditionally: only whether it is - # *narrated* depends on narrate_code, not whether it shows - # up at all. - carry_kind: ShotKind = ( - "video_muted" - if open_shot is not None and open_shot.kind.startswith("video") - else "still" - ) - carry_background = open_shot.background if open_shot is not None else None - code_shot = Shot( - kind=carry_kind, background=carry_background, - code_text=code, label="code", - ) - if (any(t.strip() for t in pending_text) - or pending_chapter is not None or pending_card is not None): - flush(code_shot) - else: - open_shot = code_shot - if options.narrate_code: - pending_text.append(_as_sentence(code)) - flush(Shot(kind=carry_kind, background=carry_background, - label="after code")) - continue - - if btype == "heading" and not options.read_chapters: - # The chapter mark was taken above; the words are just not read. - continue - - text = _block_narration(block, options) - if text.strip(): - spoken = _as_sentence(text) - # A heading is a section boundary, not another sentence of the - # paragraph above it, so it gets a pause on both sides. A card - # already has the boundary of its own shot and needs no marking. - if btype == "heading" and options.heading_pause_ms > 0: - spoken = _set_apart(spoken) - pending_text.append(spoken) - continue - - url = _media_url(block, options) - path = resolve_media_path(url, user_id, media_dir) if url else None - if path is None: - if url and not url.startswith("/media/"): - result.warnings.append(f"Skipped remote media: {url[:120]}") - elif url: - result.warnings.append(f"Skipped missing media: {url[:120]}") - # Not usable as a background — but it is still a boundary, so close - # the current shot and open a fallback-background one in its place. - flush(Shot(kind="still", background=None, label="fallback")) - continue - - if kind == "image": - flush(Shot(kind="still", background=path)) - continue + # Imported here: the importer is built on this module's helpers. + from app.video.importer import segment_via_spec - # A video clip. If it carries audio it plays whole with its own sound and - # the narration waits; the text below it then runs over the same clip - # looped silently, so the section stays visually continuous. - sounded = bool(has_audio(path)) if callable(has_audio) else False - if sounded: - flush(None) - result.shots.append(Shot(kind="video_sound", background=path, label="clip with audio")) - open_shot = Shot(kind="video_muted", background=path, label="clip continued, muted") - else: - flush(Shot(kind="video_muted", background=path)) - - flush(None) - - # Drop a trailing shot that ended up with nothing to show or say. A media - # shot with no narration is kept — it still displays for min_shot_seconds. - # So is a silent code block (narrate_code off, nothing narrated yet) — it - # still has its panel to show, even with no background of its own. - result.shots = [ - s for s in result.shots - if s.kind != "still" or s.background is not None or s.narration.strip() or s.code_text - ] - - # The section reopened after a quote or a code block is a continuation, - # not a shot of its own: when nothing followed it, it would replay the - # same background in silence, so drop it. - trimmed: List[Shot] = [] - for shot in result.shots: - previous = trimmed[-1] if trimmed else None - if (previous is not None and (previous.quote_text is not None or previous.code_text is not None) - and shot.quote_text is None and shot.code_text is None - and not shot.narration.strip() - and shot.chapter is None - and shot.kind == previous.kind - and shot.background == previous.background): - continue - trimmed.append(shot) - result.shots = trimmed - - # The intro and outro bracket the article rather than stand in for it, so a - # note with nothing of its own to show still renders nothing. - if result.shots: - intro = _bumper_shot("intro", options, user_id, media_dir, result.warnings) - outro = _bumper_shot("outro", options, user_id, media_dir, result.warnings) - if intro is not None: - result.shots.insert(0, intro) - if outro is not None: - result.shots.append(outro) - return result - - -def _bumper_shot( - which: str, options: RenderOptions, user_id: str, media_dir: str, warnings: List[str], -) -> Optional[Shot]: - """The intro or outro clip as a shot, or None when it is off or unusable. - - It is a sounded clip like any other in a note: played whole, with its own - audio when it has some and silence when it doesn't, which the renderer - probes for itself. - """ - spec = options.intro if which == "intro" else options.outro - if not spec.enabled or not spec.url: - return None - path = resolve_media_path(spec.url, user_id, media_dir) - if path is None or _media_kind(spec.url) != "video": - warnings.append(f"The {which} was skipped: that clip could not be read.") - return None - return Shot( - kind="video_sound", background=path, - chapter=which.capitalize(), bumper=which, label=which, + return segment_via_spec( + content, user_id=user_id, media_dir=media_dir, options=options, + note_title=note_title, author=author, has_audio=has_audio, ) diff --git a/backend/tests/segment_reference.py b/backend/tests/segment_reference.py new file mode 100644 index 0000000..c591afc --- /dev/null +++ b/backend/tests/segment_reference.py @@ -0,0 +1,311 @@ +"""The segmenter's walk as it was before it went through a VideoSpec — kept as +the oracle the spec path is held to. + +`segment()` now imports a note as a VideoSpec and compiles it back to shots +(app/video/importer.py, app/video/compiler.py). This is the original, verbatim, +so test_video_spec.py can keep proving that changed nothing: every segmenter +case, a generated corpus and real renders must come out identical. When a +deliberate behaviour change lands (the chapter-mark fixes in +docs/gecko-video-creator.md §3.2), it changes here too or this is retired. +""" + +import json +from typing import Any, List, Optional + +from app.video.options import RenderOptions +from app.video.segmenter import ( + Segmentation, Shot, ShotKind, _as_sentence, _block_narration, _classify, _flatten, + _inline_text, _media_kind, _media_url, _set_apart, _split_attribution, + resolve_media_path, +) + + +def segment( + content: str, + *, + user_id: str, + media_dir: str, + options: RenderOptions, + note_title: str = "", + author: str = "", + has_audio: Optional[Any] = None, +) -> Segmentation: + """Turn note content into shots. + + `has_audio` is a callable(path) -> bool used to decide whether a video clip + carries its own audio track; the renderer passes an ffprobe-backed one, and + tests pass a stub. When it is None every clip is treated as silent. + """ + result = Segmentation() + + try: + blocks = json.loads(content or "[]") + except (ValueError, TypeError): + blocks = [] + if not isinstance(blocks, list): + blocks = [] + + if options.title_card and (note_title or author): + result.shots.append(Shot( + kind="card", + card_title=note_title or "Untitled", + card_subtitle=author, + card_kind="title", + chapter=note_title or None, + label="title card", + )) + + # Text accumulated since the last media boundary, plus the shot it belongs to. + pending_text: List[str] = [] + # The shot that pending_text will be attached to when the next boundary lands. + open_shot: Optional[Shot] = None + # A heading seen since the last boundary, to hang a chapter mark on. + pending_chapter: Optional[str] = None + pending_card: Optional[Shot] = None + + def flush(next_shot: Optional[Shot]) -> None: + """Close the open shot with whatever narration has accumulated.""" + nonlocal pending_text, open_shot, pending_chapter, pending_card + text = "\n".join(t for t in pending_text if t).strip() + + if open_shot is None and text: + # Text before the first media block: give it the fallback background. + open_shot = Shot(kind="still", background=None, label="opening") + if open_shot is not None: + open_shot.narration = text + if pending_chapter and not open_shot.chapter: + open_shot.chapter = pending_chapter + if pending_card is not None: + result.shots.append(pending_card) + pending_card = None + result.shots.append(open_shot) + elif pending_card is not None: + result.shots.append(pending_card) + pending_card = None + + # A heading with nothing spoken under it (read_chapters off) has no shot + # to hang its mark on; hand it to the shot that starts here instead. + carried_chapter = None + if (pending_chapter and open_shot is None + and next_shot is not None and next_shot.kind != "card"): + carried_chapter = pending_chapter + pending_text = [] + pending_chapter = None + open_shot = next_shot + if carried_chapter and open_shot is not None and not open_shot.chapter: + open_shot.chapter = carried_chapter + + for block in _flatten(blocks): + kind = _classify(block, options) + + if kind is None: + btype = block.get("type") + if btype == "heading": + heading = _inline_text(block.get("content")).strip() + if heading: + if options.chapter_screens: + # A chapter screen interrupts: close the current shot so + # the card lands between sections rather than mid-thought. + # Nothing from this heading is recorded before the flush, + # or the section *above* it would be labelled with the + # chapter this heading is opening. + # + # The section after the card resumes on the same picture + # — cutting to the plain fallback for one screen and back + # again would read as a mistake, the same reasoning the + # quote handler below uses. A card interrupting a sounded + # clip resumes muted, matching how the clip is carried + # elsewhere; only a genuinely new image or video changes + # the background from here. + carry_kind: ShotKind = ( + "video_muted" + if open_shot is not None and open_shot.kind.startswith("video") + else "still" + ) + carry_background = open_shot.background if open_shot is not None else None + flush(Shot(kind=carry_kind, background=carry_background, + label="after chapter card")) + # The card reads its own heading. Letting the heading fall + # through to the next section instead would show it in + # silence and then speak it over the following shot, once + # the words were no longer on screen. + pending_card = Shot( + kind="card", card_title=heading, chapter=heading, + card_kind="chapter", + narration=_as_sentence(heading) if options.read_chapters else "", + label="chapter card", + ) + # The card carries the chapter mark, so the section after + # it must not claim the same one and duplicate the entry. + pending_chapter = None + continue + # Without a chapter screen the heading is read inside the + # section it introduces, so that shot carries the mark. + if pending_chapter is None: + pending_chapter = heading + + if btype == "quote" and options.quotes.enabled: + quoted = _split_attribution(_inline_text(block.get("content"))) + if quoted[0]: + # A quote gets its own screen time so the words can be on + # screen while they are read, but it keeps the background of + # the section it interrupts — cutting to a different picture + # for one sentence would read as a mistake. A sounded clip is + # carried muted, or it would replay its audio underneath. + carry_kind: ShotKind = ( + "video_muted" + if open_shot is not None and open_shot.kind.startswith("video") + else "still" + ) + carry_background = open_shot.background if open_shot is not None else None + quote_shot = Shot( + kind=carry_kind, background=carry_background, + quote_text=quoted[0], quote_attribution=quoted[1] or None, + label="quote", + ) + if (any(t.strip() for t in pending_text) + or pending_chapter is not None or pending_card is not None): + flush(quote_shot) + else: + # The open shot has nothing of its own to say, so the + # quote takes it over rather than being preceded by a + # blank copy of itself held for min_shot_seconds. + open_shot = quote_shot + # Closing the quote's own shot immediately is what keeps the + # quotation as its narration rather than whatever follows it. + pending_text.append(_as_sentence(quoted[0])) + flush(Shot(kind=carry_kind, background=carry_background, + label="after quote")) + continue + + if btype == "codeBlock": + code = _inline_text(block.get("content")) + if code.strip(): + # A code block gets its own screen time the same way a + # quote does — its own shot, over whatever background the + # section it interrupts was already showing — but unlike a + # quote it is drawn unconditionally: only whether it is + # *narrated* depends on narrate_code, not whether it shows + # up at all. + carry_kind: ShotKind = ( + "video_muted" + if open_shot is not None and open_shot.kind.startswith("video") + else "still" + ) + carry_background = open_shot.background if open_shot is not None else None + code_shot = Shot( + kind=carry_kind, background=carry_background, + code_text=code, label="code", + ) + if (any(t.strip() for t in pending_text) + or pending_chapter is not None or pending_card is not None): + flush(code_shot) + else: + open_shot = code_shot + if options.narrate_code: + pending_text.append(_as_sentence(code)) + flush(Shot(kind=carry_kind, background=carry_background, + label="after code")) + continue + + if btype == "heading" and not options.read_chapters: + # The chapter mark was taken above; the words are just not read. + continue + + text = _block_narration(block, options) + if text.strip(): + spoken = _as_sentence(text) + # A heading is a section boundary, not another sentence of the + # paragraph above it, so it gets a pause on both sides. A card + # already has the boundary of its own shot and needs no marking. + if btype == "heading" and options.heading_pause_ms > 0: + spoken = _set_apart(spoken) + pending_text.append(spoken) + continue + + url = _media_url(block, options) + path = resolve_media_path(url, user_id, media_dir) if url else None + if path is None: + if url and not url.startswith("/media/"): + result.warnings.append(f"Skipped remote media: {url[:120]}") + elif url: + result.warnings.append(f"Skipped missing media: {url[:120]}") + # Not usable as a background — but it is still a boundary, so close + # the current shot and open a fallback-background one in its place. + flush(Shot(kind="still", background=None, label="fallback")) + continue + + if kind == "image": + flush(Shot(kind="still", background=path)) + continue + + # A video clip. If it carries audio it plays whole with its own sound and + # the narration waits; the text below it then runs over the same clip + # looped silently, so the section stays visually continuous. + sounded = bool(has_audio(path)) if callable(has_audio) else False + if sounded: + flush(None) + result.shots.append(Shot(kind="video_sound", background=path, label="clip with audio")) + open_shot = Shot(kind="video_muted", background=path, label="clip continued, muted") + else: + flush(Shot(kind="video_muted", background=path)) + + flush(None) + + # Drop a trailing shot that ended up with nothing to show or say. A media + # shot with no narration is kept — it still displays for min_shot_seconds. + # So is a silent code block (narrate_code off, nothing narrated yet) — it + # still has its panel to show, even with no background of its own. + result.shots = [ + s for s in result.shots + if s.kind != "still" or s.background is not None or s.narration.strip() or s.code_text + ] + + # The section reopened after a quote or a code block is a continuation, + # not a shot of its own: when nothing followed it, it would replay the + # same background in silence, so drop it. + trimmed: List[Shot] = [] + for shot in result.shots: + previous = trimmed[-1] if trimmed else None + if (previous is not None and (previous.quote_text is not None or previous.code_text is not None) + and shot.quote_text is None and shot.code_text is None + and not shot.narration.strip() + and shot.chapter is None + and shot.kind == previous.kind + and shot.background == previous.background): + continue + trimmed.append(shot) + result.shots = trimmed + + # The intro and outro bracket the article rather than stand in for it, so a + # note with nothing of its own to show still renders nothing. + if result.shots: + intro = _bumper_shot("intro", options, user_id, media_dir, result.warnings) + outro = _bumper_shot("outro", options, user_id, media_dir, result.warnings) + if intro is not None: + result.shots.insert(0, intro) + if outro is not None: + result.shots.append(outro) + return result + + +def _bumper_shot( + which: str, options: RenderOptions, user_id: str, media_dir: str, warnings: List[str], +) -> Optional[Shot]: + """The intro or outro clip as a shot, or None when it is off or unusable. + + It is a sounded clip like any other in a note: played whole, with its own + audio when it has some and silence when it doesn't, which the renderer + probes for itself. + """ + spec = options.intro if which == "intro" else options.outro + if not spec.enabled or not spec.url: + return None + path = resolve_media_path(spec.url, user_id, media_dir) + if path is None or _media_kind(spec.url) != "video": + warnings.append(f"The {which} was skipped: that clip could not be read.") + return None + return Shot( + kind="video_sound", background=path, + chapter=which.capitalize(), bumper=which, label=which, + ) diff --git a/backend/tests/test_video_spec.py b/backend/tests/test_video_spec.py index 79317de..470ffca 100644 --- a/backend/tests/test_video_spec.py +++ b/backend/tests/test_video_spec.py @@ -1,10 +1,12 @@ -"""The VideoSpec round trip: a note imported as a spec and compiled back must be -exactly what `segment()` makes of it — the same shots, field for field, and the -same warnings — and must drive the renderer to the same ffmpeg commands. - -That equivalence is Phase 0 of docs/gecko-video-creator.md: it proves the spec -can describe everything Notes already renders before anything is built on it. -Nothing here encodes video; ffmpeg is faked as in test_video_render_resume. +"""The VideoSpec round trip: `segment()` imports a note as a spec and compiles it +back to shots, and that must be exactly what the segmenter made of the note +before it went through a spec — the same shots, field for field, and the same +warnings — and must drive the renderer to the same ffmpeg commands. + +The original walk is kept verbatim as the oracle (segment_reference.py). This is +Phase 0 of docs/gecko-video-creator.md: it proves the spec can describe +everything Notes already renders before anything is built on it. Nothing here +encodes video; ffmpeg is faked as in test_video_render_resume. """ import inspect @@ -12,17 +14,25 @@ import os import random import re +from datetime import datetime from pathlib import Path +from types import SimpleNamespace import pytest +from fastapi import HTTPException from pydantic import ValidationError +from sqlmodel import Session, SQLModel, create_engine import test_video_segmenter as segmenter_cases +from segment_reference import segment as reference_segment from test_video_render_resume import FakeFFmpeg +from app.models import Note +from app.routers import video as video_router +from app.schemas import VideoRenderRequest from app.video import renderer from app.video.compiler import IMPORT_ONLY_OPTIONS, UnsupportedSpec, compile_spec -from app.video.importer import import_note, segment_via_spec +from app.video.importer import import_note from app.video.options import RenderOptions from app.video.segmenter import segment from app.video.spec import VideoSpec @@ -31,11 +41,11 @@ def _assert_same(content, **kwargs): - want = segment(content, **kwargs) - got = segment_via_spec(content, **kwargs) + want = reference_segment(content, **kwargs) + got = segment(content, **kwargs) assert got.shots == want.shots assert got.warnings == want.warnings - return want + return got def _round_trips_as_json(spec: VideoSpec) -> None: @@ -272,6 +282,10 @@ def _render_commands(monkeypatch, media, content, options, *, via_spec): plan.warnings[:0] = imported.warnings monkeypatch.setattr(renderer, "segment", lambda *a, **k: plan) options = compiled.options + else: + # The original walk, with the options as given: how Notes rendered + # before segment() went through a spec. + monkeypatch.setattr(renderer, "segment", reference_segment) result = renderer.render( job_id="j", user_id="u1", media_dir=media, note_content=content, note_title="Salt", author="gecko", options=options, preview=True, @@ -409,3 +423,64 @@ def test_what_the_notes_renderer_cannot_draw_is_refused(change, message): def test_a_malformed_spec_is_rejected(change, message): with pytest.raises(ValidationError, match=message): _spec(**change) + + +# ── POST /api/video/spec ───────────────────────────────────────────────────── + +_SPEC_NOTE = [ + {"id": "img", "type": "image", "props": {"url": "/media/u1/a.png"}}, + {"id": "p1", "type": "paragraph", "content": [{"type": "text", "text": "Under the map."}]}, + {"id": "q1", "type": "quote", "content": [{"type": "text", "text": "Salt is gold \u2014 A Trader"}]}, +] + + +@pytest.fixture +def notes(tmp_path, monkeypatch): + monkeypatch.setattr(video_router, "MEDIA_DIR", _media_root(tmp_path)) + engine = create_engine("sqlite://", connect_args={"check_same_thread": False}) + SQLModel.metadata.create_all(engine) + with Session(engine) as session: + now = datetime.utcnow() + for note_id, owner in (("mine", "u1"), ("theirs", "u2")): + session.add(Note(id=note_id, title="Salt", content=json.dumps(_SPEC_NOTE), + category_id="cat-1", user_id=owner, created_at=now, modified_at=now)) + session.commit() + yield session + + +def _spec_route(session, note_id="mine", **fields): + payload = VideoRenderRequest(note_id=note_id, **fields) + request = SimpleNamespace(state=SimpleNamespace(user_id="u1")) + return video_router.video_spec(payload, request, session).data + + +def test_the_spec_route_returns_the_note_as_a_spec(notes): + data = _spec_route(notes, options={"title_card": False, "quotes": {"enabled": True}}) + spec = VideoSpec.model_validate(data.spec) + assert spec.source.note_id == "mine" + assert spec.title == "Salt" + [scene] = spec.segments + assert [b.block_id for b in scene.beats] == ["p1", "q1"] + assert scene.overlays[0].attribution == "A Trader" + assert data.spec["assets"][scene.visual.asset]["from"] == "notes" + assert data.warnings == [] + + +def test_the_spec_route_shapes_the_spec_as_a_render_would(notes): + selection = json.dumps(_SPEC_NOTE[2:]) + data = _spec_route(notes, options={"title_card": True}, selected_content=selection) + spec = VideoSpec.model_validate(data.spec) + assert [s.kind for s in spec.segments] == ["card", "scene"] + assert spec.segments[0].card.title == "Salt" + # Quotes are off by default, so the quotation is read as plain prose. + assert spec.segments[1].overlays == [] + + +def test_the_spec_route_refuses_another_users_note_and_bad_options(notes): + with pytest.raises(HTTPException) as missing: + _spec_route(notes, note_id="theirs") + assert missing.value.status_code == 404 + with pytest.raises(HTTPException) as invalid: + _spec_route(notes, options={"aspect": "4:3"}) + assert invalid.value.status_code == 400 + diff --git a/docs/gecko-video-creator.md b/docs/gecko-video-creator.md index 5079982..cbd1f9f 100644 --- a/docs/gecko-video-creator.md +++ b/docs/gecko-video-creator.md @@ -1,6 +1,6 @@ # Gecko Video Creator (GVC) — Product & Data Model Spec -**Status:** Draft for discussion · Phase 0 in progress (§10) +**Status:** Draft for discussion · Phase 0 in progress: renders now go through a VideoSpec (§10) **Date:** 2026-10-07 **Scope:** What GVC is, the JSON document a video is described by (the **VideoSpec**), how an AI builds and edits one, and how GVC pulls scripts from Gecko Notes and media @@ -181,7 +181,9 @@ Six existing defects should be fixed on the way in rather than carried over: and the "title + chapter" overlay as typed. Phase 0 reproduces all of these exactly, so its parity tests can hold the spec path to -today's output; each is a small follow-up fix once the spec path is the only one. +the original output. The spec path is now the only one, so each is a small follow-up fix, +made in the importer and in `tests/segment_reference.py` together (or that oracle is +retired). --- @@ -994,7 +996,7 @@ bug fixes only, each logged so it can be ported. | Phase | What | Done when | |---|---|---| -| **0 — Spec inside Notes** | **In progress.** First, alongside today's code: the VideoSpec in Pydantic ([`spec.py`](../backend/app/video/spec.py)), an importer that walks a note exactly as `segment()` does ([`importer.py`](../backend/app/video/importer.py)), and a compiler back to today's `Shot`s and `RenderOptions` ([`compiler.py`](../backend/app/video/compiler.py)), with `renderer.py` and `segment()` untouched. Then, separately: `segment()` switches to the spec path, `GET /api/video/spec?note_id=` exposes it, and the voice bug is fixed. | [`tests/test_video_spec.py`](../backend/tests/test_video_spec.py): every segmenter test case, a few thousand generated notes and the renderer's ffmpeg commands come out identical through the spec path, with no real encodes | +| **0 — Spec inside Notes** | **In progress.** Done: the VideoSpec in Pydantic ([`spec.py`](../backend/app/video/spec.py)); an importer that walks a note as the segmenter always has ([`importer.py`](../backend/app/video/importer.py)); a compiler back to the `Shot`s and `RenderOptions` the renderer uses ([`compiler.py`](../backend/app/video/compiler.py)); and `segment()` now going through them, so every render is built from a spec. `POST /api/video/spec` returns that spec for a note (same body as `/estimate`). Still to do: the voice bug and the chapter-mark quirks (§3.2). | [`tests/test_video_spec.py`](../backend/tests/test_video_spec.py): every segmenter test case, a few thousand generated notes and the renderer's ffmpeg commands come out identical to the original walk, kept verbatim as an oracle ([`segment_reference.py`](../backend/tests/segment_reference.py)), with no real encodes | | **1 — GVC renders what Notes renders** | New repo `davior/gvc` from GAM's skeleton. The engine copied from the Phase 0 merge, SSO, project storage and history, import from Notes over HTTP, speech through Notes, preview and full render, download. Output to GAM and back into the note follow once those are designed. | Any note renders in GVC the same as in Notes | | **2 — The new capabilities** | Timed overlays, multiple audio tracks, trims, per-segment voice/motion/transition, GAM search and picker, `generate` / `search` placeholders, credits | The §6 example renders | | **3 — AI editing** | The operations of §8.2, the suggestions tray, chat, staged review | "Make a video from this note" produces a reviewed, rendered video | @@ -1066,4 +1068,5 @@ already renders, it's wrong, and finding that out costs a branch, not a new app. --- *This document is a product and data-model spec. Phase 0's spec, importer and compiler -are in `backend/app/video/`; nothing a user sees has changed.* +are in `backend/app/video/`, and every render now goes through them; nothing a user sees +has changed.*