From 19ade35266a7ff59a9632da288497d9bf254d3e0 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 06:21:45 +0000 Subject: [PATCH] M2b: timed overlays, credits and Save to GAM Text, lower thirds and pictures are drawn over the stitched video in one extra pass, each in its window with alpha fades, so one can be held across a join. Their anchors default to their own scene and may name a line or a later scene; one that cannot be placed is dropped with a warning. Credits come from each GAM asset's credit, in order of first use and said once, plus extra lines: as end cards (split past a screenful) and in a description with the chapters, to paste where the video goes. A finished full render can be saved to GAM as a gvc_export with its project and render as origin (GAM-3), named and described, once. The editor gets an On screen section per scene, a Credits section in the Render tab, Save to GAM, and the description with a copy button. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016fiRJAt34ag62iPiuMkNhm --- README.md | 7 +- ...00_add_render_description_and_gam_asset.py | 33 ++ backend/app/gam/client.py | 22 +- backend/app/models/render.py | 4 + backend/app/render/compile.py | 101 +++++- backend/app/render/compose.py | 138 +++++++- backend/app/render/credits.py | 50 +++ backend/app/render/estimate.py | 5 + backend/app/render/ffmpeg.py | 53 +++ backend/app/render/media.py | 23 +- backend/app/render/renderer.py | 53 ++- backend/app/render/worker.py | 2 +- backend/app/routers/renders.py | 65 +++- backend/app/schemas_renders.py | 5 + backend/tests/fake_siblings.py | 21 +- backend/tests/test_render_compile.py | 120 ++++++- backend/tests/test_render_integration.py | 87 +++++ backend/tests/test_renders_api.py | 74 +++++ docs/plan-of-attack.md | 4 +- frontend/src/api/projects.ts | 46 +++ frontend/src/api/renders.ts | 11 + frontend/src/components/AnchorSelect.tsx | 56 ++++ frontend/src/components/CreditsSettings.tsx | 80 +++++ frontend/src/components/FootageSearch.tsx | 17 +- .../src/components/OverlayEditor.test.tsx | 89 ++++++ frontend/src/components/OverlayEditor.tsx | 301 ++++++++++++++++++ frontend/src/components/RenderPanel.test.tsx | 29 ++ frontend/src/components/RenderPanel.tsx | 67 +++- frontend/src/components/SceneSettings.tsx | 17 + frontend/src/components/SoundtrackPanel.tsx | 96 ++---- frontend/src/spec/anchors.ts | 55 ++++ frontend/src/spec/footage.ts | 2 +- frontend/src/spec/overlays.test.ts | 75 +++++ frontend/src/spec/overlays.ts | 103 ++++++ frontend/src/stores/renders.ts | 27 +- frontend/src/views/ProjectView.tsx | 6 +- 36 files changed, 1830 insertions(+), 114 deletions(-) create mode 100644 backend/alembic/versions/20261016_0700_add_render_description_and_gam_asset.py create mode 100644 backend/app/render/credits.py create mode 100644 frontend/src/components/AnchorSelect.tsx create mode 100644 frontend/src/components/CreditsSettings.tsx create mode 100644 frontend/src/components/OverlayEditor.test.tsx create mode 100644 frontend/src/components/OverlayEditor.tsx create mode 100644 frontend/src/spec/anchors.ts create mode 100644 frontend/src/spec/overlays.test.ts create mode 100644 frontend/src/spec/overlays.ts diff --git a/README.md b/README.md index 9cf6a2d..3b47b38 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,12 @@ There are three ways to start one: GAM by what is said in it. Then preview and render it: - per-scene transitions and Ken Burns moves between two frames of a picture; - any number of soundtrack tracks, crossfading between scenes; -- trimmed clips, and narration read through Gecko Notes. +- trimmed clips, and narration read through Gecko Notes; +- text, lower thirds and pictures on screen, timed to the scenes and lines they belong to; +- end credits from the library's attribution, and a description with chapters and credits. -Timed overlays, saving renders to GAM and credits are next (M2b). +A finished render can be saved back into GAM. AI drafting and the full storyboard +editor are next (M3, M4). ## Read first diff --git a/backend/alembic/versions/20261016_0700_add_render_description_and_gam_asset.py b/backend/alembic/versions/20261016_0700_add_render_description_and_gam_asset.py new file mode 100644 index 0000000..b7eb2a1 --- /dev/null +++ b/backend/alembic/versions/20261016_0700_add_render_description_and_gam_asset.py @@ -0,0 +1,33 @@ +"""Add render_job.description and render_job.gam_asset_id (M2b). + +`description`: the chapters and credits a render worked out, to paste where the video +is published. `gam_asset_id`: the GAM asset a render was saved as, so saving twice +files it once. An id in another app's database, so no foreign key. + +Revision ID: 7e2d5a91b3c6 +Revises: 1c0c7b02acd3 +Create Date: 2026-10-16 07:00:00.000000+00:00 +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = '7e2d5a91b3c6' +down_revision: Union[str, None] = '1c0c7b02acd3' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + with op.batch_alter_table('render_job', schema=None) as batch_op: + batch_op.add_column(sa.Column('description', sqlmodel.sql.sqltypes.AutoString(), nullable=True)) + batch_op.add_column(sa.Column('gam_asset_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True)) + + +def downgrade() -> None: + with op.batch_alter_table('render_job', schema=None) as batch_op: + batch_op.drop_column('gam_asset_id') + batch_op.drop_column('description') diff --git a/backend/app/gam/client.py b/backend/app/gam/client.py index fdf370f..694cc45 100644 --- a/backend/app/gam/client.py +++ b/backend/app/gam/client.py @@ -154,12 +154,20 @@ def download(self, file_url: str, sink, *, max_bytes: int) -> int: raise GamError("gam_unavailable", "Could not fetch a file from the asset library") from exc return written - def upload(self, filename: str, stream: IO[bytes], content_type: Optional[str]) -> GamAssetRead: - """Store one file in the user's library and return it.""" + def upload(self, filename: str, stream: IO[bytes], content_type: Optional[str], *, + fields: Optional[Dict[str, str]] = None, + timeout: Optional[float] = None) -> GamAssetRead: + """Store one file in the user's library and return it. + + `fields` go along as form fields: a render's `source=gvc_export` and origin ids + (GAM-3). A GAM without GAM-3 ignores them and stores an ordinary upload. + """ response = self._request( "POST", "/api/assets", files={"files": (filename, stream, content_type or "application/octet-stream")}, + data=fields or None, + **({"timeout": timeout} if timeout else {}), ) if response.status_code not in (200, 201): code, message = self._detail(response) @@ -172,3 +180,13 @@ def upload(self, filename: str, stream: IO[bytes], content_type: Optional[str]) raise GamError(rejected.get("code") or "gam_error", rejected.get("message") or "The asset library did not store the file") return to_asset(created[0]) + + def update(self, asset_id: str, changes: Dict[str, Any]) -> Optional[GamAssetRead]: + """Edit an asset's fields (its name, its description). None: it is gone.""" + response = self._request("PATCH", f"/api/assets/{asset_id}", json=changes) + if response.status_code == 404: + return None + if response.status_code != 200: + code, message = self._detail(response) + raise GamError(code, message) + return to_asset(response.json()["data"]) diff --git a/backend/app/models/render.py b/backend/app/models/render.py index 4338411..765d719 100644 --- a/backend/app/models/render.py +++ b/backend/app/models/render.py @@ -31,5 +31,9 @@ class RenderJob(SQLModel, table=True): duration: Optional[float] = None size_bytes: Optional[int] = None narration_chars: int = 0 + # Chapters and credits, to paste where the video is published (M2b). + description: Optional[str] = None + # The GAM asset this render was saved as, once it has been (GAM-3). + gam_asset_id: Optional[str] = None created_at: datetime = Field(default_factory=utcnow) updated_at: datetime = Field(default_factory=utcnow) diff --git a/backend/app/render/compile.py b/backend/app/render/compile.py index a0e546d..49c712d 100644 --- a/backend/app/render/compile.py +++ b/backend/app/render/compile.py @@ -26,8 +26,8 @@ from app.render.ffmpeg import KENBURNS_CYCLE, MIN_TRANSITION_SECONDS, XFADE_MAX_SHOTS from app.render.options import XFADE_NAMES, is_crossfade from app.spec.model import ( - AnchorRef, AudioTrack, Beat, Card, CodeOverlay, Motion, QuoteOverlay, Segment, - Transition, VideoSpec, + AnchorRef, AudioTrack, Beat, Card, CodeOverlay, ImageOverlay, LowerThirdOverlay, Motion, + QuoteOverlay, Segment, TextOverlay, Transition, VideoSpec, ) UnitKind = str # "still" | "card" | "video_muted" | "video_sound" @@ -78,6 +78,8 @@ class Unit: speed: float = 1.0 chrome: bool = True chapter: Optional[str] = None + # The end credits: a card drawn from these lines rather than from `card`. + credits: Optional[List[str]] = None def _transition_in(spec: VideoSpec, segment: Segment) -> Transition: @@ -111,10 +113,8 @@ def _panel_beats(segment: Segment, warnings: List[str]) -> Tuple[Dict[str, Panel whole: Optional[Panel] = None for overlay in segment.overlays: if not isinstance(overlay, (QuoteOverlay, CodeOverlay)): - # Timed text, lower thirds and images are M2b: a second technique - # (overlay with enable=) on top of this one. Said, never silently lost. - warnings.append(f"Scene {segment.label or segment.id}: {overlay.type} overlays " - "are not rendered yet") + # Timed text, lower thirds and images are drawn over the stitched video + # instead (`place_overlays`), since they may run across a join. continue anchor = overlay.show.from_ if overlay.show else None if isinstance(anchor, AnchorRef) and anchor.beat and anchor.segment in (None, segment.id): @@ -206,6 +206,31 @@ def plan_units(spec: VideoSpec, media: Dict[str, Media], warnings: List[str]) -> return units +CREDITS_SEGMENT = "__credits__" +CREDIT_LINES_PER_CARD = 18 + + +def credits_units(spec: VideoSpec, lines: Sequence[str]) -> List[Unit]: + """The end credits as card units after everything else, one per screenful. + + Not a segment in the spec: the lines come from GAM at render time (each asset's + credit), so the spec can't hold them, and a segment the user never made would + show up in the editor. Past a screenful they run over several cards rather than + shrinking to unreadable. + """ + if not spec.credits or not spec.credits.card or not lines: + return [] + pages = [list(lines[i:i + CREDIT_LINES_PER_CARD]) + for i in range(0, len(lines), CREDIT_LINES_PER_CARD)] + return [ + Unit(segment_id=CREDITS_SEGMENT, segment_index=len(spec.segments), + first_in_segment=position == 0, kind="card", media=None, credits=page, + transition_in=spec.style.transition, min_seconds=spec.style.cards.seconds, + chrome=False) + for position, page in enumerate(pages) + ] + + def unit_duration(unit: Unit, narration_seconds: float, cards_seconds: float) -> float: """How long a unit runs, once its narration has been measured.""" if unit.kind == "video_sound": @@ -429,3 +454,67 @@ def beat_pause_ms(spec: VideoSpec, beats: Sequence[Beat], position: int) -> int: if beat.kind == "heading" or beats[position + 1].kind == "heading": return pauses.heading_ms return pauses.paragraph_ms + + +# ── timed overlays ────────────────────────────────────────────────────────── + +TimedKind = Union[TextOverlay, LowerThirdOverlay, ImageOverlay] + +DEFAULT_OVERLAY_FADE = 0.3 +MIN_OVERLAY_SECONDS = 0.2 + + +@dataclass +class OverlayPlacement: + segment_id: str + overlay: TimedKind + start: float + end: float + fade: float + media: Optional[Media] = None # an image overlay's picture + + +def place_overlays(spec: VideoSpec, media: Dict[str, Media], where: Placement, + warnings: List[str]) -> List[OverlayPlacement]: + """Text, lower thirds and images as windows on the finished timeline. + + With no `show`, an overlay covers its scene. Its anchors default to its own scene, + so `{"edge": "start", "offset": 0.5}` means half a second into it. As with audio + tracks, one that can't be placed is dropped with a warning (§5.7): the render + still goes ahead. + """ + placed: List[OverlayPlacement] = [] + for segment in spec.segments: + span = where.segments.get(segment.id) + name = segment.label or segment.id + for overlay in segment.overlays: + if not isinstance(overlay, (TextOverlay, LowerThirdOverlay, ImageOverlay)): + continue + if span is None: + # The scene itself was left out (a clip with nothing to play). + continue + if overlay.show is None: + start, end = span + else: + try: + start = resolve_anchor(overlay.show.from_, where, segment=segment.id) + end = resolve_anchor(overlay.show.to, where, segment=segment.id) + except Orphaned as exc: + warnings.append(f"Scene {name}: an overlay is anchored to {exc}, which is " + "not in the video; left out") + continue + if end - start < MIN_OVERLAY_SECONDS: + warnings.append(f"Scene {name}: an overlay ends before it starts; left out") + continue + picture = None + if isinstance(overlay, ImageOverlay): + picture = media.get(overlay.asset) + if picture is None or picture.kind not in ("image", "slate"): + warnings.append(f"Scene {name}: an image overlay needs a picture; left out") + continue + fade = overlay.fade if overlay.fade is not None else DEFAULT_OVERLAY_FADE + # Never more than half the window each way, or the fades would cross. + fade = max(0.0, min(fade, (end - start) / 2)) + placed.append(OverlayPlacement(segment_id=segment.id, overlay=overlay, start=start, + end=end, fade=fade, media=picture)) + return placed diff --git a/backend/app/render/compose.py b/backend/app/render/compose.py index f1681d4..4b80312 100644 --- a/backend/app/render/compose.py +++ b/backend/app/render/compose.py @@ -195,6 +195,14 @@ def _anchor_xy(position: str, width: int, height: int, box_w: int, box_h: int, m return (width - box_w - margin, height - box_h - margin) +def _card_base(width: int, height: int, background: Optional[Image.Image]) -> Image.Image: + base = (background.copy() if background is not None + else gradient_background(width, height, ["#1e293b", "#0f172a"], 135)).convert("RGB") + # Blur and darken whatever is behind so the text always has contrast. + base = base.filter(ImageFilter.GaussianBlur(radius=max(6, width // 90))) + return Image.blend(base, Image.new("RGB", (width, height), (0, 0, 0)), 0.45) + + def card_image( width: int, height: int, *, title: str, subtitle: str = "", @@ -207,12 +215,7 @@ def card_image( one setting holds at every resolution and aspect ratio. """ spec = sizes or CardTextSpec() - base = (background.copy() if background is not None - else gradient_background(width, height, ["#1e293b", "#0f172a"], 135)).convert("RGB") - - # Blur and darken whatever is behind so the text always has contrast. - base = base.filter(ImageFilter.GaussianBlur(radius=max(6, width // 90))) - base = Image.blend(base, Image.new("RGB", (width, height), (0, 0, 0)), 0.45) + base = _card_base(width, height, background) layer = Image.new("RGBA", (width, height), (0, 0, 0, 0)) draw = ImageDraw.Draw(layer) @@ -551,3 +554,126 @@ def _budget(margin: int, taken: int = 0) -> int: layer.alpha_composite(mark, (max(0, x), max(0, y))) return layer + + +# ── timed overlays (GVC) ──────────────────────────────────────────────────── +# Each is a full-frame RGBA layer, like `overlay_layer`, so the video pass that shows +# it for its window is one plain `overlay=0:0` per layer with nothing to position. + +OVERLAY_MARGIN_PCT = 5.0 + + +def text_layer(width: int, height: int, *, text: str, position: str = "center", + size_pct: Optional[float] = None, color: Optional[str] = None) -> Image.Image: + """On-screen text, wrapped, with a soft shadow so it reads over any picture.""" + layer = Image.new("RGBA", (width, height), (0, 0, 0, 0)) + draw = ImageDraw.Draw(layer) + margin = int(min(width, height) * OVERLAY_MARGIN_PCT / 100) + max_width = width - 2 * margin if position == "center" else int(width * 0.6) + start = max(8, int(height * (size_pct or 6.0) / 100)) + font = fit_font(draw, text, SEMIBOLD, start, max_width) + lines = wrap_text(draw, text, font, max_width)[:8] + line_h = int(font.size * 1.25) + box_w = int(max((draw.textlength(line, font=font) for line in lines), default=0)) + x, y = _anchor_xy(position, width, height, box_w, line_h * len(lines), margin) + colour = parse_colour(color or "#ffffff") + + shadow = Image.new("RGBA", (width, height), (0, 0, 0, 0)) + shadow_draw = ImageDraw.Draw(shadow) + for position_in_block, line in enumerate(lines): + lw = draw.textlength(line, font=font) + lx = x + (box_w - lw) / 2 if position == "center" else ( + x + box_w - lw if position.endswith("right") else x) + ly = y + position_in_block * line_h + shadow_draw.text((lx, ly + max(1, font.size // 16)), line, font=font, fill=(0, 0, 0, 190)) + draw.text((lx, ly), line, font=font, fill=(*colour, 255)) + shadow = shadow.filter(ImageFilter.GaussianBlur(radius=max(2, font.size // 10))) + return Image.alpha_composite(shadow, layer) + + +def lower_third_layer(width: int, height: int, *, title: str, + subtitle: str = "") -> Image.Image: + """A name and a role, bottom left, on a translucent bar with an accent rule. + + The bar is sized to its text rather than the frame, so a short name doesn't sit + in a long empty strip. Its colours match the title cards' rule. + """ + layer = Image.new("RGBA", (width, height), (0, 0, 0, 0)) + draw = ImageDraw.Draw(layer) + margin = int(min(width, height) * OVERLAY_MARGIN_PCT / 100) + max_text = int(width * 0.7) + title_font = fit_font(draw, title, SEMIBOLD, max(10, int(height * 0.052)), max_text) + sub_font = load_font(REGULAR, max(8, int(height * 0.034))) + title = truncate_to_width(draw, title.strip(), title_font, max_text) + subtitle = truncate_to_width(draw, (subtitle or "").strip(), sub_font, max_text) + + pad_x, pad_y = int(title_font.size * 0.6), int(title_font.size * 0.4) + rule_w = max(3, height // 160) + text_w = int(max(draw.textlength(title, font=title_font), + draw.textlength(subtitle, font=sub_font) if subtitle else 0)) + title_h = int(title_font.size * 1.2) + sub_h = int(sub_font.size * 1.3) if subtitle else 0 + box_w = rule_w + pad_x * 2 + text_w + box_h = pad_y * 2 + title_h + sub_h + x0 = margin + y0 = height - margin - box_h - int(height * 0.04) + + draw.rectangle([x0, y0, x0 + box_w, y0 + box_h], fill=(15, 23, 42, 190)) + draw.rectangle([x0, y0, x0 + rule_w, y0 + box_h], fill=(129, 140, 248, 255)) + tx = x0 + rule_w + pad_x + draw.text((tx, y0 + pad_y), title, font=title_font, fill=(255, 255, 255, 255)) + if subtitle: + draw.text((tx, y0 + pad_y + title_h), subtitle, font=sub_font, fill=(203, 213, 225, 255)) + return layer + + +def image_layer(width: int, height: int, *, picture: str, position: str = "center", + scale_pct: Optional[float] = None, + opacity: Optional[float] = None) -> Optional[Image.Image]: + """A picture over the video: a logo, a map, an inset. `scale_pct` is its width as + a share of the frame's; its aspect is kept.""" + try: + with Image.open(picture) as raw: + image = ImageOps.exif_transpose(raw).convert("RGBA") + except (UnidentifiedImageError, OSError, ValueError) as exc: + logger.warning("Could not load overlay picture %s: %s", picture, exc) + return None + margin = int(min(width, height) * OVERLAY_MARGIN_PCT / 100) + target_w = max(1, int(width * (scale_pct or 30.0) / 100)) + target_h = max(1, int(image.height * target_w / max(1, image.width))) + # Never taller than the frame allows, whatever the width asked for. + if target_h > height - 2 * margin: + target_h = max(1, height - 2 * margin) + target_w = max(1, int(image.width * target_h / max(1, image.height))) + image = image.resize((target_w, target_h), Image.LANCZOS) + if opacity is not None and opacity < 1: + alpha = image.getchannel("A").point(lambda a: int(a * max(0.0, opacity))) + image.putalpha(alpha) + layer = Image.new("RGBA", (width, height), (0, 0, 0, 0)) + layer.alpha_composite(image, _anchor_xy(position, width, height, target_w, target_h, margin)) + return layer + + +def credits_card(width: int, height: int, *, lines: Sequence[str], + background: Optional[Image.Image] = None) -> Image.Image: + """The end credits: a heading over centred lines, smaller as there are more.""" + base = _card_base(width, height, background) + layer = Image.new("RGBA", (width, height), (0, 0, 0, 0)) + draw = ImageDraw.Draw(layer) + max_width = int(width * 0.82) + heading = load_font(SEMIBOLD, max(10, int(height * 0.06))) + # Sized so the whole page fits: a three-line page reads large, eighteen still fit. + size = max(8, min(int(height * 0.042), int(height * 0.7 / max(1, len(lines)) / 1.35))) + body = load_font(REGULAR, size) + wrapped = [truncate_to_width(draw, line, body, max_width) for line in lines] + head_h = int(heading.size * 1.6) + line_h = int(body.size * 1.35) + y = (height - head_h - line_h * len(wrapped)) // 2 + w = draw.textlength("Credits", font=heading) + draw.text(((width - w) / 2, y), "Credits", font=heading, fill=(255, 255, 255, 255)) + y += head_h + for line in wrapped: + w = draw.textlength(line, font=body) + draw.text(((width - w) / 2, y), line, font=body, fill=(226, 232, 240, 240)) + y += line_h + return Image.alpha_composite(base.convert("RGBA"), layer).convert("RGB") diff --git a/backend/app/render/credits.py b/backend/app/render/credits.py new file mode 100644 index 0000000..ec35685 --- /dev/null +++ b/backend/app/render/credits.py @@ -0,0 +1,50 @@ +"""What the end credits and the description say. + +Credits come from GAM: each asset's `credit`, which GAM builds from its attribution +fields (and a clip inherits from its parent). GVC keeps no attribution of its own, so +correcting a credit is done once, in the library, and every video using it follows. +""" + +from __future__ import annotations + +from typing import Dict, List, Sequence, Tuple + +from app.render.media import keys_in_order +from app.spec.model import VideoSpec + + +def credit_lines(spec: VideoSpec, credits: Dict[str, str]) -> List[str]: + """The credits, in the order their assets first appear, each said once.""" + if spec.credits is None: + return [] + lines: List[str] = [] + if spec.credits.auto: + lines += [credits.get(key, "").strip() for key in keys_in_order(spec)] + lines += [line.strip() for line in spec.credits.extra] + # One credit line for a photographer's three pictures, not three. + return list(dict.fromkeys(line for line in lines if line)) + + +def _clock(seconds: float) -> str: + whole = int(round(seconds)) + hours, rest = divmod(whole, 3600) + minutes, secs = divmod(rest, 60) + return f"{hours}:{minutes:02d}:{secs:02d}" if hours else f"{minutes}:{secs:02d}" + + +def description_text(spec: VideoSpec, lines: Sequence[str], + marks: Sequence[Tuple[float, float, str]]) -> str: + """Text to paste under the video where it is published: chapters and credits. + + Chapters are listed as `0:00 Title`, the form video sites turn into chapter + links, and only with the first at 0:00, which those sites require. + """ + parts: List[str] = [] + if spec.output.chapters and marks: + listed = [(start, title) for start, _end, title in marks] + if listed[0][0] > 0.5: + listed.insert(0, (0.0, spec.title or "Start")) + parts.append("\n".join(f"{_clock(start)} {title}" for start, title in listed)) + if spec.credits and spec.credits.description and lines: + parts.append("Credits\n" + "\n".join(lines)) + return "\n\n".join(parts) diff --git a/backend/app/render/estimate.py b/backend/app/render/estimate.py index 7cfc3c9..4af6a87 100644 --- a/backend/app/render/estimate.py +++ b/backend/app/render/estimate.py @@ -68,5 +68,10 @@ def estimate(spec: VideoSpec) -> Estimate: floor = (segment.duration.min if segment.duration and segment.duration.min else spec.style.min_segment_seconds) out.seconds += max(floor, spoken) + credits = spec.credits + if credits and credits.card and (credits.extra or (credits.auto and used)): + # Whether GAM has credits for the assets is only known at render time; a card + # is the likelier outcome, and an estimate a few seconds long is the safer one. + out.seconds += spec.style.cards.seconds out.seconds = round(out.seconds, 1) return out diff --git a/backend/app/render/ffmpeg.py b/backend/app/render/ffmpeg.py index 8367ddd..b984323 100644 --- a/backend/app/render/ffmpeg.py +++ b/backend/app/render/ffmpeg.py @@ -1120,3 +1120,56 @@ def build_mix_command( "-t", f"{max(0.1, duration):.3f}", "-movflags", "+faststart", output, ] return argv + + +@dataclass(frozen=True) +class TimedLayer: + """A full-frame RGBA PNG shown over the finished video for a window.""" + + path: str + start: float + end: float + fade: float = 0.0 + + +def build_overlay_pass( + source: str, layers: Sequence[TimedLayer], output: str, + *, options: RenderOptions, preview: bool, duration: float, +) -> List[str]: + """Draw timed layers over the stitched picture, the sound untouched. + + Per shot would have kept the shot cache, but an overlay may run across a join (a + title held through a dissolve), and a shot can't draw what lies past its end. + So this is one re-encode of the whole picture after the stitch, done only when a + video has timed overlays at all. + + Each layer is a looped still. Its fades are alpha fades at absolute times, which + line up with the video's because a looped image input starts at zero too, and + `enable` keeps it off outside its window so nothing is composited where it isn't + shown. + """ + fps = options.fps + total = max(0.1, duration) + argv = ["ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-i", source] + chains: List[str] = [] + stage = "0:v" + for position, layer in enumerate(layers, start=1): + argv += ["-loop", "1", "-framerate", str(fps), "-t", f"{total:.3f}", "-i", layer.path] + chain = f"[{position}:v]format=rgba" + if layer.fade > 0: + chain += (f",fade=t=in:st={layer.start:.3f}:d={layer.fade:.3f}:alpha=1" + f",fade=t=out:st={layer.end - layer.fade:.3f}:d={layer.fade:.3f}:alpha=1") + chains.append(chain + f"[l{position}]") + chains.append(f"[{stage}][l{position}]overlay=0:0:eof_action=pass:" + f"enable='between(t,{layer.start:.3f},{layer.end:.3f})'[v{position}]") + stage = f"v{position}" + # Only the video codec flags: the sound is copied, not re-encoded a second time. + shared = encode_args(options, preview) + video_args = shared[:shared.index("-c:a")] + argv += [ + "-filter_complex", ";".join(chains), + "-map", f"[{stage}]", "-map", "0:a?", + *video_args, "-c:a", "copy", + "-t", f"{total:.3f}", "-movflags", "+faststart", output, + ] + return argv diff --git a/backend/app/render/media.py b/backend/app/render/media.py index 41e9637..b7b5d68 100644 --- a/backend/app/render/media.py +++ b/backend/app/render/media.py @@ -28,7 +28,9 @@ from app.render import compose from app.render import ffmpeg as F from app.render.compile import Media -from app.spec.model import GamAsset, GeneratedAsset, NotesAsset, SearchAsset, VideoSpec +from app.spec.model import ( + GamAsset, GeneratedAsset, ImageOverlay, NotesAsset, SearchAsset, VideoSpec, +) logger = logging.getLogger(__name__) @@ -38,12 +40,26 @@ class Resolution: media: Dict[str, Media] = field(default_factory=dict) # Reasons the render cannot go ahead, one per asset. problems: List[str] = field(default_factory=list) + # Each GAM asset's credit line as GAM resolves it (a clip's inherited from its + # parent), for the end credits. Empty for an asset nobody has attributed. + credits: Dict[str, str] = field(default_factory=dict) + + +def keys_in_order(spec: VideoSpec) -> List[str]: + """The asset keys the video shows or plays, in the order a viewer meets them: + scene by scene (its picture, then its overlays), then the soundtrack.""" + ordered: List[str] = [] + for segment in spec.segments: + if segment.visual is not None: + ordered.append(segment.visual.asset) + ordered += [o.asset for o in segment.overlays if isinstance(o, ImageOverlay)] + ordered += [t.asset for t in spec.audio_tracks] + return list(dict.fromkeys(k for k in ordered if k in spec.assets)) def used_keys(spec: VideoSpec) -> Set[str]: """Every asset key the render will read.""" - keys = {s.visual.asset for s in spec.segments if s.visual is not None} - keys |= {t.asset for t in spec.audio_tracks} + keys = set(keys_in_order(spec)) if spec.style.background.type == "image" and spec.style.background.asset: keys.add(spec.style.background.asset) if spec.chrome.watermark and spec.chrome.watermark.asset: @@ -167,6 +183,7 @@ def resolve( if asset.out_point is not None: media.length = max(0.1, asset.out_point - (asset.in_point or 0.0)) out.media[key] = media + out.credits[key] = asset.credit or "" try: cache.prune(int(settings.render_cache_max_gb * 1024 ** 3)) diff --git a/backend/app/render/renderer.py b/backend/app/render/renderer.py index c7b06c5..923bcd8 100644 --- a/backend/app/render/renderer.py +++ b/backend/app/render/renderer.py @@ -29,9 +29,10 @@ from app.render import compose, shot_cache from app.render import ffmpeg as F from app.render.compile import ( - Unit, beat_pause_ms, chapters, layout, place_tracks, placement, plan_units, unit_duration, - xfade_name, + OverlayPlacement, Unit, beat_pause_ms, chapters, credits_units, layout, place_overlays, + place_tracks, placement, plan_units, unit_duration, xfade_name, ) +from app.render.credits import credit_lines, description_text from app.render.media import MediaCache, resolve from app.render.narration import Cue, build_silence, shift_cues, synthesize_shot, write_srt from app.render.options import ( @@ -63,6 +64,9 @@ class RenderOutput: size_bytes: int narration_chars: int warnings: List[str] = field(default_factory=list) + # Text to paste where the video is published: chapters and credits. None: nothing + # to say. + description: Optional[str] = None def base_options(spec: VideoSpec) -> RenderOptions: @@ -184,7 +188,9 @@ def _background(unit: Unit, spec: VideoSpec, options: RenderOptions, work_dir: s base = (compose.solid_background(width, height, fallback.colors[0]) if fallback.type == "solid" else compose.gradient_background(width, height, fallback.colors, fallback.angle)) - if unit.kind == "card" and unit.card: + if unit.kind == "card" and unit.credits: + image = compose.credits_card(width, height, lines=unit.credits, background=base) + elif unit.kind == "card" and unit.card: image = compose.card_image( width, height, title=unit.card.title or spec.title or "Untitled", subtitle=unit.card.subtitle or "", background=base, @@ -220,6 +226,8 @@ def render( units = plan_units(spec, media, warnings) if not units: raise RenderRefused("This video has nothing in it to render yet.") + lines = credit_lines(spec, resolved.credits) + units += credits_units(spec, lines) if len(units) > max_segments: raise RenderRefused(f"This video has {len(units)} shots, above the limit of {max_segments}.") chars = sum(len(b.text) for u in units for b in u.beats if b.speak) @@ -409,9 +417,20 @@ def argv(with_options: RenderOptions, with_path=path) -> List[str]: F.write_concat_list(os.path.join(work_dir, "shots.txt"), shot_files) F.run(F.build_concat_command("shots.txt", stitched), cwd=work_dir, timeout=1800) final = stitched + where = placement(units, timeline, [n.beats for n in narrated]) + + # ── timed overlays ─────────────────────────────────────────────────────── + timed = place_overlays(spec, media, where, warnings) + layers = _overlay_layers(timed, work_dir, width, height) + if layers: + progress("Drawing overlays", 88, + f"{len(layers)} overlay{'s' if len(layers) != 1 else ''}") + F.run(F.build_overlay_pass(final, layers, "overlaid.mp4", options=options, + preview=preview, duration=timeline.total), + cwd=work_dir, timeout=F.shot_timeout(timeline.total)) + final = "overlaid.mp4" # ── the soundtrack ─────────────────────────────────────────────────────── - where = placement(units, timeline, [n.beats for n in narrated]) tracks = place_tracks(spec, media, where, warnings) if tracks: progress("Mixing", 90, f"{len(tracks)} audio track{'s' if len(tracks) != 1 else ''}") @@ -462,9 +481,35 @@ def argv(with_options: RenderOptions, with_path=path) -> List[str]: video=video, subtitles=subtitles, poster=poster, duration=F.probe_duration(video) or timeline.total, size_bytes=os.path.getsize(video), narration_chars=speech.characters, warnings=warnings, + description=description_text(spec, lines, chapters(units, timeline)) or None, ) +def _overlay_layers(timed: List[OverlayPlacement], work_dir: str, width: int, + height: int) -> List[F.TimedLayer]: + """Each placed overlay drawn to its own PNG.""" + layers: List[F.TimedLayer] = [] + for position, item in enumerate(timed): + overlay = item.overlay + if overlay.type == "text": + image = compose.text_layer(width, height, text=overlay.text, + position=overlay.position, size_pct=overlay.size_pct, + color=overlay.color) + elif overlay.type == "lower_third": + image = compose.lower_third_layer(width, height, title=overlay.title, + subtitle=overlay.subtitle or "") + else: + image = compose.image_layer(width, height, picture=item.media.path, + position=overlay.position, scale_pct=overlay.scale_pct, + opacity=overlay.opacity) + if image is None: + continue + name = f"timed_{position:03d}.png" + image.save(os.path.join(work_dir, name), "PNG") + layers.append(F.TimedLayer(path=name, start=item.start, end=item.end, fade=item.fade)) + return layers + + def layout_without_overlaps(timeline): """The same units laid end to end, for when blends had to become cuts.""" starts, cursor = [], 0.0 diff --git a/backend/app/render/worker.py b/backend/app/render/worker.py index 95e580c..d0e7075 100644 --- a/backend/app/render/worker.py +++ b/backend/app/render/worker.py @@ -93,7 +93,7 @@ def rel(path: Optional[str]) -> Optional[str]: video_path=rel(result.video), subtitle_path=rel(result.subtitles), poster_path=rel(result.poster), duration=result.duration, size_bytes=result.size_bytes, narration_chars=result.narration_chars, - warnings=json.dumps(result.warnings)) + warnings=json.dumps(result.warnings), description=result.description) finally: gam.close() speech.close() diff --git a/backend/app/routers/renders.py b/backend/app/routers/renders.py index dfcbe48..0eef6cc 100644 --- a/backend/app/routers/renders.py +++ b/backend/app/routers/renders.py @@ -9,7 +9,9 @@ from sqlmodel import Session, select from app.auth import CallerToken, CurrentUser, sign_media_key, verify_media_signature +from app.config import settings from app.database import get_session +from app.gam.client import GamClient, GamError from app.jobs.runner import ACTIVE_STATUSES from app.media.ranged import ranged_response from app.models.render import RenderJob @@ -40,7 +42,10 @@ def _read(job: RenderJob) -> RenderRead: error=job.error_message, warnings=json.loads(job.warnings or "[]"), duration=job.duration, size_bytes=job.size_bytes, narration_chars=job.narration_chars, video_url=_signed(job, job.video_path), subtitles_url=_signed(job, job.subtitle_path), - poster_url=_signed(job, job.poster_path), + poster_url=_signed(job, job.poster_path), description=job.description, + gam_asset_id=job.gam_asset_id, + gam_asset_url=(f"{settings.gam_base_url.rstrip('/')}/a/{job.gam_asset_id}" + if job.gam_asset_id else None), created_at=job.created_at, updated_at=job.updated_at, ) @@ -115,6 +120,64 @@ def cancel_render(render_id: str, user: CurrentUser, return DataResponse(data=_read(job)) +# Generous: the file is on local disk, but a long full-quality render is gigabytes, +# and GAM probes and thumbnails it before it answers. +SAVE_TIMEOUT_SECONDS = 900 + + +@router.post("/renders/{render_id}/save-to-gam", response_model=DataResponse[RenderRead]) +def save_to_gam(render_id: str, user: CurrentUser, token: CallerToken, + session: Session = Depends(get_session)) -> DataResponse[RenderRead]: + """File a finished render in the user's GAM library (GAM-3). + + Labelled `gvc_export` with the project and render it came from, named after the + project, and described with its chapters and credits. Saving twice files it once; + if it was deleted in GAM since, it is filed again. + """ + job = _owned(session, user, render_id) + if job.preview: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, + detail={"code": "preview_not_saveable", + "message": "A preview is a small working copy. Render the " + "full video to save it to GAM."}) + path = worker.renders_dir() / job.id / (job.video_path or "") + if job.status != "done" or not job.video_path or not path.is_file(): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, + detail={"code": "render_not_done", + "message": "This render has no finished video to save"}) + project = project_service.get_owned(session, user, job.project_id) + + gam = GamClient(token) + try: + if job.gam_asset_id and gam.get_asset(job.gam_asset_id) is not None: + return DataResponse(data=_read(job)) + title = (project.title or "").strip() or "Untitled video" + with open(path, "rb") as stream: + asset = gam.upload( + f"{title}.mp4", stream, "video/mp4", + fields={"source": "gvc_export", "origin_project_id": project.id, + "origin_reference_id": job.id}, + timeout=SAVE_TIMEOUT_SECONDS, + ) + changes = {"name": f"{title} (v{job.version_seq})"[:255]} + if job.description: + changes["description"] = job.description[:20_000] + # Best effort: the file is safely filed even if naming it fails, and saving + # again would only find it. + try: + gam.update(asset.id, changes) + except GamError: + pass + finally: + gam.close() + + job.gam_asset_id = asset.id + session.add(job) + session.commit() + session.refresh(job) + return DataResponse(data=_read(job)) + + @router.delete("/renders/{render_id}", status_code=status.HTTP_204_NO_CONTENT) def delete_render(render_id: str, user: CurrentUser, session: Session = Depends(get_session)) -> Response: diff --git a/backend/app/schemas_renders.py b/backend/app/schemas_renders.py index 26c2ecc..c74d8e4 100644 --- a/backend/app/schemas_renders.py +++ b/backend/app/schemas_renders.py @@ -30,6 +30,11 @@ class RenderRead(BaseModel): video_url: Optional[str] = None subtitles_url: Optional[str] = None poster_url: Optional[str] = None + # Chapters and credits to paste where the video is published. + description: Optional[str] = None + # Set once the render is saved to GAM: the asset, and its page there. + gam_asset_id: Optional[str] = None + gam_asset_url: Optional[str] = None created_at: datetime updated_at: datetime diff --git a/backend/tests/fake_siblings.py b/backend/tests/fake_siblings.py index 320080f..f725539 100644 --- a/backend/tests/fake_siblings.py +++ b/backend/tests/fake_siblings.py @@ -96,14 +96,33 @@ def _gam(self, request: httpx.Request) -> httpx.Response: new_id = f"gam-{len(self.uploads) + 1}" asset = gam_asset(new_id, filename, "image") self.assets[new_id] = asset - self.uploads.append({"filename": filename, "size": len(body)}) + self.uploads.append({"filename": filename, "size": len(body), + "fields": _form_fields(body), + "authorization": request.headers.get("authorization")}) return httpx.Response(201, json={"created": [asset], "rejected": []}) + if path.startswith("/api/assets/") and request.method == "PATCH": + asset = self.assets.get(path.rsplit("/", 1)[-1]) + if asset is None: + return httpx.Response(404, json={"detail": {"code": "not_found", "message": "gone"}}) + asset.update(json.loads(request.content)) + return httpx.Response(200, json={"data": asset}) return httpx.Response(404) def gam_requests(self) -> List[httpx.Request]: return [r for r in self.requests if str(r.url).startswith(GAM)] +def _form_fields(body: bytes) -> Dict[str, str]: + """The plain (non-file) fields of a multipart body.""" + fields: Dict[str, str] = {} + for part in body.split(b"\r\n--"): + head, _, value = part.partition(b"\r\n\r\n") + if b'name="' in head and b"filename=" not in head: + name = head.split(b'name="', 1)[1].split(b'"', 1)[0].decode() + fields[name] = value.rsplit(b"\r\n", 1)[0].decode() + return fields + + def hit(asset: dict, *, start=None, end=None, snippet="") -> dict: data = {"asset": asset, "score": 0.03, "snippet": snippet, "start_time": start, "segment_id": "seg" if start is not None else None, "sources": ["keyword"], diff --git a/backend/tests/test_render_compile.py b/backend/tests/test_render_compile.py index 4b44ca1..3f71ffd 100644 --- a/backend/tests/test_render_compile.py +++ b/backend/tests/test_render_compile.py @@ -45,8 +45,8 @@ def test_a_scene_is_cut_at_its_quote_and_cards_and_clips_are_one_unit_each(): ("s_ruins", "still", ["b1", "b2"], None), ("s_outro", "video_sound", [], None), ] - # The lower third is M2b, and says so rather than vanishing. - assert any("lower_third" in w for w in warnings) + # The lower third doesn't cut the scene: it is drawn over the stitched video. + assert warnings == [] def test_defaults_cascade_field_by_field(): @@ -207,3 +207,119 @@ def test_pauses_after_beats(): assert beat_pause_ms(spec, beats, 0) == 800 # a heading assert beat_pause_ms(spec, beats, 1) == 350 # between paragraphs assert beat_pause_ms(spec, beats, 2) == 600 # the end of the scene + + +# ── M2b: timed overlays and credits ───────────────────────────────────────── + +from app.render.compile import CREDIT_LINES_PER_CARD, credits_units, place_overlays # noqa: E402 +from app.render.credits import credit_lines, description_text # noqa: E402 +from app.render import ffmpeg as F # noqa: E402 +from app.render.options import RenderOptions # noqa: E402 +from app.spec.model import AnchorRef, Span, TextOverlay # noqa: E402 + + +def _with_overlays(*overlays): + spec, media, units, timeline, where = _placed() + market = next(s for s in spec.segments if s.id == "s_market") + market.overlays = list(overlays) + return spec, media, where + + +def test_an_overlay_with_no_span_covers_its_scene_with_a_default_fade(): + spec, media, where = _with_overlays(TextOverlay(id="o", type="text", text="Hi")) + [placed] = [p for p in place_overlays(spec, media, where, []) if p.overlay.id == "o"] + assert (placed.start, placed.end) == where.segments["s_market"] + assert placed.fade == 0.3 + + +def test_overlay_anchors_default_to_their_own_scene_and_may_reach_into_the_next(): + span = Span.model_validate({"from": {"edge": "start", "offset": 0.5}, + "to": {"segment": "s_ruins", "edge": "start", "offset": 1.0}}) + beat = Span.model_validate({"from": {"beat": "b1", "edge": "start"}, + "to": {"beat": "b1", "edge": "end"}}) + spec, media, where = _with_overlays( + TextOverlay(id="o1", type="text", text="Hi", show=span), + TextOverlay(id="o2", type="text", text="Hi", show=beat, fade=5), + ) + placed = {p.overlay.id: p for p in place_overlays(spec, media, where, [])} + assert placed["o1"].start == pytest.approx(where.segments["s_market"][0] + 0.5) + assert placed["o1"].end == pytest.approx(where.segments["s_ruins"][0] + 1.0) + assert (placed["o2"].start, placed["o2"].end) == where.beats[("s_market", "b1")] + # A fade longer than half the window is held to half, so in and out never cross. + assert placed["o2"].fade == pytest.approx(2.0) + + +def test_an_overlay_that_cannot_be_placed_is_dropped_with_a_warning(): + orphan = Span.model_validate({"from": {"edge": "start"}, + "to": {"segment": "gone", "edge": "end"}}) + backwards = Span.model_validate({"from": {"edge": "end"}, "to": {"edge": "start"}}) + spec, media, where = _with_overlays( + TextOverlay(id="o1", type="text", text="a", show=orphan), + TextOverlay(id="o2", type="text", text="b", show=backwards), + ) + warnings = [] + placed = place_overlays(spec, media, where, warnings) + assert not [p for p in placed if p.overlay.id in ("o1", "o2")] + assert any("'gone'" in w for w in warnings) + assert any("ends before it starts" in w for w in warnings) + + +def test_an_image_overlay_needs_a_picture(): + from app.spec.model import ImageOverlay + spec, media, where = _with_overlays(ImageOverlay(id="o", type="image", asset="a_theme")) + warnings = [] + assert not [p for p in place_overlays(spec, media, where, warnings) if p.overlay.id == "o"] + assert any("needs a picture" in w for w in warnings) + + +def test_credit_lines_follow_first_use_and_are_said_once(): + spec, _ = salt_road() + assert credit_lines(spec, {}) == [] # no `credits`: none + spec = spec.model_copy(update={"credits": None}) + from app.spec.model import Credits + spec.credits = Credits(extra=[" Narrated by Rex "]) + keys = [s.visual.asset for s in spec.segments if s.visual] + credits = {keys[0]: "Footage: A", keys[1]: "Photo: B", keys[2]: "Footage: A"} + assert credit_lines(spec, credits) == ["Footage: A", "Photo: B", "Narrated by Rex"] + spec.credits = Credits(auto=False, extra=["Narrated by Rex"]) + assert credit_lines(spec, credits) == ["Narrated by Rex"] + + +def test_long_credits_run_over_several_cards_and_none_without_lines(): + spec, _ = salt_road() + from app.spec.model import Credits + spec.credits = Credits() + assert credits_units(spec, []) == [] + lines = [f"Line {i}" for i in range(CREDIT_LINES_PER_CARD + 3)] + cards = credits_units(spec, lines) + assert [len(u.credits) for u in cards] == [CREDIT_LINES_PER_CARD, 3] + assert all(u.kind == "card" and not u.chrome for u in cards) + spec.credits = Credits(card=False) + assert credits_units(spec, lines) == [] + + +def test_the_description_lists_chapters_from_zero_then_credits(): + spec, _ = salt_road() + from app.spec.model import Credits + spec.credits = Credits() + spec.output.chapters = True + text = description_text(spec, ["Photo: B"], [(4.0, 70.0, "Market"), (70.0, 200.0, "Ruins")]) + assert text == f"0:00 {spec.title}\n0:04 Market\n1:10 Ruins\n\nCredits\nPhoto: B" + spec.credits = Credits(description=False) + spec.output.chapters = False + assert description_text(spec, ["Photo: B"], [(0.0, 10.0, "x")]) == "" + + +def test_the_overlay_pass_draws_each_layer_only_in_its_window_and_copies_the_sound(): + argv = F.build_overlay_pass( + "stitched.mp4", + [F.TimedLayer("a.png", 1.0, 4.5, 0.3), F.TimedLayer("b.png", 2.0, 3.0, 0.0)], + "out.mp4", options=RenderOptions(), preview=True, duration=10.0) + graph = argv[argv.index("-filter_complex") + 1] + assert "fade=t=in:st=1.000:d=0.300:alpha=1" in graph + assert "fade=t=out:st=4.200:d=0.300:alpha=1" in graph + assert "enable='between(t,1.000,4.500)'" in graph + assert "enable='between(t,2.000,3.000)'" in graph + assert "[l2]" in graph and "fade" not in graph.split("[1:v]")[0] + assert argv[argv.index("-c:a") + 1] == "copy" + assert argv.count("-c:a") == 1 and argv[argv.index("-map") + 1] == "[v2]" diff --git a/backend/tests/test_render_integration.py b/backend/tests/test_render_integration.py index 80fc6a3..df01245 100644 --- a/backend/tests/test_render_integration.py +++ b/backend/tests/test_render_integration.py @@ -138,3 +138,90 @@ def test_a_notes_too_old_to_pin_the_voice_stops_the_render(tmp_path, library): with pytest.raises(SpeechError) as stopped: _render(tmp_path, SPEC) assert "gecko-notes#211" in str(stopped.value) + + +# ── M2b: timed overlays and credits ───────────────────────────────────────── + +OVERLAID = { + "title": "Overlays", + "output": {"chapters": True, "subtitles": "off"}, + "style": {"background": {"type": "solid", "colors": ["#000000"]}, + "transition": {"style": "none", "duration": 0.5}, "cards": {"seconds": 2.0}}, + "segments": [ + {"id": "s_one", "kind": "scene", "chapter": "One", "duration": {"fixed": 4.0}, + "overlays": [ + # Held across the dissolve into s_two: only the overlay pass can draw that. + {"id": "o_text", "type": "text", "text": "ACROSS", "size_pct": 20, + "fade": 0, + "show": {"from": {"edge": "start", "offset": 1.0}, + "to": {"segment": "s_two", "edge": "start", "offset": 1.5}}}, + {"id": "o_name", "type": "lower_third", "title": "Ada Lovelace", + "subtitle": "Mathematician", "fade": 0, + "show": {"from": {"edge": "start", "offset": 0.5}, + "to": {"edge": "start", "offset": 2.0}}}, + ]}, + {"id": "s_two", "kind": "scene", "chapter": "Two", "duration": {"fixed": 4.0}, + "transition_in": {"style": "dissolve", "duration": 1.0}}, + ], + "credits": {"auto": True, "card": True, "description": True, + "extra": ["Made with Gecko Video Creator"]}, +} + + +def _luma(video, at, box): + """Mean brightness (0-255) of a box (x0, y0, x1, y1, as fractions) at a time.""" + raw = subprocess.run( + ["ffmpeg", "-v", "error", "-ss", f"{at:.2f}", "-i", video, "-frames:v", "1", + "-f", "rawvideo", "-pix_fmt", "gray", "-"], capture_output=True, check=True).stdout + width, height = 854, 480 + x0, y0, x1, y1 = (int(box[0] * width), int(box[1] * height), + int(box[2] * width), int(box[3] * height)) + total = count = 0 + for y in range(y0, y1): + row = raw[y * width:(y + 1) * width] + total += sum(row[x0:x1]) + count += x1 - x0 + return total / max(1, count) + + +CENTRE = (0.2, 0.4, 0.8, 0.6) +LOWER_LEFT = (0.03, 0.72, 0.4, 0.92) + + +def test_timed_overlays_show_only_between_their_anchors(tmp_path, library): + result, _ = _render(tmp_path, OVERLAID) + # s_one 0-4, s_two from 3 (the 1 s dissolve), then the 2 s credits card. + assert result.duration == pytest.approx(4.0 + 4.0 - 1.0 + 2.0, abs=0.25) + assert result.warnings == [] + + # The text: on from 1.0 until 1.5 s into s_two (3.0 + 1.5 = 4.5), across the join. + assert _luma(result.video, 0.5, CENTRE) < 5 + assert _luma(result.video, 2.0, CENTRE) > 20 + assert _luma(result.video, 3.5, CENTRE) > 20 # inside the dissolve + assert _luma(result.video, 4.2, CENTRE) > 20 # into s_two + assert _luma(result.video, 5.5, CENTRE) < 5 + + # The lower third: 0.5 to 2.0 s, bottom left. + assert _luma(result.video, 0.2, LOWER_LEFT) < 5 + assert _luma(result.video, 1.2, LOWER_LEFT) > 15 + assert _luma(result.video, 3.0, LOWER_LEFT) < 5 + + +def test_credits_become_an_end_card_and_a_description(tmp_path, library): + data = json.loads(json.dumps(SPEC)) + library.assets["p"]["credit"] = "Photo: Jane Doe / CC BY 4.0" + library.assets["r"]["credit"] = "Footage: River Archive" + library.assets["a"]["credit"] = "Photo: Jane Doe / CC BY 4.0" # said once + data["credits"] = {"extra": ["Narrated by Rex"]} + plain, _ = _render(tmp_path / "plain", SPEC) + result, _ = _render(tmp_path / "credited", data) + + # The card joins with the project's fade, so it adds its own length and no overlap. + card = parse_spec(data)[0].style.cards.seconds + assert result.duration == pytest.approx(plain.duration + card, abs=0.3) + chapters, credits = result.description.split("\n\n") + assert [line.split(" ", 1)[1] for line in chapters.split("\n")] == [ + "Start", "The valley", "The river"] + assert chapters.startswith("0:00 ") + assert credits == ("Credits\nPhoto: Jane Doe / CC BY 4.0\nFootage: River Archive\n" + "Narrated by Rex") diff --git a/backend/tests/test_renders_api.py b/backend/tests/test_renders_api.py index 4f57f20..f4aa93e 100644 --- a/backend/tests/test_renders_api.py +++ b/backend/tests/test_renders_api.py @@ -151,3 +151,77 @@ def test_another_users_render_does_not_exist_for_you(auth_client, switch_user): ("get", f"/api/projects/{project['id']}/renders")): kwargs = {"json": {"preview": True}} if url.endswith("/renders") and method == "post" else {} assert getattr(auth_client, method)(url, **kwargs).status_code == 404, (method, url) + + +# ── Save to GAM (M2b, GAM-3) ──────────────────────────────────────────────── + +def _finished(auth_client, session, *, preview=False, description="0:00 Start"): + """A render marked done with a small file in place, without running ffmpeg.""" + from app.models.render import RenderJob + + project = _project(auth_client) + job = _start(auth_client, project["id"], preview=preview).json()["data"] + row = session.get(RenderJob, job["id"]) + row.status, row.video_path, row.description = "done", "video.mp4", description + session.add(row) + session.commit() + folder = worker.renders_dir() / job["id"] + folder.mkdir(parents=True, exist_ok=True) + (folder / "video.mp4").write_bytes(b"\x00" * 2048) + return project, job + + +def test_a_render_is_saved_to_gam_with_its_origin_as_the_requester(auth_client, session, siblings): + project, job = _finished(auth_client, session) + auth = {"Authorization": "Bearer the-users-token"} + + saved = auth_client.post(f"/api/renders/{job['id']}/save-to-gam", headers=auth) + assert saved.status_code == 200, saved.text + body = saved.json()["data"] + assert body["gam_asset_id"] == "gam-1" + assert body["gam_asset_url"] == f"{settings.gam_base_url}/a/gam-1" + + upload = siblings.uploads[0] + assert upload["authorization"] == "Bearer the-users-token" + assert upload["fields"] == {"source": "gvc_export", "origin_project_id": project["id"], + "origin_reference_id": job["id"]} + # Named after the project and version, described with the chapters and credits. + assert siblings.assets["gam-1"]["name"] == f"{project['title'] or 'Untitled video'} (v1)" + assert siblings.assets["gam-1"]["description"] == "0:00 Start" + + # Saving again files it once. + again = auth_client.post(f"/api/renders/{job['id']}/save-to-gam", headers=auth) + assert again.json()["data"]["gam_asset_id"] == "gam-1" + assert len(siblings.uploads) == 1 + listed = auth_client.get(f"/api/projects/{project['id']}/renders").json()["data"] + assert listed[0]["gam_asset_id"] == "gam-1" + + +def test_a_render_deleted_from_gam_is_filed_again(auth_client, session, siblings): + _, job = _finished(auth_client, session) + auth_client.post(f"/api/renders/{job['id']}/save-to-gam") + del siblings.assets["gam-1"] + again = auth_client.post(f"/api/renders/{job['id']}/save-to-gam").json()["data"] + assert again["gam_asset_id"] == "gam-2" + assert len(siblings.uploads) == 2 + + +def test_a_preview_or_an_unfinished_render_cannot_be_saved(auth_client, session, siblings): + _, preview = _finished(auth_client, session, preview=True) + refused = auth_client.post(f"/api/renders/{preview['id']}/save-to-gam") + assert refused.status_code == 400 + assert refused.json()["detail"]["code"] == "preview_not_saveable" + + project = _project(auth_client) + queued = _start(auth_client, project["id"], preview=False).json()["data"] + refused = auth_client.post(f"/api/renders/{queued['id']}/save-to-gam") + assert refused.status_code == 409 + assert refused.json()["detail"]["code"] == "render_not_done" + assert siblings.uploads == [] + + +def test_another_users_render_cannot_be_saved(auth_client, session, siblings, switch_user): + _, job = _finished(auth_client, session) + switch_user("someone-else") + assert auth_client.post(f"/api/renders/{job['id']}/save-to-gam").status_code == 404 + assert siblings.uploads == [] diff --git a/docs/plan-of-attack.md b/docs/plan-of-attack.md index 6b3f8bb..c91fae7 100644 --- a/docs/plan-of-attack.md +++ b/docs/plan-of-attack.md @@ -264,7 +264,7 @@ Specified here, applied in GAM's repository, not from this one. |---|---|---| | GAM-1 | `end_time` on `SearchHit`, from the transcript segment. **Done: [davior/gam#36](https://github.com/davior/gam/pull/36).** GVC also works without it, sizing a moment as start + 5 s | S3: a moment needs an out point | | GAM-2 | Run M5's acceptance on a real library; document that semantic search needs an embedding provider | "Find where people talk about *X*" is the feature most of this goal rests on, and it has never been checked end to end | -| GAM-3 | `origin_project_id` / `origin_reference_id` on `Asset` (migration) + upload with `source: "gvc_export"` | Filing renders back into the library | +| GAM-3 | `origin_project_id` / `origin_reference_id` on `Asset` (migration) + upload with `source: "gvc_export"`. **[davior/gam#37](https://github.com/davior/gam/pull/37).** Against a GAM without it, a save lands as an ordinary upload | Filing renders back into the library | | GAM-4 | M9: documented read API, `/picker?embed=1` with `postMessage({ assetId, inPoint, outPoint })` | Picking assets and moments by hand in the editor | | GAM-5 | `CORS_ORIGIN` / CSP for `gvc.geckopico.com`, only where GVC's browser code calls GAM directly | Server-side calls with a Bearer token need neither | @@ -287,7 +287,7 @@ Each milestone ends with something you can use, not just with code merged. |---|---|---|---| | **M0** | **Genesis** | Skeleton from GAM: `config.py`, `clock.py`, Alembic, shadow-user auth on the Notes JWT, `/api/health`, Dockerfiles (backend with ffmpeg), compose, nginx, CI. `Project` (current spec as JSON) and `ProjectVersion` (snapshot, author, message). Spec Pydantic model copied from Notes with S1–S3 applied, `0.1 → 0.2` migration on load. Projects API: blank project (title card + one empty scene), paste a spec, `POST /api/projects/import` for a Notes push, versions and revert, validation with errors by path. React shell: project list, a plain scene list (add, reorder, delete, edit narration), a JSON tab with live validation, version history. | You can sign in with your Notes account, make a video's structure by hand, paste the Notes spec doc's §6 example and see it validate (or see exactly why not), and import a spec Notes produced | | **M1** | **GAM as the library** | GAM-1 applied (gam#36). GVC's server calls GAM as the signed-in user by forwarding their own Notes token, so GAM's ownership checks hold and no service account exists. Search GAM from a scene: results with transcript snippets and the moment's time; picking a hit places it as muted b-roll or as a clip with its own sound, trimmed to what was said (padded, and sized by a placeholder's `moment`). `search` placeholders resolved the same way. Thumbnails and names looked up per page load, since GAM's signed URLs expire, and loaded straight from GAM (CSP `img-src`/`media-src`). A pushed note's media copied into GAM (D3). The byte cache keyed on asset id and checksum moved to M2: nothing reads media bytes until the renderer does. | A scene's footage is found by what is said in it ("where people talk about *X*") and lands in the spec as a trimmed GAM clip | -| **M2** | **Native renderer** | **Built (M2a).** Engine copied from Notes (`app/render/`: ffmpeg, compose, narration, pause markup, shot cache, with Notes' 359 tests). `compile.py` is the §5.8 compile step, done natively: scenes cut at quote/code beats, defaults cascading field by field, per-join transitions (blends overlap, dips draw in-shot, cuts cut), anchors resolved to seconds. Ken Burns paths between two frames (S1), trimmed and held clips, N audio tracks with fades, crossfades and ducking. Media fetched from GAM into a cache keyed on GAM id and size; placeholders drawn as slates in a preview, refusing a full render. Narration read per beat through Notes (GN-V4), so beat anchors are exact. Render jobs on GAM's runner; the requester's token held in memory only, so a restart ends a render rather than storing a credential. Finished files at signed, ranged URLs. Editor: per-scene transition and Ken Burns (with a frame editor), a Soundtrack tab, a Render tab with estimate, progress and the result. Left for M2b: timed text/lower-third/image overlays, Save to GAM (GAM-3), credits. | A spec with two music tracks crossfading at a scene cut, a dissolve into one scene, and a Ken Burns path over a chosen part of a picture renders correctly | +| **M2** | **Native renderer** | **Built (M2a).** Engine copied from Notes (`app/render/`: ffmpeg, compose, narration, pause markup, shot cache, with Notes' 359 tests). `compile.py` is the §5.8 compile step, done natively: scenes cut at quote/code beats, defaults cascading field by field, per-join transitions (blends overlap, dips draw in-shot, cuts cut), anchors resolved to seconds. Ken Burns paths between two frames (S1), trimmed and held clips, N audio tracks with fades, crossfades and ducking. Media fetched from GAM into a cache keyed on GAM id and size; placeholders drawn as slates in a preview, refusing a full render. Narration read per beat through Notes (GN-V4), so beat anchors are exact. Render jobs on GAM's runner; the requester's token held in memory only, so a restart ends a render rather than storing a credential. Finished files at signed, ranged URLs. Editor: per-scene transition and Ken Burns (with a frame editor), a Soundtrack tab, a Render tab with estimate, progress and the result. **M2b:** timed text, lower thirds and pictures, drawn over the stitched video in one extra pass (`enable=` windows with alpha fades) so they can be held across a join, anchored to a scene's edges or lines; end credits from each GAM asset's credit plus extra lines, as cards and in a description (chapters + credits) to paste where the video is published; Save to GAM (GAM-3) for full renders, once per render. | A spec with two music tracks crossfading at a scene cut, a dissolve into one scene, and a Ken Burns path over a chosen part of a picture renders correctly | | **M3** | **Editor** | The M0 scene list grows into the storyboard: a card per segment (thumbnail, label, first line), lanes for audio tracks and overlays against the segments, a per-scene panel (visual, trim, transition, motion with a drag-a-box framing tool, voice), a project defaults panel. Sliders take their ranges from the schema (the Notes spec doc, open decision 12). Undo/redo over `ProjectVersion`. | You can build and fix a whole video by hand without opening the JSON tab | | **M4** | **AI drafting** | Providers copied from GAM's M6. "New video from a prompt": brief → spec (D2) with GAM search as a tool, and placeholders for gaps, with one repair round. Also from selected GAM assets. Cost estimate before running, usage recorded. | "Make a 3-minute video about *X* using clips where people talk about it" produces a spec that previews immediately | | **M5** | **AI editing** | The Notes spec doc §8.2 operations, addressed by id. Chat beside the storyboard. Suggestions staged and accepted one by one. Generate placeholders sent to GAM's M8. | "Change the music after the second chapter to something calmer, and dissolve into the last scene" becomes reviewed, applied edits | diff --git a/frontend/src/api/projects.ts b/frontend/src/api/projects.ts index ac6c69a..d5b1828 100644 --- a/frontend/src/api/projects.ts +++ b/frontend/src/api/projects.ts @@ -95,6 +95,50 @@ export type AssetRef = label?: string } +export type Position = + 'top-left' | 'top-right' | 'center' | 'bottom-left' | 'bottom-right' + +/** When an overlay shows. Absent on the overlay: its whole scene. */ +export interface Span { + from: Anchor + to: Anchor +} + +interface OverlayBase { + id: string + show?: Span + /** Seconds each way; absent: a short default. */ + fade?: number + [key: string]: unknown +} + +export type Overlay = + | (OverlayBase & { type: 'quote'; text: string; attribution?: string }) + | (OverlayBase & { type: 'code'; text: string; language?: string }) + | (OverlayBase & { + type: 'text' + text: string + position?: Position + size_pct?: number + color?: string + }) + | (OverlayBase & { type: 'lower_third'; title: string; subtitle?: string }) + | (OverlayBase & { + type: 'image' + asset: string + position?: Position + scale_pct?: number + opacity?: number + }) + +/** The end credits. Lines come from each GAM asset's credit, then `extra`. */ +export interface Credits { + auto?: boolean + card?: boolean + description?: boolean + extra?: string[] +} + export interface Segment { id: string kind: 'scene' | 'card' | 'clip' @@ -105,6 +149,7 @@ export interface Segment { visual?: Visual narration?: { beats: Beat[]; [key: string]: unknown } transition_in?: { style?: string; duration?: number } + overlays?: Overlay[] [key: string]: unknown } @@ -116,6 +161,7 @@ export interface VideoSpec { assets?: Record segments: Segment[] audio_tracks?: AudioTrack[] + credits?: Credits | null output?: { aspect?: '16:9' | '9:16' | '1:1'; [key: string]: unknown } style?: { transition?: { style?: string; duration?: number } diff --git a/frontend/src/api/renders.ts b/frontend/src/api/renders.ts index ac8b380..8d6ccce 100644 --- a/frontend/src/api/renders.ts +++ b/frontend/src/api/renders.ts @@ -23,6 +23,11 @@ export interface Render { video_url: string | null subtitles_url: string | null poster_url: string | null + /** Chapters and credits, to paste where the video is published. */ + description: string | null + /** Set once saved to GAM: the asset, and its page there. */ + gam_asset_id: string | null + gam_asset_url: string | null created_at: string updated_at: string } @@ -77,6 +82,12 @@ export const rendersApi = { .then((r) => r.data.data) }, + saveToGam(id: string): Promise { + return client + .post>(`/renders/${id}/save-to-gam`) + .then((r) => r.data.data) + }, + remove(id: string): Promise { return client.delete(`/renders/${id}`).then(() => undefined) }, diff --git a/frontend/src/components/AnchorSelect.tsx b/frontend/src/components/AnchorSelect.tsx new file mode 100644 index 0000000..f235b04 --- /dev/null +++ b/frontend/src/components/AnchorSelect.tsx @@ -0,0 +1,56 @@ +import type { Anchor } from '@/api/projects' +import { anchorFrom, anchorValue, offsetOf, type AnchorOption } from '@/spec/anchors' + +interface Props { + label: string + value: Anchor | undefined + options: AnchorOption[] + onPick: (next: Anchor | undefined) => void + /** The label of the empty choice; none offered when absent. */ + noneLabel?: string +} + +/** + * Where something starts or stops: a scene's (or a line's) start or end, with an + * offset in seconds. Anchored to the structure rather than to a time, so rewording a + * scene moves everything placed against it along with it. + */ +export default function AnchorSelect({ + label, + value, + options, + onPick, + noneLabel, +}: Props) { + return ( + + ) +} diff --git a/frontend/src/components/CreditsSettings.tsx b/frontend/src/components/CreditsSettings.tsx new file mode 100644 index 0000000..50477c4 --- /dev/null +++ b/frontend/src/components/CreditsSettings.tsx @@ -0,0 +1,80 @@ +import type { Credits, VideoSpec } from '@/api/projects' +import { setCredits } from '@/spec/overlays' + +interface Props { + spec: VideoSpec + onChange: (next: VideoSpec) => void +} + +const DEFAULTS: Credits = { auto: true, card: true, description: true, extra: [] } + +/** + * The end credits. The lines are each GAM asset's credit, worked out when the video + * renders (the library is where a credit is corrected, once, for every video using + * it), followed by any written here. + */ +export default function CreditsSettings({ spec, onChange }: Props) { + const credits = spec.credits ?? null + const on = credits !== null + const value = { ...DEFAULTS, ...(credits ?? {}) } + const set = (patch: Partial) => + onChange(setCredits(spec, { ...value, ...patch })) + + return ( +
+ + {on && ( +
+ + + +