diff --git a/README.md b/README.md index 9cf6a2d..3b47b38 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,12 @@ There are three ways to start one: GAM by what is said in it. Then preview and render it: - per-scene transitions and Ken Burns moves between two frames of a picture; - any number of soundtrack tracks, crossfading between scenes; -- trimmed clips, and narration read through Gecko Notes. +- trimmed clips, and narration read through Gecko Notes; +- text, lower thirds and pictures on screen, timed to the scenes and lines they belong to; +- end credits from the library's attribution, and a description with chapters and credits. -Timed overlays, saving renders to GAM and credits are next (M2b). +A finished render can be saved back into GAM. AI drafting and the full storyboard +editor are next (M3, M4). ## Read first diff --git a/backend/alembic/versions/20261016_0700_add_render_description_and_gam_asset.py b/backend/alembic/versions/20261016_0700_add_render_description_and_gam_asset.py new file mode 100644 index 0000000..b7eb2a1 --- /dev/null +++ b/backend/alembic/versions/20261016_0700_add_render_description_and_gam_asset.py @@ -0,0 +1,33 @@ +"""Add render_job.description and render_job.gam_asset_id (M2b). + +`description`: the chapters and credits a render worked out, to paste where the video +is published. `gam_asset_id`: the GAM asset a render was saved as, so saving twice +files it once. An id in another app's database, so no foreign key. + +Revision ID: 7e2d5a91b3c6 +Revises: 1c0c7b02acd3 +Create Date: 2026-10-16 07:00:00.000000+00:00 +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = '7e2d5a91b3c6' +down_revision: Union[str, None] = '1c0c7b02acd3' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + with op.batch_alter_table('render_job', schema=None) as batch_op: + batch_op.add_column(sa.Column('description', sqlmodel.sql.sqltypes.AutoString(), nullable=True)) + batch_op.add_column(sa.Column('gam_asset_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True)) + + +def downgrade() -> None: + with op.batch_alter_table('render_job', schema=None) as batch_op: + batch_op.drop_column('gam_asset_id') + batch_op.drop_column('description') diff --git a/backend/app/gam/client.py b/backend/app/gam/client.py index fdf370f..694cc45 100644 --- a/backend/app/gam/client.py +++ b/backend/app/gam/client.py @@ -154,12 +154,20 @@ def download(self, file_url: str, sink, *, max_bytes: int) -> int: raise GamError("gam_unavailable", "Could not fetch a file from the asset library") from exc return written - def upload(self, filename: str, stream: IO[bytes], content_type: Optional[str]) -> GamAssetRead: - """Store one file in the user's library and return it.""" + def upload(self, filename: str, stream: IO[bytes], content_type: Optional[str], *, + fields: Optional[Dict[str, str]] = None, + timeout: Optional[float] = None) -> GamAssetRead: + """Store one file in the user's library and return it. + + `fields` go along as form fields: a render's `source=gvc_export` and origin ids + (GAM-3). A GAM without GAM-3 ignores them and stores an ordinary upload. + """ response = self._request( "POST", "/api/assets", files={"files": (filename, stream, content_type or "application/octet-stream")}, + data=fields or None, + **({"timeout": timeout} if timeout else {}), ) if response.status_code not in (200, 201): code, message = self._detail(response) @@ -172,3 +180,13 @@ def upload(self, filename: str, stream: IO[bytes], content_type: Optional[str]) raise GamError(rejected.get("code") or "gam_error", rejected.get("message") or "The asset library did not store the file") return to_asset(created[0]) + + def update(self, asset_id: str, changes: Dict[str, Any]) -> Optional[GamAssetRead]: + """Edit an asset's fields (its name, its description). None: it is gone.""" + response = self._request("PATCH", f"/api/assets/{asset_id}", json=changes) + if response.status_code == 404: + return None + if response.status_code != 200: + code, message = self._detail(response) + raise GamError(code, message) + return to_asset(response.json()["data"]) diff --git a/backend/app/models/render.py b/backend/app/models/render.py index 4338411..765d719 100644 --- a/backend/app/models/render.py +++ b/backend/app/models/render.py @@ -31,5 +31,9 @@ class RenderJob(SQLModel, table=True): duration: Optional[float] = None size_bytes: Optional[int] = None narration_chars: int = 0 + # Chapters and credits, to paste where the video is published (M2b). + description: Optional[str] = None + # The GAM asset this render was saved as, once it has been (GAM-3). + gam_asset_id: Optional[str] = None created_at: datetime = Field(default_factory=utcnow) updated_at: datetime = Field(default_factory=utcnow) diff --git a/backend/app/render/compile.py b/backend/app/render/compile.py index a0e546d..49c712d 100644 --- a/backend/app/render/compile.py +++ b/backend/app/render/compile.py @@ -26,8 +26,8 @@ from app.render.ffmpeg import KENBURNS_CYCLE, MIN_TRANSITION_SECONDS, XFADE_MAX_SHOTS from app.render.options import XFADE_NAMES, is_crossfade from app.spec.model import ( - AnchorRef, AudioTrack, Beat, Card, CodeOverlay, Motion, QuoteOverlay, Segment, - Transition, VideoSpec, + AnchorRef, AudioTrack, Beat, Card, CodeOverlay, ImageOverlay, LowerThirdOverlay, Motion, + QuoteOverlay, Segment, TextOverlay, Transition, VideoSpec, ) UnitKind = str # "still" | "card" | "video_muted" | "video_sound" @@ -78,6 +78,8 @@ class Unit: speed: float = 1.0 chrome: bool = True chapter: Optional[str] = None + # The end credits: a card drawn from these lines rather than from `card`. + credits: Optional[List[str]] = None def _transition_in(spec: VideoSpec, segment: Segment) -> Transition: @@ -111,10 +113,8 @@ def _panel_beats(segment: Segment, warnings: List[str]) -> Tuple[Dict[str, Panel whole: Optional[Panel] = None for overlay in segment.overlays: if not isinstance(overlay, (QuoteOverlay, CodeOverlay)): - # Timed text, lower thirds and images are M2b: a second technique - # (overlay with enable=) on top of this one. Said, never silently lost. - warnings.append(f"Scene {segment.label or segment.id}: {overlay.type} overlays " - "are not rendered yet") + # Timed text, lower thirds and images are drawn over the stitched video + # instead (`place_overlays`), since they may run across a join. continue anchor = overlay.show.from_ if overlay.show else None if isinstance(anchor, AnchorRef) and anchor.beat and anchor.segment in (None, segment.id): @@ -206,6 +206,31 @@ def plan_units(spec: VideoSpec, media: Dict[str, Media], warnings: List[str]) -> return units +CREDITS_SEGMENT = "__credits__" +CREDIT_LINES_PER_CARD = 18 + + +def credits_units(spec: VideoSpec, lines: Sequence[str]) -> List[Unit]: + """The end credits as card units after everything else, one per screenful. + + Not a segment in the spec: the lines come from GAM at render time (each asset's + credit), so the spec can't hold them, and a segment the user never made would + show up in the editor. Past a screenful they run over several cards rather than + shrinking to unreadable. + """ + if not spec.credits or not spec.credits.card or not lines: + return [] + pages = [list(lines[i:i + CREDIT_LINES_PER_CARD]) + for i in range(0, len(lines), CREDIT_LINES_PER_CARD)] + return [ + Unit(segment_id=CREDITS_SEGMENT, segment_index=len(spec.segments), + first_in_segment=position == 0, kind="card", media=None, credits=page, + transition_in=spec.style.transition, min_seconds=spec.style.cards.seconds, + chrome=False) + for position, page in enumerate(pages) + ] + + def unit_duration(unit: Unit, narration_seconds: float, cards_seconds: float) -> float: """How long a unit runs, once its narration has been measured.""" if unit.kind == "video_sound": @@ -429,3 +454,67 @@ def beat_pause_ms(spec: VideoSpec, beats: Sequence[Beat], position: int) -> int: if beat.kind == "heading" or beats[position + 1].kind == "heading": return pauses.heading_ms return pauses.paragraph_ms + + +# ── timed overlays ────────────────────────────────────────────────────────── + +TimedKind = Union[TextOverlay, LowerThirdOverlay, ImageOverlay] + +DEFAULT_OVERLAY_FADE = 0.3 +MIN_OVERLAY_SECONDS = 0.2 + + +@dataclass +class OverlayPlacement: + segment_id: str + overlay: TimedKind + start: float + end: float + fade: float + media: Optional[Media] = None # an image overlay's picture + + +def place_overlays(spec: VideoSpec, media: Dict[str, Media], where: Placement, + warnings: List[str]) -> List[OverlayPlacement]: + """Text, lower thirds and images as windows on the finished timeline. + + With no `show`, an overlay covers its scene. Its anchors default to its own scene, + so `{"edge": "start", "offset": 0.5}` means half a second into it. As with audio + tracks, one that can't be placed is dropped with a warning (§5.7): the render + still goes ahead. + """ + placed: List[OverlayPlacement] = [] + for segment in spec.segments: + span = where.segments.get(segment.id) + name = segment.label or segment.id + for overlay in segment.overlays: + if not isinstance(overlay, (TextOverlay, LowerThirdOverlay, ImageOverlay)): + continue + if span is None: + # The scene itself was left out (a clip with nothing to play). + continue + if overlay.show is None: + start, end = span + else: + try: + start = resolve_anchor(overlay.show.from_, where, segment=segment.id) + end = resolve_anchor(overlay.show.to, where, segment=segment.id) + except Orphaned as exc: + warnings.append(f"Scene {name}: an overlay is anchored to {exc}, which is " + "not in the video; left out") + continue + if end - start < MIN_OVERLAY_SECONDS: + warnings.append(f"Scene {name}: an overlay ends before it starts; left out") + continue + picture = None + if isinstance(overlay, ImageOverlay): + picture = media.get(overlay.asset) + if picture is None or picture.kind not in ("image", "slate"): + warnings.append(f"Scene {name}: an image overlay needs a picture; left out") + continue + fade = overlay.fade if overlay.fade is not None else DEFAULT_OVERLAY_FADE + # Never more than half the window each way, or the fades would cross. + fade = max(0.0, min(fade, (end - start) / 2)) + placed.append(OverlayPlacement(segment_id=segment.id, overlay=overlay, start=start, + end=end, fade=fade, media=picture)) + return placed diff --git a/backend/app/render/compose.py b/backend/app/render/compose.py index f1681d4..4b80312 100644 --- a/backend/app/render/compose.py +++ b/backend/app/render/compose.py @@ -195,6 +195,14 @@ def _anchor_xy(position: str, width: int, height: int, box_w: int, box_h: int, m return (width - box_w - margin, height - box_h - margin) +def _card_base(width: int, height: int, background: Optional[Image.Image]) -> Image.Image: + base = (background.copy() if background is not None + else gradient_background(width, height, ["#1e293b", "#0f172a"], 135)).convert("RGB") + # Blur and darken whatever is behind so the text always has contrast. + base = base.filter(ImageFilter.GaussianBlur(radius=max(6, width // 90))) + return Image.blend(base, Image.new("RGB", (width, height), (0, 0, 0)), 0.45) + + def card_image( width: int, height: int, *, title: str, subtitle: str = "", @@ -207,12 +215,7 @@ def card_image( one setting holds at every resolution and aspect ratio. """ spec = sizes or CardTextSpec() - base = (background.copy() if background is not None - else gradient_background(width, height, ["#1e293b", "#0f172a"], 135)).convert("RGB") - - # Blur and darken whatever is behind so the text always has contrast. - base = base.filter(ImageFilter.GaussianBlur(radius=max(6, width // 90))) - base = Image.blend(base, Image.new("RGB", (width, height), (0, 0, 0)), 0.45) + base = _card_base(width, height, background) layer = Image.new("RGBA", (width, height), (0, 0, 0, 0)) draw = ImageDraw.Draw(layer) @@ -551,3 +554,126 @@ def _budget(margin: int, taken: int = 0) -> int: layer.alpha_composite(mark, (max(0, x), max(0, y))) return layer + + +# ── timed overlays (GVC) ──────────────────────────────────────────────────── +# Each is a full-frame RGBA layer, like `overlay_layer`, so the video pass that shows +# it for its window is one plain `overlay=0:0` per layer with nothing to position. + +OVERLAY_MARGIN_PCT = 5.0 + + +def text_layer(width: int, height: int, *, text: str, position: str = "center", + size_pct: Optional[float] = None, color: Optional[str] = None) -> Image.Image: + """On-screen text, wrapped, with a soft shadow so it reads over any picture.""" + layer = Image.new("RGBA", (width, height), (0, 0, 0, 0)) + draw = ImageDraw.Draw(layer) + margin = int(min(width, height) * OVERLAY_MARGIN_PCT / 100) + max_width = width - 2 * margin if position == "center" else int(width * 0.6) + start = max(8, int(height * (size_pct or 6.0) / 100)) + font = fit_font(draw, text, SEMIBOLD, start, max_width) + lines = wrap_text(draw, text, font, max_width)[:8] + line_h = int(font.size * 1.25) + box_w = int(max((draw.textlength(line, font=font) for line in lines), default=0)) + x, y = _anchor_xy(position, width, height, box_w, line_h * len(lines), margin) + colour = parse_colour(color or "#ffffff") + + shadow = Image.new("RGBA", (width, height), (0, 0, 0, 0)) + shadow_draw = ImageDraw.Draw(shadow) + for position_in_block, line in enumerate(lines): + lw = draw.textlength(line, font=font) + lx = x + (box_w - lw) / 2 if position == "center" else ( + x + box_w - lw if position.endswith("right") else x) + ly = y + position_in_block * line_h + shadow_draw.text((lx, ly + max(1, font.size // 16)), line, font=font, fill=(0, 0, 0, 190)) + draw.text((lx, ly), line, font=font, fill=(*colour, 255)) + shadow = shadow.filter(ImageFilter.GaussianBlur(radius=max(2, font.size // 10))) + return Image.alpha_composite(shadow, layer) + + +def lower_third_layer(width: int, height: int, *, title: str, + subtitle: str = "") -> Image.Image: + """A name and a role, bottom left, on a translucent bar with an accent rule. + + The bar is sized to its text rather than the frame, so a short name doesn't sit + in a long empty strip. Its colours match the title cards' rule. + """ + layer = Image.new("RGBA", (width, height), (0, 0, 0, 0)) + draw = ImageDraw.Draw(layer) + margin = int(min(width, height) * OVERLAY_MARGIN_PCT / 100) + max_text = int(width * 0.7) + title_font = fit_font(draw, title, SEMIBOLD, max(10, int(height * 0.052)), max_text) + sub_font = load_font(REGULAR, max(8, int(height * 0.034))) + title = truncate_to_width(draw, title.strip(), title_font, max_text) + subtitle = truncate_to_width(draw, (subtitle or "").strip(), sub_font, max_text) + + pad_x, pad_y = int(title_font.size * 0.6), int(title_font.size * 0.4) + rule_w = max(3, height // 160) + text_w = int(max(draw.textlength(title, font=title_font), + draw.textlength(subtitle, font=sub_font) if subtitle else 0)) + title_h = int(title_font.size * 1.2) + sub_h = int(sub_font.size * 1.3) if subtitle else 0 + box_w = rule_w + pad_x * 2 + text_w + box_h = pad_y * 2 + title_h + sub_h + x0 = margin + y0 = height - margin - box_h - int(height * 0.04) + + draw.rectangle([x0, y0, x0 + box_w, y0 + box_h], fill=(15, 23, 42, 190)) + draw.rectangle([x0, y0, x0 + rule_w, y0 + box_h], fill=(129, 140, 248, 255)) + tx = x0 + rule_w + pad_x + draw.text((tx, y0 + pad_y), title, font=title_font, fill=(255, 255, 255, 255)) + if subtitle: + draw.text((tx, y0 + pad_y + title_h), subtitle, font=sub_font, fill=(203, 213, 225, 255)) + return layer + + +def image_layer(width: int, height: int, *, picture: str, position: str = "center", + scale_pct: Optional[float] = None, + opacity: Optional[float] = None) -> Optional[Image.Image]: + """A picture over the video: a logo, a map, an inset. `scale_pct` is its width as + a share of the frame's; its aspect is kept.""" + try: + with Image.open(picture) as raw: + image = ImageOps.exif_transpose(raw).convert("RGBA") + except (UnidentifiedImageError, OSError, ValueError) as exc: + logger.warning("Could not load overlay picture %s: %s", picture, exc) + return None + margin = int(min(width, height) * OVERLAY_MARGIN_PCT / 100) + target_w = max(1, int(width * (scale_pct or 30.0) / 100)) + target_h = max(1, int(image.height * target_w / max(1, image.width))) + # Never taller than the frame allows, whatever the width asked for. + if target_h > height - 2 * margin: + target_h = max(1, height - 2 * margin) + target_w = max(1, int(image.width * target_h / max(1, image.height))) + image = image.resize((target_w, target_h), Image.LANCZOS) + if opacity is not None and opacity < 1: + alpha = image.getchannel("A").point(lambda a: int(a * max(0.0, opacity))) + image.putalpha(alpha) + layer = Image.new("RGBA", (width, height), (0, 0, 0, 0)) + layer.alpha_composite(image, _anchor_xy(position, width, height, target_w, target_h, margin)) + return layer + + +def credits_card(width: int, height: int, *, lines: Sequence[str], + background: Optional[Image.Image] = None) -> Image.Image: + """The end credits: a heading over centred lines, smaller as there are more.""" + base = _card_base(width, height, background) + layer = Image.new("RGBA", (width, height), (0, 0, 0, 0)) + draw = ImageDraw.Draw(layer) + max_width = int(width * 0.82) + heading = load_font(SEMIBOLD, max(10, int(height * 0.06))) + # Sized so the whole page fits: a three-line page reads large, eighteen still fit. + size = max(8, min(int(height * 0.042), int(height * 0.7 / max(1, len(lines)) / 1.35))) + body = load_font(REGULAR, size) + wrapped = [truncate_to_width(draw, line, body, max_width) for line in lines] + head_h = int(heading.size * 1.6) + line_h = int(body.size * 1.35) + y = (height - head_h - line_h * len(wrapped)) // 2 + w = draw.textlength("Credits", font=heading) + draw.text(((width - w) / 2, y), "Credits", font=heading, fill=(255, 255, 255, 255)) + y += head_h + for line in wrapped: + w = draw.textlength(line, font=body) + draw.text(((width - w) / 2, y), line, font=body, fill=(226, 232, 240, 240)) + y += line_h + return Image.alpha_composite(base.convert("RGBA"), layer).convert("RGB") diff --git a/backend/app/render/credits.py b/backend/app/render/credits.py new file mode 100644 index 0000000..ec35685 --- /dev/null +++ b/backend/app/render/credits.py @@ -0,0 +1,50 @@ +"""What the end credits and the description say. + +Credits come from GAM: each asset's `credit`, which GAM builds from its attribution +fields (and a clip inherits from its parent). GVC keeps no attribution of its own, so +correcting a credit is done once, in the library, and every video using it follows. +""" + +from __future__ import annotations + +from typing import Dict, List, Sequence, Tuple + +from app.render.media import keys_in_order +from app.spec.model import VideoSpec + + +def credit_lines(spec: VideoSpec, credits: Dict[str, str]) -> List[str]: + """The credits, in the order their assets first appear, each said once.""" + if spec.credits is None: + return [] + lines: List[str] = [] + if spec.credits.auto: + lines += [credits.get(key, "").strip() for key in keys_in_order(spec)] + lines += [line.strip() for line in spec.credits.extra] + # One credit line for a photographer's three pictures, not three. + return list(dict.fromkeys(line for line in lines if line)) + + +def _clock(seconds: float) -> str: + whole = int(round(seconds)) + hours, rest = divmod(whole, 3600) + minutes, secs = divmod(rest, 60) + return f"{hours}:{minutes:02d}:{secs:02d}" if hours else f"{minutes}:{secs:02d}" + + +def description_text(spec: VideoSpec, lines: Sequence[str], + marks: Sequence[Tuple[float, float, str]]) -> str: + """Text to paste under the video where it is published: chapters and credits. + + Chapters are listed as `0:00 Title`, the form video sites turn into chapter + links, and only with the first at 0:00, which those sites require. + """ + parts: List[str] = [] + if spec.output.chapters and marks: + listed = [(start, title) for start, _end, title in marks] + if listed[0][0] > 0.5: + listed.insert(0, (0.0, spec.title or "Start")) + parts.append("\n".join(f"{_clock(start)} {title}" for start, title in listed)) + if spec.credits and spec.credits.description and lines: + parts.append("Credits\n" + "\n".join(lines)) + return "\n\n".join(parts) diff --git a/backend/app/render/estimate.py b/backend/app/render/estimate.py index 7cfc3c9..4af6a87 100644 --- a/backend/app/render/estimate.py +++ b/backend/app/render/estimate.py @@ -68,5 +68,10 @@ def estimate(spec: VideoSpec) -> Estimate: floor = (segment.duration.min if segment.duration and segment.duration.min else spec.style.min_segment_seconds) out.seconds += max(floor, spoken) + credits = spec.credits + if credits and credits.card and (credits.extra or (credits.auto and used)): + # Whether GAM has credits for the assets is only known at render time; a card + # is the likelier outcome, and an estimate a few seconds long is the safer one. + out.seconds += spec.style.cards.seconds out.seconds = round(out.seconds, 1) return out diff --git a/backend/app/render/ffmpeg.py b/backend/app/render/ffmpeg.py index 8367ddd..b984323 100644 --- a/backend/app/render/ffmpeg.py +++ b/backend/app/render/ffmpeg.py @@ -1120,3 +1120,56 @@ def build_mix_command( "-t", f"{max(0.1, duration):.3f}", "-movflags", "+faststart", output, ] return argv + + +@dataclass(frozen=True) +class TimedLayer: + """A full-frame RGBA PNG shown over the finished video for a window.""" + + path: str + start: float + end: float + fade: float = 0.0 + + +def build_overlay_pass( + source: str, layers: Sequence[TimedLayer], output: str, + *, options: RenderOptions, preview: bool, duration: float, +) -> List[str]: + """Draw timed layers over the stitched picture, the sound untouched. + + Per shot would have kept the shot cache, but an overlay may run across a join (a + title held through a dissolve), and a shot can't draw what lies past its end. + So this is one re-encode of the whole picture after the stitch, done only when a + video has timed overlays at all. + + Each layer is a looped still. Its fades are alpha fades at absolute times, which + line up with the video's because a looped image input starts at zero too, and + `enable` keeps it off outside its window so nothing is composited where it isn't + shown. + """ + fps = options.fps + total = max(0.1, duration) + argv = ["ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-i", source] + chains: List[str] = [] + stage = "0:v" + for position, layer in enumerate(layers, start=1): + argv += ["-loop", "1", "-framerate", str(fps), "-t", f"{total:.3f}", "-i", layer.path] + chain = f"[{position}:v]format=rgba" + if layer.fade > 0: + chain += (f",fade=t=in:st={layer.start:.3f}:d={layer.fade:.3f}:alpha=1" + f",fade=t=out:st={layer.end - layer.fade:.3f}:d={layer.fade:.3f}:alpha=1") + chains.append(chain + f"[l{position}]") + chains.append(f"[{stage}][l{position}]overlay=0:0:eof_action=pass:" + f"enable='between(t,{layer.start:.3f},{layer.end:.3f})'[v{position}]") + stage = f"v{position}" + # Only the video codec flags: the sound is copied, not re-encoded a second time. + shared = encode_args(options, preview) + video_args = shared[:shared.index("-c:a")] + argv += [ + "-filter_complex", ";".join(chains), + "-map", f"[{stage}]", "-map", "0:a?", + *video_args, "-c:a", "copy", + "-t", f"{total:.3f}", "-movflags", "+faststart", output, + ] + return argv diff --git a/backend/app/render/media.py b/backend/app/render/media.py index 41e9637..b7b5d68 100644 --- a/backend/app/render/media.py +++ b/backend/app/render/media.py @@ -28,7 +28,9 @@ from app.render import compose from app.render import ffmpeg as F from app.render.compile import Media -from app.spec.model import GamAsset, GeneratedAsset, NotesAsset, SearchAsset, VideoSpec +from app.spec.model import ( + GamAsset, GeneratedAsset, ImageOverlay, NotesAsset, SearchAsset, VideoSpec, +) logger = logging.getLogger(__name__) @@ -38,12 +40,26 @@ class Resolution: media: Dict[str, Media] = field(default_factory=dict) # Reasons the render cannot go ahead, one per asset. problems: List[str] = field(default_factory=list) + # Each GAM asset's credit line as GAM resolves it (a clip's inherited from its + # parent), for the end credits. Empty for an asset nobody has attributed. + credits: Dict[str, str] = field(default_factory=dict) + + +def keys_in_order(spec: VideoSpec) -> List[str]: + """The asset keys the video shows or plays, in the order a viewer meets them: + scene by scene (its picture, then its overlays), then the soundtrack.""" + ordered: List[str] = [] + for segment in spec.segments: + if segment.visual is not None: + ordered.append(segment.visual.asset) + ordered += [o.asset for o in segment.overlays if isinstance(o, ImageOverlay)] + ordered += [t.asset for t in spec.audio_tracks] + return list(dict.fromkeys(k for k in ordered if k in spec.assets)) def used_keys(spec: VideoSpec) -> Set[str]: """Every asset key the render will read.""" - keys = {s.visual.asset for s in spec.segments if s.visual is not None} - keys |= {t.asset for t in spec.audio_tracks} + keys = set(keys_in_order(spec)) if spec.style.background.type == "image" and spec.style.background.asset: keys.add(spec.style.background.asset) if spec.chrome.watermark and spec.chrome.watermark.asset: @@ -167,6 +183,7 @@ def resolve( if asset.out_point is not None: media.length = max(0.1, asset.out_point - (asset.in_point or 0.0)) out.media[key] = media + out.credits[key] = asset.credit or "" try: cache.prune(int(settings.render_cache_max_gb * 1024 ** 3)) diff --git a/backend/app/render/renderer.py b/backend/app/render/renderer.py index c7b06c5..923bcd8 100644 --- a/backend/app/render/renderer.py +++ b/backend/app/render/renderer.py @@ -29,9 +29,10 @@ from app.render import compose, shot_cache from app.render import ffmpeg as F from app.render.compile import ( - Unit, beat_pause_ms, chapters, layout, place_tracks, placement, plan_units, unit_duration, - xfade_name, + OverlayPlacement, Unit, beat_pause_ms, chapters, credits_units, layout, place_overlays, + place_tracks, placement, plan_units, unit_duration, xfade_name, ) +from app.render.credits import credit_lines, description_text from app.render.media import MediaCache, resolve from app.render.narration import Cue, build_silence, shift_cues, synthesize_shot, write_srt from app.render.options import ( @@ -63,6 +64,9 @@ class RenderOutput: size_bytes: int narration_chars: int warnings: List[str] = field(default_factory=list) + # Text to paste where the video is published: chapters and credits. None: nothing + # to say. + description: Optional[str] = None def base_options(spec: VideoSpec) -> RenderOptions: @@ -184,7 +188,9 @@ def _background(unit: Unit, spec: VideoSpec, options: RenderOptions, work_dir: s base = (compose.solid_background(width, height, fallback.colors[0]) if fallback.type == "solid" else compose.gradient_background(width, height, fallback.colors, fallback.angle)) - if unit.kind == "card" and unit.card: + if unit.kind == "card" and unit.credits: + image = compose.credits_card(width, height, lines=unit.credits, background=base) + elif unit.kind == "card" and unit.card: image = compose.card_image( width, height, title=unit.card.title or spec.title or "Untitled", subtitle=unit.card.subtitle or "", background=base, @@ -220,6 +226,8 @@ def render( units = plan_units(spec, media, warnings) if not units: raise RenderRefused("This video has nothing in it to render yet.") + lines = credit_lines(spec, resolved.credits) + units += credits_units(spec, lines) if len(units) > max_segments: raise RenderRefused(f"This video has {len(units)} shots, above the limit of {max_segments}.") chars = sum(len(b.text) for u in units for b in u.beats if b.speak) @@ -409,9 +417,20 @@ def argv(with_options: RenderOptions, with_path=path) -> List[str]: F.write_concat_list(os.path.join(work_dir, "shots.txt"), shot_files) F.run(F.build_concat_command("shots.txt", stitched), cwd=work_dir, timeout=1800) final = stitched + where = placement(units, timeline, [n.beats for n in narrated]) + + # ── timed overlays ─────────────────────────────────────────────────────── + timed = place_overlays(spec, media, where, warnings) + layers = _overlay_layers(timed, work_dir, width, height) + if layers: + progress("Drawing overlays", 88, + f"{len(layers)} overlay{'s' if len(layers) != 1 else ''}") + F.run(F.build_overlay_pass(final, layers, "overlaid.mp4", options=options, + preview=preview, duration=timeline.total), + cwd=work_dir, timeout=F.shot_timeout(timeline.total)) + final = "overlaid.mp4" # ── the soundtrack ─────────────────────────────────────────────────────── - where = placement(units, timeline, [n.beats for n in narrated]) tracks = place_tracks(spec, media, where, warnings) if tracks: progress("Mixing", 90, f"{len(tracks)} audio track{'s' if len(tracks) != 1 else ''}") @@ -462,9 +481,35 @@ def argv(with_options: RenderOptions, with_path=path) -> List[str]: video=video, subtitles=subtitles, poster=poster, duration=F.probe_duration(video) or timeline.total, size_bytes=os.path.getsize(video), narration_chars=speech.characters, warnings=warnings, + description=description_text(spec, lines, chapters(units, timeline)) or None, ) +def _overlay_layers(timed: List[OverlayPlacement], work_dir: str, width: int, + height: int) -> List[F.TimedLayer]: + """Each placed overlay drawn to its own PNG.""" + layers: List[F.TimedLayer] = [] + for position, item in enumerate(timed): + overlay = item.overlay + if overlay.type == "text": + image = compose.text_layer(width, height, text=overlay.text, + position=overlay.position, size_pct=overlay.size_pct, + color=overlay.color) + elif overlay.type == "lower_third": + image = compose.lower_third_layer(width, height, title=overlay.title, + subtitle=overlay.subtitle or "") + else: + image = compose.image_layer(width, height, picture=item.media.path, + position=overlay.position, scale_pct=overlay.scale_pct, + opacity=overlay.opacity) + if image is None: + continue + name = f"timed_{position:03d}.png" + image.save(os.path.join(work_dir, name), "PNG") + layers.append(F.TimedLayer(path=name, start=item.start, end=item.end, fade=item.fade)) + return layers + + def layout_without_overlaps(timeline): """The same units laid end to end, for when blends had to become cuts.""" starts, cursor = [], 0.0 diff --git a/backend/app/render/worker.py b/backend/app/render/worker.py index 95e580c..d0e7075 100644 --- a/backend/app/render/worker.py +++ b/backend/app/render/worker.py @@ -93,7 +93,7 @@ def rel(path: Optional[str]) -> Optional[str]: video_path=rel(result.video), subtitle_path=rel(result.subtitles), poster_path=rel(result.poster), duration=result.duration, size_bytes=result.size_bytes, narration_chars=result.narration_chars, - warnings=json.dumps(result.warnings)) + warnings=json.dumps(result.warnings), description=result.description) finally: gam.close() speech.close() diff --git a/backend/app/routers/renders.py b/backend/app/routers/renders.py index dfcbe48..0eef6cc 100644 --- a/backend/app/routers/renders.py +++ b/backend/app/routers/renders.py @@ -9,7 +9,9 @@ from sqlmodel import Session, select from app.auth import CallerToken, CurrentUser, sign_media_key, verify_media_signature +from app.config import settings from app.database import get_session +from app.gam.client import GamClient, GamError from app.jobs.runner import ACTIVE_STATUSES from app.media.ranged import ranged_response from app.models.render import RenderJob @@ -40,7 +42,10 @@ def _read(job: RenderJob) -> RenderRead: error=job.error_message, warnings=json.loads(job.warnings or "[]"), duration=job.duration, size_bytes=job.size_bytes, narration_chars=job.narration_chars, video_url=_signed(job, job.video_path), subtitles_url=_signed(job, job.subtitle_path), - poster_url=_signed(job, job.poster_path), + poster_url=_signed(job, job.poster_path), description=job.description, + gam_asset_id=job.gam_asset_id, + gam_asset_url=(f"{settings.gam_base_url.rstrip('/')}/a/{job.gam_asset_id}" + if job.gam_asset_id else None), created_at=job.created_at, updated_at=job.updated_at, ) @@ -115,6 +120,64 @@ def cancel_render(render_id: str, user: CurrentUser, return DataResponse(data=_read(job)) +# Generous: the file is on local disk, but a long full-quality render is gigabytes, +# and GAM probes and thumbnails it before it answers. +SAVE_TIMEOUT_SECONDS = 900 + + +@router.post("/renders/{render_id}/save-to-gam", response_model=DataResponse[RenderRead]) +def save_to_gam(render_id: str, user: CurrentUser, token: CallerToken, + session: Session = Depends(get_session)) -> DataResponse[RenderRead]: + """File a finished render in the user's GAM library (GAM-3). + + Labelled `gvc_export` with the project and render it came from, named after the + project, and described with its chapters and credits. Saving twice files it once; + if it was deleted in GAM since, it is filed again. + """ + job = _owned(session, user, render_id) + if job.preview: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, + detail={"code": "preview_not_saveable", + "message": "A preview is a small working copy. Render the " + "full video to save it to GAM."}) + path = worker.renders_dir() / job.id / (job.video_path or "") + if job.status != "done" or not job.video_path or not path.is_file(): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, + detail={"code": "render_not_done", + "message": "This render has no finished video to save"}) + project = project_service.get_owned(session, user, job.project_id) + + gam = GamClient(token) + try: + if job.gam_asset_id and gam.get_asset(job.gam_asset_id) is not None: + return DataResponse(data=_read(job)) + title = (project.title or "").strip() or "Untitled video" + with open(path, "rb") as stream: + asset = gam.upload( + f"{title}.mp4", stream, "video/mp4", + fields={"source": "gvc_export", "origin_project_id": project.id, + "origin_reference_id": job.id}, + timeout=SAVE_TIMEOUT_SECONDS, + ) + changes = {"name": f"{title} (v{job.version_seq})"[:255]} + if job.description: + changes["description"] = job.description[:20_000] + # Best effort: the file is safely filed even if naming it fails, and saving + # again would only find it. + try: + gam.update(asset.id, changes) + except GamError: + pass + finally: + gam.close() + + job.gam_asset_id = asset.id + session.add(job) + session.commit() + session.refresh(job) + return DataResponse(data=_read(job)) + + @router.delete("/renders/{render_id}", status_code=status.HTTP_204_NO_CONTENT) def delete_render(render_id: str, user: CurrentUser, session: Session = Depends(get_session)) -> Response: diff --git a/backend/app/schemas_renders.py b/backend/app/schemas_renders.py index 26c2ecc..c74d8e4 100644 --- a/backend/app/schemas_renders.py +++ b/backend/app/schemas_renders.py @@ -30,6 +30,11 @@ class RenderRead(BaseModel): video_url: Optional[str] = None subtitles_url: Optional[str] = None poster_url: Optional[str] = None + # Chapters and credits to paste where the video is published. + description: Optional[str] = None + # Set once the render is saved to GAM: the asset, and its page there. + gam_asset_id: Optional[str] = None + gam_asset_url: Optional[str] = None created_at: datetime updated_at: datetime diff --git a/backend/tests/fake_siblings.py b/backend/tests/fake_siblings.py index 320080f..f725539 100644 --- a/backend/tests/fake_siblings.py +++ b/backend/tests/fake_siblings.py @@ -96,14 +96,33 @@ def _gam(self, request: httpx.Request) -> httpx.Response: new_id = f"gam-{len(self.uploads) + 1}" asset = gam_asset(new_id, filename, "image") self.assets[new_id] = asset - self.uploads.append({"filename": filename, "size": len(body)}) + self.uploads.append({"filename": filename, "size": len(body), + "fields": _form_fields(body), + "authorization": request.headers.get("authorization")}) return httpx.Response(201, json={"created": [asset], "rejected": []}) + if path.startswith("/api/assets/") and request.method == "PATCH": + asset = self.assets.get(path.rsplit("/", 1)[-1]) + if asset is None: + return httpx.Response(404, json={"detail": {"code": "not_found", "message": "gone"}}) + asset.update(json.loads(request.content)) + return httpx.Response(200, json={"data": asset}) return httpx.Response(404) def gam_requests(self) -> List[httpx.Request]: return [r for r in self.requests if str(r.url).startswith(GAM)] +def _form_fields(body: bytes) -> Dict[str, str]: + """The plain (non-file) fields of a multipart body.""" + fields: Dict[str, str] = {} + for part in body.split(b"\r\n--"): + head, _, value = part.partition(b"\r\n\r\n") + if b'name="' in head and b"filename=" not in head: + name = head.split(b'name="', 1)[1].split(b'"', 1)[0].decode() + fields[name] = value.rsplit(b"\r\n", 1)[0].decode() + return fields + + def hit(asset: dict, *, start=None, end=None, snippet="") -> dict: data = {"asset": asset, "score": 0.03, "snippet": snippet, "start_time": start, "segment_id": "seg" if start is not None else None, "sources": ["keyword"], diff --git a/backend/tests/test_render_compile.py b/backend/tests/test_render_compile.py index 4b44ca1..3f71ffd 100644 --- a/backend/tests/test_render_compile.py +++ b/backend/tests/test_render_compile.py @@ -45,8 +45,8 @@ def test_a_scene_is_cut_at_its_quote_and_cards_and_clips_are_one_unit_each(): ("s_ruins", "still", ["b1", "b2"], None), ("s_outro", "video_sound", [], None), ] - # The lower third is M2b, and says so rather than vanishing. - assert any("lower_third" in w for w in warnings) + # The lower third doesn't cut the scene: it is drawn over the stitched video. + assert warnings == [] def test_defaults_cascade_field_by_field(): @@ -207,3 +207,119 @@ def test_pauses_after_beats(): assert beat_pause_ms(spec, beats, 0) == 800 # a heading assert beat_pause_ms(spec, beats, 1) == 350 # between paragraphs assert beat_pause_ms(spec, beats, 2) == 600 # the end of the scene + + +# ── M2b: timed overlays and credits ───────────────────────────────────────── + +from app.render.compile import CREDIT_LINES_PER_CARD, credits_units, place_overlays # noqa: E402 +from app.render.credits import credit_lines, description_text # noqa: E402 +from app.render import ffmpeg as F # noqa: E402 +from app.render.options import RenderOptions # noqa: E402 +from app.spec.model import AnchorRef, Span, TextOverlay # noqa: E402 + + +def _with_overlays(*overlays): + spec, media, units, timeline, where = _placed() + market = next(s for s in spec.segments if s.id == "s_market") + market.overlays = list(overlays) + return spec, media, where + + +def test_an_overlay_with_no_span_covers_its_scene_with_a_default_fade(): + spec, media, where = _with_overlays(TextOverlay(id="o", type="text", text="Hi")) + [placed] = [p for p in place_overlays(spec, media, where, []) if p.overlay.id == "o"] + assert (placed.start, placed.end) == where.segments["s_market"] + assert placed.fade == 0.3 + + +def test_overlay_anchors_default_to_their_own_scene_and_may_reach_into_the_next(): + span = Span.model_validate({"from": {"edge": "start", "offset": 0.5}, + "to": {"segment": "s_ruins", "edge": "start", "offset": 1.0}}) + beat = Span.model_validate({"from": {"beat": "b1", "edge": "start"}, + "to": {"beat": "b1", "edge": "end"}}) + spec, media, where = _with_overlays( + TextOverlay(id="o1", type="text", text="Hi", show=span), + TextOverlay(id="o2", type="text", text="Hi", show=beat, fade=5), + ) + placed = {p.overlay.id: p for p in place_overlays(spec, media, where, [])} + assert placed["o1"].start == pytest.approx(where.segments["s_market"][0] + 0.5) + assert placed["o1"].end == pytest.approx(where.segments["s_ruins"][0] + 1.0) + assert (placed["o2"].start, placed["o2"].end) == where.beats[("s_market", "b1")] + # A fade longer than half the window is held to half, so in and out never cross. + assert placed["o2"].fade == pytest.approx(2.0) + + +def test_an_overlay_that_cannot_be_placed_is_dropped_with_a_warning(): + orphan = Span.model_validate({"from": {"edge": "start"}, + "to": {"segment": "gone", "edge": "end"}}) + backwards = Span.model_validate({"from": {"edge": "end"}, "to": {"edge": "start"}}) + spec, media, where = _with_overlays( + TextOverlay(id="o1", type="text", text="a", show=orphan), + TextOverlay(id="o2", type="text", text="b", show=backwards), + ) + warnings = [] + placed = place_overlays(spec, media, where, warnings) + assert not [p for p in placed if p.overlay.id in ("o1", "o2")] + assert any("'gone'" in w for w in warnings) + assert any("ends before it starts" in w for w in warnings) + + +def test_an_image_overlay_needs_a_picture(): + from app.spec.model import ImageOverlay + spec, media, where = _with_overlays(ImageOverlay(id="o", type="image", asset="a_theme")) + warnings = [] + assert not [p for p in place_overlays(spec, media, where, warnings) if p.overlay.id == "o"] + assert any("needs a picture" in w for w in warnings) + + +def test_credit_lines_follow_first_use_and_are_said_once(): + spec, _ = salt_road() + assert credit_lines(spec, {}) == [] # no `credits`: none + spec = spec.model_copy(update={"credits": None}) + from app.spec.model import Credits + spec.credits = Credits(extra=[" Narrated by Rex "]) + keys = [s.visual.asset for s in spec.segments if s.visual] + credits = {keys[0]: "Footage: A", keys[1]: "Photo: B", keys[2]: "Footage: A"} + assert credit_lines(spec, credits) == ["Footage: A", "Photo: B", "Narrated by Rex"] + spec.credits = Credits(auto=False, extra=["Narrated by Rex"]) + assert credit_lines(spec, credits) == ["Narrated by Rex"] + + +def test_long_credits_run_over_several_cards_and_none_without_lines(): + spec, _ = salt_road() + from app.spec.model import Credits + spec.credits = Credits() + assert credits_units(spec, []) == [] + lines = [f"Line {i}" for i in range(CREDIT_LINES_PER_CARD + 3)] + cards = credits_units(spec, lines) + assert [len(u.credits) for u in cards] == [CREDIT_LINES_PER_CARD, 3] + assert all(u.kind == "card" and not u.chrome for u in cards) + spec.credits = Credits(card=False) + assert credits_units(spec, lines) == [] + + +def test_the_description_lists_chapters_from_zero_then_credits(): + spec, _ = salt_road() + from app.spec.model import Credits + spec.credits = Credits() + spec.output.chapters = True + text = description_text(spec, ["Photo: B"], [(4.0, 70.0, "Market"), (70.0, 200.0, "Ruins")]) + assert text == f"0:00 {spec.title}\n0:04 Market\n1:10 Ruins\n\nCredits\nPhoto: B" + spec.credits = Credits(description=False) + spec.output.chapters = False + assert description_text(spec, ["Photo: B"], [(0.0, 10.0, "x")]) == "" + + +def test_the_overlay_pass_draws_each_layer_only_in_its_window_and_copies_the_sound(): + argv = F.build_overlay_pass( + "stitched.mp4", + [F.TimedLayer("a.png", 1.0, 4.5, 0.3), F.TimedLayer("b.png", 2.0, 3.0, 0.0)], + "out.mp4", options=RenderOptions(), preview=True, duration=10.0) + graph = argv[argv.index("-filter_complex") + 1] + assert "fade=t=in:st=1.000:d=0.300:alpha=1" in graph + assert "fade=t=out:st=4.200:d=0.300:alpha=1" in graph + assert "enable='between(t,1.000,4.500)'" in graph + assert "enable='between(t,2.000,3.000)'" in graph + assert "[l2]" in graph and "fade" not in graph.split("[1:v]")[0] + assert argv[argv.index("-c:a") + 1] == "copy" + assert argv.count("-c:a") == 1 and argv[argv.index("-map") + 1] == "[v2]" diff --git a/backend/tests/test_render_integration.py b/backend/tests/test_render_integration.py index 80fc6a3..df01245 100644 --- a/backend/tests/test_render_integration.py +++ b/backend/tests/test_render_integration.py @@ -138,3 +138,90 @@ def test_a_notes_too_old_to_pin_the_voice_stops_the_render(tmp_path, library): with pytest.raises(SpeechError) as stopped: _render(tmp_path, SPEC) assert "gecko-notes#211" in str(stopped.value) + + +# ── M2b: timed overlays and credits ───────────────────────────────────────── + +OVERLAID = { + "title": "Overlays", + "output": {"chapters": True, "subtitles": "off"}, + "style": {"background": {"type": "solid", "colors": ["#000000"]}, + "transition": {"style": "none", "duration": 0.5}, "cards": {"seconds": 2.0}}, + "segments": [ + {"id": "s_one", "kind": "scene", "chapter": "One", "duration": {"fixed": 4.0}, + "overlays": [ + # Held across the dissolve into s_two: only the overlay pass can draw that. + {"id": "o_text", "type": "text", "text": "ACROSS", "size_pct": 20, + "fade": 0, + "show": {"from": {"edge": "start", "offset": 1.0}, + "to": {"segment": "s_two", "edge": "start", "offset": 1.5}}}, + {"id": "o_name", "type": "lower_third", "title": "Ada Lovelace", + "subtitle": "Mathematician", "fade": 0, + "show": {"from": {"edge": "start", "offset": 0.5}, + "to": {"edge": "start", "offset": 2.0}}}, + ]}, + {"id": "s_two", "kind": "scene", "chapter": "Two", "duration": {"fixed": 4.0}, + "transition_in": {"style": "dissolve", "duration": 1.0}}, + ], + "credits": {"auto": True, "card": True, "description": True, + "extra": ["Made with Gecko Video Creator"]}, +} + + +def _luma(video, at, box): + """Mean brightness (0-255) of a box (x0, y0, x1, y1, as fractions) at a time.""" + raw = subprocess.run( + ["ffmpeg", "-v", "error", "-ss", f"{at:.2f}", "-i", video, "-frames:v", "1", + "-f", "rawvideo", "-pix_fmt", "gray", "-"], capture_output=True, check=True).stdout + width, height = 854, 480 + x0, y0, x1, y1 = (int(box[0] * width), int(box[1] * height), + int(box[2] * width), int(box[3] * height)) + total = count = 0 + for y in range(y0, y1): + row = raw[y * width:(y + 1) * width] + total += sum(row[x0:x1]) + count += x1 - x0 + return total / max(1, count) + + +CENTRE = (0.2, 0.4, 0.8, 0.6) +LOWER_LEFT = (0.03, 0.72, 0.4, 0.92) + + +def test_timed_overlays_show_only_between_their_anchors(tmp_path, library): + result, _ = _render(tmp_path, OVERLAID) + # s_one 0-4, s_two from 3 (the 1 s dissolve), then the 2 s credits card. + assert result.duration == pytest.approx(4.0 + 4.0 - 1.0 + 2.0, abs=0.25) + assert result.warnings == [] + + # The text: on from 1.0 until 1.5 s into s_two (3.0 + 1.5 = 4.5), across the join. + assert _luma(result.video, 0.5, CENTRE) < 5 + assert _luma(result.video, 2.0, CENTRE) > 20 + assert _luma(result.video, 3.5, CENTRE) > 20 # inside the dissolve + assert _luma(result.video, 4.2, CENTRE) > 20 # into s_two + assert _luma(result.video, 5.5, CENTRE) < 5 + + # The lower third: 0.5 to 2.0 s, bottom left. + assert _luma(result.video, 0.2, LOWER_LEFT) < 5 + assert _luma(result.video, 1.2, LOWER_LEFT) > 15 + assert _luma(result.video, 3.0, LOWER_LEFT) < 5 + + +def test_credits_become_an_end_card_and_a_description(tmp_path, library): + data = json.loads(json.dumps(SPEC)) + library.assets["p"]["credit"] = "Photo: Jane Doe / CC BY 4.0" + library.assets["r"]["credit"] = "Footage: River Archive" + library.assets["a"]["credit"] = "Photo: Jane Doe / CC BY 4.0" # said once + data["credits"] = {"extra": ["Narrated by Rex"]} + plain, _ = _render(tmp_path / "plain", SPEC) + result, _ = _render(tmp_path / "credited", data) + + # The card joins with the project's fade, so it adds its own length and no overlap. + card = parse_spec(data)[0].style.cards.seconds + assert result.duration == pytest.approx(plain.duration + card, abs=0.3) + chapters, credits = result.description.split("\n\n") + assert [line.split(" ", 1)[1] for line in chapters.split("\n")] == [ + "Start", "The valley", "The river"] + assert chapters.startswith("0:00 ") + assert credits == ("Credits\nPhoto: Jane Doe / CC BY 4.0\nFootage: River Archive\n" + "Narrated by Rex") diff --git a/backend/tests/test_renders_api.py b/backend/tests/test_renders_api.py index 4f57f20..f4aa93e 100644 --- a/backend/tests/test_renders_api.py +++ b/backend/tests/test_renders_api.py @@ -151,3 +151,77 @@ def test_another_users_render_does_not_exist_for_you(auth_client, switch_user): ("get", f"/api/projects/{project['id']}/renders")): kwargs = {"json": {"preview": True}} if url.endswith("/renders") and method == "post" else {} assert getattr(auth_client, method)(url, **kwargs).status_code == 404, (method, url) + + +# ── Save to GAM (M2b, GAM-3) ──────────────────────────────────────────────── + +def _finished(auth_client, session, *, preview=False, description="0:00 Start"): + """A render marked done with a small file in place, without running ffmpeg.""" + from app.models.render import RenderJob + + project = _project(auth_client) + job = _start(auth_client, project["id"], preview=preview).json()["data"] + row = session.get(RenderJob, job["id"]) + row.status, row.video_path, row.description = "done", "video.mp4", description + session.add(row) + session.commit() + folder = worker.renders_dir() / job["id"] + folder.mkdir(parents=True, exist_ok=True) + (folder / "video.mp4").write_bytes(b"\x00" * 2048) + return project, job + + +def test_a_render_is_saved_to_gam_with_its_origin_as_the_requester(auth_client, session, siblings): + project, job = _finished(auth_client, session) + auth = {"Authorization": "Bearer the-users-token"} + + saved = auth_client.post(f"/api/renders/{job['id']}/save-to-gam", headers=auth) + assert saved.status_code == 200, saved.text + body = saved.json()["data"] + assert body["gam_asset_id"] == "gam-1" + assert body["gam_asset_url"] == f"{settings.gam_base_url}/a/gam-1" + + upload = siblings.uploads[0] + assert upload["authorization"] == "Bearer the-users-token" + assert upload["fields"] == {"source": "gvc_export", "origin_project_id": project["id"], + "origin_reference_id": job["id"]} + # Named after the project and version, described with the chapters and credits. + assert siblings.assets["gam-1"]["name"] == f"{project['title'] or 'Untitled video'} (v1)" + assert siblings.assets["gam-1"]["description"] == "0:00 Start" + + # Saving again files it once. + again = auth_client.post(f"/api/renders/{job['id']}/save-to-gam", headers=auth) + assert again.json()["data"]["gam_asset_id"] == "gam-1" + assert len(siblings.uploads) == 1 + listed = auth_client.get(f"/api/projects/{project['id']}/renders").json()["data"] + assert listed[0]["gam_asset_id"] == "gam-1" + + +def test_a_render_deleted_from_gam_is_filed_again(auth_client, session, siblings): + _, job = _finished(auth_client, session) + auth_client.post(f"/api/renders/{job['id']}/save-to-gam") + del siblings.assets["gam-1"] + again = auth_client.post(f"/api/renders/{job['id']}/save-to-gam").json()["data"] + assert again["gam_asset_id"] == "gam-2" + assert len(siblings.uploads) == 2 + + +def test_a_preview_or_an_unfinished_render_cannot_be_saved(auth_client, session, siblings): + _, preview = _finished(auth_client, session, preview=True) + refused = auth_client.post(f"/api/renders/{preview['id']}/save-to-gam") + assert refused.status_code == 400 + assert refused.json()["detail"]["code"] == "preview_not_saveable" + + project = _project(auth_client) + queued = _start(auth_client, project["id"], preview=False).json()["data"] + refused = auth_client.post(f"/api/renders/{queued['id']}/save-to-gam") + assert refused.status_code == 409 + assert refused.json()["detail"]["code"] == "render_not_done" + assert siblings.uploads == [] + + +def test_another_users_render_cannot_be_saved(auth_client, session, siblings, switch_user): + _, job = _finished(auth_client, session) + switch_user("someone-else") + assert auth_client.post(f"/api/renders/{job['id']}/save-to-gam").status_code == 404 + assert siblings.uploads == [] diff --git a/docs/plan-of-attack.md b/docs/plan-of-attack.md index 6b3f8bb..c91fae7 100644 --- a/docs/plan-of-attack.md +++ b/docs/plan-of-attack.md @@ -264,7 +264,7 @@ Specified here, applied in GAM's repository, not from this one. |---|---|---| | GAM-1 | `end_time` on `SearchHit`, from the transcript segment. **Done: [davior/gam#36](https://github.com/davior/gam/pull/36).** GVC also works without it, sizing a moment as start + 5 s | S3: a moment needs an out point | | GAM-2 | Run M5's acceptance on a real library; document that semantic search needs an embedding provider | "Find where people talk about *X*" is the feature most of this goal rests on, and it has never been checked end to end | -| GAM-3 | `origin_project_id` / `origin_reference_id` on `Asset` (migration) + upload with `source: "gvc_export"` | Filing renders back into the library | +| GAM-3 | `origin_project_id` / `origin_reference_id` on `Asset` (migration) + upload with `source: "gvc_export"`. **[davior/gam#37](https://github.com/davior/gam/pull/37).** Against a GAM without it, a save lands as an ordinary upload | Filing renders back into the library | | GAM-4 | M9: documented read API, `/picker?embed=1` with `postMessage({ assetId, inPoint, outPoint })` | Picking assets and moments by hand in the editor | | GAM-5 | `CORS_ORIGIN` / CSP for `gvc.geckopico.com`, only where GVC's browser code calls GAM directly | Server-side calls with a Bearer token need neither | @@ -287,7 +287,7 @@ Each milestone ends with something you can use, not just with code merged. |---|---|---|---| | **M0** | **Genesis** | Skeleton from GAM: `config.py`, `clock.py`, Alembic, shadow-user auth on the Notes JWT, `/api/health`, Dockerfiles (backend with ffmpeg), compose, nginx, CI. `Project` (current spec as JSON) and `ProjectVersion` (snapshot, author, message). Spec Pydantic model copied from Notes with S1–S3 applied, `0.1 → 0.2` migration on load. Projects API: blank project (title card + one empty scene), paste a spec, `POST /api/projects/import` for a Notes push, versions and revert, validation with errors by path. React shell: project list, a plain scene list (add, reorder, delete, edit narration), a JSON tab with live validation, version history. | You can sign in with your Notes account, make a video's structure by hand, paste the Notes spec doc's §6 example and see it validate (or see exactly why not), and import a spec Notes produced | | **M1** | **GAM as the library** | GAM-1 applied (gam#36). GVC's server calls GAM as the signed-in user by forwarding their own Notes token, so GAM's ownership checks hold and no service account exists. Search GAM from a scene: results with transcript snippets and the moment's time; picking a hit places it as muted b-roll or as a clip with its own sound, trimmed to what was said (padded, and sized by a placeholder's `moment`). `search` placeholders resolved the same way. Thumbnails and names looked up per page load, since GAM's signed URLs expire, and loaded straight from GAM (CSP `img-src`/`media-src`). A pushed note's media copied into GAM (D3). The byte cache keyed on asset id and checksum moved to M2: nothing reads media bytes until the renderer does. | A scene's footage is found by what is said in it ("where people talk about *X*") and lands in the spec as a trimmed GAM clip | -| **M2** | **Native renderer** | **Built (M2a).** Engine copied from Notes (`app/render/`: ffmpeg, compose, narration, pause markup, shot cache, with Notes' 359 tests). `compile.py` is the §5.8 compile step, done natively: scenes cut at quote/code beats, defaults cascading field by field, per-join transitions (blends overlap, dips draw in-shot, cuts cut), anchors resolved to seconds. Ken Burns paths between two frames (S1), trimmed and held clips, N audio tracks with fades, crossfades and ducking. Media fetched from GAM into a cache keyed on GAM id and size; placeholders drawn as slates in a preview, refusing a full render. Narration read per beat through Notes (GN-V4), so beat anchors are exact. Render jobs on GAM's runner; the requester's token held in memory only, so a restart ends a render rather than storing a credential. Finished files at signed, ranged URLs. Editor: per-scene transition and Ken Burns (with a frame editor), a Soundtrack tab, a Render tab with estimate, progress and the result. Left for M2b: timed text/lower-third/image overlays, Save to GAM (GAM-3), credits. | A spec with two music tracks crossfading at a scene cut, a dissolve into one scene, and a Ken Burns path over a chosen part of a picture renders correctly | +| **M2** | **Native renderer** | **Built (M2a).** Engine copied from Notes (`app/render/`: ffmpeg, compose, narration, pause markup, shot cache, with Notes' 359 tests). `compile.py` is the §5.8 compile step, done natively: scenes cut at quote/code beats, defaults cascading field by field, per-join transitions (blends overlap, dips draw in-shot, cuts cut), anchors resolved to seconds. Ken Burns paths between two frames (S1), trimmed and held clips, N audio tracks with fades, crossfades and ducking. Media fetched from GAM into a cache keyed on GAM id and size; placeholders drawn as slates in a preview, refusing a full render. Narration read per beat through Notes (GN-V4), so beat anchors are exact. Render jobs on GAM's runner; the requester's token held in memory only, so a restart ends a render rather than storing a credential. Finished files at signed, ranged URLs. Editor: per-scene transition and Ken Burns (with a frame editor), a Soundtrack tab, a Render tab with estimate, progress and the result. **M2b:** timed text, lower thirds and pictures, drawn over the stitched video in one extra pass (`enable=` windows with alpha fades) so they can be held across a join, anchored to a scene's edges or lines; end credits from each GAM asset's credit plus extra lines, as cards and in a description (chapters + credits) to paste where the video is published; Save to GAM (GAM-3) for full renders, once per render. | A spec with two music tracks crossfading at a scene cut, a dissolve into one scene, and a Ken Burns path over a chosen part of a picture renders correctly | | **M3** | **Editor** | The M0 scene list grows into the storyboard: a card per segment (thumbnail, label, first line), lanes for audio tracks and overlays against the segments, a per-scene panel (visual, trim, transition, motion with a drag-a-box framing tool, voice), a project defaults panel. Sliders take their ranges from the schema (the Notes spec doc, open decision 12). Undo/redo over `ProjectVersion`. | You can build and fix a whole video by hand without opening the JSON tab | | **M4** | **AI drafting** | Providers copied from GAM's M6. "New video from a prompt": brief → spec (D2) with GAM search as a tool, and placeholders for gaps, with one repair round. Also from selected GAM assets. Cost estimate before running, usage recorded. | "Make a 3-minute video about *X* using clips where people talk about it" produces a spec that previews immediately | | **M5** | **AI editing** | The Notes spec doc §8.2 operations, addressed by id. Chat beside the storyboard. Suggestions staged and accepted one by one. Generate placeholders sent to GAM's M8. | "Change the music after the second chapter to something calmer, and dissolve into the last scene" becomes reviewed, applied edits | diff --git a/frontend/src/api/projects.ts b/frontend/src/api/projects.ts index ac6c69a..d5b1828 100644 --- a/frontend/src/api/projects.ts +++ b/frontend/src/api/projects.ts @@ -95,6 +95,50 @@ export type AssetRef = label?: string } +export type Position = + 'top-left' | 'top-right' | 'center' | 'bottom-left' | 'bottom-right' + +/** When an overlay shows. Absent on the overlay: its whole scene. */ +export interface Span { + from: Anchor + to: Anchor +} + +interface OverlayBase { + id: string + show?: Span + /** Seconds each way; absent: a short default. */ + fade?: number + [key: string]: unknown +} + +export type Overlay = + | (OverlayBase & { type: 'quote'; text: string; attribution?: string }) + | (OverlayBase & { type: 'code'; text: string; language?: string }) + | (OverlayBase & { + type: 'text' + text: string + position?: Position + size_pct?: number + color?: string + }) + | (OverlayBase & { type: 'lower_third'; title: string; subtitle?: string }) + | (OverlayBase & { + type: 'image' + asset: string + position?: Position + scale_pct?: number + opacity?: number + }) + +/** The end credits. Lines come from each GAM asset's credit, then `extra`. */ +export interface Credits { + auto?: boolean + card?: boolean + description?: boolean + extra?: string[] +} + export interface Segment { id: string kind: 'scene' | 'card' | 'clip' @@ -105,6 +149,7 @@ export interface Segment { visual?: Visual narration?: { beats: Beat[]; [key: string]: unknown } transition_in?: { style?: string; duration?: number } + overlays?: Overlay[] [key: string]: unknown } @@ -116,6 +161,7 @@ export interface VideoSpec { assets?: Record segments: Segment[] audio_tracks?: AudioTrack[] + credits?: Credits | null output?: { aspect?: '16:9' | '9:16' | '1:1'; [key: string]: unknown } style?: { transition?: { style?: string; duration?: number } diff --git a/frontend/src/api/renders.ts b/frontend/src/api/renders.ts index ac8b380..8d6ccce 100644 --- a/frontend/src/api/renders.ts +++ b/frontend/src/api/renders.ts @@ -23,6 +23,11 @@ export interface Render { video_url: string | null subtitles_url: string | null poster_url: string | null + /** Chapters and credits, to paste where the video is published. */ + description: string | null + /** Set once saved to GAM: the asset, and its page there. */ + gam_asset_id: string | null + gam_asset_url: string | null created_at: string updated_at: string } @@ -77,6 +82,12 @@ export const rendersApi = { .then((r) => r.data.data) }, + saveToGam(id: string): Promise { + return client + .post>(`/renders/${id}/save-to-gam`) + .then((r) => r.data.data) + }, + remove(id: string): Promise { return client.delete(`/renders/${id}`).then(() => undefined) }, diff --git a/frontend/src/components/AnchorSelect.tsx b/frontend/src/components/AnchorSelect.tsx new file mode 100644 index 0000000..f235b04 --- /dev/null +++ b/frontend/src/components/AnchorSelect.tsx @@ -0,0 +1,56 @@ +import type { Anchor } from '@/api/projects' +import { anchorFrom, anchorValue, offsetOf, type AnchorOption } from '@/spec/anchors' + +interface Props { + label: string + value: Anchor | undefined + options: AnchorOption[] + onPick: (next: Anchor | undefined) => void + /** The label of the empty choice; none offered when absent. */ + noneLabel?: string +} + +/** + * Where something starts or stops: a scene's (or a line's) start or end, with an + * offset in seconds. Anchored to the structure rather than to a time, so rewording a + * scene moves everything placed against it along with it. + */ +export default function AnchorSelect({ + label, + value, + options, + onPick, + noneLabel, +}: Props) { + return ( + + ) +} diff --git a/frontend/src/components/CreditsSettings.tsx b/frontend/src/components/CreditsSettings.tsx new file mode 100644 index 0000000..50477c4 --- /dev/null +++ b/frontend/src/components/CreditsSettings.tsx @@ -0,0 +1,80 @@ +import type { Credits, VideoSpec } from '@/api/projects' +import { setCredits } from '@/spec/overlays' + +interface Props { + spec: VideoSpec + onChange: (next: VideoSpec) => void +} + +const DEFAULTS: Credits = { auto: true, card: true, description: true, extra: [] } + +/** + * The end credits. The lines are each GAM asset's credit, worked out when the video + * renders (the library is where a credit is corrected, once, for every video using + * it), followed by any written here. + */ +export default function CreditsSettings({ spec, onChange }: Props) { + const credits = spec.credits ?? null + const on = credits !== null + const value = { ...DEFAULTS, ...(credits ?? {}) } + const set = (patch: Partial) => + onChange(setCredits(spec, { ...value, ...patch })) + + return ( +
+ + {on && ( +
+ + + +