diff --git a/backend/app/assistant/prompt.py b/backend/app/assistant/prompt.py index 09fa5a7..68fd3dc 100644 --- a/backend/app/assistant/prompt.py +++ b/backend/app/assistant/prompt.py @@ -42,13 +42,15 @@ - `card`: a title or chapter screen: `{"card":{"variant":"title"|"chapter","title":...,\ "subtitle":...}}`. May have narration too. - `clip`: a video played with its own sound and no narration: `visual` with `trim` and \ -`"audio":"play"`. +`"audio":"play"`. Its sound is matched to the narration's loudness automatically; \ +`visual.volume_db` (-30 to 20) moves it from there, only when asked. - Optional: `chapter` (a chapter mark at its start), `transition_in`, `duration` \ (`min` or `fixed` seconds), `overlays`, `role` (title / intro / outro). - `audio_tracks`: music, sound effects or extra voice, each from a `start` anchor to an \ `end` anchor (omit `end` to play once to its natural end), with `volume` (0-1; music \ usually 0.12-0.2 under narration), `fade_in` / `fade_out` seconds, `duck_under_narration`, \ -`loop`. +`loop`, and `during_clips`: omit it to treat clips like narration, `"mute"` to fade the \ +music out while a clip plays, `"play"` to keep it playing at its level. ## Time is anchored to structure, not seconds An anchor is `"video_start"`, `"video_end"`, or `{"segment":id,"beat":id?,"edge":"start"|\ diff --git a/backend/app/render/compile.py b/backend/app/render/compile.py index be8e5a8..542d9f9 100644 --- a/backend/app/render/compile.py +++ b/backend/app/render/compile.py @@ -68,6 +68,7 @@ class Unit: # Seconds into the file and how long, for a video; None plays it whole. trim: Optional[Tuple[float, float]] = None loop: bool = True + volume_db: float = 0.0 # a clip's own sound, on top of matching the narration panel: Optional[Panel] = None motion: Optional[Motion] = None # resolved, `alternate` already cycled fit: str = "blur" @@ -173,7 +174,8 @@ def plan_units(spec: VideoSpec, media: Dict[str, Media], warnings: List[str]) -> warnings.append(f"Clip {segment.label or segment.id} has no video to play; left out") continue units.append(Unit(**common, first_in_segment=True, kind="video_sound", media=found, - trim=trim, loop=False, chapter=segment.chapter)) + trim=trim, loop=False, chapter=segment.chapter, + volume_db=(visual.volume_db or 0.0) if visual else 0.0)) continue if found and found.kind == "video": @@ -399,6 +401,7 @@ class TrackPlacement: loop: bool duck: bool trim_in: float + during_clips: Optional[str] = None def place_tracks(spec: VideoSpec, media: Dict[str, Media], where: Placement, @@ -441,7 +444,7 @@ def place_tracks(spec: VideoSpec, media: Dict[str, Media], where: Placement, track=track, media=found, start=start, length=length, loop=loop, duck=(track.duck_under_narration if track.duck_under_narration is not None else track.kind == "music"), - trim_in=trim_in, + trim_in=trim_in, during_clips=track.during_clips, )) return placed diff --git a/backend/app/render/ffmpeg.py b/backend/app/render/ffmpeg.py index bf73988..d7390c6 100644 --- a/backend/app/render/ffmpeg.py +++ b/backend/app/render/ffmpeg.py @@ -164,6 +164,56 @@ def probe_has_audio(path: str) -> bool: return False +# Spoken word sits about here; used as the target when a video has no narration. +DEFAULT_LOUDNESS_LUFS = -16.0 +# A clip quieter than this is silence or room tone, and matching it to the voice +# would only raise the noise. +SILENT_LUFS = -60.0 +CLIP_GAIN_LIMITS_DB = (-30.0, 24.0) + + +def measure_loudness(path: str, length: Optional[float] = None) -> Optional[float]: + """Integrated loudness (EBU R128, LUFS) of a file's sound, or of its first + `length` seconds; None when it has none worth measuring.""" + argv = ["ffmpeg", "-hide_banner", "-nostats", "-i", path] + if length is not None: + argv += ["-t", f"{max(0.1, length):.3f}"] + argv += ["-vn", "-af", "ebur128=framelog=quiet", "-f", "null", "-"] + try: + out = subprocess.run(argv, capture_output=True, timeout=PROBE_TIMEOUT_SECONDS * 10) + except (OSError, subprocess.SubprocessError): + return None + # The summary comes last; its "I:" line is the integrated loudness. + found = None + for line in out.stderr.decode(errors="replace").splitlines(): + line = line.strip() + if line.startswith("I:") and line.endswith("LUFS"): + try: + found = float(line[2:-4]) + except ValueError: + found = None + if found is None or found < SILENT_LUFS: + return None + return found + + +def combined_loudness(levels: Sequence[Tuple[float, float]]) -> Optional[float]: + """The loudness of several pieces played one after another, from each one's + (LUFS, seconds): their energy averaged by time, as R128 would over the whole.""" + weighted = [(10 ** (lufs / 10), seconds) for lufs, seconds in levels if seconds > 0] + total = sum(seconds for _, seconds in weighted) + if not weighted or total <= 0: + return None + return 10 * math.log10(sum(energy * seconds for energy, seconds in weighted) / total) + + +def clip_gain(clip_lufs: Optional[float], target_lufs: float, adjust_db: float = 0.0) -> float: + """dB to bring a clip to the narration's loudness, then move it by `adjust_db`.""" + match = 0.0 if clip_lufs is None else target_lufs - clip_lufs + low, high = CLIP_GAIN_LIMITS_DB + return max(low, min(high, match + adjust_db)) + + def encode_args(options: RenderOptions, preview: bool) -> List[str]: """Codec flags shared by every shot. Identical output = concat can `-c copy`.""" preset, crf = encoder_tier(options.quality, preview) @@ -480,6 +530,7 @@ def build_shot_command( out_transition: Optional[TransitionSpec] = None, loop: bool = True, path: Optional["KenBurnsPath"] = None, + audio_gain_db: float = 0.0, ) -> List[str]: """Full argv for rendering one shot to its own MP4. @@ -495,6 +546,7 @@ def build_shot_command( - `loop` False on a video holds its last frame instead of repeating it. - `path`: a Ken Burns move between two frames of the picture (S1), in place of the preset drift. + - `audio_gain_db`: a clip's own sound brought to the narration's loudness (S5). """ width, height = frame_size(options.aspect, options.resolution, preview) argv: List[str] = ["ffmpeg", "-y", "-hide_banner", "-loglevel", "error"] @@ -611,7 +663,16 @@ def build_shot_command( # Pad the audio so a shot always reaches its target length even when the # narration is shorter (a media block with little text under it). - audio_chain = f"[{audio_main}]aresample=48000,aformat=channel_layouts=stereo,apad" + audio_chain = f"[{audio_main}]aresample=48000,aformat=channel_layouts=stereo" + if abs(audio_gain_db) >= 0.05: + audio_chain += f",volume={audio_gain_db:.1f}dB" + if audio_gain_db > 0: + # Raising a quiet clip can push its loudest moments past full scale, so + # they are limited at -1 dBFS rather than clipped. `level=0` stops the + # limiter normalising the whole clip again, and `latency` takes back + # the few milliseconds its lookahead would put between sound and picture. + audio_chain += ",alimiter=limit=0.891:level=0:latency=1" + audio_chain += ",apad" if dip > 0: # Only the tail is faded, to match the picture dipping to colour there — # and it lands in the trailing hold `shot_end_pause_ms` leaves after the @@ -996,7 +1057,7 @@ def build_extract_command( def build_stitch_command( shot_files: Sequence[str], durations: Sequence[float], output: str, *, joins: Sequence[Optional[Tuple[str, float]]], options: RenderOptions, preview: bool, - start_frame: int = 0, end_frame: Optional[int] = None, + start_frame: int = 0, end_frame: Optional[int] = None, streams: str = "both", ) -> List[str]: """`build_xfade_command` with a transition of its own at every join. @@ -1008,6 +1069,10 @@ def build_stitch_command( `start_frame` drops that many frames off the front of the first shot and `end_frame` stops the last one at that frame: one run of a chunked stitch (see `plan_stitch_chunks`), whose neighbours show the rest of the shot. + + `streams` is "video" or "audio" to build only that half: the partial stitch + (see `plan_partial_stitch`) encodes the picture only around each blend, and + the sound for the whole video in one pass of its own. """ fps = max(1, options.fps) durations = list(durations) @@ -1016,24 +1081,36 @@ def build_stitch_command( durations[last] = end_frame / fps if start_frame: durations[0] -= start_frame / fps + picture, sound = streams in ("both", "video"), streams in ("both", "audio") + # The picture alone can seek its first shot to `start_frame` rather than decode + # its way there, which for a blend piece is most of the shot. Half a frame early, + # so the frame-accurate seek keeps that frame however its time rounds. The sound + # can't: a seek lands on an AAC packet, not a sample. + seek = start_frame if streams == "video" else 0 argv = ["ffmpeg", "-y", "-hide_banner", "-loglevel", "error"] - for name in shot_files: + for index, name in enumerate(shot_files): # Every input is open, with its decoder, for the whole pass. A decoder's # default is a thread per core and a frame buffer per thread, which made # this pass's memory the shot count times the core count. - argv += ["-threads", "1", "-i", name] + argv += ["-threads", "1"] + ([] if sound else ["-an"]) + ([] if picture else ["-vn"]) + if index == 0 and seek: + argv += ["-ss", f"{(seek - 0.5) / fps:.6f}"] + argv += ["-i", name] # Every input on one timebase first. `concat` re-times its output to # AV_TIME_BASE (1/1000000) while the shots carry 1/90000, and `xfade` refuses two # inputs that disagree — so a cut followed by a blend failed to build at all. - chains: List[str] = [] + vchains: List[str] = [] + achains: List[str] = [] sounds: List[str] = [] for i in range(len(shot_files)): - lo = start_frame if i == 0 else 0 + lo = start_frame - seek if i == 0 else 0 hi = end_frame if i == last else None + if hi is not None and i == 0: + hi -= seek if not lo and hi is None: - chains.append(f"[{i}:v]settb=AVTB[s{i}]") + vchains.append(f"[{i}:v]settb=AVTB[s{i}]") sounds.append(f"{i}:a") continue # Trimmed by frame and by sample rather than by seconds, so the picture and @@ -1047,33 +1124,46 @@ def build_stitch_command( # ffmpeg 7 leaves a trimmed stream with no frame rate (1/0), and `xfade` # refuses one, so it is set back: the frames are already at it, so `fps` # neither drops nor repeats any. - chains.append(f"[{i}:v]trim={':'.join(bounds)},setpts=PTS-STARTPTS," - f"fps={fps},settb=AVTB[s{i}]") - chains.append(f"[{i}:a]atrim={':'.join(samples)},asetpts=PTS-STARTPTS[st{i}]") + vchains.append(f"[{i}:v]trim={':'.join(bounds)},setpts=PTS-STARTPTS," + f"fps={fps},settb=AVTB[s{i}]") + achains.append(f"[{i}:a]atrim={':'.join(samples)},asetpts=PTS-STARTPTS[st{i}]") sounds.append(f"st{i}") video, audio = "s0", sounds[0] total = durations[0] for index in range(1, len(shot_files)): join = joins[index] if index < len(joins) else None if join is None: - chains.append(f"[{video}][s{index}]concat=n=2:v=1:a=0[vx{index}]") - chains.append(f"[{audio}][{sounds[index]}]concat=n=2:v=0:a=1[ax{index}]") + vchains.append(f"[{video}][s{index}]concat=n=2:v=1:a=0[vx{index}]") + achains.append(f"[{audio}][{sounds[index]}]concat=n=2:v=0:a=1[ax{index}]") total += durations[index] else: name, overlap = join offset = max(0.0, total - overlap) - chains.append( + vchains.append( f"[{video}][s{index}]xfade=transition={name}:" f"duration={overlap:.3f}:offset={offset:.3f}[vx{index}]" ) - chains.append(f"[{audio}]atrim=end={offset:.3f},asetpts=PTS-STARTPTS[at{index}]") - chains.append(f"[at{index}][{sounds[index]}]concat=n=2:v=0:a=1[ax{index}]") + achains.append(f"[{audio}]atrim=end={offset:.3f},asetpts=PTS-STARTPTS[at{index}]") + achains.append(f"[at{index}][{sounds[index]}]concat=n=2:v=0:a=1[ax{index}]") total += durations[index] - overlap video, audio = f"vx{index}", f"ax{index}" + vchains.append(f"[{video}]setsar=1,format=yuv420p[v]") - chains.append(f"[{video}]setsar=1,format=yuv420p[v]") - argv += ["-filter_complex", ";".join(chains), "-map", "[v]", "-map", f"[{audio}]"] - argv += encode_args(options, preview) + shared = encode_args(options, preview) + split = shared.index("-c:a") + if not sound: + argv += ["-filter_complex", ";".join(vchains), "-map", "[v]", *shared[:split], "-an"] + elif not picture: + # A lone input's sound needs no graph at all, and a graph with nothing in it + # is an error, so the one-shot case maps the track straight through. + if achains: + argv += ["-filter_complex", ";".join(achains), "-map", f"[{audio}]"] + else: + argv += ["-map", f"{audio}"] + argv += [*shared[split:], "-vn"] + else: + argv += ["-filter_complex", ";".join(vchains + achains), + "-map", "[v]", "-map", f"[{audio}]", *shared] argv += ["-movflags", "+faststart", output] return argv @@ -1147,6 +1237,144 @@ def plan_stitch_chunks( return chunks +@dataclass(frozen=True) +class StitchPiece: + """One stretch of a partial stitch's picture, in output order. + + `blend` False: frames `start_frame`..`end_frame` of shot `first` (== `last`), + copied as they were encoded. True: shots `first`..`last` encoded together, from + `start_frame` of the first to `end_frame` of the last, with their blends. An + `end_frame` of None runs to the end of the shot. + """ + + blend: bool + first: int + last: int + start_frame: int = 0 + end_frame: Optional[int] = None + + +def probe_keyframes(path: str, fps: int) -> Tuple[List[int], int]: + """Frame numbers of a video's keyframes, and how many frames it has. + + Read from the packets, not the frames, so nothing is decoded and a long shot + probes in a moment. + """ + out = subprocess.run( + ["ffprobe", "-v", "error", "-select_streams", "v:0", + "-show_entries", "packet=pts_time,flags", "-of", "csv=p=0", path], + capture_output=True, timeout=PROBE_TIMEOUT_SECONDS, + ) + if out.returncode != 0: + raise FFmpegError(f"ffprobe could not read the keyframes of {path}") + frames = set() + count = 0 + for line in out.stdout.decode(errors="replace").splitlines(): + when, _, flags = line.partition(",") + count += 1 + if "K" in flags: + try: + frames.add(round(float(when) * fps)) + except ValueError: + continue + return sorted(frames), count + + +def plan_partial_stitch( + durations: Sequence[float], joins: Sequence[Optional[Tuple[str, float]]], + keyframes: Sequence[Sequence[int]], fps: int, +) -> List[StitchPiece]: + """Lay a blended stitch out as copied stretches and the blends between them. + + A blend needs both shots decoded and encoded again, but only where they + overlap. Every shot was encoded with `encode_args`, so between two of its + keyframes it can be copied as it is and remuxed beside freshly encoded frames. + Each blend is therefore re-encoded from the last keyframe of the outgoing shot + before the blend begins to the first keyframe of the incoming one after it + ends: a second or two a join rather than the whole video, which at 1080p on + the `medium` preset is minutes against seconds. + + A shot with no keyframe clear of the blends at both ends (a short one, blended + in and out) has nothing to copy, so the blend before it and the one after it + become one encode through it. + """ + count = len(durations) + fps = max(1, fps) + + def overlap(index: int) -> float: + return _join_overlap(joins, index) if 0 < index < count else 0.0 + + pieces: List[StitchPiece] = [] + group: Optional[Tuple[int, int]] = None # an open blend: (first shot, start frame) + for i in range(count): + frames_in, frames_out = overlap(i), overlap(i + 1) + keys = keyframes[i] if i < len(keyframes) else [0] + if frames_in > 0: + after = [k for k in keys if k >= math.ceil(frames_in * fps)] + lo: Optional[int] = after[0] if after else None + else: + lo = 0 + hi: Optional[int] = None + if frames_out > 0: + before = [k for k in keys if k <= math.floor((durations[i] - frames_out) * fps)] + hi = before[-1] if before else 0 + body = lo is not None and (hi is None or hi > lo) + if group is not None: + if not body: + if frames_out <= 0: # the blend ends inside this shot, and so does it + pieces.append(StitchPiece(True, group[0], i, group[1], None)) + group = None + continue + pieces.append(StitchPiece(True, group[0], i, group[1], lo)) + group = None + start = lo or 0 + if hi is None: + pieces.append(StitchPiece(False, i, i, start, None)) + else: + if hi > start: + pieces.append(StitchPiece(False, i, i, start, hi)) + group = (i, hi) + if group is not None: + pieces.append(StitchPiece(True, group[0], count - 1, group[1], None)) + return pieces + + +def build_cut_command(source: str, output_pattern: str, at_frames: Sequence[int], + fps: int) -> List[str]: + """Split a shot's picture at keyframes, copied: `output_pattern` takes a %d. + + The segment muxer cuts at the first keyframe at or after each time, so each is + asked for half a frame early: the cut lands on that keyframe however its time + rounds. Audio is left out; the partial stitch encodes the sound separately. + """ + times = ",".join(f"{(frame - 0.5) / max(1, fps):.6f}" for frame in at_frames) + return [ + "ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-i", source, + "-map", "0:v", "-c", "copy", "-an", "-f", "segment", "-segment_format", "mp4", + # The timescale every encode writes (see `encode_args`). The segment muxer + # would otherwise pick 1/15360, and ffmpeg 6's concat, given pieces on two + # timebases, laid them out a minute apart. + "-segment_format_options", "video_track_timescale=90000", + "-segment_times", times, "-reset_timestamps", "1", output_pattern, + ] + + +def build_video_copy_command(source: str, output: str) -> List[str]: + """A shot's picture alone, copied: every file the concat joins has the same + streams, and the partial stitch's pieces carry no sound.""" + return ["ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-i", source, + "-map", "0:v", "-c", "copy", "-an", "-video_track_timescale", "90000", output] + + +def build_join_av_command(video_list: str, audio: str, output: str) -> List[str]: + """Remux the partial stitch's picture pieces and its sound into one file.""" + return [ + "ffmpeg", "-y", "-hide_banner", "-loglevel", "error", + "-f", "concat", "-safe", "0", "-i", video_list, "-i", audio, + "-map", "0:v", "-map", "1:a", "-c", "copy", "-movflags", "+faststart", output, + ] + + @dataclass(frozen=True) class MixTrack: """One audio track placed on the finished video's timeline.""" @@ -1160,11 +1388,27 @@ class MixTrack: fade_in: float = 0.0 fade_out: float = 0.0 duck: bool = True # pushed down under the narration + during_clips: Optional[str] = None # None, "mute" or "play": see AudioTrack + + +# How long the score takes to leave before a clip and come back after it. +CLIP_FADE_SECONDS = 0.6 + + +def _quiet_during(windows: Sequence[Tuple[float, float]], fade: float) -> str: + """A `volume` expression that is 1, except 0 inside each window, ramping over + `fade` seconds either side of it so the sound leaves and returns rather than + cutting. Quoted, since an expression's commas would otherwise split the graph.""" + fade = max(0.01, fade) + terms = [f"(1-max(0,min(1,min((t-{start - fade:.3f})/{fade:.3f}," + f"({end + fade:.3f}-t)/{fade:.3f}))))" for start, end in windows] + return "'" + "*".join(terms) + "'" def build_mix_command( source: str, tracks: Sequence[MixTrack], output: str, *, duration: float, duck_available: bool = True, + clip_windows: Sequence[Tuple[float, float]] = (), ) -> List[str]: """Mix any number of tracks under the finished video, the picture untouched. @@ -1178,10 +1422,16 @@ def build_mix_command( A looping track restarts from the top of its file, not from `trim_in`: that is what `-stream_loop` does with an input seek, and a bed short enough to loop is rarely trimmed as well. + + `clip_windows` are where a clip plays its own sound. A track set to "mute" + there fades out around each one. The narration's key carries the clips too, + which is what dips a ducked bed under them; a ducked track set to "play" is + compressed against a key with the clips taken out, so only the voice moves it. """ argv = ["ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-i", source] chains: List[str] = [] ducked: List[str] = [] + spared: List[str] = [] flat: List[str] = [] for position, track in enumerate(tracks, start=1): if track.trim_in > 0: @@ -1201,22 +1451,39 @@ def build_mix_command( delay = int(round(max(0.0, track.start) * 1000)) if delay: chain += f",adelay={delay}|{delay}" - chain += f",apad[t{position}]" - chains.append(chain) - (ducked if track.duck and duck_available else flat).append(f"t{position}") + chain += ",apad" + if track.during_clips == "mute" and clip_windows: + # After the delay, so `t` is the video's own clock. + chain += f",volume={_quiet_during(clip_windows, CLIP_FADE_SECONDS)}:eval=frame" + chains.append(chain + f"[t{position}]") + if not (track.duck and duck_available): + flat.append(f"t{position}") + elif track.during_clips == "play" and clip_windows: + spared.append(f"t{position}") + else: + ducked.append(f"t{position}") mix_inputs = ["nar"] - if ducked: - if len(ducked) == 1: - bed = ducked[0] - else: - chains.append("".join(f"[{t}]" for t in ducked) - + f"amix=inputs={len(ducked)}:normalize=0[bedmix]") - bed = "bedmix" - chains.append("[0:a]asplit=2[nar][key]") - chains.append(f"[{bed}][key]sidechaincompress=" - "threshold=0.060:ratio=8:attack=15:release=350[bed]") - mix_inputs.append("bed") + beds = [(name, group) for name, group in (("bed", ducked), ("spared", spared)) if group] + if beds: + keys = (["key"] if ducked else []) + (["keyvoice"] if spared else []) + chains.append(f"[0:a]asplit={1 + len(keys)}[nar]" + "".join(f"[{k}]" for k in keys)) + if spared: + # The clips cut out of the key, quickly, so the bed is not ducked + # across them; the voice either side still is. + chains.append(f"[keyvoice]volume={_quiet_during(clip_windows, 0.05)}:eval=frame" + "[keyspared]") + for name, group in beds: + if len(group) == 1: + source_label = group[0] + else: + chains.append("".join(f"[{t}]" for t in group) + + f"amix=inputs={len(group)}:normalize=0[{name}mix]") + source_label = f"{name}mix" + key = "key" if name == "bed" else "keyspared" + chains.append(f"[{source_label}][{key}]sidechaincompress=" + f"threshold=0.060:ratio=8:attack=15:release=350[{name}]") + mix_inputs.append(name) else: chains.append("[0:a]anull[nar]") mix_inputs += flat diff --git a/backend/app/render/renderer.py b/backend/app/render/renderer.py index cd0cbdc..77e51cc 100644 --- a/backend/app/render/renderer.py +++ b/backend/app/render/renderer.py @@ -225,6 +225,86 @@ def _stitch_blended( F.run(F.build_concat_command("stitch.txt", output), cwd=work_dir, timeout=1800) +def _stitch_partial( + shot_files: List[str], durations: List[float], joins: list, output: str, + *, options: RenderOptions, preview: bool, work_dir: str, +) -> None: + """Encode the picture again only around each blend, and copy the rest of it. + + See `plan_partial_stitch`. The sound is built for the whole video in one pass, + exactly as the full stitch builds it, and muxed back in, so it is sample-exact + and never cut at a packet boundary. + """ + fps = options.fps + probed = [F.probe_keyframes(os.path.join(work_dir, name), fps) for name in shot_files] + keyframes, counts = [k for k, _ in probed], [c for _, c in probed] + pieces = F.plan_partial_stitch(durations, joins, keyframes, fps) + # Where each shot starts on the timeline, which is where the sound puts it. + starts = [0.0] + for index in range(1, len(durations)): + join = joins[index] if index < len(joins) else None + starts.append(starts[-1] + durations[index - 1] - (join[1] if join else 0.0)) + emitted = 0 # frames of picture laid down so far + + cuts: Dict[int, List[int]] = {} + for piece in pieces: + if not piece.blend: + at = cuts.setdefault(piece.first, []) + at += [f for f in (piece.start_frame or None, piece.end_frame) if f is not None] + for shot, frames in cuts.items(): + if frames: + cuts[shot] = sorted(set(frames)) + F.run(F.build_cut_command(shot_files[shot], f"cut_{shot:04d}_%03d.mp4", + cuts[shot], fps), cwd=work_dir, timeout=600) + + names: List[str] = [] + for number, piece in enumerate(pieces): + if not piece.blend: + frames = cuts.get(piece.first) or [] + if not frames: + name = f"cut_{piece.first:04d}_000.mp4" + F.run(F.build_video_copy_command(shot_files[piece.first], name), + cwd=work_dir, timeout=600) + names.append(name) + else: + names.append(f"cut_{piece.first:04d}_" + f"{sum(1 for f in frames if f <= piece.start_frame):03d}.mp4") + end = counts[piece.first] if piece.end_frame is None else piece.end_frame + emitted += end - piece.start_frame + continue + span = slice(piece.first, piece.last + 1) + sub_joins = [None] + list(joins[piece.first + 1:piece.last + 1]) + # A run of short shots blended into each other is one blend piece, and as + # long as the video in the worst case: it is chunked like the full stitch. + chunks = F.plan_stitch_chunks(durations[span], sub_joins, fps) + for part, chunk in enumerate(chunks): + start = piece.start_frame if part == 0 else chunk.start_frame + end = piece.end_frame if part == len(chunks) - 1 else chunk.end_frame + first, last = piece.first + chunk.first, piece.first + chunk.last + # The copied stretches are whole frames, while the timeline is not: a + # shot planned at 13.242 s has 397 frames. Left alone, each blend would + # start up to a frame late and the error would add up along the video, + # where the full stitch measures every blend from the timeline. So the + # first shot's length here is what puts its blend back on the timeline. + lengths = list(durations[first:last + 1]) + lengths[0] += (starts[first] + start / fps) - emitted / fps + name = f"blend_{number:03d}_{part:02d}.mp4" + F.run(F.build_stitch_command( + shot_files[first:last + 1], lengths, name, + joins=[None] + list(joins[first + 1:last + 1]), options=options, + preview=preview, start_frame=start, end_frame=end, streams="video"), + cwd=work_dir, timeout=F.shot_timeout(sum(durations[first:last + 1]))) + names.append(name) + emitted += F.probe_keyframes(os.path.join(work_dir, name), fps)[1] + + F.run(F.build_stitch_command(shot_files, durations, "stitch_audio.m4a", joins=joins, + options=options, preview=preview, streams="audio"), + cwd=work_dir, timeout=1800) + F.write_concat_list(os.path.join(work_dir, "stitch_video.txt"), names) + F.run(F.build_join_av_command("stitch_video.txt", "stitch_audio.m4a", output), + cwd=work_dir, timeout=1800) + + def render( *, spec: VideoSpec, preview: bool, work_dir: str, out_dir: str, data_dir: str, user_id: str, gam: GamClient, speech: NotesSpeech, progress: Progress, @@ -280,6 +360,19 @@ def render( for u, n in zip(units, narrated)] timeline = layout(units, durations, warnings) + # ── how loud the voice is, for clips to be matched to (S5) ───────────────── + # Measured, not assumed: voices and providers differ by several dB. R128 gates + # out the pauses, so the holds between lines don't pull it down. + narration_lufs = F.DEFAULT_LOUDNESS_LUFS + if any(u.kind == "video_sound" for u in units): + levels = [] + for narration in narrated: + if narration.path and narration.duration > 0: + lufs = F.measure_loudness(os.path.join(work_dir, narration.path)) + if lufs is not None: + levels.append((lufs, narration.duration)) + narration_lufs = F.combined_loudness(levels) or F.DEFAULT_LOUDNESS_LUFS + # ── encode each unit ───────────────────────────────────────────────────── cache = shot_cache.ShotCache(data_dir, user_id) try: @@ -341,6 +434,15 @@ def render( cwd=work_dir, timeout=F.shot_timeout(length)) background = clip + # A clip's sound at the voice's loudness, then moved by its own setting: + # footage arrives anywhere from a whisper to a broadcast master. Measured + # over what plays, so a loud intro the trim cuts off doesn't count. + gain = 0.0 + if unit.kind == "video_sound" and has_audio: + gain = round(F.clip_gain(F.measure_loudness(os.path.join(work_dir, background), + duration), + narration_lufs, unit.volume_db), 1) + # Watermark, overlay text and a quote or code panel, as one layer. if unit.chapter: current_chapter = unit.chapter @@ -390,7 +492,7 @@ def argv(with_options: RenderOptions, with_path=path) -> List[str]: duration=duration, output=output, options=with_options, preview=preview, overlay_png=overlay_png, subtitle_file=shot_srt, background_has_audio=has_audio, index=index, in_transition=in_transition, out_transition=out_transition, - loop=unit.loop, path=with_path, + loop=unit.loop, path=with_path, audio_gain_db=gain, ) if narration.path is None and unit.kind != "video_sound": @@ -424,8 +526,15 @@ def argv(with_options: RenderOptions, with_path=path) -> List[str]: stitched = "stitched.mp4" if any(blends): try: - _stitch_blended(shot_files, timeline.durations, blends, stitched, - options=options, preview=preview, work_dir=work_dir) + try: + _stitch_partial(shot_files, timeline.durations, blends, stitched, + options=options, preview=preview, work_dir=work_dir) + except F.FFmpegError as exc: + # The same blends with every frame encoded again: several times + # slower at full quality, but it relies on no keyframes or remuxing. + logger.warning("Partial stitch failed (%s); encoding it whole", exc) + _stitch_blended(shot_files, timeline.durations, blends, stitched, + options=options, preview=preview, work_dir=work_dir) except F.FFmpegError as exc: # The blends cost a re-encode; if that fails, the shots still join by cuts, # and the timeline is re-laid without their overlaps to match. A clip's @@ -460,9 +569,14 @@ def argv(with_options: RenderOptions, with_path=path) -> List[str]: progress("Mixing", 90, f"{len(tracks)} audio track{'s' if len(tracks) != 1 else ''}") mix = [F.MixTrack(path=t.media.path, start=t.start, length=t.length, trim_in=t.trim_in, loop=t.loop, volume=t.track.volume, fade_in=t.track.fade_in, - fade_out=t.track.fade_out, duck=t.duck) for t in tracks] + fade_out=t.track.fade_out, duck=t.duck, + during_clips=t.during_clips) for t in tracks] + # Where a clip plays its own sound, for a track that leaves for clips (S5). + clip_windows = [(timeline.starts[i], timeline.end(i)) for i, u in enumerate(units) + if u.kind == "video_sound" and u.media and u.media.has_audio] F.run(F.build_mix_command(final, mix, "scored.mp4", duration=timeline.total, - duck_available="sidechaincompress" in F.available_filters()), + duck_available="sidechaincompress" in F.available_filters(), + clip_windows=clip_windows), cwd=work_dir, timeout=1800) final = "scored.mp4" diff --git a/backend/app/spec/model.py b/backend/app/spec/model.py index b8fb4b2..4d57759 100644 --- a/backend/app/spec/model.py +++ b/backend/app/spec/model.py @@ -397,6 +397,9 @@ class Visual(_Model): trim: Optional[Trim] = None # videos only audio: Literal["mute", "play"] = "mute" # "play" only on a clip loop: bool = True # shorter than its segment: loop, or hold + # A clip's sound is matched to the narration's loudness by the renderer; this + # moves it from there, in dB (S5). Ignored where the footage plays muted. + volume_db: Optional[float] = Field(default=None, ge=-30, le=20) class Beat(_Model): @@ -519,6 +522,10 @@ class AudioTrack(_Model): loop: Optional[bool] = None # default: on for music volume: Unit = 0.18 duck_under_narration: Optional[bool] = None # default: on for music + # While a clip plays its own sound (S5). None: as under narration (a ducked + # track dips, one that isn't plays on). "mute" fades it out for the clip and + # back in after; "play" keeps it at its level, ducked under narration only. + during_clips: Optional[Literal["mute", "play"]] = None fade_in: float = Field(default=0.0, ge=0, le=15) fade_out: float = Field(default=0.0, ge=0, le=15) diff --git a/backend/tests/test_render_compile.py b/backend/tests/test_render_compile.py index 731ad87..0bc8863 100644 --- a/backend/tests/test_render_compile.py +++ b/backend/tests/test_render_compile.py @@ -357,3 +357,32 @@ def test_the_overlay_pass_draws_each_layer_only_in_its_window_and_copies_the_sou assert "[l2]" in graph and "fade" not in graph.split("[1:v]")[0] assert argv[argv.index("-c:a") + 1] == "copy" assert argv.count("-c:a") == 1 and argv[argv.index("-map") + 1] == "[v2]" + + +# ── clip sound levels and the score during clips (S5) ─────────────────────── + +def test_a_clips_own_level_and_a_tracks_choice_during_clips_reach_the_render(): + spec = spec_of({ + "assets": {"v": {"from": "gam", "id": "g1", "type": "video"}, + "m": {"from": "gam", "id": "g2", "type": "audio"}}, + "segments": [ + {"id": "a", "kind": "clip", "visual": {"asset": "v", "audio": "play", + "volume_db": -4.5}}, + {"id": "b", "kind": "clip", "visual": {"asset": "v", "audio": "play"}}, + ], + "audio_tracks": [{"id": "t", "asset": "m", "start": "video_start", + "end": "video_end", "during_clips": "mute"}]}) + units = plan_units(spec, {"v": VIDEO, "m": SONG}, []) + assert [u.volume_db for u in units] == [-4.5, 0.0] + timeline = layout(units, [6.0, 6.0], []) + where = placement(units, timeline, [{} for _ in units]) + [track] = place_tracks(spec, {"v": VIDEO, "m": SONG}, where, []) + assert track.during_clips == "mute" + + +def test_a_clip_level_outside_its_range_is_refused(): + _spec, issues = parse_spec({ + "assets": {"v": {"from": "gam", "id": "g1", "type": "video"}}, + "segments": [{"id": "a", "kind": "clip", + "visual": {"asset": "v", "audio": "play", "volume_db": 40}}]}) + assert _spec is None and issues diff --git a/backend/tests/test_render_ffmpeg.py b/backend/tests/test_render_ffmpeg.py index 1912106..ed03073 100644 --- a/backend/tests/test_render_ffmpeg.py +++ b/backend/tests/test_render_ffmpeg.py @@ -1360,3 +1360,163 @@ def test_each_overlay_layer_is_one_frame_repeated_for_its_window_only(): assert "-itsoffset" not in argv and "-loop" not in argv assert ("[1:v]format=rgba,loop=loop=-1:size=1,trim=end_frame=105,settb=AVTB," "setpts=N/(30*TB)+1.000/TB,") in _graph(argv) + + +# ── the partial stitch: only the blends are encoded again ─────────────────── + +def _keys(*lists): + return [list(k) for k in lists] + + +def test_a_blend_is_encoded_from_the_keyframe_before_it_to_the_one_after_it(): + # Two 4 s shots at 30 fps, keyframes every 2 s, blended for 0.5 s. + pieces = F.plan_partial_stitch([4.0, 4.0], [None, ("fade", 0.5)], + _keys([0, 60], [0, 60]), 30) + assert pieces == [ + F.StitchPiece(False, 0, 0, 0, 60), # shot 0 up to its keyframe before 3.5 s + F.StitchPiece(True, 0, 1, 60, 60), # the blend, to shot 1's keyframe after 0.5 s + F.StitchPiece(False, 1, 1, 60, None), # the rest of shot 1, copied + ] + + +def test_a_cut_or_a_dip_needs_nothing_encoded(): + pieces = F.plan_partial_stitch([4.0, 4.0, 4.0], [None, None, None], + _keys([0, 60], [0, 60], [0, 60]), 30) + assert all(not p.blend and p.end_frame is None for p in pieces) and len(pieces) == 3 + + +def test_a_short_shot_blended_at_both_ends_joins_the_two_blends_into_one(): + # Shot 1 has no keyframe after 0.5 s and before its own blend out at 1.5 s. + pieces = F.plan_partial_stitch([4.0, 2.0, 4.0], [None, ("fade", 0.5), ("fade", 0.5)], + _keys([0, 60], [0], [0, 60]), 30) + assert pieces == [ + F.StitchPiece(False, 0, 0, 0, 60), + F.StitchPiece(True, 0, 2, 60, 60), + F.StitchPiece(False, 2, 2, 60, None), + ] + + +def test_a_blend_into_a_shot_with_no_later_keyframe_runs_to_its_end(): + pieces = F.plan_partial_stitch([4.0, 1.0, 4.0], [None, ("fade", 0.5), None], + _keys([0, 60], [0], [0, 60]), 30) + assert pieces == [ + F.StitchPiece(False, 0, 0, 0, 60), + F.StitchPiece(True, 0, 1, 60, None), + F.StitchPiece(False, 2, 2, 0, None), + ] + + +def test_a_shot_whose_only_keyframe_is_its_first_frame_is_blended_from_the_start(): + pieces = F.plan_partial_stitch([2.0, 4.0], [None, ("fade", 0.5)], _keys([0], [0, 60]), 30) + assert pieces[0] == F.StitchPiece(True, 0, 1, 0, 60) + + +def test_the_pieces_cover_every_frame_once(): + durations = [4.0, 2.0, 6.0, 3.0, 5.0] + joins = [None, ("fade", 0.5), ("fade", 0.6), None, ("fade", 0.5)] + keys = _keys([0, 60], [0], [0, 60, 120], [0, 60], [0, 60, 120]) + pieces = F.plan_partial_stitch(durations, joins, keys, 30) + for before, after in zip(pieces, pieces[1:]): + if before.end_frame is None: + assert after.first == before.last + 1 and after.start_frame == 0 + else: + assert after.first == before.last and after.start_frame == before.end_frame + assert pieces[0].start_frame == 0 and pieces[-1].end_frame is None + + +def test_the_partial_stitch_encodes_the_picture_and_the_sound_apart(): + video = F.build_stitch_command(["a.mp4", "b.mp4"], [4.0, 4.0], "v.mp4", + joins=_blend(2), options=RenderOptions(), preview=True, + start_frame=60, end_frame=60, streams="video") + assert "-an" in video and "-c:a" not in video and "atrim" not in _graph(video) + audio = F.build_stitch_command(["a.mp4", "b.mp4"], [4.0, 4.0], "a.m4a", + joins=_blend(2), options=RenderOptions(), preview=True, + streams="audio") + assert "-vn" in audio and "-c:v" not in audio and "xfade" not in _graph(audio) + lone = F.build_stitch_command(["a.mp4"], [4.0], "a.m4a", joins=[None], + options=RenderOptions(), preview=True, streams="audio") + assert "-filter_complex" not in lone and lone[lone.index("-map") + 1] == "0:a" + + +def test_a_shot_is_cut_half_a_frame_before_each_keyframe(): + argv = F.build_cut_command("s.mp4", "cut_%03d.mp4", [60, 120], 30) + assert argv[argv.index("-segment_times") + 1] == "1.983333,3.983333" + assert argv[argv.index("-c") + 1] == "copy" and "-an" in argv + + +def test_a_blend_piece_seeks_to_its_first_frame_rather_than_decoding_up_to_it(): + argv = F.build_stitch_command(["a.mp4", "b.mp4"], [4.0, 4.0], "v.mp4", joins=_blend(2), + options=RenderOptions(), preview=True, + start_frame=60, end_frame=60, streams="video") + assert argv[argv.index("a.mp4") - 3:argv.index("a.mp4")] == ["-ss", "1.983333", "-i"] + assert "[0:v]settb=AVTB[s0]" in _graph(argv) and "trim=end_frame=60" in _graph(argv) + # Inside one shot, its end is counted from where the seek landed. + lone = F.build_stitch_command(["a.mp4"], [4.0], "v.mp4", joins=[None], + options=RenderOptions(), preview=True, + start_frame=60, end_frame=90, streams="video") + assert "[0:v]trim=end_frame=30," in _graph(lone) + + +# ── clip sound levels and the score during clips (S5) ─────────────────────── + +def test_a_clip_is_brought_to_the_voices_loudness_then_moved_by_its_own_setting(): + assert F.clip_gain(-30.0, -18.0) == 12.0 + assert F.clip_gain(-10.0, -18.0) == -8.0 + assert F.clip_gain(-30.0, -18.0, adjust_db=-6.0) == 6.0 + # Silence (unmeasurable) is left alone, but still follows its own setting. + assert F.clip_gain(None, -18.0) == 0.0 and F.clip_gain(None, -18.0, 3.0) == 3.0 + # A near-silent clip is not raised without limit: that would only raise the noise. + assert F.clip_gain(-70.0, -16.0) == F.CLIP_GAIN_LIMITS_DB[1] + + +def test_the_voices_loudness_is_its_pieces_averaged_by_energy_and_time(): + assert F.combined_loudness([(-18.0, 10.0), (-18.0, 30.0)]) == pytest.approx(-18.0) + # Twice the time at one level weighs twice as much; levels add as energy, not dB. + assert F.combined_loudness([(-20.0, 10.0), (-10.0, 10.0)]) == pytest.approx(-12.6, abs=0.1) + assert F.combined_loudness([]) is None + + +def test_a_raised_clip_is_limited_and_a_lowered_one_is_not(): + def audio(gain): + argv = F.build_shot_command( + kind="video_sound", background="clip.mp4", audio=None, duration=6.0, + output="s.mp4", options=RenderOptions(), preview=True, + background_has_audio=True, audio_gain_db=gain) + return [c for c in _graph(argv).split(";") if c.endswith("[a]")][0] + assert "volume=12.0dB,alimiter=limit=0.891:level=0:latency=1,apad" in audio(12.0) + assert "volume=-8.0dB,apad" in audio(-8.0) and "alimiter" not in audio(-8.0) + assert "volume" not in audio(0.0) + + +def test_a_track_can_leave_for_clips_and_come_back_after(): + track = F.MixTrack(path="m.mp3", start=0.0, length=30.0, during_clips="mute") + graph = _graph(F.build_mix_command("in.mp4", [track], "o.mp4", duration=30.0, + clip_windows=[(8.0, 15.0)])) + assert ("volume='(1-max(0,min(1,min((t-7.400)/0.600,(15.600-t)/0.600))))':eval=frame" + in graph) + # Still ducked under the narration as before. + assert "[t1][key]sidechaincompress=" in graph + + +def test_a_track_can_play_on_through_clips_and_still_duck_under_the_voice(): + plain = F.MixTrack(path="a.mp3", start=0.0, length=30.0) + spared = F.MixTrack(path="b.mp3", start=0.0, length=30.0, during_clips="play") + graph = _graph(F.build_mix_command("in.mp4", [plain, spared], "o.mp4", duration=30.0, + clip_windows=[(8.0, 15.0)])) + assert "[0:a]asplit=3[nar][key][keyvoice]" in graph + assert "[keyvoice]volume='" in graph and "[keyspared]" in graph + assert "[t1][key]sidechaincompress=" in graph + assert "[t2][keyspared]sidechaincompress=" in graph + assert "[nar][bed][spared]amix=inputs=3" in graph + # Alone, it needs no key for a bed that isn't there. + alone = _graph(F.build_mix_command("in.mp4", [spared], "o.mp4", duration=30.0, + clip_windows=[(8.0, 15.0)])) + assert "[0:a]asplit=2[nar][keyvoice]" in alone + + +def test_without_clips_a_tracks_choice_during_clips_changes_nothing(): + for mode in ("mute", "play"): + graph = _graph(F.build_mix_command( + "in.mp4", [F.MixTrack(path="m.mp3", start=0.0, length=30.0, during_clips=mode)], + "o.mp4", duration=30.0)) + assert "eval=frame" not in graph and "[t1][key]sidechaincompress=" in graph diff --git a/docs/plan-of-attack.md b/docs/plan-of-attack.md index 0530637..49b1185 100644 --- a/docs/plan-of-attack.md +++ b/docs/plan-of-attack.md @@ -313,6 +313,24 @@ This plan takes **field by field**, so `{ "transition_in": { "style": "dissolve" keeps the project's duration, because that is what the AI and the editor will both expect. +### S5. Clip sound levels, and the score during clips + +Footage arrives at any level, from a phone in a room to a broadcast master, so a clip +playing its own sound is matched to the narration by the renderer: each clip's +integrated loudness (EBU R128) over the part that plays, against the narration's, +one static gain per clip (dynamics kept, a limiter only when raising), capped at ++24 dB so a near-silent clip doesn't become loud hiss. Two optional fields then move +it from there, so specs without them are unchanged and stay `"0.2"`: + +```ts +visual.volume_db?: number // -30..20; a clip's level from the match +audio_tracks[].during_clips?: "mute" | "play" // absent: as under narration +``` + +A ducked track already dips under clips, since the narration's key carries their +sound. `"mute"` fades the track out around each clip and back after; `"play"` keeps +it at its level through clips, ducked under the voice alone. + ## Changes needed in GAM Specified here, applied in GAM's repository, not from this one. diff --git a/frontend/src/api/projects.ts b/frontend/src/api/projects.ts index 56119f8..5805260 100644 --- a/frontend/src/api/projects.ts +++ b/frontend/src/api/projects.ts @@ -47,6 +47,8 @@ export interface Visual { trim?: Trim audio?: 'mute' | 'play' loop?: boolean + /** A clip's sound, in dB from the narration's loudness it is matched to. */ + volume_db?: number fit?: string motion?: Motion [key: string]: unknown @@ -68,6 +70,8 @@ export interface AudioTrack { loop?: boolean volume?: number duck_under_narration?: boolean + /** While a clip plays its own sound. Absent: as under narration. */ + during_clips?: 'mute' | 'play' fade_in?: number fade_out?: number [key: string]: unknown diff --git a/frontend/src/components/ClipPlayer.tsx b/frontend/src/components/ClipPlayer.tsx index f49110e..ad29232 100644 --- a/frontend/src/components/ClipPlayer.tsx +++ b/frontend/src/components/ClipPlayer.tsx @@ -10,6 +10,7 @@ import { setLength, setLoop, setTrim, + setVolume, timingNote, trimAt, } from '@/spec/clip' @@ -183,6 +184,27 @@ export default function ClipPlayer({ {edge('in')} {edge('out')} + {visual.audio === 'play' && ( + + )} )} {segment.kind === 'scene' && ( diff --git a/frontend/src/components/SoundtrackPanel.tsx b/frontend/src/components/SoundtrackPanel.tsx index 7e4ef90..b1c348e 100644 --- a/frontend/src/components/SoundtrackPanel.tsx +++ b/frontend/src/components/SoundtrackPanel.tsx @@ -139,6 +139,24 @@ export default function SoundtrackPanel({ spec, gamAssets, onChange }: Props) { /> Quieter under narration +