From d862aaba8e3fb2f5382d812e3308dfd9dbeaaef5 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 04:00:46 +0000 Subject: [PATCH 1/2] Encode the picture again only around each blend A blended stitch re-encoded the whole video to draw its dissolves. At 1080p on the medium preset that was most of a render: 12 minutes of a half-hour job for a 14-minute video with 53 shots. Every shot is encoded with encode_args, so between two keyframes it can be copied and remuxed beside freshly encoded frames. plan_partial_stitch lays the picture out as copied stretches and, for each blend, a re-encode from the outgoing shot's last keyframe before the blend to the incoming shot's first after it (a short shot with no keyframe clear of both blends joins them into one encode, chunked like the full stitch). Each blend is anchored to the timeline, since the copied stretches are whole frames and the timeline is not; left alone the error added up to 25 frames over 38 blends. The sound is built for the whole video in one pass, exactly as before, and muxed back in, so it is never cut at an AAC packet boundary. The copied pieces are written on the 90 kHz timescale every encode uses: on 1/15360, ffmpeg 6's concat laid them out a minute apart. Checked against the full stitch on ffmpeg 6.1 and 7.1.1, 45 shots and 38 blends: same frame count, min 36 dB (the encode generations), and the sound on the planned timeline to the sample, where the full stitch ran 20 ms late. At 1080p medium, 12 shots and 10 blends: 38 s -> 22 s, of which 16 s is the blends, a cost that follows the number of blends rather than the length of the video. If the partial stitch fails, the full one runs, and then the cuts. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WJkfhgbVRktPf2Lg6XXfNs --- backend/app/render/ffmpeg.py | 201 +++++++++++++++++++++++++--- backend/app/render/renderer.py | 91 ++++++++++++- backend/tests/test_render_ffmpeg.py | 95 +++++++++++++ 3 files changed, 368 insertions(+), 19 deletions(-) diff --git a/backend/app/render/ffmpeg.py b/backend/app/render/ffmpeg.py index bf73988..fff7472 100644 --- a/backend/app/render/ffmpeg.py +++ b/backend/app/render/ffmpeg.py @@ -996,7 +996,7 @@ def build_extract_command( def build_stitch_command( shot_files: Sequence[str], durations: Sequence[float], output: str, *, joins: Sequence[Optional[Tuple[str, float]]], options: RenderOptions, preview: bool, - start_frame: int = 0, end_frame: Optional[int] = None, + start_frame: int = 0, end_frame: Optional[int] = None, streams: str = "both", ) -> List[str]: """`build_xfade_command` with a transition of its own at every join. @@ -1008,6 +1008,10 @@ def build_stitch_command( `start_frame` drops that many frames off the front of the first shot and `end_frame` stops the last one at that frame: one run of a chunked stitch (see `plan_stitch_chunks`), whose neighbours show the rest of the shot. + + `streams` is "video" or "audio" to build only that half: the partial stitch + (see `plan_partial_stitch`) encodes the picture only around each blend, and + the sound for the whole video in one pass of its own. """ fps = max(1, options.fps) durations = list(durations) @@ -1016,24 +1020,36 @@ def build_stitch_command( durations[last] = end_frame / fps if start_frame: durations[0] -= start_frame / fps + picture, sound = streams in ("both", "video"), streams in ("both", "audio") + # The picture alone can seek its first shot to `start_frame` rather than decode + # its way there, which for a blend piece is most of the shot. Half a frame early, + # so the frame-accurate seek keeps that frame however its time rounds. The sound + # can't: a seek lands on an AAC packet, not a sample. + seek = start_frame if streams == "video" else 0 argv = ["ffmpeg", "-y", "-hide_banner", "-loglevel", "error"] - for name in shot_files: + for index, name in enumerate(shot_files): # Every input is open, with its decoder, for the whole pass. A decoder's # default is a thread per core and a frame buffer per thread, which made # this pass's memory the shot count times the core count. - argv += ["-threads", "1", "-i", name] + argv += ["-threads", "1"] + ([] if sound else ["-an"]) + ([] if picture else ["-vn"]) + if index == 0 and seek: + argv += ["-ss", f"{(seek - 0.5) / fps:.6f}"] + argv += ["-i", name] # Every input on one timebase first. `concat` re-times its output to # AV_TIME_BASE (1/1000000) while the shots carry 1/90000, and `xfade` refuses two # inputs that disagree — so a cut followed by a blend failed to build at all. - chains: List[str] = [] + vchains: List[str] = [] + achains: List[str] = [] sounds: List[str] = [] for i in range(len(shot_files)): - lo = start_frame if i == 0 else 0 + lo = start_frame - seek if i == 0 else 0 hi = end_frame if i == last else None + if hi is not None and i == 0: + hi -= seek if not lo and hi is None: - chains.append(f"[{i}:v]settb=AVTB[s{i}]") + vchains.append(f"[{i}:v]settb=AVTB[s{i}]") sounds.append(f"{i}:a") continue # Trimmed by frame and by sample rather than by seconds, so the picture and @@ -1047,33 +1063,46 @@ def build_stitch_command( # ffmpeg 7 leaves a trimmed stream with no frame rate (1/0), and `xfade` # refuses one, so it is set back: the frames are already at it, so `fps` # neither drops nor repeats any. - chains.append(f"[{i}:v]trim={':'.join(bounds)},setpts=PTS-STARTPTS," - f"fps={fps},settb=AVTB[s{i}]") - chains.append(f"[{i}:a]atrim={':'.join(samples)},asetpts=PTS-STARTPTS[st{i}]") + vchains.append(f"[{i}:v]trim={':'.join(bounds)},setpts=PTS-STARTPTS," + f"fps={fps},settb=AVTB[s{i}]") + achains.append(f"[{i}:a]atrim={':'.join(samples)},asetpts=PTS-STARTPTS[st{i}]") sounds.append(f"st{i}") video, audio = "s0", sounds[0] total = durations[0] for index in range(1, len(shot_files)): join = joins[index] if index < len(joins) else None if join is None: - chains.append(f"[{video}][s{index}]concat=n=2:v=1:a=0[vx{index}]") - chains.append(f"[{audio}][{sounds[index]}]concat=n=2:v=0:a=1[ax{index}]") + vchains.append(f"[{video}][s{index}]concat=n=2:v=1:a=0[vx{index}]") + achains.append(f"[{audio}][{sounds[index]}]concat=n=2:v=0:a=1[ax{index}]") total += durations[index] else: name, overlap = join offset = max(0.0, total - overlap) - chains.append( + vchains.append( f"[{video}][s{index}]xfade=transition={name}:" f"duration={overlap:.3f}:offset={offset:.3f}[vx{index}]" ) - chains.append(f"[{audio}]atrim=end={offset:.3f},asetpts=PTS-STARTPTS[at{index}]") - chains.append(f"[at{index}][{sounds[index]}]concat=n=2:v=0:a=1[ax{index}]") + achains.append(f"[{audio}]atrim=end={offset:.3f},asetpts=PTS-STARTPTS[at{index}]") + achains.append(f"[at{index}][{sounds[index]}]concat=n=2:v=0:a=1[ax{index}]") total += durations[index] - overlap video, audio = f"vx{index}", f"ax{index}" + vchains.append(f"[{video}]setsar=1,format=yuv420p[v]") - chains.append(f"[{video}]setsar=1,format=yuv420p[v]") - argv += ["-filter_complex", ";".join(chains), "-map", "[v]", "-map", f"[{audio}]"] - argv += encode_args(options, preview) + shared = encode_args(options, preview) + split = shared.index("-c:a") + if not sound: + argv += ["-filter_complex", ";".join(vchains), "-map", "[v]", *shared[:split], "-an"] + elif not picture: + # A lone input's sound needs no graph at all, and a graph with nothing in it + # is an error, so the one-shot case maps the track straight through. + if achains: + argv += ["-filter_complex", ";".join(achains), "-map", f"[{audio}]"] + else: + argv += ["-map", f"{audio}"] + argv += [*shared[split:], "-vn"] + else: + argv += ["-filter_complex", ";".join(vchains + achains), + "-map", "[v]", "-map", f"[{audio}]", *shared] argv += ["-movflags", "+faststart", output] return argv @@ -1147,6 +1176,144 @@ def plan_stitch_chunks( return chunks +@dataclass(frozen=True) +class StitchPiece: + """One stretch of a partial stitch's picture, in output order. + + `blend` False: frames `start_frame`..`end_frame` of shot `first` (== `last`), + copied as they were encoded. True: shots `first`..`last` encoded together, from + `start_frame` of the first to `end_frame` of the last, with their blends. An + `end_frame` of None runs to the end of the shot. + """ + + blend: bool + first: int + last: int + start_frame: int = 0 + end_frame: Optional[int] = None + + +def probe_keyframes(path: str, fps: int) -> Tuple[List[int], int]: + """Frame numbers of a video's keyframes, and how many frames it has. + + Read from the packets, not the frames, so nothing is decoded and a long shot + probes in a moment. + """ + out = subprocess.run( + ["ffprobe", "-v", "error", "-select_streams", "v:0", + "-show_entries", "packet=pts_time,flags", "-of", "csv=p=0", path], + capture_output=True, timeout=PROBE_TIMEOUT_SECONDS, + ) + if out.returncode != 0: + raise FFmpegError(f"ffprobe could not read the keyframes of {path}") + frames = set() + count = 0 + for line in out.stdout.decode(errors="replace").splitlines(): + when, _, flags = line.partition(",") + count += 1 + if "K" in flags: + try: + frames.add(round(float(when) * fps)) + except ValueError: + continue + return sorted(frames), count + + +def plan_partial_stitch( + durations: Sequence[float], joins: Sequence[Optional[Tuple[str, float]]], + keyframes: Sequence[Sequence[int]], fps: int, +) -> List[StitchPiece]: + """Lay a blended stitch out as copied stretches and the blends between them. + + A blend needs both shots decoded and encoded again, but only where they + overlap. Every shot was encoded with `encode_args`, so between two of its + keyframes it can be copied as it is and remuxed beside freshly encoded frames. + Each blend is therefore re-encoded from the last keyframe of the outgoing shot + before the blend begins to the first keyframe of the incoming one after it + ends: a second or two a join rather than the whole video, which at 1080p on + the `medium` preset is minutes against seconds. + + A shot with no keyframe clear of the blends at both ends (a short one, blended + in and out) has nothing to copy, so the blend before it and the one after it + become one encode through it. + """ + count = len(durations) + fps = max(1, fps) + + def overlap(index: int) -> float: + return _join_overlap(joins, index) if 0 < index < count else 0.0 + + pieces: List[StitchPiece] = [] + group: Optional[Tuple[int, int]] = None # an open blend: (first shot, start frame) + for i in range(count): + frames_in, frames_out = overlap(i), overlap(i + 1) + keys = keyframes[i] if i < len(keyframes) else [0] + if frames_in > 0: + after = [k for k in keys if k >= math.ceil(frames_in * fps)] + lo: Optional[int] = after[0] if after else None + else: + lo = 0 + hi: Optional[int] = None + if frames_out > 0: + before = [k for k in keys if k <= math.floor((durations[i] - frames_out) * fps)] + hi = before[-1] if before else 0 + body = lo is not None and (hi is None or hi > lo) + if group is not None: + if not body: + if frames_out <= 0: # the blend ends inside this shot, and so does it + pieces.append(StitchPiece(True, group[0], i, group[1], None)) + group = None + continue + pieces.append(StitchPiece(True, group[0], i, group[1], lo)) + group = None + start = lo or 0 + if hi is None: + pieces.append(StitchPiece(False, i, i, start, None)) + else: + if hi > start: + pieces.append(StitchPiece(False, i, i, start, hi)) + group = (i, hi) + if group is not None: + pieces.append(StitchPiece(True, group[0], count - 1, group[1], None)) + return pieces + + +def build_cut_command(source: str, output_pattern: str, at_frames: Sequence[int], + fps: int) -> List[str]: + """Split a shot's picture at keyframes, copied: `output_pattern` takes a %d. + + The segment muxer cuts at the first keyframe at or after each time, so each is + asked for half a frame early: the cut lands on that keyframe however its time + rounds. Audio is left out; the partial stitch encodes the sound separately. + """ + times = ",".join(f"{(frame - 0.5) / max(1, fps):.6f}" for frame in at_frames) + return [ + "ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-i", source, + "-map", "0:v", "-c", "copy", "-an", "-f", "segment", "-segment_format", "mp4", + # The timescale every encode writes (see `encode_args`). The segment muxer + # would otherwise pick 1/15360, and ffmpeg 6's concat, given pieces on two + # timebases, laid them out a minute apart. + "-segment_format_options", "video_track_timescale=90000", + "-segment_times", times, "-reset_timestamps", "1", output_pattern, + ] + + +def build_video_copy_command(source: str, output: str) -> List[str]: + """A shot's picture alone, copied: every file the concat joins has the same + streams, and the partial stitch's pieces carry no sound.""" + return ["ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-i", source, + "-map", "0:v", "-c", "copy", "-an", "-video_track_timescale", "90000", output] + + +def build_join_av_command(video_list: str, audio: str, output: str) -> List[str]: + """Remux the partial stitch's picture pieces and its sound into one file.""" + return [ + "ffmpeg", "-y", "-hide_banner", "-loglevel", "error", + "-f", "concat", "-safe", "0", "-i", video_list, "-i", audio, + "-map", "0:v", "-map", "1:a", "-c", "copy", "-movflags", "+faststart", output, + ] + + @dataclass(frozen=True) class MixTrack: """One audio track placed on the finished video's timeline.""" diff --git a/backend/app/render/renderer.py b/backend/app/render/renderer.py index cd0cbdc..736b1eb 100644 --- a/backend/app/render/renderer.py +++ b/backend/app/render/renderer.py @@ -225,6 +225,86 @@ def _stitch_blended( F.run(F.build_concat_command("stitch.txt", output), cwd=work_dir, timeout=1800) +def _stitch_partial( + shot_files: List[str], durations: List[float], joins: list, output: str, + *, options: RenderOptions, preview: bool, work_dir: str, +) -> None: + """Encode the picture again only around each blend, and copy the rest of it. + + See `plan_partial_stitch`. The sound is built for the whole video in one pass, + exactly as the full stitch builds it, and muxed back in, so it is sample-exact + and never cut at a packet boundary. + """ + fps = options.fps + probed = [F.probe_keyframes(os.path.join(work_dir, name), fps) for name in shot_files] + keyframes, counts = [k for k, _ in probed], [c for _, c in probed] + pieces = F.plan_partial_stitch(durations, joins, keyframes, fps) + # Where each shot starts on the timeline, which is where the sound puts it. + starts = [0.0] + for index in range(1, len(durations)): + join = joins[index] if index < len(joins) else None + starts.append(starts[-1] + durations[index - 1] - (join[1] if join else 0.0)) + emitted = 0 # frames of picture laid down so far + + cuts: Dict[int, List[int]] = {} + for piece in pieces: + if not piece.blend: + at = cuts.setdefault(piece.first, []) + at += [f for f in (piece.start_frame or None, piece.end_frame) if f is not None] + for shot, frames in cuts.items(): + if frames: + cuts[shot] = sorted(set(frames)) + F.run(F.build_cut_command(shot_files[shot], f"cut_{shot:04d}_%03d.mp4", + cuts[shot], fps), cwd=work_dir, timeout=600) + + names: List[str] = [] + for number, piece in enumerate(pieces): + if not piece.blend: + frames = cuts.get(piece.first) or [] + if not frames: + name = f"cut_{piece.first:04d}_000.mp4" + F.run(F.build_video_copy_command(shot_files[piece.first], name), + cwd=work_dir, timeout=600) + names.append(name) + else: + names.append(f"cut_{piece.first:04d}_" + f"{sum(1 for f in frames if f <= piece.start_frame):03d}.mp4") + end = counts[piece.first] if piece.end_frame is None else piece.end_frame + emitted += end - piece.start_frame + continue + span = slice(piece.first, piece.last + 1) + sub_joins = [None] + list(joins[piece.first + 1:piece.last + 1]) + # A run of short shots blended into each other is one blend piece, and as + # long as the video in the worst case: it is chunked like the full stitch. + chunks = F.plan_stitch_chunks(durations[span], sub_joins, fps) + for part, chunk in enumerate(chunks): + start = piece.start_frame if part == 0 else chunk.start_frame + end = piece.end_frame if part == len(chunks) - 1 else chunk.end_frame + first, last = piece.first + chunk.first, piece.first + chunk.last + # The copied stretches are whole frames, while the timeline is not: a + # shot planned at 13.242 s has 397 frames. Left alone, each blend would + # start up to a frame late and the error would add up along the video, + # where the full stitch measures every blend from the timeline. So the + # first shot's length here is what puts its blend back on the timeline. + lengths = list(durations[first:last + 1]) + lengths[0] += (starts[first] + start / fps) - emitted / fps + name = f"blend_{number:03d}_{part:02d}.mp4" + F.run(F.build_stitch_command( + shot_files[first:last + 1], lengths, name, + joins=[None] + list(joins[first + 1:last + 1]), options=options, + preview=preview, start_frame=start, end_frame=end, streams="video"), + cwd=work_dir, timeout=F.shot_timeout(sum(durations[first:last + 1]))) + names.append(name) + emitted += F.probe_keyframes(os.path.join(work_dir, name), fps)[1] + + F.run(F.build_stitch_command(shot_files, durations, "stitch_audio.m4a", joins=joins, + options=options, preview=preview, streams="audio"), + cwd=work_dir, timeout=1800) + F.write_concat_list(os.path.join(work_dir, "stitch_video.txt"), names) + F.run(F.build_join_av_command("stitch_video.txt", "stitch_audio.m4a", output), + cwd=work_dir, timeout=1800) + + def render( *, spec: VideoSpec, preview: bool, work_dir: str, out_dir: str, data_dir: str, user_id: str, gam: GamClient, speech: NotesSpeech, progress: Progress, @@ -424,8 +504,15 @@ def argv(with_options: RenderOptions, with_path=path) -> List[str]: stitched = "stitched.mp4" if any(blends): try: - _stitch_blended(shot_files, timeline.durations, blends, stitched, - options=options, preview=preview, work_dir=work_dir) + try: + _stitch_partial(shot_files, timeline.durations, blends, stitched, + options=options, preview=preview, work_dir=work_dir) + except F.FFmpegError as exc: + # The same blends with every frame encoded again: several times + # slower at full quality, but it relies on no keyframes or remuxing. + logger.warning("Partial stitch failed (%s); encoding it whole", exc) + _stitch_blended(shot_files, timeline.durations, blends, stitched, + options=options, preview=preview, work_dir=work_dir) except F.FFmpegError as exc: # The blends cost a re-encode; if that fails, the shots still join by cuts, # and the timeline is re-laid without their overlaps to match. A clip's diff --git a/backend/tests/test_render_ffmpeg.py b/backend/tests/test_render_ffmpeg.py index 1912106..d491e90 100644 --- a/backend/tests/test_render_ffmpeg.py +++ b/backend/tests/test_render_ffmpeg.py @@ -1360,3 +1360,98 @@ def test_each_overlay_layer_is_one_frame_repeated_for_its_window_only(): assert "-itsoffset" not in argv and "-loop" not in argv assert ("[1:v]format=rgba,loop=loop=-1:size=1,trim=end_frame=105,settb=AVTB," "setpts=N/(30*TB)+1.000/TB,") in _graph(argv) + + +# ── the partial stitch: only the blends are encoded again ─────────────────── + +def _keys(*lists): + return [list(k) for k in lists] + + +def test_a_blend_is_encoded_from_the_keyframe_before_it_to_the_one_after_it(): + # Two 4 s shots at 30 fps, keyframes every 2 s, blended for 0.5 s. + pieces = F.plan_partial_stitch([4.0, 4.0], [None, ("fade", 0.5)], + _keys([0, 60], [0, 60]), 30) + assert pieces == [ + F.StitchPiece(False, 0, 0, 0, 60), # shot 0 up to its keyframe before 3.5 s + F.StitchPiece(True, 0, 1, 60, 60), # the blend, to shot 1's keyframe after 0.5 s + F.StitchPiece(False, 1, 1, 60, None), # the rest of shot 1, copied + ] + + +def test_a_cut_or_a_dip_needs_nothing_encoded(): + pieces = F.plan_partial_stitch([4.0, 4.0, 4.0], [None, None, None], + _keys([0, 60], [0, 60], [0, 60]), 30) + assert all(not p.blend and p.end_frame is None for p in pieces) and len(pieces) == 3 + + +def test_a_short_shot_blended_at_both_ends_joins_the_two_blends_into_one(): + # Shot 1 has no keyframe after 0.5 s and before its own blend out at 1.5 s. + pieces = F.plan_partial_stitch([4.0, 2.0, 4.0], [None, ("fade", 0.5), ("fade", 0.5)], + _keys([0, 60], [0], [0, 60]), 30) + assert pieces == [ + F.StitchPiece(False, 0, 0, 0, 60), + F.StitchPiece(True, 0, 2, 60, 60), + F.StitchPiece(False, 2, 2, 60, None), + ] + + +def test_a_blend_into_a_shot_with_no_later_keyframe_runs_to_its_end(): + pieces = F.plan_partial_stitch([4.0, 1.0, 4.0], [None, ("fade", 0.5), None], + _keys([0, 60], [0], [0, 60]), 30) + assert pieces == [ + F.StitchPiece(False, 0, 0, 0, 60), + F.StitchPiece(True, 0, 1, 60, None), + F.StitchPiece(False, 2, 2, 0, None), + ] + + +def test_a_shot_whose_only_keyframe_is_its_first_frame_is_blended_from_the_start(): + pieces = F.plan_partial_stitch([2.0, 4.0], [None, ("fade", 0.5)], _keys([0], [0, 60]), 30) + assert pieces[0] == F.StitchPiece(True, 0, 1, 0, 60) + + +def test_the_pieces_cover_every_frame_once(): + durations = [4.0, 2.0, 6.0, 3.0, 5.0] + joins = [None, ("fade", 0.5), ("fade", 0.6), None, ("fade", 0.5)] + keys = _keys([0, 60], [0], [0, 60, 120], [0, 60], [0, 60, 120]) + pieces = F.plan_partial_stitch(durations, joins, keys, 30) + for before, after in zip(pieces, pieces[1:]): + if before.end_frame is None: + assert after.first == before.last + 1 and after.start_frame == 0 + else: + assert after.first == before.last and after.start_frame == before.end_frame + assert pieces[0].start_frame == 0 and pieces[-1].end_frame is None + + +def test_the_partial_stitch_encodes_the_picture_and_the_sound_apart(): + video = F.build_stitch_command(["a.mp4", "b.mp4"], [4.0, 4.0], "v.mp4", + joins=_blend(2), options=RenderOptions(), preview=True, + start_frame=60, end_frame=60, streams="video") + assert "-an" in video and "-c:a" not in video and "atrim" not in _graph(video) + audio = F.build_stitch_command(["a.mp4", "b.mp4"], [4.0, 4.0], "a.m4a", + joins=_blend(2), options=RenderOptions(), preview=True, + streams="audio") + assert "-vn" in audio and "-c:v" not in audio and "xfade" not in _graph(audio) + lone = F.build_stitch_command(["a.mp4"], [4.0], "a.m4a", joins=[None], + options=RenderOptions(), preview=True, streams="audio") + assert "-filter_complex" not in lone and lone[lone.index("-map") + 1] == "0:a" + + +def test_a_shot_is_cut_half_a_frame_before_each_keyframe(): + argv = F.build_cut_command("s.mp4", "cut_%03d.mp4", [60, 120], 30) + assert argv[argv.index("-segment_times") + 1] == "1.983333,3.983333" + assert argv[argv.index("-c") + 1] == "copy" and "-an" in argv + + +def test_a_blend_piece_seeks_to_its_first_frame_rather_than_decoding_up_to_it(): + argv = F.build_stitch_command(["a.mp4", "b.mp4"], [4.0, 4.0], "v.mp4", joins=_blend(2), + options=RenderOptions(), preview=True, + start_frame=60, end_frame=60, streams="video") + assert argv[argv.index("a.mp4") - 3:argv.index("a.mp4")] == ["-ss", "1.983333", "-i"] + assert "[0:v]settb=AVTB[s0]" in _graph(argv) and "trim=end_frame=60" in _graph(argv) + # Inside one shot, its end is counted from where the seek landed. + lone = F.build_stitch_command(["a.mp4"], [4.0], "v.mp4", joins=[None], + options=RenderOptions(), preview=True, + start_frame=60, end_frame=90, streams="video") + assert "[0:v]trim=end_frame=30," in _graph(lone) From 8e58c6d07622fa3fa866ed2d94010516d4d4738d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 04:13:29 +0000 Subject: [PATCH 2/2] Match clips' sound to the narration, and choose what the score does during clips Clips played their sound at whatever level the footage had, so one interview was a whisper and the next drowned the voice. Each clip is now measured (EBU R128, over the part that plays) against the narration, measured once per render, and given one static gain to match it: the clip's own dynamics kept, a limiter at -1 dBFS only when it is raised, and a +24 dB ceiling so near-silence doesn't become loud hiss. Measured on ffmpeg 6.1 and 7.1: a clip at -18.4 LUFS lands at -31.4 against narration at -31.3. Two optional spec fields (S5), so existing specs are unchanged: - visual.volume_db, -30..20: a clip's level moved from that match. The clip player has a slider for it. - audio_tracks[].during_clips: absent keeps today's behaviour (a ducked track dips under clips as under the voice); "mute" fades the track out around each clip and back after; "play" keeps it at its level through clips, ducked under the voice alone by compressing it against a key with the clips taken out. The soundtrack panel has a select for it. The assistant's prompt names both fields. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WJkfhgbVRktPf2Lg6XXfNs --- backend/app/assistant/prompt.py | 6 +- backend/app/render/compile.py | 7 +- backend/app/render/ffmpeg.py | 130 +++++++++++++++++--- backend/app/render/renderer.py | 33 ++++- backend/app/spec/model.py | 7 ++ backend/tests/test_render_compile.py | 29 +++++ backend/tests/test_render_ffmpeg.py | 65 ++++++++++ docs/plan-of-attack.md | 18 +++ frontend/src/api/projects.ts | 4 + frontend/src/components/ClipPlayer.tsx | 22 ++++ frontend/src/components/SoundtrackPanel.tsx | 18 +++ frontend/src/spec/clip.test.ts | 7 ++ frontend/src/spec/clip.ts | 15 ++- 13 files changed, 338 insertions(+), 23 deletions(-) diff --git a/backend/app/assistant/prompt.py b/backend/app/assistant/prompt.py index 09fa5a7..68fd3dc 100644 --- a/backend/app/assistant/prompt.py +++ b/backend/app/assistant/prompt.py @@ -42,13 +42,15 @@ - `card`: a title or chapter screen: `{"card":{"variant":"title"|"chapter","title":...,\ "subtitle":...}}`. May have narration too. - `clip`: a video played with its own sound and no narration: `visual` with `trim` and \ -`"audio":"play"`. +`"audio":"play"`. Its sound is matched to the narration's loudness automatically; \ +`visual.volume_db` (-30 to 20) moves it from there, only when asked. - Optional: `chapter` (a chapter mark at its start), `transition_in`, `duration` \ (`min` or `fixed` seconds), `overlays`, `role` (title / intro / outro). - `audio_tracks`: music, sound effects or extra voice, each from a `start` anchor to an \ `end` anchor (omit `end` to play once to its natural end), with `volume` (0-1; music \ usually 0.12-0.2 under narration), `fade_in` / `fade_out` seconds, `duck_under_narration`, \ -`loop`. +`loop`, and `during_clips`: omit it to treat clips like narration, `"mute"` to fade the \ +music out while a clip plays, `"play"` to keep it playing at its level. ## Time is anchored to structure, not seconds An anchor is `"video_start"`, `"video_end"`, or `{"segment":id,"beat":id?,"edge":"start"|\ diff --git a/backend/app/render/compile.py b/backend/app/render/compile.py index be8e5a8..542d9f9 100644 --- a/backend/app/render/compile.py +++ b/backend/app/render/compile.py @@ -68,6 +68,7 @@ class Unit: # Seconds into the file and how long, for a video; None plays it whole. trim: Optional[Tuple[float, float]] = None loop: bool = True + volume_db: float = 0.0 # a clip's own sound, on top of matching the narration panel: Optional[Panel] = None motion: Optional[Motion] = None # resolved, `alternate` already cycled fit: str = "blur" @@ -173,7 +174,8 @@ def plan_units(spec: VideoSpec, media: Dict[str, Media], warnings: List[str]) -> warnings.append(f"Clip {segment.label or segment.id} has no video to play; left out") continue units.append(Unit(**common, first_in_segment=True, kind="video_sound", media=found, - trim=trim, loop=False, chapter=segment.chapter)) + trim=trim, loop=False, chapter=segment.chapter, + volume_db=(visual.volume_db or 0.0) if visual else 0.0)) continue if found and found.kind == "video": @@ -399,6 +401,7 @@ class TrackPlacement: loop: bool duck: bool trim_in: float + during_clips: Optional[str] = None def place_tracks(spec: VideoSpec, media: Dict[str, Media], where: Placement, @@ -441,7 +444,7 @@ def place_tracks(spec: VideoSpec, media: Dict[str, Media], where: Placement, track=track, media=found, start=start, length=length, loop=loop, duck=(track.duck_under_narration if track.duck_under_narration is not None else track.kind == "music"), - trim_in=trim_in, + trim_in=trim_in, during_clips=track.during_clips, )) return placed diff --git a/backend/app/render/ffmpeg.py b/backend/app/render/ffmpeg.py index fff7472..d7390c6 100644 --- a/backend/app/render/ffmpeg.py +++ b/backend/app/render/ffmpeg.py @@ -164,6 +164,56 @@ def probe_has_audio(path: str) -> bool: return False +# Spoken word sits about here; used as the target when a video has no narration. +DEFAULT_LOUDNESS_LUFS = -16.0 +# A clip quieter than this is silence or room tone, and matching it to the voice +# would only raise the noise. +SILENT_LUFS = -60.0 +CLIP_GAIN_LIMITS_DB = (-30.0, 24.0) + + +def measure_loudness(path: str, length: Optional[float] = None) -> Optional[float]: + """Integrated loudness (EBU R128, LUFS) of a file's sound, or of its first + `length` seconds; None when it has none worth measuring.""" + argv = ["ffmpeg", "-hide_banner", "-nostats", "-i", path] + if length is not None: + argv += ["-t", f"{max(0.1, length):.3f}"] + argv += ["-vn", "-af", "ebur128=framelog=quiet", "-f", "null", "-"] + try: + out = subprocess.run(argv, capture_output=True, timeout=PROBE_TIMEOUT_SECONDS * 10) + except (OSError, subprocess.SubprocessError): + return None + # The summary comes last; its "I:" line is the integrated loudness. + found = None + for line in out.stderr.decode(errors="replace").splitlines(): + line = line.strip() + if line.startswith("I:") and line.endswith("LUFS"): + try: + found = float(line[2:-4]) + except ValueError: + found = None + if found is None or found < SILENT_LUFS: + return None + return found + + +def combined_loudness(levels: Sequence[Tuple[float, float]]) -> Optional[float]: + """The loudness of several pieces played one after another, from each one's + (LUFS, seconds): their energy averaged by time, as R128 would over the whole.""" + weighted = [(10 ** (lufs / 10), seconds) for lufs, seconds in levels if seconds > 0] + total = sum(seconds for _, seconds in weighted) + if not weighted or total <= 0: + return None + return 10 * math.log10(sum(energy * seconds for energy, seconds in weighted) / total) + + +def clip_gain(clip_lufs: Optional[float], target_lufs: float, adjust_db: float = 0.0) -> float: + """dB to bring a clip to the narration's loudness, then move it by `adjust_db`.""" + match = 0.0 if clip_lufs is None else target_lufs - clip_lufs + low, high = CLIP_GAIN_LIMITS_DB + return max(low, min(high, match + adjust_db)) + + def encode_args(options: RenderOptions, preview: bool) -> List[str]: """Codec flags shared by every shot. Identical output = concat can `-c copy`.""" preset, crf = encoder_tier(options.quality, preview) @@ -480,6 +530,7 @@ def build_shot_command( out_transition: Optional[TransitionSpec] = None, loop: bool = True, path: Optional["KenBurnsPath"] = None, + audio_gain_db: float = 0.0, ) -> List[str]: """Full argv for rendering one shot to its own MP4. @@ -495,6 +546,7 @@ def build_shot_command( - `loop` False on a video holds its last frame instead of repeating it. - `path`: a Ken Burns move between two frames of the picture (S1), in place of the preset drift. + - `audio_gain_db`: a clip's own sound brought to the narration's loudness (S5). """ width, height = frame_size(options.aspect, options.resolution, preview) argv: List[str] = ["ffmpeg", "-y", "-hide_banner", "-loglevel", "error"] @@ -611,7 +663,16 @@ def build_shot_command( # Pad the audio so a shot always reaches its target length even when the # narration is shorter (a media block with little text under it). - audio_chain = f"[{audio_main}]aresample=48000,aformat=channel_layouts=stereo,apad" + audio_chain = f"[{audio_main}]aresample=48000,aformat=channel_layouts=stereo" + if abs(audio_gain_db) >= 0.05: + audio_chain += f",volume={audio_gain_db:.1f}dB" + if audio_gain_db > 0: + # Raising a quiet clip can push its loudest moments past full scale, so + # they are limited at -1 dBFS rather than clipped. `level=0` stops the + # limiter normalising the whole clip again, and `latency` takes back + # the few milliseconds its lookahead would put between sound and picture. + audio_chain += ",alimiter=limit=0.891:level=0:latency=1" + audio_chain += ",apad" if dip > 0: # Only the tail is faded, to match the picture dipping to colour there — # and it lands in the trailing hold `shot_end_pause_ms` leaves after the @@ -1327,11 +1388,27 @@ class MixTrack: fade_in: float = 0.0 fade_out: float = 0.0 duck: bool = True # pushed down under the narration + during_clips: Optional[str] = None # None, "mute" or "play": see AudioTrack + + +# How long the score takes to leave before a clip and come back after it. +CLIP_FADE_SECONDS = 0.6 + + +def _quiet_during(windows: Sequence[Tuple[float, float]], fade: float) -> str: + """A `volume` expression that is 1, except 0 inside each window, ramping over + `fade` seconds either side of it so the sound leaves and returns rather than + cutting. Quoted, since an expression's commas would otherwise split the graph.""" + fade = max(0.01, fade) + terms = [f"(1-max(0,min(1,min((t-{start - fade:.3f})/{fade:.3f}," + f"({end + fade:.3f}-t)/{fade:.3f}))))" for start, end in windows] + return "'" + "*".join(terms) + "'" def build_mix_command( source: str, tracks: Sequence[MixTrack], output: str, *, duration: float, duck_available: bool = True, + clip_windows: Sequence[Tuple[float, float]] = (), ) -> List[str]: """Mix any number of tracks under the finished video, the picture untouched. @@ -1345,10 +1422,16 @@ def build_mix_command( A looping track restarts from the top of its file, not from `trim_in`: that is what `-stream_loop` does with an input seek, and a bed short enough to loop is rarely trimmed as well. + + `clip_windows` are where a clip plays its own sound. A track set to "mute" + there fades out around each one. The narration's key carries the clips too, + which is what dips a ducked bed under them; a ducked track set to "play" is + compressed against a key with the clips taken out, so only the voice moves it. """ argv = ["ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-i", source] chains: List[str] = [] ducked: List[str] = [] + spared: List[str] = [] flat: List[str] = [] for position, track in enumerate(tracks, start=1): if track.trim_in > 0: @@ -1368,22 +1451,39 @@ def build_mix_command( delay = int(round(max(0.0, track.start) * 1000)) if delay: chain += f",adelay={delay}|{delay}" - chain += f",apad[t{position}]" - chains.append(chain) - (ducked if track.duck and duck_available else flat).append(f"t{position}") + chain += ",apad" + if track.during_clips == "mute" and clip_windows: + # After the delay, so `t` is the video's own clock. + chain += f",volume={_quiet_during(clip_windows, CLIP_FADE_SECONDS)}:eval=frame" + chains.append(chain + f"[t{position}]") + if not (track.duck and duck_available): + flat.append(f"t{position}") + elif track.during_clips == "play" and clip_windows: + spared.append(f"t{position}") + else: + ducked.append(f"t{position}") mix_inputs = ["nar"] - if ducked: - if len(ducked) == 1: - bed = ducked[0] - else: - chains.append("".join(f"[{t}]" for t in ducked) - + f"amix=inputs={len(ducked)}:normalize=0[bedmix]") - bed = "bedmix" - chains.append("[0:a]asplit=2[nar][key]") - chains.append(f"[{bed}][key]sidechaincompress=" - "threshold=0.060:ratio=8:attack=15:release=350[bed]") - mix_inputs.append("bed") + beds = [(name, group) for name, group in (("bed", ducked), ("spared", spared)) if group] + if beds: + keys = (["key"] if ducked else []) + (["keyvoice"] if spared else []) + chains.append(f"[0:a]asplit={1 + len(keys)}[nar]" + "".join(f"[{k}]" for k in keys)) + if spared: + # The clips cut out of the key, quickly, so the bed is not ducked + # across them; the voice either side still is. + chains.append(f"[keyvoice]volume={_quiet_during(clip_windows, 0.05)}:eval=frame" + "[keyspared]") + for name, group in beds: + if len(group) == 1: + source_label = group[0] + else: + chains.append("".join(f"[{t}]" for t in group) + + f"amix=inputs={len(group)}:normalize=0[{name}mix]") + source_label = f"{name}mix" + key = "key" if name == "bed" else "keyspared" + chains.append(f"[{source_label}][{key}]sidechaincompress=" + f"threshold=0.060:ratio=8:attack=15:release=350[{name}]") + mix_inputs.append(name) else: chains.append("[0:a]anull[nar]") mix_inputs += flat diff --git a/backend/app/render/renderer.py b/backend/app/render/renderer.py index 736b1eb..77e51cc 100644 --- a/backend/app/render/renderer.py +++ b/backend/app/render/renderer.py @@ -360,6 +360,19 @@ def render( for u, n in zip(units, narrated)] timeline = layout(units, durations, warnings) + # ── how loud the voice is, for clips to be matched to (S5) ───────────────── + # Measured, not assumed: voices and providers differ by several dB. R128 gates + # out the pauses, so the holds between lines don't pull it down. + narration_lufs = F.DEFAULT_LOUDNESS_LUFS + if any(u.kind == "video_sound" for u in units): + levels = [] + for narration in narrated: + if narration.path and narration.duration > 0: + lufs = F.measure_loudness(os.path.join(work_dir, narration.path)) + if lufs is not None: + levels.append((lufs, narration.duration)) + narration_lufs = F.combined_loudness(levels) or F.DEFAULT_LOUDNESS_LUFS + # ── encode each unit ───────────────────────────────────────────────────── cache = shot_cache.ShotCache(data_dir, user_id) try: @@ -421,6 +434,15 @@ def render( cwd=work_dir, timeout=F.shot_timeout(length)) background = clip + # A clip's sound at the voice's loudness, then moved by its own setting: + # footage arrives anywhere from a whisper to a broadcast master. Measured + # over what plays, so a loud intro the trim cuts off doesn't count. + gain = 0.0 + if unit.kind == "video_sound" and has_audio: + gain = round(F.clip_gain(F.measure_loudness(os.path.join(work_dir, background), + duration), + narration_lufs, unit.volume_db), 1) + # Watermark, overlay text and a quote or code panel, as one layer. if unit.chapter: current_chapter = unit.chapter @@ -470,7 +492,7 @@ def argv(with_options: RenderOptions, with_path=path) -> List[str]: duration=duration, output=output, options=with_options, preview=preview, overlay_png=overlay_png, subtitle_file=shot_srt, background_has_audio=has_audio, index=index, in_transition=in_transition, out_transition=out_transition, - loop=unit.loop, path=with_path, + loop=unit.loop, path=with_path, audio_gain_db=gain, ) if narration.path is None and unit.kind != "video_sound": @@ -547,9 +569,14 @@ def argv(with_options: RenderOptions, with_path=path) -> List[str]: progress("Mixing", 90, f"{len(tracks)} audio track{'s' if len(tracks) != 1 else ''}") mix = [F.MixTrack(path=t.media.path, start=t.start, length=t.length, trim_in=t.trim_in, loop=t.loop, volume=t.track.volume, fade_in=t.track.fade_in, - fade_out=t.track.fade_out, duck=t.duck) for t in tracks] + fade_out=t.track.fade_out, duck=t.duck, + during_clips=t.during_clips) for t in tracks] + # Where a clip plays its own sound, for a track that leaves for clips (S5). + clip_windows = [(timeline.starts[i], timeline.end(i)) for i, u in enumerate(units) + if u.kind == "video_sound" and u.media and u.media.has_audio] F.run(F.build_mix_command(final, mix, "scored.mp4", duration=timeline.total, - duck_available="sidechaincompress" in F.available_filters()), + duck_available="sidechaincompress" in F.available_filters(), + clip_windows=clip_windows), cwd=work_dir, timeout=1800) final = "scored.mp4" diff --git a/backend/app/spec/model.py b/backend/app/spec/model.py index b8fb4b2..4d57759 100644 --- a/backend/app/spec/model.py +++ b/backend/app/spec/model.py @@ -397,6 +397,9 @@ class Visual(_Model): trim: Optional[Trim] = None # videos only audio: Literal["mute", "play"] = "mute" # "play" only on a clip loop: bool = True # shorter than its segment: loop, or hold + # A clip's sound is matched to the narration's loudness by the renderer; this + # moves it from there, in dB (S5). Ignored where the footage plays muted. + volume_db: Optional[float] = Field(default=None, ge=-30, le=20) class Beat(_Model): @@ -519,6 +522,10 @@ class AudioTrack(_Model): loop: Optional[bool] = None # default: on for music volume: Unit = 0.18 duck_under_narration: Optional[bool] = None # default: on for music + # While a clip plays its own sound (S5). None: as under narration (a ducked + # track dips, one that isn't plays on). "mute" fades it out for the clip and + # back in after; "play" keeps it at its level, ducked under narration only. + during_clips: Optional[Literal["mute", "play"]] = None fade_in: float = Field(default=0.0, ge=0, le=15) fade_out: float = Field(default=0.0, ge=0, le=15) diff --git a/backend/tests/test_render_compile.py b/backend/tests/test_render_compile.py index 731ad87..0bc8863 100644 --- a/backend/tests/test_render_compile.py +++ b/backend/tests/test_render_compile.py @@ -357,3 +357,32 @@ def test_the_overlay_pass_draws_each_layer_only_in_its_window_and_copies_the_sou assert "[l2]" in graph and "fade" not in graph.split("[1:v]")[0] assert argv[argv.index("-c:a") + 1] == "copy" assert argv.count("-c:a") == 1 and argv[argv.index("-map") + 1] == "[v2]" + + +# ── clip sound levels and the score during clips (S5) ─────────────────────── + +def test_a_clips_own_level_and_a_tracks_choice_during_clips_reach_the_render(): + spec = spec_of({ + "assets": {"v": {"from": "gam", "id": "g1", "type": "video"}, + "m": {"from": "gam", "id": "g2", "type": "audio"}}, + "segments": [ + {"id": "a", "kind": "clip", "visual": {"asset": "v", "audio": "play", + "volume_db": -4.5}}, + {"id": "b", "kind": "clip", "visual": {"asset": "v", "audio": "play"}}, + ], + "audio_tracks": [{"id": "t", "asset": "m", "start": "video_start", + "end": "video_end", "during_clips": "mute"}]}) + units = plan_units(spec, {"v": VIDEO, "m": SONG}, []) + assert [u.volume_db for u in units] == [-4.5, 0.0] + timeline = layout(units, [6.0, 6.0], []) + where = placement(units, timeline, [{} for _ in units]) + [track] = place_tracks(spec, {"v": VIDEO, "m": SONG}, where, []) + assert track.during_clips == "mute" + + +def test_a_clip_level_outside_its_range_is_refused(): + _spec, issues = parse_spec({ + "assets": {"v": {"from": "gam", "id": "g1", "type": "video"}}, + "segments": [{"id": "a", "kind": "clip", + "visual": {"asset": "v", "audio": "play", "volume_db": 40}}]}) + assert _spec is None and issues diff --git a/backend/tests/test_render_ffmpeg.py b/backend/tests/test_render_ffmpeg.py index d491e90..ed03073 100644 --- a/backend/tests/test_render_ffmpeg.py +++ b/backend/tests/test_render_ffmpeg.py @@ -1455,3 +1455,68 @@ def test_a_blend_piece_seeks_to_its_first_frame_rather_than_decoding_up_to_it(): options=RenderOptions(), preview=True, start_frame=60, end_frame=90, streams="video") assert "[0:v]trim=end_frame=30," in _graph(lone) + + +# ── clip sound levels and the score during clips (S5) ─────────────────────── + +def test_a_clip_is_brought_to_the_voices_loudness_then_moved_by_its_own_setting(): + assert F.clip_gain(-30.0, -18.0) == 12.0 + assert F.clip_gain(-10.0, -18.0) == -8.0 + assert F.clip_gain(-30.0, -18.0, adjust_db=-6.0) == 6.0 + # Silence (unmeasurable) is left alone, but still follows its own setting. + assert F.clip_gain(None, -18.0) == 0.0 and F.clip_gain(None, -18.0, 3.0) == 3.0 + # A near-silent clip is not raised without limit: that would only raise the noise. + assert F.clip_gain(-70.0, -16.0) == F.CLIP_GAIN_LIMITS_DB[1] + + +def test_the_voices_loudness_is_its_pieces_averaged_by_energy_and_time(): + assert F.combined_loudness([(-18.0, 10.0), (-18.0, 30.0)]) == pytest.approx(-18.0) + # Twice the time at one level weighs twice as much; levels add as energy, not dB. + assert F.combined_loudness([(-20.0, 10.0), (-10.0, 10.0)]) == pytest.approx(-12.6, abs=0.1) + assert F.combined_loudness([]) is None + + +def test_a_raised_clip_is_limited_and_a_lowered_one_is_not(): + def audio(gain): + argv = F.build_shot_command( + kind="video_sound", background="clip.mp4", audio=None, duration=6.0, + output="s.mp4", options=RenderOptions(), preview=True, + background_has_audio=True, audio_gain_db=gain) + return [c for c in _graph(argv).split(";") if c.endswith("[a]")][0] + assert "volume=12.0dB,alimiter=limit=0.891:level=0:latency=1,apad" in audio(12.0) + assert "volume=-8.0dB,apad" in audio(-8.0) and "alimiter" not in audio(-8.0) + assert "volume" not in audio(0.0) + + +def test_a_track_can_leave_for_clips_and_come_back_after(): + track = F.MixTrack(path="m.mp3", start=0.0, length=30.0, during_clips="mute") + graph = _graph(F.build_mix_command("in.mp4", [track], "o.mp4", duration=30.0, + clip_windows=[(8.0, 15.0)])) + assert ("volume='(1-max(0,min(1,min((t-7.400)/0.600,(15.600-t)/0.600))))':eval=frame" + in graph) + # Still ducked under the narration as before. + assert "[t1][key]sidechaincompress=" in graph + + +def test_a_track_can_play_on_through_clips_and_still_duck_under_the_voice(): + plain = F.MixTrack(path="a.mp3", start=0.0, length=30.0) + spared = F.MixTrack(path="b.mp3", start=0.0, length=30.0, during_clips="play") + graph = _graph(F.build_mix_command("in.mp4", [plain, spared], "o.mp4", duration=30.0, + clip_windows=[(8.0, 15.0)])) + assert "[0:a]asplit=3[nar][key][keyvoice]" in graph + assert "[keyvoice]volume='" in graph and "[keyspared]" in graph + assert "[t1][key]sidechaincompress=" in graph + assert "[t2][keyspared]sidechaincompress=" in graph + assert "[nar][bed][spared]amix=inputs=3" in graph + # Alone, it needs no key for a bed that isn't there. + alone = _graph(F.build_mix_command("in.mp4", [spared], "o.mp4", duration=30.0, + clip_windows=[(8.0, 15.0)])) + assert "[0:a]asplit=2[nar][keyvoice]" in alone + + +def test_without_clips_a_tracks_choice_during_clips_changes_nothing(): + for mode in ("mute", "play"): + graph = _graph(F.build_mix_command( + "in.mp4", [F.MixTrack(path="m.mp3", start=0.0, length=30.0, during_clips=mode)], + "o.mp4", duration=30.0)) + assert "eval=frame" not in graph and "[t1][key]sidechaincompress=" in graph diff --git a/docs/plan-of-attack.md b/docs/plan-of-attack.md index 0530637..49b1185 100644 --- a/docs/plan-of-attack.md +++ b/docs/plan-of-attack.md @@ -313,6 +313,24 @@ This plan takes **field by field**, so `{ "transition_in": { "style": "dissolve" keeps the project's duration, because that is what the AI and the editor will both expect. +### S5. Clip sound levels, and the score during clips + +Footage arrives at any level, from a phone in a room to a broadcast master, so a clip +playing its own sound is matched to the narration by the renderer: each clip's +integrated loudness (EBU R128) over the part that plays, against the narration's, +one static gain per clip (dynamics kept, a limiter only when raising), capped at ++24 dB so a near-silent clip doesn't become loud hiss. Two optional fields then move +it from there, so specs without them are unchanged and stay `"0.2"`: + +```ts +visual.volume_db?: number // -30..20; a clip's level from the match +audio_tracks[].during_clips?: "mute" | "play" // absent: as under narration +``` + +A ducked track already dips under clips, since the narration's key carries their +sound. `"mute"` fades the track out around each clip and back after; `"play"` keeps +it at its level through clips, ducked under the voice alone. + ## Changes needed in GAM Specified here, applied in GAM's repository, not from this one. diff --git a/frontend/src/api/projects.ts b/frontend/src/api/projects.ts index 56119f8..5805260 100644 --- a/frontend/src/api/projects.ts +++ b/frontend/src/api/projects.ts @@ -47,6 +47,8 @@ export interface Visual { trim?: Trim audio?: 'mute' | 'play' loop?: boolean + /** A clip's sound, in dB from the narration's loudness it is matched to. */ + volume_db?: number fit?: string motion?: Motion [key: string]: unknown @@ -68,6 +70,8 @@ export interface AudioTrack { loop?: boolean volume?: number duck_under_narration?: boolean + /** While a clip plays its own sound. Absent: as under narration. */ + during_clips?: 'mute' | 'play' fade_in?: number fade_out?: number [key: string]: unknown diff --git a/frontend/src/components/ClipPlayer.tsx b/frontend/src/components/ClipPlayer.tsx index f49110e..ad29232 100644 --- a/frontend/src/components/ClipPlayer.tsx +++ b/frontend/src/components/ClipPlayer.tsx @@ -10,6 +10,7 @@ import { setLength, setLoop, setTrim, + setVolume, timingNote, trimAt, } from '@/spec/clip' @@ -183,6 +184,27 @@ export default function ClipPlayer({ {edge('in')} {edge('out')} + {visual.audio === 'play' && ( + + )} )} {segment.kind === 'scene' && ( diff --git a/frontend/src/components/SoundtrackPanel.tsx b/frontend/src/components/SoundtrackPanel.tsx index 7e4ef90..b1c348e 100644 --- a/frontend/src/components/SoundtrackPanel.tsx +++ b/frontend/src/components/SoundtrackPanel.tsx @@ -139,6 +139,24 @@ export default function SoundtrackPanel({ spec, gamAssets, onChange }: Props) { /> Quieter under narration +