Play a scene's footage, time it by hand, and keep dissolves off clip audio

backend/app/render/compile.py

@@ -80,6 +80,9 @@ class Unit:
     chapter: Optional[str] = None
     # The end credits: a card drawn from these lines rather than from `card`.
     credits: Optional[List[str]] = None
+    # Seconds a clip with its sound runs past its out point, under a blend out of it
+    # (`layout`). Footage after the out point where the file has it, else a held frame.
+    tail: float = 0.0
 
 
 def _transition_in(spec: VideoSpec, segment: Segment) -> Transition:
@@ -281,7 +284,13 @@ def layout(units: Sequence[Unit], durations: Sequence[float], warnings: List[str
     project's transition, as every quote has in Notes. A blend overlaps its two units
     by its duration, held to half the shorter one so neither is ever entirely under a
     blend; too short to see, and it becomes a cut.
+
+    A clip playing its own sound is lengthened by the blend out of it rather than
+    overlapped: the stitch ends the outgoing shot's audio where the blend starts, so
+    otherwise the last words before its out point would be cut. Its `tail` says by how
+    much, for the renderer to extract that much more.
     """
+    durations = list(durations)
     joins: List[Optional[Join]] = [None]
     for index in range(1, len(units)):
         unit = units[index]
@@ -304,6 +313,13 @@ def layout(units: Sequence[Unit], durations: Sequence[float], warnings: List[str
             if join and join.is_blend:
                 join.style, join.overlap = "fade", 0.0
 
+    for index, join in enumerate(joins):
+        before = units[index - 1] if index else None
+        if join is not None and join.is_blend and before is not None \
+                and before.kind == "video_sound":
+            before.tail = join.overlap
+            durations[index - 1] += join.overlap
+
     starts: List[float] = []
     cursor = 0.0
     for index, duration in enumerate(durations):

backend/app/render/ffmpeg.py

@@ -492,7 +492,7 @@ def build_shot_command(
     - `in_transition` / `out_transition`: the join at each end, which in GVC differ
       per scene (a scene's `transition_in` is its incoming join and the previous
       scene's outgoing one). Unset, both are `options.transition`.
-    - `loop` False on a muted video holds its last frame instead of repeating it.
+    - `loop` False on a video holds its last frame instead of repeating it.
     - `path`: a Ken Burns move between two frames of the picture (S1), in place of
       the preset drift.
     """
@@ -565,8 +565,10 @@ def build_shot_command(
         chains = [fit_chain(video_in, "fit", width, height, options.fit)]
         stage = "fit"
 
-    if kind == "video_muted" and not loop:
-        # Shorter than its scene and not to be repeated: hold the last frame.
+    if kind in ("video_muted", "video_sound") and not loop:
+        # Shorter than its scene and not to be repeated: hold the last frame. A clip
+        # with its sound reaches here when a blend out of it asks for more than the
+        # file has left after its out point (`compile.layout`'s tail).
         chains.append(f"[{stage}]tpad=stop_mode=clone:stop_duration={duration:.3f}[hold]")
         stage = "hold"
 

backend/app/render/renderer.py

@@ -311,6 +311,8 @@ def render(
                 start, length = unit.trim
                 if length is None:
                     length = max(0.1, (unit.media.duration or 0.0) - start)
+                # Past the file's end `-t` yields less, and the shot holds its last frame.
+                length += unit.tail
                 clip = f"clip_{index:04d}.mp4"
                 F.run(F.build_extract_command(background, clip, start=start, length=length,
                                               with_audio=unit.kind == "video_sound"),
@@ -405,7 +407,8 @@ def argv(with_options: RenderOptions, with_path=path) -> List[str]:
                   cwd=work_dir, timeout=F.shot_timeout(sum(timeline.durations)))
         except F.FFmpegError as exc:
             # The blends cost a re-encode; if that fails, the shots still join by cuts,
-            # and the timeline is re-laid without their overlaps to match.
+            # and the timeline is re-laid without their overlaps to match. A clip's
+            # tail stays, as a moment more of it before the cut.
             logger.warning("Blended stitch failed (%s); joining with cuts", exc)
             warnings.append("Blends were joined as cuts: blending the shots together failed.")
             for join in timeline.joins:

backend/tests/test_render_compile.py

@@ -139,6 +139,40 @@ def test_a_blend_never_covers_more_than_half_a_short_unit():
     assert timeline.total == 20.5
 
 
+def _clip_then(style):
+    spec = spec_of({
+        "assets": {"v": {"from": "gam", "id": "g1", "type": "video"}},
+        "segments": [
+            {"id": "a", "kind": "clip", "visual": {"asset": "v", "audio": "play",
+                                                    "trim": {"in": 10, "out": 16}}},
+            {"id": "b", "kind": "scene", "transition_in": {"style": style, "duration": 0.6}},
+            {"id": "c", "kind": "scene", "transition_in": {"style": "dissolve",
+                                                          "duration": 0.6}},
+        ]})
+    return plan_units(spec, {"v": VIDEO}, [])
+
+
+def test_a_blend_out_of_a_clip_lengthens_it_rather_than_cutting_its_last_words():
+    units = _clip_then("dissolve")
+    durations = [6.0, 10.0, 10.0]
+    timeline = layout(units, durations, [])
+    assert units[0].tail == pytest.approx(0.6)
+    assert timeline.durations[0] == pytest.approx(6.6)
+    # The blend starts once the clip has reached its out point.
+    assert timeline.starts[1] == pytest.approx(6.0)
+    # A muted scene into a blend is overlapped as before.
+    assert units[1].tail == 0.0 and timeline.durations[1] == 10.0
+    assert timeline.starts[2] == pytest.approx(15.4)
+    assert durations == [6.0, 10.0, 10.0]
+
+
+def test_a_cut_out_of_a_clip_adds_nothing():
+    units = _clip_then("none")
+    timeline = layout(units, [6.0, 10.0, 10.0], [])
+    assert units[0].tail == 0.0
+    assert timeline.durations[0] == 6.0 and timeline.starts[1] == 6.0
+
+
 def _placed():
     spec, media = salt_road()
     units = plan_units(spec, media, [])

backend/tests/test_render_ffmpeg.py

@@ -119,6 +119,15 @@ def test_sounded_clip_uses_its_own_audio():
     assert "[0:a]" in _graph(argv)
 
 
+def test_a_sounded_clip_holds_its_last_frame_rather_than_running_short():
+    """A blend's tail can ask for more than the file has after the out point."""
+    argv = F.build_shot_command(kind="video_sound", background="c.mp4", audio=None, duration=2,
+                                output="s.mp4", options=RenderOptions(), preview=False,
+                                background_has_audio=True, loop=False)
+    assert "tpad=stop_mode=clone" in _graph(argv)
+    assert "-stream_loop" not in argv
+
+
 def test_sounded_clip_without_an_audio_track_gets_silence():
     """The concat demuxer needs every part to carry an audio stream."""
     argv = F.build_shot_command(kind="video_sound", background="c.mp4", audio=None, duration=2,

frontend/src/api/projects.ts

@@ -154,6 +154,8 @@ export interface Segment {
   narration?: { beats: Beat[]; [key: string]: unknown }
   transition_in?: { style?: string; duration?: number }
   overlays?: Overlay[]
+  /** Seconds: `fixed` sets the scene's length, `min` floors one that follows narration. */
+  duration?: { min?: number; fixed?: number }
   [key: string]: unknown
 }
 

frontend/src/components/ClipPlayer.test.tsx

@@ -0,0 +1,84 @@
+import { render, screen } from '@testing-library/react'
+import userEvent from '@testing-library/user-event'
+import { describe, expect, it, vi } from 'vitest'
+import type { GamAsset } from '@/api/gam'
+import type { VideoSpec } from '@/api/projects'
+import ClipPlayer from '@/components/ClipPlayer'
+
+const reel: GamAsset = {
+  id: 'g1',
+  name: 'Interview',
+  asset_type: 'video',
+  duration_seconds: 60,
+  width: null,
+  height: null,
+  file_url: 'https://gam.test/f',
+  thumb_url: null,
+  credit: '',
+  parent_asset_id: null,
+  in_point: null,
+  out_point: null,
+  missing: false,
+}
+
+const spec: VideoSpec = {
+  title: 'T',
+  assets: { v: { from: 'gam', id: 'g1' } },
+  segments: [
+    {
+      id: 's1',
+      kind: 'scene',
+      visual: { asset: 'v', trim: { in: 10, out: 16 } },
+      narration: { beats: [{ id: 'b1', text: 'Hello' }] },
+    },
+  ],
+}
+
+function player(onChange = vi.fn()) {
+  render(
+    <ClipPlayer
+      spec={spec}
+      index={0}
+      segment={spec.segments[0]}
+      asset={reel}
+      onChange={onChange}
+      onClose={vi.fn()}
+    />
+  )
+  return onChange
+}
+
+describe('ClipPlayer', () => {
+  it('shows the trimmed window and its length', () => {
+    player()
+    expect(screen.getByLabelText('in point')).toHaveTextContent('0:10.0')
+    expect(screen.getByLabelText('out point')).toHaveTextContent('0:16.0')
+    expect(screen.getByText(/6 s of footage/)).toBeInTheDocument()
+  })
+
+  it('nudges the out point', async () => {
+    const onChange = player()
+    await userEvent.click(screen.getByRole('button', { name: 'out +0.1 s' }))
+    expect(onChange.mock.calls[0][0].segments[0].visual.trim).toEqual({
+      in: 10,
+      out: 16.1,
+    })
+  })
+
+  it('plays the whole clip again', async () => {
+    const onChange = player()
+    await userEvent.click(screen.getByRole('button', { name: 'Whole clip' }))
+    expect(onChange.mock.calls[0][0].segments[0].visual.trim).toBeUndefined()
+  })
+
+  it('fixes the scene to the footage’s length', async () => {
+    const onChange = player()
+    await userEvent.click(screen.getByRole('button', { name: 'Match footage' }))
+    expect(onChange.mock.calls[0][0].segments[0].duration).toEqual({ fixed: 6 })
+  })
+
+  it('says the footage is cut short when the narration is shorter', () => {
+    player()
+    expect(screen.getByRole('status')).toHaveTextContent('so its last 3.5 s aren’t shown')
+  })
+})