From 9dbf78624d110e531529b8df8662fc60b80dacdf Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 11:19:59 +0000 Subject: [PATCH] Play a scene's footage, time it by hand, and keep dissolves off clip audio MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clicking a scene's thumbnail opens its footage in a player that stops at the out point. Under it: in/out points with ±0.1 s / ±1 s nudges and "set here", "Whole clip", the scene's length (follow narration, fixed, match footage), loop or hold for short footage, and one line saying when the footage will loop, hold or be cut short by its narration. A blend out of a clip that plays its own sound used to cut its last words: the stitch ends the outgoing shot's audio where the blend starts. The layout now lengthens such a clip by the blend's overlap (its `tail`), played from the footage after the out point, or as a held last frame where the file ends. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ko6G3pNhKu51bkYyUceSyg --- backend/app/render/compile.py | 16 ++ backend/app/render/ffmpeg.py | 8 +- backend/app/render/renderer.py | 5 +- backend/tests/test_render_compile.py | 34 +++ backend/tests/test_render_ffmpeg.py | 9 + frontend/src/api/projects.ts | 2 + frontend/src/components/ClipPlayer.test.tsx | 84 ++++++ frontend/src/components/ClipPlayer.tsx | 258 +++++++++++++++++++ frontend/src/components/SceneList.tsx | 236 +++++++++-------- frontend/src/components/VisualStrip.test.tsx | 16 ++ frontend/src/components/VisualStrip.tsx | 19 +- frontend/src/spec/clip.test.ts | 151 +++++++++++ frontend/src/spec/clip.ts | 221 ++++++++++++++++ 13 files changed, 945 insertions(+), 114 deletions(-) create mode 100644 frontend/src/components/ClipPlayer.test.tsx create mode 100644 frontend/src/components/ClipPlayer.tsx create mode 100644 frontend/src/spec/clip.test.ts create mode 100644 frontend/src/spec/clip.ts diff --git a/backend/app/render/compile.py b/backend/app/render/compile.py index 49c712d..be8e5a8 100644 --- a/backend/app/render/compile.py +++ b/backend/app/render/compile.py @@ -80,6 +80,9 @@ class Unit: chapter: Optional[str] = None # The end credits: a card drawn from these lines rather than from `card`. credits: Optional[List[str]] = None + # Seconds a clip with its sound runs past its out point, under a blend out of it + # (`layout`). Footage after the out point where the file has it, else a held frame. + tail: float = 0.0 def _transition_in(spec: VideoSpec, segment: Segment) -> Transition: @@ -281,7 +284,13 @@ def layout(units: Sequence[Unit], durations: Sequence[float], warnings: List[str project's transition, as every quote has in Notes. A blend overlaps its two units by its duration, held to half the shorter one so neither is ever entirely under a blend; too short to see, and it becomes a cut. + + A clip playing its own sound is lengthened by the blend out of it rather than + overlapped: the stitch ends the outgoing shot's audio where the blend starts, so + otherwise the last words before its out point would be cut. Its `tail` says by how + much, for the renderer to extract that much more. """ + durations = list(durations) joins: List[Optional[Join]] = [None] for index in range(1, len(units)): unit = units[index] @@ -304,6 +313,13 @@ def layout(units: Sequence[Unit], durations: Sequence[float], warnings: List[str if join and join.is_blend: join.style, join.overlap = "fade", 0.0 + for index, join in enumerate(joins): + before = units[index - 1] if index else None + if join is not None and join.is_blend and before is not None \ + and before.kind == "video_sound": + before.tail = join.overlap + durations[index - 1] += join.overlap + starts: List[float] = [] cursor = 0.0 for index, duration in enumerate(durations): diff --git a/backend/app/render/ffmpeg.py b/backend/app/render/ffmpeg.py index b984323..f03b228 100644 --- a/backend/app/render/ffmpeg.py +++ b/backend/app/render/ffmpeg.py @@ -492,7 +492,7 @@ def build_shot_command( - `in_transition` / `out_transition`: the join at each end, which in GVC differ per scene (a scene's `transition_in` is its incoming join and the previous scene's outgoing one). Unset, both are `options.transition`. - - `loop` False on a muted video holds its last frame instead of repeating it. + - `loop` False on a video holds its last frame instead of repeating it. - `path`: a Ken Burns move between two frames of the picture (S1), in place of the preset drift. """ @@ -565,8 +565,10 @@ def build_shot_command( chains = [fit_chain(video_in, "fit", width, height, options.fit)] stage = "fit" - if kind == "video_muted" and not loop: - # Shorter than its scene and not to be repeated: hold the last frame. + if kind in ("video_muted", "video_sound") and not loop: + # Shorter than its scene and not to be repeated: hold the last frame. A clip + # with its sound reaches here when a blend out of it asks for more than the + # file has left after its out point (`compile.layout`'s tail). chains.append(f"[{stage}]tpad=stop_mode=clone:stop_duration={duration:.3f}[hold]") stage = "hold" diff --git a/backend/app/render/renderer.py b/backend/app/render/renderer.py index 923bcd8..3a8a71f 100644 --- a/backend/app/render/renderer.py +++ b/backend/app/render/renderer.py @@ -311,6 +311,8 @@ def render( start, length = unit.trim if length is None: length = max(0.1, (unit.media.duration or 0.0) - start) + # Past the file's end `-t` yields less, and the shot holds its last frame. + length += unit.tail clip = f"clip_{index:04d}.mp4" F.run(F.build_extract_command(background, clip, start=start, length=length, with_audio=unit.kind == "video_sound"), @@ -405,7 +407,8 @@ def argv(with_options: RenderOptions, with_path=path) -> List[str]: cwd=work_dir, timeout=F.shot_timeout(sum(timeline.durations))) except F.FFmpegError as exc: # The blends cost a re-encode; if that fails, the shots still join by cuts, - # and the timeline is re-laid without their overlaps to match. + # and the timeline is re-laid without their overlaps to match. A clip's + # tail stays, as a moment more of it before the cut. logger.warning("Blended stitch failed (%s); joining with cuts", exc) warnings.append("Blends were joined as cuts: blending the shots together failed.") for join in timeline.joins: diff --git a/backend/tests/test_render_compile.py b/backend/tests/test_render_compile.py index 3f71ffd..731ad87 100644 --- a/backend/tests/test_render_compile.py +++ b/backend/tests/test_render_compile.py @@ -139,6 +139,40 @@ def test_a_blend_never_covers_more_than_half_a_short_unit(): assert timeline.total == 20.5 +def _clip_then(style): + spec = spec_of({ + "assets": {"v": {"from": "gam", "id": "g1", "type": "video"}}, + "segments": [ + {"id": "a", "kind": "clip", "visual": {"asset": "v", "audio": "play", + "trim": {"in": 10, "out": 16}}}, + {"id": "b", "kind": "scene", "transition_in": {"style": style, "duration": 0.6}}, + {"id": "c", "kind": "scene", "transition_in": {"style": "dissolve", + "duration": 0.6}}, + ]}) + return plan_units(spec, {"v": VIDEO}, []) + + +def test_a_blend_out_of_a_clip_lengthens_it_rather_than_cutting_its_last_words(): + units = _clip_then("dissolve") + durations = [6.0, 10.0, 10.0] + timeline = layout(units, durations, []) + assert units[0].tail == pytest.approx(0.6) + assert timeline.durations[0] == pytest.approx(6.6) + # The blend starts once the clip has reached its out point. + assert timeline.starts[1] == pytest.approx(6.0) + # A muted scene into a blend is overlapped as before. + assert units[1].tail == 0.0 and timeline.durations[1] == 10.0 + assert timeline.starts[2] == pytest.approx(15.4) + assert durations == [6.0, 10.0, 10.0] + + +def test_a_cut_out_of_a_clip_adds_nothing(): + units = _clip_then("none") + timeline = layout(units, [6.0, 10.0, 10.0], []) + assert units[0].tail == 0.0 + assert timeline.durations[0] == 6.0 and timeline.starts[1] == 6.0 + + def _placed(): spec, media = salt_road() units = plan_units(spec, media, []) diff --git a/backend/tests/test_render_ffmpeg.py b/backend/tests/test_render_ffmpeg.py index 4db74c9..bdafb13 100644 --- a/backend/tests/test_render_ffmpeg.py +++ b/backend/tests/test_render_ffmpeg.py @@ -119,6 +119,15 @@ def test_sounded_clip_uses_its_own_audio(): assert "[0:a]" in _graph(argv) +def test_a_sounded_clip_holds_its_last_frame_rather_than_running_short(): + """A blend's tail can ask for more than the file has after the out point.""" + argv = F.build_shot_command(kind="video_sound", background="c.mp4", audio=None, duration=2, + output="s.mp4", options=RenderOptions(), preview=False, + background_has_audio=True, loop=False) + assert "tpad=stop_mode=clone" in _graph(argv) + assert "-stream_loop" not in argv + + def test_sounded_clip_without_an_audio_track_gets_silence(): """The concat demuxer needs every part to carry an audio stream.""" argv = F.build_shot_command(kind="video_sound", background="c.mp4", audio=None, duration=2, diff --git a/frontend/src/api/projects.ts b/frontend/src/api/projects.ts index 40a4104..56119f8 100644 --- a/frontend/src/api/projects.ts +++ b/frontend/src/api/projects.ts @@ -154,6 +154,8 @@ export interface Segment { narration?: { beats: Beat[]; [key: string]: unknown } transition_in?: { style?: string; duration?: number } overlays?: Overlay[] + /** Seconds: `fixed` sets the scene's length, `min` floors one that follows narration. */ + duration?: { min?: number; fixed?: number } [key: string]: unknown } diff --git a/frontend/src/components/ClipPlayer.test.tsx b/frontend/src/components/ClipPlayer.test.tsx new file mode 100644 index 0000000..dd201b9 --- /dev/null +++ b/frontend/src/components/ClipPlayer.test.tsx @@ -0,0 +1,84 @@ +import { render, screen } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { describe, expect, it, vi } from 'vitest' +import type { GamAsset } from '@/api/gam' +import type { VideoSpec } from '@/api/projects' +import ClipPlayer from '@/components/ClipPlayer' + +const reel: GamAsset = { + id: 'g1', + name: 'Interview', + asset_type: 'video', + duration_seconds: 60, + width: null, + height: null, + file_url: 'https://gam.test/f', + thumb_url: null, + credit: '', + parent_asset_id: null, + in_point: null, + out_point: null, + missing: false, +} + +const spec: VideoSpec = { + title: 'T', + assets: { v: { from: 'gam', id: 'g1' } }, + segments: [ + { + id: 's1', + kind: 'scene', + visual: { asset: 'v', trim: { in: 10, out: 16 } }, + narration: { beats: [{ id: 'b1', text: 'Hello' }] }, + }, + ], +} + +function player(onChange = vi.fn()) { + render( + + ) + return onChange +} + +describe('ClipPlayer', () => { + it('shows the trimmed window and its length', () => { + player() + expect(screen.getByLabelText('in point')).toHaveTextContent('0:10.0') + expect(screen.getByLabelText('out point')).toHaveTextContent('0:16.0') + expect(screen.getByText(/6 s of footage/)).toBeInTheDocument() + }) + + it('nudges the out point', async () => { + const onChange = player() + await userEvent.click(screen.getByRole('button', { name: 'out +0.1 s' })) + expect(onChange.mock.calls[0][0].segments[0].visual.trim).toEqual({ + in: 10, + out: 16.1, + }) + }) + + it('plays the whole clip again', async () => { + const onChange = player() + await userEvent.click(screen.getByRole('button', { name: 'Whole clip' })) + expect(onChange.mock.calls[0][0].segments[0].visual.trim).toBeUndefined() + }) + + it('fixes the scene to the footage’s length', async () => { + const onChange = player() + await userEvent.click(screen.getByRole('button', { name: 'Match footage' })) + expect(onChange.mock.calls[0][0].segments[0].duration).toEqual({ fixed: 6 }) + }) + + it('says the footage is cut short when the narration is shorter', () => { + player() + expect(screen.getByRole('status')).toHaveTextContent('so its last 3.5 s aren’t shown') + }) +}) diff --git a/frontend/src/components/ClipPlayer.tsx b/frontend/src/components/ClipPlayer.tsx new file mode 100644 index 0000000..f49110e --- /dev/null +++ b/frontend/src/components/ClipPlayer.tsx @@ -0,0 +1,258 @@ +import { useEffect, useRef, useState } from 'react' +import { Play, X } from 'lucide-react' +import type { GamAsset } from '@/api/gam' +import type { Segment, Trim, VideoSpec } from '@/api/projects' +import { + assetLength, + footageSeconds, + nudge, + playWindow, + setLength, + setLoop, + setTrim, + timingNote, + trimAt, +} from '@/spec/clip' +import { clock } from '@/spec/footage' + +interface Props { + spec: VideoSpec + index: number + segment: Segment + asset: GamAsset + onChange: (next: VideoSpec) => void + onClose: () => void +} + +/** Clock time with tenths: a trim is set to the tenth, so it is shown to the tenth. */ +function fine(seconds: number): string { + const tenths = Math.round(seconds * 10) + return `${clock(Math.floor(tenths / 10))}.${tenths % 10}` +} + +const STEPS = [-1, -0.1, 0.1, 1] + +/** + * A scene's footage, played as the video will play it, and its timing set while + * watching: in and out points, how long the scene runs, and what short footage does. + * + * The player stops at the out point itself rather than through a `#t=` fragment, + * which browsers stop honouring once you seek. Muted unless the scene plays the + * footage's own sound, so what you hear is what the video will have. + */ +export default function ClipPlayer({ + spec, + index, + segment, + asset, + onChange, + onClose, +}: Props) { + const video = useRef(null) + const [now, setNow] = useState(0) + const visual = segment.visual + const trim = visual?.trim + const length = assetLength(asset) + const span = playWindow(asset, trim) + const offset = span.start - (trim?.in ?? 0) + const footage = footageSeconds(asset, trim) + const fixed = segment.duration?.fixed + const note = timingNote(spec, index, asset) + const isVideo = asset.asset_type === 'video' + + // Jump to the new in point whenever it moves, so a nudge is seen at once. + useEffect(() => { + if (video.current) video.current.currentTime = span.start + }, [span.start]) + + if (!visual) return null + + function play() { + const el = video.current + if (!el) return + el.currentTime = span.start + void el.play()?.catch(() => {}) + } + + function onTime() { + const el = video.current + if (!el) return + setNow(Math.max(0, el.currentTime - offset)) + if (span.end !== null && el.currentTime >= span.end) el.pause() + } + + const change = (next: Trim | undefined) => onChange(setTrim(spec, index, next)) + + const edge = (which: 'in' | 'out') => ( +
+ {which === 'in' ? 'In' : 'Out'} + + {which === 'in' + ? fine(trim?.in ?? 0) + : trim?.out !== undefined + ? fine(trim.out) + : length !== null + ? fine(length) + : 'end'} + + {STEPS.map((step) => ( + + ))} + +
+ ) + + return ( +
+
+ + {asset.name} + + +
+ {isVideo ? ( +
+ ) +} diff --git a/frontend/src/components/SceneList.tsx b/frontend/src/components/SceneList.tsx index 9a9640c..ac51c36 100644 --- a/frontend/src/components/SceneList.tsx +++ b/frontend/src/components/SceneList.tsx @@ -1,8 +1,11 @@ +import { useState } from 'react' import { ArrowDown, ArrowUp, Plus, Trash2 } from 'lucide-react' import type { GamAsset } from '@/api/gam' import type { Segment, VideoSpec } from '@/api/projects' +import ClipPlayer from '@/components/ClipPlayer' import SceneSettings from '@/components/SceneSettings' import VisualStrip from '@/components/VisualStrip' +import { footageOf } from '@/spec/clip' import { removeVisual } from '@/spec/footage' import { addScene, @@ -35,8 +38,8 @@ const KIND_LABEL: Record = { * * The seed of M3's storyboard: today it edits labels, card titles, narration and each * scene's picture or footage (found in GAM), and adds, moves and removes scenes. - * Everything else a segment carries (transition, Ken Burns move) is kept untouched and - * edited in the JSON tab until the per-scene panel exists. + * Clicking a scene's footage opens it in a player, where its trim and length are set + * while watching. */ export default function SceneList({ spec, @@ -46,124 +49,141 @@ export default function SceneList({ onGenerate, }: Props) { const segments = spec.segments + // One player at a time, by segment id so it follows its scene through a reorder. + const [playing, setPlaying] = useState(null) const setFocus = useAssistantStore((s) => s.setFocus) const edit = (index: number, change: (s: Segment) => Segment) => onChange(updateSegment(spec, index, change)) return (
- {segments.map((segment, index) => ( -
setFocus(segment.id)} - > -
- {index + 1} - - {KIND_LABEL[segment.kind]} - - edit(index, (s) => ({ ...s, label: e.target.value }))} - /> - - - -
+ {segments.map((segment, index) => { + const footage = footageOf(spec, segment, gamAssets) + return ( +
setFocus(segment.id)} + > +
+ {index + 1} + + {KIND_LABEL[segment.kind]} + + edit(index, (s) => ({ ...s, label: e.target.value }))} + /> + + + +
- {segment.kind === 'card' && segment.card && ( - - edit(index, (s) => ({ - ...s, - card: { ...s.card!, title: e.target.value }, - })) - } - /> - )} + {segment.kind === 'card' && segment.card && ( + + edit(index, (s) => ({ + ...s, + card: { ...s.card!, title: e.target.value }, + })) + } + /> + )} - {segment.kind !== 'card' && onFindFootage && ( - onFindFootage(index)} - onRemove={() => onChange(removeVisual(spec, index))} - onGenerate={onGenerate} - /> - )} + {segment.kind !== 'card' && onFindFootage && ( + onFindFootage(index)} + onRemove={() => onChange(removeVisual(spec, index))} + onOpen={footage ? () => setPlaying(segment.id) : undefined} + onGenerate={onGenerate} + /> + )} - {onFindFootage && ( - - )} + {playing === segment.id && footage && ( + setPlaying(null)} + /> + )} - {segment.kind === 'scene' && ( -