diff --git a/backend/app/render/compile.py b/backend/app/render/compile.py index 49c712d..be8e5a8 100644 --- a/backend/app/render/compile.py +++ b/backend/app/render/compile.py @@ -80,6 +80,9 @@ class Unit: chapter: Optional[str] = None # The end credits: a card drawn from these lines rather than from `card`. credits: Optional[List[str]] = None + # Seconds a clip with its sound runs past its out point, under a blend out of it + # (`layout`). Footage after the out point where the file has it, else a held frame. + tail: float = 0.0 def _transition_in(spec: VideoSpec, segment: Segment) -> Transition: @@ -281,7 +284,13 @@ def layout(units: Sequence[Unit], durations: Sequence[float], warnings: List[str project's transition, as every quote has in Notes. A blend overlaps its two units by its duration, held to half the shorter one so neither is ever entirely under a blend; too short to see, and it becomes a cut. + + A clip playing its own sound is lengthened by the blend out of it rather than + overlapped: the stitch ends the outgoing shot's audio where the blend starts, so + otherwise the last words before its out point would be cut. Its `tail` says by how + much, for the renderer to extract that much more. """ + durations = list(durations) joins: List[Optional[Join]] = [None] for index in range(1, len(units)): unit = units[index] @@ -304,6 +313,13 @@ def layout(units: Sequence[Unit], durations: Sequence[float], warnings: List[str if join and join.is_blend: join.style, join.overlap = "fade", 0.0 + for index, join in enumerate(joins): + before = units[index - 1] if index else None + if join is not None and join.is_blend and before is not None \ + and before.kind == "video_sound": + before.tail = join.overlap + durations[index - 1] += join.overlap + starts: List[float] = [] cursor = 0.0 for index, duration in enumerate(durations): diff --git a/backend/app/render/ffmpeg.py b/backend/app/render/ffmpeg.py index b984323..f03b228 100644 --- a/backend/app/render/ffmpeg.py +++ b/backend/app/render/ffmpeg.py @@ -492,7 +492,7 @@ def build_shot_command( - `in_transition` / `out_transition`: the join at each end, which in GVC differ per scene (a scene's `transition_in` is its incoming join and the previous scene's outgoing one). Unset, both are `options.transition`. - - `loop` False on a muted video holds its last frame instead of repeating it. + - `loop` False on a video holds its last frame instead of repeating it. - `path`: a Ken Burns move between two frames of the picture (S1), in place of the preset drift. """ @@ -565,8 +565,10 @@ def build_shot_command( chains = [fit_chain(video_in, "fit", width, height, options.fit)] stage = "fit" - if kind == "video_muted" and not loop: - # Shorter than its scene and not to be repeated: hold the last frame. + if kind in ("video_muted", "video_sound") and not loop: + # Shorter than its scene and not to be repeated: hold the last frame. A clip + # with its sound reaches here when a blend out of it asks for more than the + # file has left after its out point (`compile.layout`'s tail). chains.append(f"[{stage}]tpad=stop_mode=clone:stop_duration={duration:.3f}[hold]") stage = "hold" diff --git a/backend/app/render/renderer.py b/backend/app/render/renderer.py index 923bcd8..3a8a71f 100644 --- a/backend/app/render/renderer.py +++ b/backend/app/render/renderer.py @@ -311,6 +311,8 @@ def render( start, length = unit.trim if length is None: length = max(0.1, (unit.media.duration or 0.0) - start) + # Past the file's end `-t` yields less, and the shot holds its last frame. + length += unit.tail clip = f"clip_{index:04d}.mp4" F.run(F.build_extract_command(background, clip, start=start, length=length, with_audio=unit.kind == "video_sound"), @@ -405,7 +407,8 @@ def argv(with_options: RenderOptions, with_path=path) -> List[str]: cwd=work_dir, timeout=F.shot_timeout(sum(timeline.durations))) except F.FFmpegError as exc: # The blends cost a re-encode; if that fails, the shots still join by cuts, - # and the timeline is re-laid without their overlaps to match. + # and the timeline is re-laid without their overlaps to match. A clip's + # tail stays, as a moment more of it before the cut. logger.warning("Blended stitch failed (%s); joining with cuts", exc) warnings.append("Blends were joined as cuts: blending the shots together failed.") for join in timeline.joins: diff --git a/backend/tests/test_render_compile.py b/backend/tests/test_render_compile.py index 3f71ffd..731ad87 100644 --- a/backend/tests/test_render_compile.py +++ b/backend/tests/test_render_compile.py @@ -139,6 +139,40 @@ def test_a_blend_never_covers_more_than_half_a_short_unit(): assert timeline.total == 20.5 +def _clip_then(style): + spec = spec_of({ + "assets": {"v": {"from": "gam", "id": "g1", "type": "video"}}, + "segments": [ + {"id": "a", "kind": "clip", "visual": {"asset": "v", "audio": "play", + "trim": {"in": 10, "out": 16}}}, + {"id": "b", "kind": "scene", "transition_in": {"style": style, "duration": 0.6}}, + {"id": "c", "kind": "scene", "transition_in": {"style": "dissolve", + "duration": 0.6}}, + ]}) + return plan_units(spec, {"v": VIDEO}, []) + + +def test_a_blend_out_of_a_clip_lengthens_it_rather_than_cutting_its_last_words(): + units = _clip_then("dissolve") + durations = [6.0, 10.0, 10.0] + timeline = layout(units, durations, []) + assert units[0].tail == pytest.approx(0.6) + assert timeline.durations[0] == pytest.approx(6.6) + # The blend starts once the clip has reached its out point. + assert timeline.starts[1] == pytest.approx(6.0) + # A muted scene into a blend is overlapped as before. + assert units[1].tail == 0.0 and timeline.durations[1] == 10.0 + assert timeline.starts[2] == pytest.approx(15.4) + assert durations == [6.0, 10.0, 10.0] + + +def test_a_cut_out_of_a_clip_adds_nothing(): + units = _clip_then("none") + timeline = layout(units, [6.0, 10.0, 10.0], []) + assert units[0].tail == 0.0 + assert timeline.durations[0] == 6.0 and timeline.starts[1] == 6.0 + + def _placed(): spec, media = salt_road() units = plan_units(spec, media, []) diff --git a/backend/tests/test_render_ffmpeg.py b/backend/tests/test_render_ffmpeg.py index 4db74c9..bdafb13 100644 --- a/backend/tests/test_render_ffmpeg.py +++ b/backend/tests/test_render_ffmpeg.py @@ -119,6 +119,15 @@ def test_sounded_clip_uses_its_own_audio(): assert "[0:a]" in _graph(argv) +def test_a_sounded_clip_holds_its_last_frame_rather_than_running_short(): + """A blend's tail can ask for more than the file has after the out point.""" + argv = F.build_shot_command(kind="video_sound", background="c.mp4", audio=None, duration=2, + output="s.mp4", options=RenderOptions(), preview=False, + background_has_audio=True, loop=False) + assert "tpad=stop_mode=clone" in _graph(argv) + assert "-stream_loop" not in argv + + def test_sounded_clip_without_an_audio_track_gets_silence(): """The concat demuxer needs every part to carry an audio stream.""" argv = F.build_shot_command(kind="video_sound", background="c.mp4", audio=None, duration=2, diff --git a/frontend/src/api/projects.ts b/frontend/src/api/projects.ts index 40a4104..56119f8 100644 --- a/frontend/src/api/projects.ts +++ b/frontend/src/api/projects.ts @@ -154,6 +154,8 @@ export interface Segment { narration?: { beats: Beat[]; [key: string]: unknown } transition_in?: { style?: string; duration?: number } overlays?: Overlay[] + /** Seconds: `fixed` sets the scene's length, `min` floors one that follows narration. */ + duration?: { min?: number; fixed?: number } [key: string]: unknown } diff --git a/frontend/src/components/ClipPlayer.test.tsx b/frontend/src/components/ClipPlayer.test.tsx new file mode 100644 index 0000000..dd201b9 --- /dev/null +++ b/frontend/src/components/ClipPlayer.test.tsx @@ -0,0 +1,84 @@ +import { render, screen } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { describe, expect, it, vi } from 'vitest' +import type { GamAsset } from '@/api/gam' +import type { VideoSpec } from '@/api/projects' +import ClipPlayer from '@/components/ClipPlayer' + +const reel: GamAsset = { + id: 'g1', + name: 'Interview', + asset_type: 'video', + duration_seconds: 60, + width: null, + height: null, + file_url: 'https://gam.test/f', + thumb_url: null, + credit: '', + parent_asset_id: null, + in_point: null, + out_point: null, + missing: false, +} + +const spec: VideoSpec = { + title: 'T', + assets: { v: { from: 'gam', id: 'g1' } }, + segments: [ + { + id: 's1', + kind: 'scene', + visual: { asset: 'v', trim: { in: 10, out: 16 } }, + narration: { beats: [{ id: 'b1', text: 'Hello' }] }, + }, + ], +} + +function player(onChange = vi.fn()) { + render( + + ) + return onChange +} + +describe('ClipPlayer', () => { + it('shows the trimmed window and its length', () => { + player() + expect(screen.getByLabelText('in point')).toHaveTextContent('0:10.0') + expect(screen.getByLabelText('out point')).toHaveTextContent('0:16.0') + expect(screen.getByText(/6 s of footage/)).toBeInTheDocument() + }) + + it('nudges the out point', async () => { + const onChange = player() + await userEvent.click(screen.getByRole('button', { name: 'out +0.1 s' })) + expect(onChange.mock.calls[0][0].segments[0].visual.trim).toEqual({ + in: 10, + out: 16.1, + }) + }) + + it('plays the whole clip again', async () => { + const onChange = player() + await userEvent.click(screen.getByRole('button', { name: 'Whole clip' })) + expect(onChange.mock.calls[0][0].segments[0].visual.trim).toBeUndefined() + }) + + it('fixes the scene to the footage’s length', async () => { + const onChange = player() + await userEvent.click(screen.getByRole('button', { name: 'Match footage' })) + expect(onChange.mock.calls[0][0].segments[0].duration).toEqual({ fixed: 6 }) + }) + + it('says the footage is cut short when the narration is shorter', () => { + player() + expect(screen.getByRole('status')).toHaveTextContent('so its last 3.5 s aren’t shown') + }) +}) diff --git a/frontend/src/components/ClipPlayer.tsx b/frontend/src/components/ClipPlayer.tsx new file mode 100644 index 0000000..f49110e --- /dev/null +++ b/frontend/src/components/ClipPlayer.tsx @@ -0,0 +1,258 @@ +import { useEffect, useRef, useState } from 'react' +import { Play, X } from 'lucide-react' +import type { GamAsset } from '@/api/gam' +import type { Segment, Trim, VideoSpec } from '@/api/projects' +import { + assetLength, + footageSeconds, + nudge, + playWindow, + setLength, + setLoop, + setTrim, + timingNote, + trimAt, +} from '@/spec/clip' +import { clock } from '@/spec/footage' + +interface Props { + spec: VideoSpec + index: number + segment: Segment + asset: GamAsset + onChange: (next: VideoSpec) => void + onClose: () => void +} + +/** Clock time with tenths: a trim is set to the tenth, so it is shown to the tenth. */ +function fine(seconds: number): string { + const tenths = Math.round(seconds * 10) + return `${clock(Math.floor(tenths / 10))}.${tenths % 10}` +} + +const STEPS = [-1, -0.1, 0.1, 1] + +/** + * A scene's footage, played as the video will play it, and its timing set while + * watching: in and out points, how long the scene runs, and what short footage does. + * + * The player stops at the out point itself rather than through a `#t=` fragment, + * which browsers stop honouring once you seek. Muted unless the scene plays the + * footage's own sound, so what you hear is what the video will have. + */ +export default function ClipPlayer({ + spec, + index, + segment, + asset, + onChange, + onClose, +}: Props) { + const video = useRef(null) + const [now, setNow] = useState(0) + const visual = segment.visual + const trim = visual?.trim + const length = assetLength(asset) + const span = playWindow(asset, trim) + const offset = span.start - (trim?.in ?? 0) + const footage = footageSeconds(asset, trim) + const fixed = segment.duration?.fixed + const note = timingNote(spec, index, asset) + const isVideo = asset.asset_type === 'video' + + // Jump to the new in point whenever it moves, so a nudge is seen at once. + useEffect(() => { + if (video.current) video.current.currentTime = span.start + }, [span.start]) + + if (!visual) return null + + function play() { + const el = video.current + if (!el) return + el.currentTime = span.start + void el.play()?.catch(() => {}) + } + + function onTime() { + const el = video.current + if (!el) return + setNow(Math.max(0, el.currentTime - offset)) + if (span.end !== null && el.currentTime >= span.end) el.pause() + } + + const change = (next: Trim | undefined) => onChange(setTrim(spec, index, next)) + + const edge = (which: 'in' | 'out') => ( +
+ {which === 'in' ? 'In' : 'Out'} + + {which === 'in' + ? fine(trim?.in ?? 0) + : trim?.out !== undefined + ? fine(trim.out) + : length !== null + ? fine(length) + : 'end'} + + {STEPS.map((step) => ( + + ))} + +
+ ) + + return ( +
+
+ + {asset.name} + + +
+ {isVideo ? ( +
+ ) +} diff --git a/frontend/src/components/SceneList.tsx b/frontend/src/components/SceneList.tsx index 9a9640c..ac51c36 100644 --- a/frontend/src/components/SceneList.tsx +++ b/frontend/src/components/SceneList.tsx @@ -1,8 +1,11 @@ +import { useState } from 'react' import { ArrowDown, ArrowUp, Plus, Trash2 } from 'lucide-react' import type { GamAsset } from '@/api/gam' import type { Segment, VideoSpec } from '@/api/projects' +import ClipPlayer from '@/components/ClipPlayer' import SceneSettings from '@/components/SceneSettings' import VisualStrip from '@/components/VisualStrip' +import { footageOf } from '@/spec/clip' import { removeVisual } from '@/spec/footage' import { addScene, @@ -35,8 +38,8 @@ const KIND_LABEL: Record = { * * The seed of M3's storyboard: today it edits labels, card titles, narration and each * scene's picture or footage (found in GAM), and adds, moves and removes scenes. - * Everything else a segment carries (transition, Ken Burns move) is kept untouched and - * edited in the JSON tab until the per-scene panel exists. + * Clicking a scene's footage opens it in a player, where its trim and length are set + * while watching. */ export default function SceneList({ spec, @@ -46,124 +49,141 @@ export default function SceneList({ onGenerate, }: Props) { const segments = spec.segments + // One player at a time, by segment id so it follows its scene through a reorder. + const [playing, setPlaying] = useState(null) const setFocus = useAssistantStore((s) => s.setFocus) const edit = (index: number, change: (s: Segment) => Segment) => onChange(updateSegment(spec, index, change)) return (
- {segments.map((segment, index) => ( -
setFocus(segment.id)} - > -
- {index + 1} - - {KIND_LABEL[segment.kind]} - - edit(index, (s) => ({ ...s, label: e.target.value }))} - /> - - - -
+ {segments.map((segment, index) => { + const footage = footageOf(spec, segment, gamAssets) + return ( +
setFocus(segment.id)} + > +
+ {index + 1} + + {KIND_LABEL[segment.kind]} + + edit(index, (s) => ({ ...s, label: e.target.value }))} + /> + + + +
- {segment.kind === 'card' && segment.card && ( - - edit(index, (s) => ({ - ...s, - card: { ...s.card!, title: e.target.value }, - })) - } - /> - )} + {segment.kind === 'card' && segment.card && ( + + edit(index, (s) => ({ + ...s, + card: { ...s.card!, title: e.target.value }, + })) + } + /> + )} - {segment.kind !== 'card' && onFindFootage && ( - onFindFootage(index)} - onRemove={() => onChange(removeVisual(spec, index))} - onGenerate={onGenerate} - /> - )} + {segment.kind !== 'card' && onFindFootage && ( + onFindFootage(index)} + onRemove={() => onChange(removeVisual(spec, index))} + onOpen={footage ? () => setPlaying(segment.id) : undefined} + onGenerate={onGenerate} + /> + )} - {onFindFootage && ( - - )} + {playing === segment.id && footage && ( + setPlaying(null)} + /> + )} - {segment.kind === 'scene' && ( -