diff --git a/backend/app/render/compile.py b/backend/app/render/compile.py
index 49c712d..be8e5a8 100644
--- a/backend/app/render/compile.py
+++ b/backend/app/render/compile.py
@@ -80,6 +80,9 @@ class Unit:
chapter: Optional[str] = None
# The end credits: a card drawn from these lines rather than from `card`.
credits: Optional[List[str]] = None
+ # Seconds a clip with its sound runs past its out point, under a blend out of it
+ # (`layout`). Footage after the out point where the file has it, else a held frame.
+ tail: float = 0.0
def _transition_in(spec: VideoSpec, segment: Segment) -> Transition:
@@ -281,7 +284,13 @@ def layout(units: Sequence[Unit], durations: Sequence[float], warnings: List[str
project's transition, as every quote has in Notes. A blend overlaps its two units
by its duration, held to half the shorter one so neither is ever entirely under a
blend; too short to see, and it becomes a cut.
+
+ A clip playing its own sound is lengthened by the blend out of it rather than
+ overlapped: the stitch ends the outgoing shot's audio where the blend starts, so
+ otherwise the last words before its out point would be cut. Its `tail` says by how
+ much, for the renderer to extract that much more.
"""
+ durations = list(durations)
joins: List[Optional[Join]] = [None]
for index in range(1, len(units)):
unit = units[index]
@@ -304,6 +313,13 @@ def layout(units: Sequence[Unit], durations: Sequence[float], warnings: List[str
if join and join.is_blend:
join.style, join.overlap = "fade", 0.0
+ for index, join in enumerate(joins):
+ before = units[index - 1] if index else None
+ if join is not None and join.is_blend and before is not None \
+ and before.kind == "video_sound":
+ before.tail = join.overlap
+ durations[index - 1] += join.overlap
+
starts: List[float] = []
cursor = 0.0
for index, duration in enumerate(durations):
diff --git a/backend/app/render/ffmpeg.py b/backend/app/render/ffmpeg.py
index b984323..f03b228 100644
--- a/backend/app/render/ffmpeg.py
+++ b/backend/app/render/ffmpeg.py
@@ -492,7 +492,7 @@ def build_shot_command(
- `in_transition` / `out_transition`: the join at each end, which in GVC differ
per scene (a scene's `transition_in` is its incoming join and the previous
scene's outgoing one). Unset, both are `options.transition`.
- - `loop` False on a muted video holds its last frame instead of repeating it.
+ - `loop` False on a video holds its last frame instead of repeating it.
- `path`: a Ken Burns move between two frames of the picture (S1), in place of
the preset drift.
"""
@@ -565,8 +565,10 @@ def build_shot_command(
chains = [fit_chain(video_in, "fit", width, height, options.fit)]
stage = "fit"
- if kind == "video_muted" and not loop:
- # Shorter than its scene and not to be repeated: hold the last frame.
+ if kind in ("video_muted", "video_sound") and not loop:
+ # Shorter than its scene and not to be repeated: hold the last frame. A clip
+ # with its sound reaches here when a blend out of it asks for more than the
+ # file has left after its out point (`compile.layout`'s tail).
chains.append(f"[{stage}]tpad=stop_mode=clone:stop_duration={duration:.3f}[hold]")
stage = "hold"
diff --git a/backend/app/render/renderer.py b/backend/app/render/renderer.py
index 923bcd8..3a8a71f 100644
--- a/backend/app/render/renderer.py
+++ b/backend/app/render/renderer.py
@@ -311,6 +311,8 @@ def render(
start, length = unit.trim
if length is None:
length = max(0.1, (unit.media.duration or 0.0) - start)
+ # Past the file's end `-t` yields less, and the shot holds its last frame.
+ length += unit.tail
clip = f"clip_{index:04d}.mp4"
F.run(F.build_extract_command(background, clip, start=start, length=length,
with_audio=unit.kind == "video_sound"),
@@ -405,7 +407,8 @@ def argv(with_options: RenderOptions, with_path=path) -> List[str]:
cwd=work_dir, timeout=F.shot_timeout(sum(timeline.durations)))
except F.FFmpegError as exc:
# The blends cost a re-encode; if that fails, the shots still join by cuts,
- # and the timeline is re-laid without their overlaps to match.
+ # and the timeline is re-laid without their overlaps to match. A clip's
+ # tail stays, as a moment more of it before the cut.
logger.warning("Blended stitch failed (%s); joining with cuts", exc)
warnings.append("Blends were joined as cuts: blending the shots together failed.")
for join in timeline.joins:
diff --git a/backend/tests/test_render_compile.py b/backend/tests/test_render_compile.py
index 3f71ffd..731ad87 100644
--- a/backend/tests/test_render_compile.py
+++ b/backend/tests/test_render_compile.py
@@ -139,6 +139,40 @@ def test_a_blend_never_covers_more_than_half_a_short_unit():
assert timeline.total == 20.5
+def _clip_then(style):
+ spec = spec_of({
+ "assets": {"v": {"from": "gam", "id": "g1", "type": "video"}},
+ "segments": [
+ {"id": "a", "kind": "clip", "visual": {"asset": "v", "audio": "play",
+ "trim": {"in": 10, "out": 16}}},
+ {"id": "b", "kind": "scene", "transition_in": {"style": style, "duration": 0.6}},
+ {"id": "c", "kind": "scene", "transition_in": {"style": "dissolve",
+ "duration": 0.6}},
+ ]})
+ return plan_units(spec, {"v": VIDEO}, [])
+
+
+def test_a_blend_out_of_a_clip_lengthens_it_rather_than_cutting_its_last_words():
+ units = _clip_then("dissolve")
+ durations = [6.0, 10.0, 10.0]
+ timeline = layout(units, durations, [])
+ assert units[0].tail == pytest.approx(0.6)
+ assert timeline.durations[0] == pytest.approx(6.6)
+ # The blend starts once the clip has reached its out point.
+ assert timeline.starts[1] == pytest.approx(6.0)
+ # A muted scene into a blend is overlapped as before.
+ assert units[1].tail == 0.0 and timeline.durations[1] == 10.0
+ assert timeline.starts[2] == pytest.approx(15.4)
+ assert durations == [6.0, 10.0, 10.0]
+
+
+def test_a_cut_out_of_a_clip_adds_nothing():
+ units = _clip_then("none")
+ timeline = layout(units, [6.0, 10.0, 10.0], [])
+ assert units[0].tail == 0.0
+ assert timeline.durations[0] == 6.0 and timeline.starts[1] == 6.0
+
+
def _placed():
spec, media = salt_road()
units = plan_units(spec, media, [])
diff --git a/backend/tests/test_render_ffmpeg.py b/backend/tests/test_render_ffmpeg.py
index 4db74c9..bdafb13 100644
--- a/backend/tests/test_render_ffmpeg.py
+++ b/backend/tests/test_render_ffmpeg.py
@@ -119,6 +119,15 @@ def test_sounded_clip_uses_its_own_audio():
assert "[0:a]" in _graph(argv)
+def test_a_sounded_clip_holds_its_last_frame_rather_than_running_short():
+ """A blend's tail can ask for more than the file has after the out point."""
+ argv = F.build_shot_command(kind="video_sound", background="c.mp4", audio=None, duration=2,
+ output="s.mp4", options=RenderOptions(), preview=False,
+ background_has_audio=True, loop=False)
+ assert "tpad=stop_mode=clone" in _graph(argv)
+ assert "-stream_loop" not in argv
+
+
def test_sounded_clip_without_an_audio_track_gets_silence():
"""The concat demuxer needs every part to carry an audio stream."""
argv = F.build_shot_command(kind="video_sound", background="c.mp4", audio=None, duration=2,
diff --git a/frontend/src/api/projects.ts b/frontend/src/api/projects.ts
index 40a4104..56119f8 100644
--- a/frontend/src/api/projects.ts
+++ b/frontend/src/api/projects.ts
@@ -154,6 +154,8 @@ export interface Segment {
narration?: { beats: Beat[]; [key: string]: unknown }
transition_in?: { style?: string; duration?: number }
overlays?: Overlay[]
+ /** Seconds: `fixed` sets the scene's length, `min` floors one that follows narration. */
+ duration?: { min?: number; fixed?: number }
[key: string]: unknown
}
diff --git a/frontend/src/components/ClipPlayer.test.tsx b/frontend/src/components/ClipPlayer.test.tsx
new file mode 100644
index 0000000..dd201b9
--- /dev/null
+++ b/frontend/src/components/ClipPlayer.test.tsx
@@ -0,0 +1,84 @@
+import { render, screen } from '@testing-library/react'
+import userEvent from '@testing-library/user-event'
+import { describe, expect, it, vi } from 'vitest'
+import type { GamAsset } from '@/api/gam'
+import type { VideoSpec } from '@/api/projects'
+import ClipPlayer from '@/components/ClipPlayer'
+
+const reel: GamAsset = {
+ id: 'g1',
+ name: 'Interview',
+ asset_type: 'video',
+ duration_seconds: 60,
+ width: null,
+ height: null,
+ file_url: 'https://gam.test/f',
+ thumb_url: null,
+ credit: '',
+ parent_asset_id: null,
+ in_point: null,
+ out_point: null,
+ missing: false,
+}
+
+const spec: VideoSpec = {
+ title: 'T',
+ assets: { v: { from: 'gam', id: 'g1' } },
+ segments: [
+ {
+ id: 's1',
+ kind: 'scene',
+ visual: { asset: 'v', trim: { in: 10, out: 16 } },
+ narration: { beats: [{ id: 'b1', text: 'Hello' }] },
+ },
+ ],
+}
+
+function player(onChange = vi.fn()) {
+ render(
+
+ )
+ return onChange
+}
+
+describe('ClipPlayer', () => {
+ it('shows the trimmed window and its length', () => {
+ player()
+ expect(screen.getByLabelText('in point')).toHaveTextContent('0:10.0')
+ expect(screen.getByLabelText('out point')).toHaveTextContent('0:16.0')
+ expect(screen.getByText(/6 s of footage/)).toBeInTheDocument()
+ })
+
+ it('nudges the out point', async () => {
+ const onChange = player()
+ await userEvent.click(screen.getByRole('button', { name: 'out +0.1 s' }))
+ expect(onChange.mock.calls[0][0].segments[0].visual.trim).toEqual({
+ in: 10,
+ out: 16.1,
+ })
+ })
+
+ it('plays the whole clip again', async () => {
+ const onChange = player()
+ await userEvent.click(screen.getByRole('button', { name: 'Whole clip' }))
+ expect(onChange.mock.calls[0][0].segments[0].visual.trim).toBeUndefined()
+ })
+
+ it('fixes the scene to the footage’s length', async () => {
+ const onChange = player()
+ await userEvent.click(screen.getByRole('button', { name: 'Match footage' }))
+ expect(onChange.mock.calls[0][0].segments[0].duration).toEqual({ fixed: 6 })
+ })
+
+ it('says the footage is cut short when the narration is shorter', () => {
+ player()
+ expect(screen.getByRole('status')).toHaveTextContent('so its last 3.5 s aren’t shown')
+ })
+})
diff --git a/frontend/src/components/ClipPlayer.tsx b/frontend/src/components/ClipPlayer.tsx
new file mode 100644
index 0000000..f49110e
--- /dev/null
+++ b/frontend/src/components/ClipPlayer.tsx
@@ -0,0 +1,258 @@
+import { useEffect, useRef, useState } from 'react'
+import { Play, X } from 'lucide-react'
+import type { GamAsset } from '@/api/gam'
+import type { Segment, Trim, VideoSpec } from '@/api/projects'
+import {
+ assetLength,
+ footageSeconds,
+ nudge,
+ playWindow,
+ setLength,
+ setLoop,
+ setTrim,
+ timingNote,
+ trimAt,
+} from '@/spec/clip'
+import { clock } from '@/spec/footage'
+
+interface Props {
+ spec: VideoSpec
+ index: number
+ segment: Segment
+ asset: GamAsset
+ onChange: (next: VideoSpec) => void
+ onClose: () => void
+}
+
+/** Clock time with tenths: a trim is set to the tenth, so it is shown to the tenth. */
+function fine(seconds: number): string {
+ const tenths = Math.round(seconds * 10)
+ return `${clock(Math.floor(tenths / 10))}.${tenths % 10}`
+}
+
+const STEPS = [-1, -0.1, 0.1, 1]
+
+/**
+ * A scene's footage, played as the video will play it, and its timing set while
+ * watching: in and out points, how long the scene runs, and what short footage does.
+ *
+ * The player stops at the out point itself rather than through a `#t=` fragment,
+ * which browsers stop honouring once you seek. Muted unless the scene plays the
+ * footage's own sound, so what you hear is what the video will have.
+ */
+export default function ClipPlayer({
+ spec,
+ index,
+ segment,
+ asset,
+ onChange,
+ onClose,
+}: Props) {
+ const video = useRef(null)
+ const [now, setNow] = useState(0)
+ const visual = segment.visual
+ const trim = visual?.trim
+ const length = assetLength(asset)
+ const span = playWindow(asset, trim)
+ const offset = span.start - (trim?.in ?? 0)
+ const footage = footageSeconds(asset, trim)
+ const fixed = segment.duration?.fixed
+ const note = timingNote(spec, index, asset)
+ const isVideo = asset.asset_type === 'video'
+
+ // Jump to the new in point whenever it moves, so a nudge is seen at once.
+ useEffect(() => {
+ if (video.current) video.current.currentTime = span.start
+ }, [span.start])
+
+ if (!visual) return null
+
+ function play() {
+ const el = video.current
+ if (!el) return
+ el.currentTime = span.start
+ void el.play()?.catch(() => {})
+ }
+
+ function onTime() {
+ const el = video.current
+ if (!el) return
+ setNow(Math.max(0, el.currentTime - offset))
+ if (span.end !== null && el.currentTime >= span.end) el.pause()
+ }
+
+ const change = (next: Trim | undefined) => onChange(setTrim(spec, index, next))
+
+ const edge = (which: 'in' | 'out') => (
+
+ )
+}
diff --git a/frontend/src/components/SceneList.tsx b/frontend/src/components/SceneList.tsx
index 9a9640c..ac51c36 100644
--- a/frontend/src/components/SceneList.tsx
+++ b/frontend/src/components/SceneList.tsx
@@ -1,8 +1,11 @@
+import { useState } from 'react'
import { ArrowDown, ArrowUp, Plus, Trash2 } from 'lucide-react'
import type { GamAsset } from '@/api/gam'
import type { Segment, VideoSpec } from '@/api/projects'
+import ClipPlayer from '@/components/ClipPlayer'
import SceneSettings from '@/components/SceneSettings'
import VisualStrip from '@/components/VisualStrip'
+import { footageOf } from '@/spec/clip'
import { removeVisual } from '@/spec/footage'
import {
addScene,
@@ -35,8 +38,8 @@ const KIND_LABEL: Record = {
*
* The seed of M3's storyboard: today it edits labels, card titles, narration and each
* scene's picture or footage (found in GAM), and adds, moves and removes scenes.
- * Everything else a segment carries (transition, Ken Burns move) is kept untouched and
- * edited in the JSON tab until the per-scene panel exists.
+ * Clicking a scene's footage opens it in a player, where its trim and length are set
+ * while watching.
*/
export default function SceneList({
spec,
@@ -46,124 +49,141 @@ export default function SceneList({
onGenerate,
}: Props) {
const segments = spec.segments
+ // One player at a time, by segment id so it follows its scene through a reorder.
+ const [playing, setPlaying] = useState(null)
const setFocus = useAssistantStore((s) => s.setFocus)
const edit = (index: number, change: (s: Segment) => Segment) =>
onChange(updateSegment(spec, index, change))
return (