diff --git a/crates/compositor/src/compositor_linux.rs b/crates/compositor/src/compositor_linux.rs index e35a4b3c41..3e8003b434 100644 --- a/crates/compositor/src/compositor_linux.rs +++ b/crates/compositor/src/compositor_linux.rs @@ -2926,10 +2926,7 @@ impl Compositor { && g.shape_fade > 0.0 && webcam_draw.is_some() && !is_cutout - && !matches!( - g.scene_preset.as_deref(), - Some("dual-frame") | Some("vertical-stack") - )) + && g.webcam_shadow > 0.0) .then(|| { let cb = self.shadow_cb( g.w_dst, @@ -2937,7 +2934,7 @@ impl Compositor { g.w_radius, crate::frame_geometry::WEBCAM_SHADOW_SPREAD_FRAC * g.frame_min_px, [0.0, crate::frame_geometry::WEBCAM_SHADOW_OFFSET_FRAC * g.frame_min_px], - crate::frame_geometry::WEBCAM_SHADOW_OPACITY * g.shape_fade, + crate::frame_geometry::WEBCAM_SHADOW_OPACITY * g.shape_fade * g.webcam_shadow, ); self.make_bind(&cb, None, &dummy) }); diff --git a/crates/compositor/src/compositor_macos.rs b/crates/compositor/src/compositor_macos.rs index c5e12e9fc4..c1072cbf1b 100644 --- a/crates/compositor/src/compositor_macos.rs +++ b/crates/compositor/src/compositor_macos.rs @@ -2986,10 +2986,6 @@ impl Compositor { g.w_px[0] / g.w_px[1].max(0.0001), ); let (u0, u1) = if lp.webcam_mirror { (cu1, cu0) } else { (cu0, cu1) }; - let webcam_is_block = matches!( - g.scene_preset.as_deref(), - Some("dual-frame") | Some("vertical-stack") - ); // Effet d'arrière-plan : le mode vient de la scène, le masque par pixel de // l'inférence. Les DEUX sont requis — un mode sans masque rendrait la webcam // invisible en détourage, donc tant que rien n'a été segmenté on dessine la @@ -3008,7 +3004,7 @@ impl Compositor { // part lui aussi en détourage ci-dessous, mais sa bulle, elle, est bien peinte et // garde donc son ombre. let is_cutout = matches!(effect, Some((code, _)) if code == 1.0); - if cfg.shadow && !webcam_is_block && !is_cutout && g.shape_fade > 0.0 { + if cfg.shadow && g.webcam_shadow > 0.0 && !is_cutout && g.shape_fade > 0.0 { self.draw_shadow( enc, g.w_dst, @@ -3016,7 +3012,7 @@ impl Compositor { g.w_radius, WEBCAM_SHADOW_SPREAD_FRAC * g.frame_min_px, [0.0, WEBCAM_SHADOW_OFFSET_FRAC * g.frame_min_px], - WEBCAM_SHADOW_OPACITY * g.shape_fade, + WEBCAM_SHADOW_OPACITY * g.shape_fade * g.webcam_shadow, ); } diff --git a/crates/compositor/src/compositor_windows.rs b/crates/compositor/src/compositor_windows.rs index d386472634..586cfc19b3 100644 --- a/crates/compositor/src/compositor_windows.rs +++ b/crates/compositor/src/compositor_windows.rs @@ -2379,7 +2379,6 @@ impl Compositor { programme_time: *self.programme_time.borrow(), }); self.footage.set(Some(g.footage_quad([self.rw(), self.rh()]))); - let scene_preset = g.scene_preset.clone(); let mb_taps = g.mb_taps; let mb_amount = g.mb_amount; let source_t = g.source_t; @@ -2813,18 +2812,14 @@ impl Compositor { // que l'écran (`WEBCAM_SHADOW_OPACITY`, pas `shadow_scale`) — et propre au PiP : les // blocs side-by-side / top-bottom soudent la caméra à l'écran et n'en portent aucune // (parité `preset.shadow` web, `null` hors PiP dans `compositeLayout.ts`). - let webcam_is_block = matches!( - scene_preset.as_deref(), - Some("dual-frame") | Some("vertical-stack"), - ); // L'ombre appartient à la bulle PiP. En détourage il n'y a plus de bulle — une // ombre portée par un rectangle invisible se lit comme un artefact. let is_cutout = matches!( scene_ref.as_ref().and_then(|s| s.webcam_effect.as_ref()), Some(e) if e.shader_code() == 1.0 ) && self.webcam_mask.borrow().is_some(); - if cfg.shadow && !webcam_is_block && !is_cutout && shape_fade > 0.0 { - let strength = WEBCAM_SHADOW_OPACITY * shape_fade; + if cfg.shadow && g.webcam_shadow > 0.0 && !is_cutout && shape_fade > 0.0 { + let strength = WEBCAM_SHADOW_OPACITY * shape_fade * g.webcam_shadow; self.draw_shadow( w_dst, w_px, diff --git a/crates/compositor/src/frame_geometry.rs b/crates/compositor/src/frame_geometry.rs index f189f6f379..e7410ad558 100644 --- a/crates/compositor/src/frame_geometry.rs +++ b/crates/compositor/src/frame_geometry.rs @@ -1906,6 +1906,8 @@ pub struct FrameGeometryInput<'a> { /// qui traversent. pub struct FrameGeometry { pub scene_preset: Option, + /// Continuous PiP shadow weight through a layout change. + pub webcam_shadow: f32, pub mb_taps: f32, pub mb_amount: f32, pub source_t: f32, @@ -3034,26 +3036,37 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { let (wcw, wch) = (input.webcam_visible_px[0], input.webcam_visible_px[1]); let (u_max, v_max) = (input.u_max, input.v_max); let (frame, cfg) = (input.frame, input.cfg); - let lp = input.live; + let mut lp = input.live; let scene = input.scene; let cursor = input.cursor; + let layout_time = input.timeline_t_override.unwrap_or(frame / FPS); + let sampled_layout = scene.and_then(|s| s.layout_at(layout_time)); + // A cut has no layout velocity. Keep the previous zoom sample, but never trail + // the old layout into a new passage whose transition was intentionally disabled. + let previous_layout_time = if sampled_layout.as_ref().is_some_and(|s| s.hard_cut) { + layout_time + } else { layout_time - 1.0 / FPS }; + let previous_layout = scene.and_then(|s| s.layout_at(previous_layout_time)); + if let Some(shape) = sampled_layout.as_ref().and_then(|s| s.layout.webcam_shape.as_ref()) { + lp.webcam_shape = webcam_shape_code(shape); + } // Scène de l'app présente → placements du layout preset (ou, mieux, le rect résolu par // l'app dans `layout.webcam_rect`) ; sinon planning fixture (bench). let scene_preset: Option = - scene.map(|s| s.layout.preset.clone()); + sampled_layout.as_ref().map(|s| s.preset.to_owned()).or_else(|| scene.map(|s| s.layout.preset.clone())); // Webcam rect résolu par l'app (= `computeCompositeLayout`, source de vérité unique // entre preview et natif) : quand il est présent ET que la scène est posée, on l'utilise // COMME placement de base. Sinon, fallback sur `preset_placements` historique (PiP // codé en dur à 320 px + 40 px de marge — l'arrangement qui dérivait de la preview). - let app_webcam_rect: Option<[f32; 4]> = scene - .and_then(|s| s.layout.webcam_rect) + let app_webcam_rect: Option<[f32; 4]> = sampled_layout.as_ref().map(|s| s.layout.webcam_rect) + .unwrap_or_else(|| scene.and_then(|s| s.layout.webcam_rect)) .map(|r| [r.x, r.y, r.width, r.height]); // Idem pour l'écran. Les deux rects viennent du MÊME appel `computeCompositeLayout`, donc // les consommer ensemble est la seule façon de garder le bloc écran+caméra cohérent : // n'en prendre qu'un revenait à mélanger la géométrie de l'app et un placement fixture. - let app_screen_rect: Option<[f32; 4]> = scene - .and_then(|s| s.layout.screen_rect) + let app_screen_rect: Option<[f32; 4]> = sampled_layout.as_ref().map(|s| Some(s.layout.screen_rect)) + .unwrap_or_else(|| scene.and_then(|s| s.layout.screen_rect)) .map(|r| [r.x, r.y, r.width, r.height]); let (mut p, mut pp) = match &scene_preset { Some(preset) => { @@ -3079,6 +3092,15 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { } None => (timeline(frame, cfg), timeline(frame - 1.0, cfg)), }; + if let Some(previous) = previous_layout.as_ref() { + let r = previous.layout.screen_rect; + pp.screen.dst = [r.x, r.y, r.width, r.height]; + if let Some(r) = previous.layout.webcam_rect { + pp.webcam.dst = [r.x, r.y, r.width, r.height]; + } else { + pp.webcam.dst = [0.0; 4]; + } + } // Motion blur écran : quand la scène (contrat de l'app) est posée, c'est elle qui pilote // (parité inspector : 1.0 + motion_blur*15 taps), sinon on retombe sur `cfg.mblur_n` // (le bench fixture continue d'utiliser ses taps explicites). @@ -3096,7 +3118,9 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { let zoom_regions = scene.map(|s| &s.zoom_regions).unwrap_or(&empty_zoom); let cam_regions = scene.map(|s| &s.camera_fullscreen_regions).unwrap_or(&empty_cam); - let webcam_reactive = scene.map(|s| s.layout.webcam_reactive_zoom).unwrap_or(false); + let webcam_reactive = sampled_layout.as_ref().map(|s| s.webcam_reactive_zoom) + .unwrap_or_else(|| scene.map(|s| s.layout.webcam_reactive_zoom as u8 as f32).unwrap_or(0.0)); + let webcam_reactive_prev = previous_layout.as_ref().map(|s| s.webcam_reactive_zoom).unwrap_or(webcam_reactive); let source_t = input.timeline_t_override.unwrap_or(frame / FPS); // Les transitions se mesurent à l'écran (`ScreenClock`), la frame précédente aussi : une // frame d'écran plus tôt, ce qui donne au flou de mouvement du zoom la même traînée qu'à 1×, @@ -3190,9 +3214,9 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { // avec le ease-in du zoom, tient 0,7 sur le span, remonte avec le ease-out. Une région à // échelle 1 ne zoome rien : elle laisse la caméra à sa taille. Ignoré pendant Full Camera // (voir ci-dessus). - let reactive_scale = |strength: f32, zoom: f32, progress: f32| -> f32 { - if webcam_reactive && progress <= 0.0 && zoom > 1.0 && strength.is_finite() { - (1.0 - 0.3 * strength.clamp(0.0, 1.0)).clamp(0.7, 1.0) + let reactive_scale = |strength: f32, zoom: f32, progress: f32, layout_weight: f32| -> f32 { + if layout_weight > 0.0 && progress <= 0.0 && zoom > 1.0 && strength.is_finite() { + (1.0 - 0.3 * strength.clamp(0.0, 1.0) * layout_weight).clamp(0.7, 1.0) } else { 1.0 } @@ -3206,9 +3230,9 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { // Seul `reactive_scale` (rétrécissement pendant un zoom, une valeur ANIMÉE par frame que // le rect statique de l'app ne capture pas) doit encore s'appliquer dans ce cas. let base_size_scale = if app_webcam_rect.is_some() { 1.0 } else { lp.webcam_size_scale }; - let webcam_size_scale = base_size_scale * reactive_scale(zoom_strength, p.zoom, cam_progress); + let webcam_size_scale = base_size_scale * reactive_scale(zoom_strength, p.zoom, cam_progress, webcam_reactive); let webcam_size_scale_prev = - base_size_scale * reactive_scale(zoom_strength_prev, pp.zoom, cam_progress_prev); + base_size_scale * reactive_scale(zoom_strength_prev, pp.zoom, cam_progress_prev, webcam_reactive_prev); // padding : échelle globale du layout autour du centre du cadre (parité web frameRenderer : // paddingScale = 1 - padding*0.4 → padding 0 = plein cadre). S'applique à TOUS les presets : @@ -3342,11 +3366,18 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { // ratio de boîte se lit sur `s_base` : `remap_box` met les deux axes à la même // échelle, donc la boîte finale a le même ratio et le cover ne dépend pas d'elle // (ce qui casserait la circularité coupe → boîte → coupe). - let cover_box_ar = scene.and_then(|s| { - s.layout - .screen_cover - .then_some((s_base[2] * rw) / (s_base[3] * rh).max(0.0001)) - }); + // A zoom already running inside a block keeps that viewport until it settles. + // Releasing its clip halfway into PiP would inflate the frame and then shrink it + // during the same zoom-out. This is derived from time, so seeks and exports agree. + let keep_screen_clip = zoom_strength > 0.0 && sampled_layout.as_ref() + .and_then(|s| s.keep_screen_clip_from_sec) + .is_some_and(|from| crate::regions::zoom_continues_since(zoom_regions, from, source_t, &clock)); + let keep_screen_clip_prev = zoom_strength_prev > 0.0 && previous_layout.as_ref() + .and_then(|s| s.keep_screen_clip_from_sec) + .is_some_and(|from| crate::regions::zoom_continues_since(zoom_regions, from, source_t_prev, &clock)); + let screen_cover = keep_screen_clip || sampled_layout.as_ref().map(|s| s.layout.screen_cover) + .unwrap_or_else(|| scene.map(|s| s.layout.screen_cover).unwrap_or(false)); + let cover_box_ar = screen_cover.then_some((s_base[2] * rw) / (s_base[3] * rh).max(0.0001)); // Remplissage du format (`screen_follow`) : la fenêtre du cover suit le curseur lissé, // en UV. Sans piste, elle reste centrée. let follow = cover_box_ar.is_some() && scene.is_some_and(|s| s.layout.screen_follow); @@ -3354,13 +3385,17 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { let (x, y) = cursor.filter(|_| follow)?.follow_at(t)?; Some([x * u_max, y * v_max]) }; - let cover = |uv: [f32; 4], at: Option<[f32; 2]>| -> [f32; 4] { - match (cover_box_ar, at) { + let previous_cover_box_ar = (keep_screen_clip_prev || previous_layout.as_ref() + .map(|sample| sample.layout.screen_cover).unwrap_or(screen_cover)) + .then_some((s_base_prev[2] * rw) / (s_base_prev[3] * rh).max(0.0001)); + let cover_for = |uv: [f32; 4], at: Option<[f32; 2]>, box_ar: Option| -> [f32; 4] { + match (box_ar, at) { (Some(ar), Some(at)) => follow_cover_uv_rect(uv, [stw as f32, sth as f32], ar, at), (Some(ar), None) => cover_uv_rect(uv, [stw as f32, sth as f32], ar), (None, _) => uv, } }; + let cover = |uv, at| cover_for(uv, at, cover_box_ar); // La coupe RÉFÉRENCE (zoom entier) est celle qui remplissait la boîte paddée avant // ce correctif ; la coupe DESSINÉE ne porte plus que le crop. `remap_box` reporte la // seconde à travers le mapping de la première, ce qui conserve le cadrage exact. @@ -3376,9 +3411,9 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { let focus = if fill_tilt && zoom_camera > 0.0 { zoom_aim } else { p.focus }; let cut_ref = cover(screen_source_rect(u_max, v_max, active_crop, p.zoom, focus), follow_at(source_t)); - let cut_ref_prev = cover( + let cut_ref_prev = cover_for( screen_source_rect(u_max, v_max, active_crop, pp.zoom, focus), - follow_at(source_t_prev), + follow_at(source_t_prev), previous_cover_box_ar, ); // Sous le remplissage, la coupe dessinée est le crop ENTIER : la fenêtre bouge, et une // coupe dessinée qui ne la contiendrait plus laisserait un trou dans le slot. Le masque du @@ -3527,7 +3562,8 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { // la boîte, pas suivre le cadre. let frame_min_px = rw.min(rh); let s_px = [s_dst[2] * rw, s_dst[3] * rh]; - let app_screen_radius_frac = scene.and_then(|s| s.layout.screen_radius_frac); + let app_screen_radius_frac = sampled_layout.as_ref().map(|s| s.layout.screen_radius_frac) + .unwrap_or_else(|| scene.and_then(|s| s.layout.screen_radius_frac)); let scene_roundness_frac = scene.map(|s| s.effects.roundness_frac); // ---- Le rayon des coins, en UN seul endroit ---- // @@ -3545,7 +3581,7 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { // chaque bordure garde son épaisseur tout autour de chaque coin. Sous le chrome de // fenêtre, les coins HAUTS du métrage restent carrés, à ras de la barre : l'arrondi du // haut se fait une seule fois, par le cadre (`screen_square_top`, `screen_top_lift_px`). - let outer_radius_of = |s_px: [f32; 2]| match (cfg.rounded, app_screen_radius_frac, scene_roundness_frac) { + let radius_for = |s_px: [f32; 2], app_radius: Option| match (cfg.rounded, app_radius, scene_roundness_frac) { (false, _, _) => 0.0, (true, _, _) if frame_kind != crate::scene::SceneFrame::None => { let t = scene.map(roundness_slider_position).unwrap_or(0.0); @@ -3559,6 +3595,13 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { // Fixture/bench (pas de scène) : chemin inspector historique, inchangé. (true, None, None) => p.screen.radius * lp.radius_scale, }; + let outer_radius_of = |s_px: [f32; 2]| { + let target = radius_for(s_px, app_screen_radius_frac); + sampled_layout.as_ref().and_then(|sample| sample.from_layout.map(|from| { + let initial = radius_for(s_px, from.screen_radius_frac); + initial + (target - initial) * sample.progress + })).unwrap_or(target) + }; let s_radius = screen_corner_radius_px(outer_radius_of(s_px), s_px); // Layouts en bloc : le slot masque l'écran (`ScreenMask`), avec les coins de l'écran AU // REPOS — la boîte, elle, grandit avec le zoom. `screen_cover` est le drapeau du slot : @@ -3566,11 +3609,19 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { // remplissage du format, qui exclut les cadres (`formatFillAvailability`). Un cadre sous // un slot est donc toujours dans un bloc : il y devient le conteneur, au repos autour. // Le remplissage incliné n'a pas de masque : sa boîte est le slot, et la carte penche entière. - let screen_mask = (scene.is_some_and(|s| s.layout.screen_cover) && !fill_tilt).then(|| { + let screen_mask = (screen_cover && !fill_tilt).then(|| { let slot_px = [s_base[2] * rw, s_base[3] * rh]; + let cover_mix = if keep_screen_clip { 1.0 } + else { sampled_layout.as_ref().map(|s| s.screen_cover_mix).unwrap_or(1.0) }; + let slot_radius = screen_corner_radius_px(outer_radius_of(slot_px), slot_px); ScreenMask { - rect: s_base, - radius_px: screen_corner_radius_px(outer_radius_of(slot_px), slot_px), + // A free PiP has no clipping; its equivalent mask is the zoomed box. + // Morph between that box and the fixed block slot before dropping the mask. + // Keep settled geometry bit-exact: a + (b - a) can round away from b. + rect: if cover_mix >= 1.0 { s_base } else if cover_mix <= 0.0 { s_dst } + else { lerp4(s_dst, s_base, cover_mix) }, + radius_px: if cover_mix >= 1.0 { slot_radius } else if cover_mix <= 0.0 { s_radius } + else { lerp(s_radius, slot_radius, cover_mix) }, square_top: frame_kind == crate::scene::SceneFrame::Window, } }); @@ -3603,7 +3654,8 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { // l'écran consommant déjà `screen_radius_frac` du même calcul. La table ci-dessous en // était une SECONDE, indépendante — fraction différente (0.12 vs 0.06 côté web) et sans // bornes — donc écran et caméra ne pouvaient pas s'accorder. - let app_webcam_radius_frac = scene.and_then(|s| s.layout.webcam_radius_frac); + let app_webcam_radius_frac = sampled_layout.as_ref().map(|s| s.layout.webcam_radius_frac) + .unwrap_or_else(|| scene.and_then(|s| s.layout.webcam_radius_frac)); // Full Camera dissout la forme en prenant le cadre : le rayon fond vers 0, donc le cercle // devient un rect à coins de plus en plus francs puis un plein cadre net — aucun masque // ne survit au plein écran. Il fond sur sa propre courbe, en retard sur le rect @@ -3624,6 +3676,8 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { }; FrameGeometry { + webcam_shadow: sampled_layout.as_ref().map(|s| s.webcam_shadow) + .unwrap_or_else(|| if matches!(scene_preset.as_deref(), Some("dual-frame") | Some("vertical-stack") | Some("camera-only")) { 0.0 } else { 1.0 }), scene_preset, mb_taps, mb_amount, @@ -4802,6 +4856,295 @@ mod tests { use super::*; + fn layout_region(preset: &str, start: f32, end: f32, camera: Option) -> crate::scene::SceneLayoutRegion { + crate::scene::SceneLayoutRegion { + clip_index: 0, start_sec: start, end_sec: end, preset: preset.to_owned(), + layout: crate::scene::ResolvedClipLayout { + screen_rect: crate::scene::SceneRect { x: 0.1, y: 0.1, width: 0.8, height: 0.8 }, + webcam_rect: camera, screen_radius_frac: None, webcam_radius_frac: Some(0.0), + webcam_shape: Some("rectangle".to_owned()), screen_cover: false, + }, + webcam_reactive_zoom: false, from_layout: None, from_preset: None, from_webcam_reactive_zoom: None, + keep_screen_clip_from_sec: None, transition_start_sec: start, transition_sec: 0.0, + } + } + + #[test] + fn layout_partition_controls_real_frame_geometry() { + let mut scene = golden_scene(); + scene.output.width = 1170; + scene.output.height = 658; + let full = crate::scene::SceneRect { x: 0.0, y: 0.0, width: 1.0, height: 1.0 }; + scene.layout_regions = vec![layout_region("camera-only", 0.0, 5.0, Some(full)), + layout_region("no-webcam", 5.0, 10.0, None)]; + let cfg = crate::config::all().pop().expect("compositor config"); + let mut input = golden_input(&scene, &cfg); + input.timeline_t_override = Some(2.0); + let camera = plan_frame(&input); + assert_eq!(camera.scene_preset.as_deref(), Some("camera-only")); + assert!((camera.w_dst[2] - 1.0).abs() < 0.001); + assert!((camera.w_dst[3] - 1.0).abs() < 0.001); + assert_eq!(camera.w_radius, 0.0); + input.timeline_t_override = Some(5.0); + let screen = plan_frame(&input); + assert_eq!(screen.scene_preset.as_deref(), Some("no-webcam")); + assert_eq!(screen.w_dst[2], 0.0); + assert_eq!(screen.w_dst[3], 0.0); + } + + #[test] + fn a_layout_passage_keeps_zoom_inside_its_slot_and_restores_pip_overflow() { + for preset in ["dual-frame", "vertical-stack"] { + for frame in [crate::scene::SceneFrame::None, crate::scene::SceneFrame::Window, + crate::scene::SceneFrame::Laptop, crate::scene::SceneFrame::Phone, + crate::scene::SceneFrame::Monitor] { + let mut scene = zoomed_golden_scene(); + scene.effects.frame = frame; + let mut block = layout_region(preset, 0.0, 3.0, None); + block.layout.screen_cover = true; + let pip = layout_region("picture-in-picture", 3.0, 10.0, None); + let slot = block.layout.screen_rect; + scene.layout_regions = vec![block, pip]; + let cfg = crate::config::all().pop().expect("compositor config"); + let mut input = golden_input(&scene, &cfg); + input.timeline_t_override = Some(1.5); + let inside = plan_frame(&input); + let mask = inside.screen_mask.expect("the active block passage owns the mask"); + assert_eq!(mask.rect, [slot.x, slot.y, slot.width, slot.height]); + let pixels = [inside.s_dst[2] * input.render_px[0], inside.s_dst[3] * input.render_px[1]]; + let (drawn, _, _, _) = inside.mask_flat_screen(inside.s_dst, inside.cut, pixels, inside.s_radius, input.render_px); + assert!(contains(mask.rect, drawn, 1e-5), "{preset}: zoom escaped the slot"); + input.timeline_t_override = Some(3.5); + let outside = plan_frame(&input); + assert!(outside.screen_mask.is_none(), "PiP must restore zoom overflow"); + } + } + } + + #[test] + fn layout_transition_is_source_anchored_and_never_crosses_clip_identity() { + let mut scene = golden_scene(); + let full = crate::scene::SceneRect { x: 0.0, y: 0.0, width: 1.0, height: 1.0 }; + let mut region = layout_region("camera-only", 3.0, 10.0, Some(full)); + let mut from = region.layout.clone(); + from.webcam_rect = Some(crate::scene::SceneRect { x: 0.8, y: 0.8, width: 0.2, height: 0.2 }); + region.from_layout = Some(from); + region.transition_start_sec = 3.0; + region.transition_sec = 0.2; + scene.layout_regions = vec![region]; + assert!((scene.layout_at(3.1).unwrap().layout.webcam_rect.unwrap().width - 0.6).abs() < 0.001); + assert_eq!(scene.layout_at(3.5).unwrap().layout.webcam_rect.unwrap().width, 1.0); + assert!(scene.layout_at(10.0).is_none()); + let other = scene.for_clip_window(1, 0.0, 10.0); + assert!(other.layout_at(3.1).is_none()); + let trimmed = scene.for_clip_window(0, 3.5, 7.0); + assert_eq!(trimmed.layout_at(3.5).unwrap().layout.webcam_rect.unwrap().width, 1.0); + } + + #[test] + fn hard_layout_cuts_do_not_trail_the_previous_layout() { + use crate::scene::SceneRect; + let cfg = crate::config::all().pop().expect("compositor config"); + let full = SceneRect { x: 0.0, y: 0.0, width: 1.0, height: 1.0 }; + for explicit_from in [false, true] { + let mut scene = golden_scene(); + scene.effects.motion_blur = 1.0; + let source = layout_region("picture-in-picture", 0.0, 3.0, + Some(SceneRect { x: 0.7, y: 0.7, width: 0.2, height: 0.2 })); + let mut target = layout_region("camera-only", 3.0, 10.0, Some(full)); + target.layout.screen_rect = full; + target.from_layout = explicit_from.then(|| source.layout.clone()); + target.transition_sec = 0.0; + scene.layout_regions = vec![source, target]; + assert!(scene.layout_at(3.0).unwrap().hard_cut); + let mut input = golden_input(&scene, &cfg); + input.timeline_t_override = Some(3.0); + let g = plan_frame(&input); + assert!(g.mb_amount > 0.0); + assert_eq!(g.s_dst, g.s_dst_prev, "a hard cut must not smear the old screen"); + assert_eq!(g.w_dst, g.w_dst_prev, "a hard cut must not smear the old camera"); + } + } + + #[test] + fn animated_layout_changes_keep_the_previous_frame_for_motion_blur() { + use crate::scene::SceneRect; + let cfg = crate::config::all().pop().expect("compositor config"); + let mut scene = golden_scene(); + scene.effects.motion_blur = 1.0; + let full = SceneRect { x: 0.0, y: 0.0, width: 1.0, height: 1.0 }; + let source = layout_region("picture-in-picture", 0.0, 3.0, + Some(SceneRect { x: 0.7, y: 0.7, width: 0.2, height: 0.2 })); + let mut target = layout_region("camera-only", 3.0, 10.0, Some(full)); + target.layout.screen_rect = full; + target.from_layout = Some(source.layout.clone()); + target.transition_sec = 0.2; + scene.layout_regions = vec![source, target]; + assert!(!scene.layout_at(3.1).unwrap().hard_cut); + let mut input = golden_input(&scene, &cfg); + input.timeline_t_override = Some(3.1); + let g = plan_frame(&input); + assert_ne!(g.s_dst, g.s_dst_prev, "an animated screen keeps its motion trail"); + assert_ne!(g.w_dst, g.w_dst_prev, "an animated camera keeps its motion trail"); + } + + #[test] + fn layout_animation_does_not_puff_the_footage_during_an_existing_zoom() { + use crate::scene::SceneRect; + let mut scene = golden_scene(); + scene.effects.frame = crate::scene::SceneFrame::Phone; + scene.crop_by_clip = vec![None]; + let mut zoom = zoomed_golden_scene().zoom_regions.remove(0); + zoom.start_sec = 0.963; + zoom.end_sec = 1.301; + zoom.scale = 1.8; + let mut later_zoom = zoom.clone(); + later_zoom.start_sec = 5.0; + later_zoom.end_sec = 6.0; + scene.zoom_regions = vec![zoom, later_zoom]; + let screen = SceneRect { x: 0.114792, y: 0.077604, width: 0.770416, height: 0.292708 }; + let mut block = layout_region("vertical-stack", 0.0, 1.129, None); + block.layout.screen_rect = screen; + block.layout.screen_cover = true; + let mut pip = layout_region("picture-in-picture", 1.129, 2.163, None); + pip.layout.screen_rect = SceneRect { y: 0.353646, ..screen }; + pip.from_layout = Some(block.layout.clone()); + pip.transition_sec = 0.2; + pip.keep_screen_clip_from_sec = Some(1.129); + let mut no_camera = layout_region("no-webcam", 2.163, 10.0, None); + no_camera.layout.screen_rect = pip.layout.screen_rect; + no_camera.from_layout = Some(pip.layout.clone()); + no_camera.transition_sec = 0.1345; + no_camera.keep_screen_clip_from_sec = Some(1.129); + scene.layout_regions = vec![block, pip, no_camera]; + let cfg = crate::config::all().pop().expect("compositor config"); + let sample = |t| { + let mut input = golden_input(&scene, &cfg); + input.render_px = [1298.0, 1920.0]; + input.timeline_t_override = Some(t); + plan_frame(&input) + }; + // Same footage size at both endpoints: moving from top to centre must not inflate it. + for frame in 0..=180 { + let t = 1.129 + frame as f32 / 120.0; + let g = sample(t); + assert!((g.frame_box()[2] - screen.width).abs() < 0.0001, + "at {t}: the footage grew to {} instead of {}", g.frame_box()[2], screen.width); + assert!((g.frame_box()[3] - screen.height).abs() < 0.0001, + "at {t}: the footage height pulsed"); + } + assert!(sample(1.25).s_dst[2] > screen.width + 0.1, "the existing zoom still runs inside"); + let next = sample(5.5); + assert!(next.screen_mask.is_none(), "a later independent PiP zoom must stay free"); + assert!(next.s_dst[2] > screen.width + 0.1); + } + + #[test] + fn layout_animation_preserves_camera_roundness_while_it_disappears() { + use crate::scene::SceneRect; + let cfg = crate::config::all().pop().expect("compositor config"); + for radius in [0.0, 0.06, 0.3, 0.5] { + for appearing in [false, true] { + let mut scene = golden_scene(); + let mut camera = layout_region("picture-in-picture", 0.0, 3.0, + Some(SceneRect { x: 0.7, y: 0.6, width: 0.2, height: 0.2 * 16.0 / 9.0 })); + camera.layout.webcam_radius_frac = Some(radius); + let mut hidden = layout_region("no-webcam", 0.0, 3.0, None); + hidden.layout.webcam_radius_frac = None; + let (source, mut target) = if appearing { (hidden, camera) } else { (camera, hidden) }; + target.start_sec = 3.0; + target.end_sec = 10.0; + target.transition_start_sec = 3.0; + target.transition_sec = 0.2; + target.from_layout = Some(source.layout.clone()); + scene.layout_regions = vec![source, target]; + for progress in [0.05, 0.25, 0.5, 0.75, 0.95] { + let mut input = golden_input(&scene, &cfg); + input.render_px = [1920.0, 1080.0]; + input.timeline_t_override = Some(3.0 + 0.2 * progress); + let g = plan_frame(&input); + let side = (g.w_dst[2] * 1920.0).min(g.w_dst[3] * 1080.0); + assert!(side > 0.0); + assert!((g.w_radius / side - radius).abs() < 0.0001, + "roundness {radius}, appearing {appearing}, progress {progress}: got {}", g.w_radius / side); + } + } + } + } + + #[test] + fn every_layout_pair_has_continuous_footage_at_both_transition_edges() { + use crate::scene::{SceneFrame, SceneRect}; + let presets = ["picture-in-picture", "dual-frame", "vertical-stack", "camera-only", "no-webcam"]; + let resolved = |preset: &str, start: f32, end: f32| { + let block = matches!(preset, "dual-frame" | "vertical-stack"); + let camera = match preset { + "no-webcam" => None, + "camera-only" => Some(SceneRect { x: 0.0, y: 0.0, width: 1.0, height: 1.0 }), + "picture-in-picture" => Some(SceneRect { x: 0.7, y: 0.65, width: 0.15, height: 0.15 * 16.0 / 9.0 }), + _ => Some(SceneRect { x: 0.65, y: 0.2, width: 0.25, height: 0.6 }), + }; + let mut region = layout_region(preset, start, end, camera); + region.layout.screen_rect = if block { + SceneRect { x: 0.1, y: 0.2, width: 0.45, height: 0.6 } + } else { SceneRect { x: 0.1, y: 0.1, width: 0.8, height: 0.8 } }; + region.layout.screen_cover = block; + region.layout.screen_radius_frac = block.then_some(0.06); + region.layout.webcam_radius_frac = Some(if preset == "picture-in-picture" { 0.5 } else { 0.0 }); + region.webcam_reactive_zoom = preset == "picture-in-picture"; + region + }; + let cfg = crate::config::all().pop().expect("compositor config"); + for from in presets { + for to in presets { + if from == to { continue; } + for frame in [SceneFrame::None, SceneFrame::Window, SceneFrame::Laptop] { + for zoom in [1.0, 2.0] { + let mut scene = zoomed_golden_scene(); + scene.effects.frame = frame; + scene.zoom_regions[0].scale = zoom; + let source = resolved(from, 0.0, 3.0); + let mut target = resolved(to, 3.0, 10.0); + target.from_layout = Some(source.layout.clone()); + target.from_preset = Some(from.to_owned()); + target.from_webcam_reactive_zoom = Some(source.webcam_reactive_zoom); + target.transition_sec = 0.2; + scene.layout_regions = vec![source, target]; + let sample = |t| { + let mut input = golden_input(&scene, &cfg); + input.render_px = [1920.0, 1080.0]; + input.timeline_t_override = Some(t); + plan_frame(&input) + }; + for edge in [3.0, 3.2] { + let a = sample(edge - 0.00001); + let b = sample(edge + 0.00001); + let visible = |g: &FrameGeometry| { + g.mask_flat_screen(g.s_dst, g.cut, + [g.s_dst[2] * 1920.0, g.s_dst[3] * 1080.0], g.s_radius, [1920.0, 1080.0]) + }; + let (ad, au, _, ar) = visible(&a); + let (bd, bu, _, br) = visible(&b); + for k in 0..4 { + assert!((ad[k] - bd[k]).abs() < 0.0001, + "{from} → {to}, {frame:?}, zoom {zoom}, edge {edge}: footage jumped {ad:?} → {bd:?}"); + assert!((au[k] - bu[k]).abs() < 0.0001, + "{from} → {to}, edge {edge}: crop jumped {au:?} → {bu:?}"); + if a.w_dst[2].max(b.w_dst[2]) > 0.0001 { + assert!((a.w_dst[k] - b.w_dst[k]).abs() < 0.0001, + "{from} → {to}, edge {edge}: camera jumped"); + } + } + assert!((ar - br).abs() < 0.1, "{from} → {to}, edge {edge}: corners jumped {ar} → {br}"); + assert!((a.webcam_shadow - b.webcam_shadow).abs() < 0.0001, + "{from} → {to}: camera shadow jumped"); + } + } + } + } + } + } + /// La scène de référence du golden : un cas qui exerce le padding, le crop, le zoom, /// une caméra PiP décalée, un rayon et une inclinaison nulle. fn golden_scene() -> Scene { @@ -8142,6 +8485,7 @@ mod tests { fn full_frame_geometry() -> FrameGeometry { FrameGeometry { scene_preset: None, + webcam_shadow: 1.0, mb_taps: 1.0, mb_amount: 0.0, source_t: 0.0, diff --git a/crates/compositor/src/regions.rs b/crates/compositor/src/regions.rs index c4791cdecb..a923dde780 100644 --- a/crates/compositor/src/regions.rs +++ b/crates/compositor/src/regions.rs @@ -835,6 +835,37 @@ fn connected_pairs( pairs } +/// Whether the same uninterrupted zoom is still running after `since` (source time). +/// Uses the zoom sampler's envelope duration, screen clock and connected-pan graph. +/// Once the image settles, a later independent zoom must not inherit a former layout's clip. +pub fn zoom_continues_since( + regions: &[SceneZoomRegion], + since: f32, + t: f32, + clock: &ScreenClock, +) -> bool { + if regions.is_empty() || t < since { return false; } + let (since, t) = (clock.at(since), clock.at(t)); + let mut spans: Vec<(f32, f32)> = regions.iter().map(|r| { + let window = if r.under_trim { 0.0 } else { zoom_transition_s(r.scale) }; + (clock.at(r.start_sec as f32) - window, clock.at(r.end_sec as f32) + window) + }).collect(); + spans.extend(connected_pairs(regions, clock).into_iter().map(|(_, ni, start, end)| + (start, end.max(clock.at(regions[ni].start_sec as f32))))); + spans.sort_by(|a, b| a.0.total_cmp(&b.0)); + let mut covered_to: Option = None; + for (start, end) in spans { + match covered_to { + Some(until) if start > until => return false, + Some(until) => covered_to = Some(until.max(end)), + None if start < since && since < end => covered_to = Some(end), + None => continue, + } + if covered_to.is_some_and(|until| t < until) { return true; } + } + false +} + /// `zoom_state_in` sans speed region ni piste pour l'orbite : ce que les tests comparent. #[cfg(test)] pub fn zoom_state_at( @@ -1703,6 +1734,53 @@ mod zoom_focus_tests { } } + #[test] + fn layout_zoom_continuity_stops_before_a_later_independent_zoom() { + let first = region(1.8, 0.5); + let mut later = region(1.8, 0.5); + later.start_sec = 14.0; + later.end_sec = 16.0; + let window = zoom_transition_s(first.scale); + let regions = [first, later]; + let clock = ScreenClock::default(); + assert!(zoom_continues_since(®ions, 4.0, 8.0 + window * 0.9, &clock)); + assert!(!zoom_continues_since(®ions, 4.0, 8.0 + window, &clock)); + assert!(!zoom_continues_since(®ions, 4.0, 15.0, &clock)); + assert!(!zoom_continues_since(®ions, 4.0, 3.0, &clock)); + assert!(!zoom_continues_since(®ions, 0.0, 4.0, &clock)); + } + + #[test] + fn layout_zoom_continuity_includes_the_connected_pan_and_hold() { + let mut first = region(1.25, 0.5); + first.end_sec = 3.0; + let mut next = region(1.25, 0.7); + next.start_sec = 4.49; + next.end_sec = 6.0; + let regions = [first, next]; + let clock = ScreenClock::default(); + // The independent envelopes leave a hole; the connected pan fills it. + let middle = 3.745; + assert_eq!(zoom_region_strength(®ions[0], middle, &clock), 0.0); + assert_eq!(zoom_region_strength(®ions[1], middle, &clock), 0.0); + assert!(zoom_state_at(®ions, middle, None).strength > 0.0); + assert!(zoom_continues_since(®ions, 2.5, middle, &clock)); + assert!(zoom_continues_since(®ions, 2.5, 6.2, &clock)); + } + + #[test] + fn layout_zoom_continuity_uses_the_same_speed_clock_and_trim_gate() { + let zoom = region(1.8, 0.5); + let window = zoom_transition_s(zoom.scale); + let speeds = [SceneSpeedRegion { start_sec: 0.0, end_sec: 30.0, speed: 2.0, clip_index: Some(0) }]; + let clock = ScreenClock::new(&speeds, 0); + assert!(zoom_continues_since(&[zoom.clone()], 4.0, 8.0 + window * 1.5, &clock)); + assert!(!zoom_continues_since(&[zoom.clone()], 4.0, 8.0 + window * 2.0 + 0.001, &clock)); + let trimmed = SceneZoomRegion { under_trim: true, ..zoom }; + assert!(zoom_continues_since(&[trimmed.clone()], 4.0, 7.99, &clock)); + assert!(!zoom_continues_since(&[trimmed], 4.0, 8.01, &clock)); + } + /// Le focus automatique ne « perd » jamais la piste : hors de ses bornes, il tient la /// dernière (ou la première) position suivie au lieu de retomber sur le point statique de la /// région. Sans ça, le plan net sauterait d'un bout de l'écran à l'autre à la fin de la diff --git a/crates/compositor/src/scene.rs b/crates/compositor/src/scene.rs index 94dee1d593..2a479c8dd8 100644 --- a/crates/compositor/src/scene.rs +++ b/crates/compositor/src/scene.rs @@ -158,6 +158,95 @@ pub struct ResolvedClipLayout { pub screen_cover: bool, } +/// One source-time fragment of the layout partition, resolved by the same TS geometry as +/// the default per-clip layout. Boundaries survive trims without restarting transitions. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SceneLayoutRegion { + pub clip_index: usize, + pub start_sec: f32, + pub end_sec: f32, + pub preset: String, + pub layout: ResolvedClipLayout, + pub webcam_reactive_zoom: bool, + #[serde(default)] + pub from_layout: Option, + #[serde(default)] + pub from_preset: Option, + #[serde(default)] + pub from_webcam_reactive_zoom: Option, + /// Last contained-layout exit in source time. A zoom crossing it keeps its viewport. + #[serde(default)] + pub keep_screen_clip_from_sec: Option, + pub transition_start_sec: f32, + pub transition_sec: f32, +} + +pub struct SampledLayout<'a> { + pub preset: &'a str, + pub layout: ResolvedClipLayout, + pub from_layout: Option<&'a ResolvedClipLayout>, + pub hard_cut: bool, + pub progress: f32, + pub screen_cover_mix: f32, + pub webcam_reactive_zoom: f32, + pub webcam_shadow: f32, + pub keep_screen_clip_from_sec: Option, +} + +impl SceneLayoutRegion { + pub fn sample(&self, t: f32) -> SampledLayout<'_> { + let linear = if self.from_layout.is_some() && self.transition_sec > 0.0 { + ((t - self.transition_start_sec) / self.transition_sec).clamp(0.0, 1.0) + } else { 1.0 }; + let progress = linear * linear * (3.0 - 2.0 * linear); + let mix = |a: f32, b: f32| if progress <= 0.0 { a } else if progress >= 1.0 { b } + else { a + (b - a) * progress }; + let enabled = |on: bool| on as u8 as f32; + let shadow = |preset: &str| if preset == "picture-in-picture" { 1.0 } else { 0.0 }; + let mut layout = self.layout.clone(); + let mut screen_cover_mix = enabled(layout.screen_cover); + let mut reactive = enabled(self.webcam_reactive_zoom); + let mut webcam_shadow = shadow(&self.preset); + if let Some(from) = self.from_layout.as_ref() { + screen_cover_mix = mix(enabled(from.screen_cover), screen_cover_mix); + reactive = mix(enabled(self.from_webcam_reactive_zoom.unwrap_or(self.webcam_reactive_zoom)), reactive); + webcam_shadow = mix(shadow(self.from_preset.as_deref().unwrap_or(&self.preset)), webcam_shadow); + if linear < 1.0 { + let rect = |a: SceneRect, b: SceneRect| SceneRect { + x: mix(a.x, b.x), y: mix(a.y, b.y), + width: mix(a.width, b.width), height: mix(a.height, b.height), + }; + let collapsed = |r: SceneRect| SceneRect { x: r.x + r.width / 2.0, + y: r.y + r.height / 2.0, width: 0.0, height: 0.0 }; + layout.screen_rect = rect(from.screen_rect, layout.screen_rect); + layout.webcam_rect = match (from.webcam_rect, layout.webcam_rect) { + (Some(a), Some(b)) => Some(rect(a, b)), + (None, Some(b)) => Some(rect(collapsed(b), b)), + (Some(a), None) => Some(rect(a, collapsed(a))), + _ => None, + }; + // Keep the crop policy through the morph. Its mask is blended separately; + // switching the flag at the boundary exposes the entire zoomed footage. + layout.screen_cover = from.screen_cover || layout.screen_cover; + // A missing camera is a scaled-down endpoint of the visible one, not a + // square mask. Keep its relative roundness while its rectangle collapses. + layout.webcam_radius_frac = match (from.webcam_rect, self.layout.webcam_rect) { + (Some(_), None) => from.webcam_radius_frac, + (None, Some(_)) => self.layout.webcam_radius_frac, + _ => Some(mix(from.webcam_radius_frac.unwrap_or(0.0), + layout.webcam_radius_frac.unwrap_or(0.0))), + }; + layout.webcam_shape = Some("rectangle".to_owned()); + } + } + SampledLayout { preset: &self.preset, layout, from_layout: self.from_layout.as_ref(), + hard_cut: self.from_layout.is_none() || self.transition_sec <= 0.0, + keep_screen_clip_from_sec: self.keep_screen_clip_from_sec, + progress, screen_cover_mix, webcam_reactive_zoom: reactive, webcam_shadow } + } +} + /// Rect normalisé 0..1 du cadre de sortie : x, y en haut-gauche ; width, height. #[derive(Debug, Clone, Copy, Deserialize)] #[serde(rename_all = "camelCase")] @@ -847,6 +936,8 @@ pub struct Scene { /// `#[serde(default)]` : champ ajouté après coup, absent des JSON de test existants. #[serde(default)] pub camera_fullscreen_regions: Vec, + #[serde(default)] + pub layout_regions: Vec, pub cursor: SceneCursor, /// Global audio finishing. Default keeps old scene payloads bit-for-bit compatible. #[serde(default)] @@ -875,6 +966,10 @@ pub struct Scene { } impl Scene { + pub fn layout_at(&self, t: f32) -> Option> { + self.layout_regions.iter().find(|r| r.clip_index == self.active_clip_index + && t >= r.start_sec && t < r.end_sec).map(|r| r.sample(t)) + } /// Parse le JSON produit par `buildSceneDescription` (TS). pub fn from_json(json: &str) -> anyhow::Result { Ok(serde_json::from_str(json)?) @@ -912,6 +1007,7 @@ impl Scene { scene.camera_fullscreen_regions.retain(|region| { belongs(region.clip_index, region.start_sec, region.end_sec) }); + scene.layout_regions.retain(|region| belongs(Some(region.clip_index), region.start_sec as f64, region.end_sec as f64)); scene.annotations.retain(|annotation| { belongs(annotation.clip_index, annotation.start_sec, annotation.end_sec) }); diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.ts index fffd23feb2..44fa595e36 100644 --- a/electron/ai-edition/agent-tools.test.ts +++ b/electron/ai-edition/agent-tools.test.ts @@ -11,6 +11,7 @@ import { zoomScaleLimit } from "../../src/native/sceneDescription"; import { type AgentToolOptions, addZoomArgs, + documentSnapshotForModel, executeAgentTool, isMutatingTool, MUTATING_TOOL_NAMES, @@ -196,6 +197,7 @@ describe("the mutating-tool table", () => { "addAudio", "addCameraFullscreen", "addEffect", + "addLayoutChange", "addSpeed", "addTrim", "addTrims", @@ -205,6 +207,7 @@ describe("the mutating-tool table", () => { "duplicateClip", "insertClip", "moveClip", + "moveLayoutBoundary", "removeClip", "removeModifier", "removeTrim", @@ -218,6 +221,7 @@ describe("the mutating-tool table", () => { "setEffect", "setEffectDefinition", "setEditorSettings", + "setLayoutPreset", "setSpeed", "setTrim", "setWordText", @@ -241,6 +245,109 @@ describe("the mutating-tool table", () => { }); describe("executeAgentTool", () => { + it("refuses layout edits that would not change the document", () => { + const doc = withCameraTrack(fixtureDocument()); + const unchanged = executeAgentTool( + doc, + "addLayoutChange", + JSON.stringify({ + timeSec: 5, + preset: "picture-in-picture", + }), + ); + expect(unchanged.ok).toBe(false); + expect(unchanged.document).toBeUndefined(); + expect(JSON.parse(unchanged.resultJson).error).toMatch(/nothing to change/i); + + const added = executeAgentTool( + doc, + "addLayoutChange", + JSON.stringify({ + timeSec: 5, + preset: "dual-frame", + }), + ); + expect(added.ok).toBe(true); + const next = added.document as AxcutDocument; + const layoutId = next.layoutRegions?.find((r) => r.preset === "dual-frame")?.id; + expect(layoutId).toBeDefined(); + const samePassage = executeAgentTool( + next, + "setLayoutPreset", + JSON.stringify({ + layoutId, + preset: "dual-frame", + }), + ); + expect(samePassage.ok).toBe(false); + expect(samePassage.document).toBeUndefined(); + expect(JSON.parse(samePassage.resultJson).error).toMatch(/nothing to change/i); + const sameChange = executeAgentTool( + next, + "addLayoutChange", + JSON.stringify({ + timeSec: 6, + preset: "dual-frame", + }), + ); + expect(sameChange.ok).toBe(false); + expect(sameChange.document).toBeUndefined(); + }); + + it("edits the same continuous layouts as the timeline and protects the first passage", () => { + const first = executeAgentTool( + withCameraTrack(fixtureDocument()), + "addLayoutChange", + JSON.stringify({ timeSec: 5, preset: "dual-frame" }), + ); + expect(first.ok).toBe(true); + expect(first.document?.legacyEditor?.cameraFullscreenRegions).toBeUndefined(); + const second = executeAgentTool( + first.document as AxcutDocument, + "addLayoutChange", + JSON.stringify({ timeSec: 10, preset: "no-webcam" }), + ); + const segments = documentSnapshotForModel(second.document as AxcutDocument) + .layoutSegments as Array<{ id: string; startSec: number; endSec: number; preset: string }>; + expect(segments.slice(0, 2).map((r) => [r.startSec, r.endSec, r.preset])).toEqual([ + [0, 5, "picture-in-picture"], + [5, 10, "dual-frame"], + ]); + const moved = executeAgentTool( + second.document as AxcutDocument, + "moveLayoutBoundary", + JSON.stringify({ layoutId: segments[1].id, timeSec: 8 }), + ); + expect(moved.ok).toBe(true); + const changed = executeAgentTool( + moved.document as AxcutDocument, + "setLayoutPreset", + JSON.stringify({ layoutId: segments[1].id, preset: "camera-only" }), + ); + expect(changed.ok).toBe(true); + const updated = documentSnapshotForModel(changed.document as AxcutDocument) + .layoutSegments as typeof segments; + expect(updated[1]).toMatchObject({ startSec: 8, endSec: 10, preset: "camera-only" }); + expect( + executeAgentTool( + changed.document as AxcutDocument, + "removeModifier", + JSON.stringify({ id: updated[0].id }), + ).ok, + ).toBe(false); + const removed = executeAgentTool( + changed.document as AxcutDocument, + "removeModifier", + JSON.stringify({ id: updated[1].id }), + ); + expect(removed.ok).toBe(true); + expect( + ( + documentSnapshotForModel(removed.document as AxcutDocument) + .layoutSegments as typeof segments + )[0], + ).toMatchObject({ startSec: 0, endSec: 10, preset: "picture-in-picture" }); + }); it("getCurrentDocument returns a compact snapshot with ids and times", () => { const result = executeAgentTool(fixtureDocument(), "getCurrentDocument", ""); expect(result.ok).toBe(true); @@ -1018,10 +1125,7 @@ describe("executeAgentTool", () => { JSON.stringify({ startSec: 9, endSec: 5 }), ); expect(added.ok).toBe(true); - const legacy = added.document?.legacyEditor as Record; - const region = ( - legacy.cameraFullscreenRegions as Array<{ id: string; startMs: number; endMs: number }> - ).at(-1); + const region = added.document?.layoutRegions?.find((r) => r.preset === "camera-only"); // bounds normalised (start ≤ end). expect(region).toMatchObject({ startMs: 5000, endMs: 9000 }); expect(() => documentSchema.parse(added.document)).not.toThrow(); @@ -1029,7 +1133,9 @@ describe("executeAgentTool", () => { const snapshot = JSON.parse( executeAgentTool(added.document as AxcutDocument, "getCurrentDocument", "").resultJson, ); - expect(snapshot.cameraFullscreenRegions[0]).toMatchObject({ startSec: 5, endSec: 9 }); + expect( + snapshot.layoutSegments.find((r: { preset: string }) => r.preset === "camera-only"), + ).toMatchObject({ startSec: 5, endSec: 9 }); expect(snapshot.timeBaseNote).toMatch(/cameraFullscreen/i); const moved = executeAgentTool( @@ -1038,12 +1144,7 @@ describe("executeAgentTool", () => { JSON.stringify({ cameraFullscreenId: region?.id, startSec: 1, endSec: 2 }), ); expect(moved.ok).toBe(true); - const movedRegion = ( - (moved.document?.legacyEditor as Record).cameraFullscreenRegions as Array<{ - startMs: number; - endMs: number; - }> - )[0]; + const movedRegion = moved.document?.layoutRegions?.find((r) => r.preset === "camera-only"); expect(movedRegion).toMatchObject({ startMs: 1000, endMs: 2000 }); expect( executeAgentTool( @@ -1449,8 +1550,9 @@ describe("a full-camera region needs a camera", () => { JSON.stringify({ startSec: 0, endSec: 5 }), ); expect(result.ok).toBe(true); - const legacy = result.document?.legacyEditor as Record; - expect(legacy.cameraFullscreenRegions).toHaveLength(1); + expect(result.document?.layoutRegions?.filter((r) => r.preset === "camera-only")).toHaveLength( + 1, + ); }); it("refuses to MOVE a region onto footage with no webcam", () => { @@ -1482,11 +1584,7 @@ describe("a full-camera region needs a camera", () => { JSON.stringify({ startSec: 0, endSec: 5 }), ); expect(added.ok).toBe(true); - const regionId = ( - (added.document?.legacyEditor as Record).cameraFullscreenRegions as Array<{ - id: string; - }> - )[0].id; + const regionId = added.document?.layoutRegions?.find((r) => r.preset === "camera-only")?.id; const moved = executeAgentTool( added.document as AxcutDocument, "setCameraFullscreen", diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index 2a485ce384..3cbccdb011 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -30,6 +30,14 @@ import { trackGroupId, } from "../../src/lib/ai-edition/document/audioTracks"; import { createId } from "../../src/lib/ai-edition/document/ids"; +import { + addLayoutChange, + applyLayoutSpan, + defaultLayoutPreset, + layoutRuns, + moveLayoutBoundary, + setLayoutPreset, +} from "../../src/lib/ai-edition/document/layout"; import { resolveAspectRatioValue } from "../../src/lib/ai-edition/document/outputFormat"; import { duplicateClip, @@ -96,6 +104,7 @@ import { transitionCutsMs, zoomTransitions, } from "../../src/lib/ai-edition/timeline/zoom-transitions"; +import { LAYOUT_PRESETS } from "../../src/lib/layoutTypes"; import { SETTING_BOUNDS } from "../../src/lib/projectDefaults"; import type { SegmentationSupport } from "../../src/native/contracts"; import { annotationFootageRect, zoomScaleLimit } from "../../src/native/sceneDescription"; @@ -142,6 +151,8 @@ function modifierIds(document: AxcutDocument, kind: ModifierKind): string[] { return ((legacy.cameraFullscreenRegions as Array<{ id: string }> | undefined) ?? []).map( (region) => region.id, ); + case "layout": + return layoutRuns(document).flatMap((run) => run.ids); case "audio": // Fragments of one user-visible track share `trackId` and render as a single pill, // so the id that disappears is the group key once, not one per fragment. @@ -603,6 +614,7 @@ function modifierIdsOf(document: AxcutDocument): string[] { (legacy.cameraFullscreenRegions as Array<{ id: string }> | undefined) ?? []; return [ ...document.zoomRanges.map((r) => r.id), + ...(document.layoutRegions ?? []).map((r) => r.id), ...document.annotations.map((r) => r.id), ...speedRegions.map((r) => r.id), ...cameraFullscreenRegions.map((r) => r.id), @@ -1070,6 +1082,19 @@ export const removeTrimArgs = z.object({ trimRangeId: z.string().min(1), }); +export const addLayoutChangeArgs = z.object({ + timeSec: secondsSchema, + preset: z.enum(LAYOUT_PRESETS), +}); +export const setLayoutPresetArgs = z.object({ + layoutId: z.string().min(1), + preset: z.enum(LAYOUT_PRESETS), +}); +export const moveLayoutBoundaryArgs = z.object({ + layoutId: z.string().min(1), + timeSec: secondsSchema, +}); + export const removeModifierArgs = z.object({ id: z.string().min(1), }); @@ -1121,8 +1146,9 @@ export const OPENSCREEN_TOOL_NAMES = [ "setSpeed", "addAnnotation", "setAnnotation", - "addCameraFullscreen", - "setCameraFullscreen", + "addLayoutChange", + "setLayoutPreset", + "moveLayoutBoundary", "addEffect", "setEffect", "setEffectDefinition", @@ -1203,6 +1229,9 @@ export const MUTATING_TOOL_NAMES: ReadonlySet = new Set([ "setSpeed", "addAnnotation", "setAnnotation", + "addLayoutChange", + "setLayoutPreset", + "moveLayoutBoundary", "addCameraFullscreen", "setCameraFullscreen", "addEffect", @@ -1366,6 +1395,14 @@ export function documentSnapshotForModel( annotationNote: "Each annotation reports its box the way addAnnotation and setAnnotation take it: x/y its centre, width/height its size, in percent of `box`. box 'frame' is the output frame (text and arrows, which padding and zooms never move); box 'footage' is the recording's picture (a blur, which stays on what it hides when the picture moves). A text also reports its look (fontSize, color, background, bold, animation), an arrow (type 'figure') its figureData, a blur its blurData.", project: { id: document.project.id, title: document.project.title }, + layoutSegments: layoutRuns(document).map((r) => ({ + id: r.ids[0], + startSec: r.start, + endSec: r.end, + preset: r.member.preset, + })), + layoutNote: + "layoutSegments use RAW document-timeline seconds, matching addLayoutChange.timeSec and moveLayoutBoundary.timeSec. Layouts form a continuous partition. A change lasts until the next one. Removing a change extends the preceding layout. Camera presets require linked webcam footage (assets[].hasCameraTrack).", primaryAssetId: document.project.primaryAssetId ?? document.assets[0]?.id ?? null, autoFocusAll, hasAnyCamera: hasAnyClipWithCamera(document.assets, document.timeline.clips), @@ -3335,13 +3372,70 @@ export function executeAgentTool( }; } + case "addLayoutChange": { + const parsed = addLayoutChangeArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const { timeSec, preset } = parsed.data; + const run = layoutRuns(document).find((r) => timeSec >= r.start && timeSec < r.end); + if (!run) return failure("The layout change must be inside a clip."); + if (preset !== "no-webcam") { + const blind = noCameraUnderSpan(document, timeSec, Math.min(run.end, timeSec + 0.001)); + if (blind) return blind; + } + const next = addLayoutChange(document, timeSec, preset); + if (next === document) return failure("This layout already applies here; nothing to change."); + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + layoutSegments: documentSnapshotForModel(next).layoutSegments, + }), + summary: `layout ${preset} from ${formatSec(timeSec)} until ${formatSec(run.end)}`, + }; + } + case "setLayoutPreset": { + const parsed = setLayoutPresetArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const run = layoutRuns(document).find((r) => r.ids.includes(parsed.data.layoutId)); + if (!run) return failure(`Unknown layout: ${parsed.data.layoutId}`); + if (parsed.data.preset !== "no-webcam") { + const blind = noCameraUnderSpan(document, run.start, run.end); + if (blind) return blind; + } + const next = setLayoutPreset(document, parsed.data.layoutId, parsed.data.preset); + if (next === document) + return failure("This passage already uses this layout; nothing to change."); + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + layoutSegments: documentSnapshotForModel(next).layoutSegments, + }), + summary: `changed layout to ${parsed.data.preset}`, + }; + } + case "moveLayoutBoundary": { + const parsed = moveLayoutBoundaryArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const runs = layoutRuns(document); + const index = runs.findIndex((r) => r.ids.includes(parsed.data.layoutId)); + if (index <= 0) + return failure("Only an existing change after the first passage can be moved."); + const next = moveLayoutBoundary(document, parsed.data.layoutId, parsed.data.timeSec); + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + layoutSegments: documentSnapshotForModel(next).layoutSegments, + }), + summary: "moved layout change", + }; + } case "addCameraFullscreen": { const parsed = addCameraFullscreenArgs.safeParse(args); if (!parsed.success) return failure(parsed.error.message); const startMs = toMs(Math.min(parsed.data.startSec, parsed.data.endSec)); const endMs = toMs(Math.max(parsed.data.startSec, parsed.data.endSec)); - const legacy = (document.legacyEditor as Record) ?? {}; - const prev = (legacy.cameraFullscreenRegions as unknown[] | undefined) ?? []; const region = { id: createId("camfull"), startMs, endMs }; const placed = anchorForAgent(region, document, "camfull"); const landing = landingOf(placed, document); @@ -3350,15 +3444,14 @@ export function executeAgentTool( } const blind = noCameraUnderSpan(document, landing.startSec, landing.endSec); if (blind) return blind; - const next: AxcutDocument = { - ...document, - legacyEditor: { ...legacy, cameraFullscreenRegions: [...prev, ...placed] }, - }; + const next = applyLayoutSpan(document, landing.startSec, landing.endSec, "camera-only"); return { ok: true, document: next, resultJson: JSON.stringify({ - cameraFullscreenId: landing.ids[0], + cameraFullscreenId: layoutRuns(next).find( + (r) => r.member.preset === "camera-only" && r.end > landing.startSec, + )?.ids[0], ...landingReport(landing, startMs / 1000, endMs / 1000), }), summary: @@ -3569,44 +3662,39 @@ export function executeAgentTool( case "setCameraFullscreen": { const parsed = setCameraFullscreenArgs.safeParse(args); if (!parsed.success) return failure(parsed.error.message); - const legacy = (document.legacyEditor as Record) ?? {}; - const prev = - (legacy.cameraFullscreenRegions as - | Array<{ id: string; startMs: number; endMs: number }> - | undefined) ?? []; - const existing = prev.find((r) => r.id === parsed.data.cameraFullscreenId); + const runs = layoutRuns(document); + const index = runs.findIndex( + (r) => r.member.preset === "camera-only" && r.ids.includes(parsed.data.cameraFullscreenId), + ); + const existing = runs[index]; if (!existing) return failure(`Unknown full-camera region: ${parsed.data.cameraFullscreenId}`); - const { startMs, endMs } = resolveSpanMs(existing, parsed.data.startSec, parsed.data.endSec); - const blocked = refuseIfBlocked("full-camera region", prev, parsed.data.cameraFullscreenId, { - startMs, - endMs, - }); - if (blocked) return blocked; - const camPill = new Set(resolvePillIds(prev, parsed.data.cameraFullscreenId)); - const rebuiltCamera = replacePillSpan( - prev, - parsed.data.cameraFullscreenId, - startMs, - endMs, - document.timeline.clips, - () => createId("camfull"), + const { startMs, endMs } = resolveSpanMs( + { startMs: existing.start * 1000, endMs: existing.end * 1000 }, + parsed.data.startSec, + parsed.data.endSec, ); - const landing = landingAfterPillEdit(prev, rebuiltCamera, camPill, document); + const placed = anchorForAgent({ id: createId("layout"), startMs, endMs }, document, "layout"); + const landing = landingOf(placed, document); if (!landing.anchored) { return coversNoClip("full-camera region", startMs / 1000, endMs / 1000, document); } const blindMove = noCameraUnderSpan(document, landing.startSec, landing.endSec); if (blindMove) return blindMove; - const next: AxcutDocument = { - ...document, - legacyEditor: { ...legacy, cameraFullscreenRegions: rebuiltCamera }, - }; + const base = applyLayoutSpan( + document, + existing.start, + existing.end, + runs[index - 1]?.member.preset ?? defaultLayoutPreset(document), + ); + const next = applyLayoutSpan(base, landing.startSec, landing.endSec, "camera-only"); return { ok: true, document: next, resultJson: JSON.stringify({ - cameraFullscreenId: landing.ids[0] ?? parsed.data.cameraFullscreenId, + cameraFullscreenId: layoutRuns(next).find( + (r) => r.member.preset === "camera-only" && r.end > landing.startSec, + )?.ids[0], ...landingReport(landing, startMs / 1000, endMs / 1000), }), summary: @@ -3809,6 +3897,10 @@ export function executeAgentTool( else if (cameraFullscreenRegions.some((c) => c.id === id)) kind = "cameraFullscreen"; else if (document.audioTracks.some((t) => trackGroupId(t) === id)) kind = "audio"; else if ((document.shaderEffectRanges ?? []).some((r) => r.id === id)) kind = "shaderEffect"; + else if (layoutRuns(document).some((r) => r.ids.includes(id))) kind = "layout"; + if (kind === "layout" && layoutRuns(document)[0]?.ids.includes(id)) { + return failure("The first layout cannot be removed. Change its preset instead."); + } if (!kind) { return failure( `No zoom / speed / annotation / full-camera / audio / effect modifier with id ${id}. ` + diff --git a/electron/ai-edition/deep-agent/service.test.ts b/electron/ai-edition/deep-agent/service.test.ts index 0a368e95b4..a82b14681d 100644 --- a/electron/ai-edition/deep-agent/service.test.ts +++ b/electron/ai-edition/deep-agent/service.test.ts @@ -100,6 +100,9 @@ const ARGS: Record = { // the honest one to pin: the agent can place imported audio, never import it. addAudio: { assetId: "audio_nope", startSec: 1, endSec: 2 }, setAudio: { audioId: "audio_nope" }, + addLayoutChange: { timeSec: 1, preset: "no-webcam" }, + setLayoutPreset: { layoutId: "layout_nope", preset: "no-webcam" }, + moveLayoutBoundary: { layoutId: "layout_nope", timeSec: 2 }, removeTrim: { trimRangeId: "trim_1" }, removeModifier: { id: "nope" }, removeClip: { clipId: "clip_1" }, @@ -467,7 +470,7 @@ describe("what the descriptions say about zoom strength", () => { }); it("tells the model a full-camera region needs a linked webcam", () => { - expect(TOOL_DESCRIPTIONS.addCameraFullscreen).toMatch(/hasCameraTrack/); + expect(TOOL_DESCRIPTIONS.addLayoutChange).toMatch(/hasCameraTrack/); expect(SYSTEM_PROMPT).toMatch(/hasCameraTrack/); }); diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index 332808a085..aea2aaf828 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -55,8 +55,8 @@ import { addAnnotationArgs, addAudioArgs, addAutoZoomsArgs, - addCameraFullscreenArgs, addEffectArgs, + addLayoutChangeArgs, addSpeedArgs, addTrimArgs, addTrimsArgs, @@ -72,6 +72,7 @@ import { insertClipArgs, isMutatingTool, moveClipArgs, + moveLayoutBoundaryArgs, removeClipArgs, removeModifierArgs, removeTrimArgs, @@ -81,10 +82,10 @@ import { setAnnotationArgs, setAspectRatioArgs, setAudioArgs, - setCameraFullscreenArgs, setClipRangeArgs, setEffectArgs, setEffectDefinitionArgs, + setLayoutPresetArgs, setSpeedArgs, setTrimArgs, setWordTextArgs, @@ -171,7 +172,7 @@ const BASE_SYSTEM_PROMPT = [ "- Cutting specific spoken words (fillers, a repeated or flubbed phrase, a false start, a duplicate take) is removeWords: read getTranscriptWords, choose the occurrences from their surrounding speech, then pass that assetId and ONLY those word IDs. Word IDs restart for each asset, so keep the read's assetId. Consecutive words become one cut. Do not cut every occurrence of a word just because its text matches one you were asked to remove.", "- Changing where a clip starts or ends within its source is setClipRange — the clip's in/out, distinct from a trim. It also crops the clip (cropRegion).", "- insertClip places footage of any video asset at a position on the timeline; duplicateClip copies a clip right after itself. addAutoZooms plans zooms on the recorded clicks, as the timeline's Automatic zooms does.", - `- addZoom takes a virtual-timeline span (depth is an ordinal 1–6 selecting from a fixed table — ${ZOOM_DEPTH_LEGEND} — never a multiplier; focus in 0–1 frame fractions). addSpeed changes pacing over a span. addAnnotation puts text, an arrow or a privacy blur on screen: a blur is how an email address, a token or any private detail the recording shows is hidden. addCameraFullscreen enlarges the webcam, and only does something where assets[].hasCameraTrack is true.`, + `- addZoom takes a virtual-timeline span (depth is an ordinal 1–6 selecting from a fixed table — ${ZOOM_DEPTH_LEGEND} — never a multiplier; focus in 0–1 frame fractions). addSpeed changes pacing over a span. addAnnotation puts text, an arrow or a privacy blur on screen: a blur is how an email address, a token or any private detail the recording shows is hidden. addLayoutChange switches layouts until the next change. Camera presets require assets[].hasCameraTrack.`, "- setEditorSettings sets the look of the whole video: the background (wallpaper, its motion and blur), padding, rounded corners, shadow, a device frame, motion blur, depth of field, the shape of the video (aspectRatio: auto follows the recording) and how a recording of another shape fills it, the camera's layout (picture-in-picture, side by side, stacked, its shape, size, position, framing and background cutout), the cursor, and the master volume. getCurrentDocument returns all of them under settings, in the shape this tool takes, and settingsContext.inactive names each one the editor hides in the current state and why: setting one of those changes nothing, so say so instead of reporting the look as changed. setAspectRatio is the same write for the shape alone.", "- A zoom animates in just before its span and out just after it (zoomInFromSec, zoomOutUntilSec in getCurrentDocument). Leave those windows untrimmed, or the export jumps at the cut.", "- addAudio lays an imported voiceover or music file over a span. It plays an asset the project already has (kind 'audio'); importing or recording one is the editor's job, not a tool you have — so when the project has none, say so rather than naming an id that does not exist.", @@ -223,7 +224,7 @@ const CUT_TRANSITIONS_RESULT = `When the result cuts a zoom's move in or out, it export const TOOL_DESCRIPTIONS: Record = { getCurrentDocument: - "Read a compact snapshot of the current project: assets (with durations), timeline clips and trim ranges (source-time), and the zoom / speed / annotation / camera-fullscreen / audio / effect regions (virtual, edited-timeline time), each with the settings its writes take: a zoom's level, focus mode and hidden cursor, an annotation's type, box, look and arrow or blur data, an audio track's level and fades. It also gives the project's look under `settings`, in the shape setEditorSettings takes, with `settingsContext`: the formats, wallpapers and cursor themes on offer, and under `inactive` each setting the editor hides in the current state and why. Call this before editing if the snapshot in the system prompt may be stale. The AxcutDocument is the single source of truth — your edits should preserve the user's placed clips and any timeline state they have already set up.", + "Read a compact snapshot of the current project: assets (with durations), timeline clips and trim ranges (source-time), and the zoom / speed / annotation / layout change / audio / effect regions (virtual, edited-timeline time), each with the settings its writes take: a zoom's level, focus mode and hidden cursor, an annotation's type, box, look and arrow or blur data, an audio track's level and fades. It also gives the project's look under `settings`, in the shape setEditorSettings takes, with `settingsContext`: the formats, wallpapers and cursor themes on offer, and under `inactive` each setting the editor hides in the current state and why. Call this before editing if the snapshot in the system prompt may be stale. The AxcutDocument is the single source of truth — your edits should preserve the user's placed clips and any timeline state they have already set up.", getTranscript: "Read the transcript segments (speech and silence, with start/end seconds and text) for an asset. Omit assetId to read the primary asset's transcript.", getCursorTrack: @@ -265,8 +266,6 @@ export const TOOL_DESCRIPTIONS: Record = { "Set the shape of the exported video: 16:9 for X, YouTube or a slide, 9:16 for a vertical story or short, 1:1 or 4:5 for a square feed, auto to follow the recording, or one of the clips' own shapes (getCurrentDocument's settingsContext.originalAspectRatios). Annotations on the frame keep their place and shape. The same as setEditorSettings with aspectRatio alone.", setEditorSettings: 'Change the project\'s look: pass only the settings to change, named as getCurrentDocument returns them under `settings`. Background: wallpaper (a bundled path from settingsContext.bundledWallpapers, a CSS colour or gradient, or a base64 PNG/JPEG data URL; never a file path or web URL; an embedded image reads back as a marker, leave the key out to keep it), wallpaperMotion (moves a gradient or image only), backgroundBlur 0–1. Frame: aspectRatio ("auto", a preset, or an originalAspectRatios token; a fixed format picked for the first time fills the frame with a window that follows the cursor, which formatFollowCursor turns off when settingsContext.formatFill is "available"), padding 0–100 %, borderRadius 0–64 px, shadowIntensity 0–1, frame (none, window, laptop, phone, monitor) and frameTheme (light, dark), motionBlurAmount 0–1, depthOfField (blurs 3D-tilted zooms only). Camera: webcamLayoutPreset, webcamMirrored, webcamReactiveZoom (shrinks on zoom), webcamMaskShape (square or rectangle, the camera\'s own proportions; an untouched roundness follows it), webcamRoundness 0–1, webcamSizePreset 15–60, webcamAnchor (a corner or an edge middle), webcamCropRegion (the framing: a square share of the picture, width = height ≥ 1/3) and webcamCropPan (where that window sits, 0–1 per axis, 0.5 centred), webcamBackgroundMode (none, transparent, blur, custom: needs settingsContext.cameraSegmentation ready), webcamBlurIntensity 0–1, webcamWallpaper (as wallpaper). Cursor, under `cursor`: show, autoHide, theme (settingsContext.cursorThemes), asArrow (the recorded cursor kinds drawn as the arrow instead), size 1.5–6, smoothing 0–1, motionBlur 0–1, clickBounce 0–2, clickImpact, clickSound (only heard when the take recorded clicks) and clickSoundGainDb −24–0, model3d. Also autoFocusAll (every zoom follows the cursor) and audioGainDb −12–12 dB, the master volume. A value out of range or a key not listed is refused with the reason, and nothing changes. The result lists what was applied, including what the editor changes with it, and `inactive` for any of those the current state leaves without effect.', - addCameraFullscreen: - "Add a camera-fullscreen region over a span of the edited timeline (virtual seconds): the webcam fills the frame for that span. This only does something when the footage under that span comes from an asset with a linked webcam — check assets[].hasCameraTrack (or hasAnyCamera) in getCurrentDocument first. On footage with no camera the call is refused rather than storing a region that would render nothing; say so instead of retrying.", addEffect: "Write a visual effect as a shader and pose it over a span of the edited timeline (virtual seconds), or pose one of this project's effects again (effectId). Use it for a look the settings do not offer. `effect.code` is WGSL that defines `fn effect(uv: vec2) -> vec4`, called for every pixel; uv is 0..1 with (0,0) top-left. Available inside it: `frame(uv)` samples the composed picture; `footage(p)` samples the recording itself at its own resolution at point p of its surface (the space of `fx_to_footage`), transparent outside 0..1, with no cursor or webcam on it: read it for anything that magnifies, frame() for the rest; `params.` reads each declared parameter; `ctx.time` is programme seconds; `ctx.region.x` goes 0 to 1 across the region (y its length in s, z seconds since it began); `ctx.resolution` and `ctx.aspect`; `ctx.cursor` (xy in uv, z = 1 when on screen); `ctx.clicks[0..3]`, the last four clicks, newest first (xy in uv where it happened, z seconds since, w = 1 when the slot holds one); `ctx.footage` and `ctx.camera`, the recording's and the webcam's rects (x, y, w, h in uv; camera zero-sized when absent) — all already placed through zoom and layout; `fx_to_footage(uv)` and `fx_from_footage(p)` map between output uv and the recording's own surface (0..1 across the recording) through the warp the compositor draws it with, zoom, layout and 3D camera included, and `ctx.footage_aspect` is its width over height: the effect works in whichever of the two spaces fits it; `fx_px(n)`, n pixels of a 1080p frame as a fraction of the height; `fx_hash(vec2)`, a random number in [0,1) identical on every GPU. Rules: the effect is a pure function of those inputs (no state between frames), so the preview and the export show the same pixels; reason in uv or fx_px, never raw pixels; every loop is a `for` with literal bounds (`for (var i = 0; i < 16; i++)`), at most 4096 iterations per pixel — the one hard rule. `params` declares up to 8 (recommended) parameters, each `{type: number|toggle|color|point, name, label, default}` (number also `min`, `max`); declare one only when moving it visibly changes the effect over its whole range. `target`: `image` (default: the picture under the annotations and captions), `text` (the annotations and captions alone, premultiplied alpha), `both` (everything); use text or both only when asked. One effect runs at a time: to combine two looks, write them in one shader. The result gives the cost tier, in frame() and footage() reads per pixel, with what a 1080p frame pays on a fast discrete GPU (RTX 4070 Ti) and on an integrated one (Apple M1): light ≤ 8, up to 0.06 ms and 1.3 ms; medium ≤ 32, 0.14 ms and 4.3 ms; heavy ≤ 128, 0.54 ms and 16 ms; very heavy beyond, 2 ms and 49 ms at 512 reads. It also gives a recommendation when the tier runs long: light anywhere, medium up to a minute, heavy up to 10 s, very heavy a few seconds. It is a recommendation, never a refusal. Code that does not compile is refused with the compiler's error at the line of your code: fix it and call again. Check the result with getFrames.", setEffect: @@ -275,14 +274,18 @@ export const TOOL_DESCRIPTIONS: Record = { "Change an effect itself by effectId — its name, description, WGSL `code` or `params` — and with it every region that uses it. New code is compiled first and refused with the compiler's error if it does not compile. A removed parameter's values drop; the bounds re-clamp the others. Read the current code with getEffect before rewriting it.", getEffect: "Read one effect of the project in full: its WGSL code, parameters, description, cost and the regions that use it. getCurrentDocument lists effects without their code.", - setCameraFullscreen: - "Move or resize an existing camera-fullscreen region by id (virtual-timeline seconds). Only the fields you pass are changed. Refused if the new span lands on footage with no linked webcam.", addAudio: `Lay an ALREADY-IMPORTED audio file over the recording across a span of the edited timeline (virtual seconds): a voiceover, or a music bed. assetId must name an asset whose kind is 'audio' — getCurrentDocument lists them. If there is none, a file has to be imported first (importMedia, where it is offered): never guess an id. Omit endSec to play the whole file from offsetSec. kind picks the lane ('voiceover' or 'music'). offsetSec is where in the FILE playback starts, gainDb its level (0 unchanged, negative ducks it), fadeInMs / fadeOutMs its fades (0 to ${AUDIO_FADE_MAX_MS} ms, default none; a track too short for both shares its span between them). A voiceover-lane track is also what gets transcribed, so the lane is not only cosmetic.`, setAudio: `Move, resize, re-level, re-lane, mute, loop, fade or re-point an existing audio track by id (virtual-timeline seconds). Only the fields you pass are changed. Use it to duck a bed under narration (gainDb), to fade it in or out (fadeInMs / fadeOutMs, 0 to ${AUDIO_FADE_MAX_MS} ms), to shift what part of the file plays (offsetSec), or to move it between the voiceover and music lanes (kind). The whole track is edited, not one fragment of it, so a track split across a cut stays one thing.`, removeTrim: "Delete a trim range by id — the cut is undone and that span plays/exports again. This is how you 'remove a trim'; never re-add a trim to undo one.", removeModifier: - "Delete a modifier (zoom / speed / annotation / camera-fullscreen / audio / effect) by id; the kind is resolved from the id. Touching rows with the same styling render as one pill and are deleted together, so read removedIds in the result for the complete set that disappeared. This is how you 'remove'/'delete' one — never neutralise it (span 0, speed 1×), which leaves it in the document. For a trim use removeTrim; for a clip use removeClip.", + "Delete a modifier (zoom / speed / annotation / layout change / audio / effect) by id; the kind is resolved from the id. Touching rows with the same styling render as one pill and are deleted together, so read removedIds in the result for the complete set that disappeared. This is how you 'remove'/'delete' one — never neutralise it (span 0, speed 1×), which leaves it in the document. Removing a layout change extends its predecessor; the first passage cannot be removed. For a trim use removeTrim; for a clip use removeClip.", + addLayoutChange: + "Change layout at timeSec (RAW timeline seconds) until the next change or the end. Presets: picture-in-picture, dual-frame, vertical-stack, camera-only, no-webcam (screen only). Camera presets require assets[].hasCameraTrack. For a temporary passage, add its layout at the beginning and the previous preset at the end.", + setLayoutPreset: + "Change an existing layout passage by layoutId from getCurrentDocument.layoutSegments. Edits only that passage. Adjacent equal layouts merge. Camera presets require linked webcam footage.", + moveLayoutBoundary: + "Move the start boundary of a layout passage by layoutId and timeSec (RAW seconds). Updates both neighbours atomically, with no gaps or overlaps. Clamps inside them. The first passage cannot move.", removeClip: "Delete a placed clip by id; remaining clips close the gap and effects anchored to it are dropped. Use only when the user asks to remove a clip — to shorten one, use setClipRange.", getCaptions: @@ -569,8 +572,9 @@ export const TOOL_ARG_SCHEMAS: ReadonlyArray = [ ["setSpeed", setSpeedArgs], ["addAnnotation", addAnnotationArgs], ["setAnnotation", setAnnotationArgs], - ["addCameraFullscreen", addCameraFullscreenArgs], - ["setCameraFullscreen", setCameraFullscreenArgs], + ["addLayoutChange", addLayoutChangeArgs], + ["setLayoutPreset", setLayoutPresetArgs], + ["moveLayoutBoundary", moveLayoutBoundaryArgs], ["addEffect", addEffectArgs], ["setEffect", setEffectArgs], ["setEffectDefinition", setEffectDefinitionArgs], diff --git a/electron/ai-edition/document-service.test.ts b/electron/ai-edition/document-service.test.ts index 577bf690e2..b7d0dd58ce 100644 --- a/electron/ai-edition/document-service.test.ts +++ b/electron/ai-edition/document-service.test.ts @@ -47,7 +47,7 @@ describe("DocumentService", () => { describe("createProject", () => { it("creates a v8 doc with the given title and writes it to disk", async () => { const doc = await service.createProject("Demo Project"); - expect(doc.schemaVersion).toBe(8); + expect(doc.schemaVersion).toBe(9); expect(doc.project.title).toBe("Demo Project"); expect(doc.project.id).toMatch(/^proj_/); expect(doc.assets).toEqual([]); diff --git a/electron/ai-edition/editor-settings-tool.ts b/electron/ai-edition/editor-settings-tool.ts index 5d828406aa..4ed6bdf708 100644 --- a/electron/ai-edition/editor-settings-tool.ts +++ b/electron/ai-edition/editor-settings-tool.ts @@ -1,3 +1,5 @@ +import { layoutRuns } from "../../src/lib/ai-edition/document/layout"; +import { LAYOUT_PRESETS } from "../../src/lib/layoutTypes"; // The project's look for the agent: `setEditorSettings`, which writes it, and the `settings` // block `getCurrentDocument` reads it back through (#1180, batch A). // @@ -33,7 +35,6 @@ import { hasAnyClipWithCamera } from "../../src/lib/ai-edition/timeline/camera"; import { resolveWebcamLayoutPreset, supportsWebcamReactiveZoom, - type WebcamLayoutPreset, } from "../../src/lib/compositeLayout"; import { CURSOR_KIND_IDS, @@ -55,12 +56,7 @@ import { ASPECT_RATIO_PRESETS, type AspectRatio } from "../../src/utils/aspectRa import type { AgentToolExecution } from "./agent-tools"; import { measureTextInShippedFonts } from "./text-measure"; -const WEBCAM_LAYOUT_PRESETS = [ - "picture-in-picture", - "dual-frame", - "vertical-stack", - "no-webcam", -] as const satisfies readonly WebcamLayoutPreset[]; +const WEBCAM_LAYOUT_PRESETS = LAYOUT_PRESETS; /** The tightest the camera framing goes: a third of the picture (the pane's `MIN_CROP_SIZE`). */ const MIN_WEBCAM_CROP_SIZE = 1 / 3; @@ -263,7 +259,10 @@ export function inactiveSettings( for (const key of WEBCAM_KEYS) out[key] = "No clip on the timeline has a camera."; } else { const preset = resolveWebcamLayoutPreset(s.webcamLayoutPreset, true); - if (preset !== "picture-in-picture") { + const hasPipPassage = layoutRuns(document).some( + (run) => run.member.preset === "picture-in-picture", + ); + if (preset !== "picture-in-picture" && !hasPipPassage) { const why = `Only the picture-in-picture layout has it; the layout is ${preset}.`; for (const key of [ "webcamMaskShape", @@ -274,7 +273,7 @@ export function inactiveSettings( out[key] = why; } } - if (!supportsWebcamReactiveZoom(preset)) { + if (!supportsWebcamReactiveZoom(preset) && !hasPipPassage) { out.webcamReactiveZoom = `Only a picture-in-picture camera shrinks on zoom; the layout is ${preset}.`; } if (segmentation !== null && segmentation !== "ready") { diff --git a/electron/cli/projectCommands.test.ts b/electron/cli/projectCommands.test.ts index e0d6ef2132..9ec4aa8884 100644 --- a/electron/cli/projectCommands.test.ts +++ b/electron/cli/projectCommands.test.ts @@ -185,7 +185,7 @@ describe("a project the editor saved", () => { const out = recorder(); expect(await runInfoCommand(project, true, out.write)).toBe(0); expect(JSON.parse(out.text())).toMatchObject({ - version: 8, + version: 9, screenVideoPath: screen, screenVideoExists: true, exportFormat: "gif", diff --git a/electron/mcp/openscreen-mcp-server.test.ts b/electron/mcp/openscreen-mcp-server.test.ts index bcdde4b26d..c33d4ce778 100644 --- a/electron/mcp/openscreen-mcp-server.test.ts +++ b/electron/mcp/openscreen-mcp-server.test.ts @@ -10,11 +10,20 @@ import path from "node:path"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + addLayoutChange, + layoutRuns, + moveLayoutBoundary, + removeLayoutChange, + setLayoutPreset, +} from "../../src/lib/ai-edition/document/layout"; import { type AxcutDocument, createEmptyDocument, documentSchema, } from "../../src/lib/ai-edition/schema"; +import { LAYOUT_PRESETS } from "../../src/lib/layoutTypes"; +import { buildSceneDescription } from "../../src/native/sceneDescription"; import { OPENSCREEN_TOOL_NAMES } from "../ai-edition/agent-tools"; import { TOOL_DESCRIPTIONS } from "../ai-edition/deep-agent/service"; import { DocumentService } from "../ai-edition/document-service"; @@ -192,6 +201,113 @@ function resultText(result: Awaited>): string { } describe("the MCP tool surface", () => { + it("edits layout passages through the same document operations and exposes their result", async () => { + const editor = new FakeEditor(); + editor.document = documentSchema.parse({ + ...editor.document, + assets: editor.document?.assets.map((asset) => ({ + ...asset, + cameraTrack: { sourcePath: "/tmp/camera.mp4", startMs: 0, offsetMs: 0, visible: true }, + })), + }); + let expected = editor.document; + const sameDocumentAndScene = () => { + const actual = editor.document; + if (!actual) throw new Error("The editor lost its document"); + const passages = (doc: AxcutDocument) => + layoutRuns(doc).map((r) => ({ + start: r.start, + end: r.end, + preset: r.member.preset, + })); + expect(passages(actual)).toEqual(passages(expected)); + const anchors = (doc: AxcutDocument) => + (doc.layoutRegions ?? []).map(({ id: _id, ...region }) => region); + expect(anchors(actual)).toEqual(anchors(expected)); + expect(actual.legacyEditor?.cameraFullscreenRegions).toBeUndefined(); + expect(buildSceneDescription(actual).layoutRegions).toEqual( + buildSceneDescription(expected).layoutRegions, + ); + }; + const mcp = await connect(editor); + const changed = await mcp.callTool({ + name: "addLayoutChange", + arguments: { timeSec: 5, preset: "dual-frame" }, + }); + expect(changed.isError).not.toBe(true); + expect(editor.applied).toHaveLength(1); + expected = addLayoutChange(expected, 5, "dual-frame"); + sameDocumentAndScene(); + const layoutId = JSON.parse(resultText(changed)).layoutSegments[1].id as string; + const moved = await mcp.callTool({ + name: "moveLayoutBoundary", + arguments: { layoutId, timeSec: 8 }, + }); + expect(moved.isError).not.toBe(true); + expected = moveLayoutBoundary(expected, layoutRuns(expected)[1].ids[0], 8); + sameDocumentAndScene(); + const restyled = await mcp.callTool({ + name: "setLayoutPreset", + arguments: { layoutId, preset: "vertical-stack" }, + }); + expect(restyled.isError).not.toBe(true); + expected = setLayoutPreset(expected, layoutRuns(expected)[1].ids[0], "vertical-stack"); + sameDocumentAndScene(); + const noOp = await mcp.callTool({ + name: "setLayoutPreset", + arguments: { layoutId, preset: "vertical-stack" }, + }); + expect(noOp.isError).toBe(true); + expect(editor.applied).toHaveLength(3); + const snapshot = await mcp.callTool({ name: "getCurrentDocument", arguments: {} }); + const segments = JSON.parse(resultText(snapshot)).layoutSegments; + expect( + segments.map((r: { startSec: number; preset: string }) => [r.startSec, r.preset]), + ).toEqual([ + [0, "picture-in-picture"], + [8, "vertical-stack"], + ]); + const removed = await mcp.callTool({ + name: "removeModifier", + arguments: { id: segments[1].id }, + }); + expect(removed.isError).not.toBe(true); + expected = removeLayoutChange(expected, layoutRuns(expected)[1].ids[0]); + sameDocumentAndScene(); + expect(editor.document?.layoutRegions).toEqual([]); + const { tools } = await mcp.listTools(); + expect(tools.map((t) => t.name)).toContain("moveLayoutBoundary"); + expect(tools.map((t) => t.name)).not.toContain("addCameraFullscreen"); + expect(tools.map((t) => t.name)).not.toContain("setCameraFullscreen"); + }); + + it("advertises the shared presets and marks all layout tools as mutations", async () => { + const mcp = await connect(new FakeEditor()); + const { tools } = await mcp.listTools(); + for (const name of ["addLayoutChange", "setLayoutPreset", "moveLayoutBoundary"]) { + const tool = tools.find((t) => t.name === name); + expect(tool?.description).toBe(TOOL_DESCRIPTIONS[name]); + expect(tool?.annotations?.readOnlyHint).toBe(false); + if (name !== "moveLayoutBoundary") + expect(tool?.inputSchema.properties?.preset).toMatchObject({ enum: [...LAYOUT_PRESETS] }); + } + }); + + it.each([ + { name: "addLayoutChange", args: { timeSec: 5, preset: "no-webcam" } }, + { name: "setLayoutPreset", args: { layoutId: "layout", preset: "no-webcam" } }, + { name: "moveLayoutBoundary", args: { layoutId: "layout", timeSec: 8 } }, + ])("guards $name with MCP edit consent", async ({ name, args }) => { + const editor = new FakeEditor(); + const before = editor.document; + const mcp = await connect(editor, { editsAllowed: false }); + const result = await mcp.callTool({ name, arguments: args }); + expect(result.isError).toBe(true); + expect(resultText(result)).toMatch(/consent_required/); + expect(editor.document).toBe(before); + expect(editor.applied).toHaveLength(0); + }); + it("is listProjects, then exactly the in-app agent's tools with its descriptions, then the checkpoints and the media tools", async () => { const mcp = await connect(new FakeEditor()); const { tools } = await mcp.listTools(); diff --git a/src/cli/exportProject.test.ts b/src/cli/exportProject.test.ts index da41568826..174308011c 100644 --- a/src/cli/exportProject.test.ts +++ b/src/cli/exportProject.test.ts @@ -74,7 +74,7 @@ describe("loadDocumentProject", () => { const raw = savedDocument({}, 7); delete (raw.legacyEditor as Record).aspectRatio; const project = await loadDocumentProject(raw, noProbe); - expect(project.document.schemaVersion).toBe(8); + expect(project.document.schemaVersion).toBe(9); expect(project.document.legacyEditor?.aspectRatio).toBe("16:9"); }); diff --git a/src/components/ai-edition/CaptionsPane.gating.test.tsx b/src/components/ai-edition/CaptionsPane.gating.test.tsx index 6c45674b59..407d85deb0 100644 --- a/src/components/ai-edition/CaptionsPane.gating.test.tsx +++ b/src/components/ai-edition/CaptionsPane.gating.test.tsx @@ -23,7 +23,7 @@ vi.mock("sonner", () => ({ toast: { success: vi.fn(), error: vi.fn() } })); function documentWith(asset: AxcutAsset): AxcutDocument { return { - schemaVersion: 8, + schemaVersion: 9, project: { id: "proj_1", title: "Test", diff --git a/src/components/ai-edition/CaptionsPane.placement.test.tsx b/src/components/ai-edition/CaptionsPane.placement.test.tsx index 62fe218a86..05f46f36fe 100644 --- a/src/components/ai-edition/CaptionsPane.placement.test.tsx +++ b/src/components/ai-edition/CaptionsPane.placement.test.tsx @@ -33,7 +33,7 @@ const ASSET: AxcutAsset = { function documentWith(captions: Record): AxcutDocument { return { - schemaVersion: 8, + schemaVersion: 9, project: { id: "proj_1", title: "Test", diff --git a/src/components/ai-edition/EditorEmptyState.test.tsx b/src/components/ai-edition/EditorEmptyState.test.tsx index 49183fa599..fd1d5b6793 100644 --- a/src/components/ai-edition/EditorEmptyState.test.tsx +++ b/src/components/ai-edition/EditorEmptyState.test.tsx @@ -33,7 +33,7 @@ const sampleDoc = vi.hoisted( // imports are initialised, so reading the constant here throws // "Cannot access ... before initialization". The typecheck gate catches it // if it ever falls behind the schema. - schemaVersion: 8, + schemaVersion: 9, project: { id: "proj_test", title: "Test", diff --git a/src/components/ai-edition/NewEditorShell.tsx b/src/components/ai-edition/NewEditorShell.tsx index d87a215eaa..d4e52187ec 100644 --- a/src/components/ai-edition/NewEditorShell.tsx +++ b/src/components/ai-edition/NewEditorShell.tsx @@ -1059,6 +1059,20 @@ export function NewEditorShell() { const { pasteClipboard } = await import("@/lib/ai-edition/store/regionClipboard"); const snapshot = pasteClipboard(); if (!snapshot) return; + if (snapshot.kind === "cameraFullscreen") { + const { applyLayoutSpan } = await import("@/lib/ai-edition/document/layout"); + const time = useProjectStore.getState().currentTimeSec; + const duration = (Number(snapshot.region.endMs) - Number(snapshot.region.startMs)) / 1000; + await saveDocument(applyLayoutSpan(doc, time, time + duration, "camera-only"), { + history: true, + }); + return; + } + if (snapshot.kind === "layout") { + const { isLayoutPreset } = await import("@/lib/layoutTypes"); + if (isLayoutPreset(snapshot.region.preset)) await tl.addLayoutChange(snapshot.region.preset); + return; + } // A copied trim is just a length: recreate one of that length at the // playhead, through the same call the toolbar's cut button uses (which @@ -1163,8 +1177,7 @@ export function NewEditorShell() { } await saveDocument(next, { history: true }); } else { - // speed and cameraFullscreen are both plain spans on legacyEditor. - const key = snapshot.kind === "speed" ? "speedRegions" : "cameraFullscreenRegions"; + const key = "speedRegions"; const legacy = (doc.legacyEditor as Record) ?? {}; const prev = (legacy[key] as unknown[]) ?? []; await saveDocument( @@ -1189,6 +1202,14 @@ export function NewEditorShell() { const sel = tl.selection; if (!sel) return; const { copyRegion } = await import("@/lib/ai-edition/store/regionClipboard"); + if (sel.kind === "layout") { + const run = tl.layoutSegments.find((r) => r.ids.includes(sel.id)); + if (run) { + copyRegion({ kind: "layout", region: { preset: run.member.preset } }); + setCopiedClipId(null); + } + return; + } // A trim is stored in SOURCE time against a clip anchor, so there is no // row to clone — but there is nothing to clone either: what the user means @@ -1433,7 +1454,7 @@ export function NewEditorShell() { } if (matchesShortcut(e, shortcuts.addCameraFullscreen, isMac)) { e.preventDefault(); - void tl.addCameraFullscreen(newRegionDurationSec()); + window.dispatchEvent(new Event("openscreen:change-layout")); return; } @@ -1480,6 +1501,7 @@ export function NewEditorShell() { isMac, togglePlay, handleSeek, + openVoiceoverFlow, ]); const showTimeline = mode !== "rec"; diff --git a/src/components/ai-edition/PreviewCanvas.tsx b/src/components/ai-edition/PreviewCanvas.tsx index 557593f4c9..8a0693a43d 100644 --- a/src/components/ai-edition/PreviewCanvas.tsx +++ b/src/components/ai-edition/PreviewCanvas.tsx @@ -33,6 +33,7 @@ import { type ZoomFocus, } from "@/components/video-editor/types"; import { useScopedT } from "@/contexts/I18nContext"; +import { materializeLayouts, sampleLayoutRegionsAt } from "@/lib/ai-edition/document/layout"; import { isFormatFillActive, resolveAspectRatioValue, @@ -53,6 +54,7 @@ import { locateVirtualPosition } from "@/lib/ai-edition/timeline/virtual-preview import { computeCameraFullscreenRect, computeCompositeLayout, + interpolateCompositeLayout, paddedContentSize, resolveWebcamLayoutPreset, type WebcamCompositeLayout, @@ -255,16 +257,19 @@ export function PreviewCanvas(props: PreviewCanvasProps) { ), [activeCameraTrack, settings.webcamCropRegion, webcamSizeRevision], ); + const layoutRegions = useMemo(() => (document ? materializeLayouts(document) : []), [document]); + const sample = useMemo( + () => sampleLayoutRegionsAt(layoutRegions, props.currentTimeSec, settings.webcamLayoutPreset), + [layoutRegions, props.currentTimeSec, settings.webcamLayoutPreset], + ); + const activePreset = resolveWebcamLayoutPreset(sample.preset, activeClipHasCamera); const formatFill = useMemo(() => (document ? isFormatFillActive(document) : false), [document]); const layout = useMemo(() => { // A clip with no camera lays out as "no-webcam", whatever the panel says. Hiding // only the webcam slot is not enough: the block presets size the SCREEN off the // block, so the screen stayed squeezed into its half with nothing beside it. - const preset = resolveWebcamLayoutPreset( - settings.webcamLayoutPreset as WebcamLayoutPreset, - activeClipHasCamera, - ); + const preset = activePreset; const mask = settings.webcamMaskShape as WebcamMaskShape; // Padding shrinks the available content area for ALL layouts (PiP/dual/stack), // through the same `paddedContentSize` the scene description uses. @@ -282,27 +287,33 @@ export function PreviewCanvas(props: PreviewCanvasProps) { width: Math.max(1, Math.round(fullScreenSize.width * cropRegion.width)), height: Math.max(1, Math.round(fullScreenSize.height * cropRegion.height)), }; - return computeCompositeLayout({ - canvasSize: frameSize, - maxContentSize, - // Same box as the scene: a filled format gives the screen the whole padded area. - screenSize: formatFill ? maxContentSize : croppedScreenSize, - webcamSize: preset === "no-webcam" ? null : webcamSourceSize, - layoutPreset: preset, - webcamSizePreset: settings.webcamSizePreset, - // Picture-in-picture only: the block layouts place and round their own camera. - webcamAnchor: settings.webcamAnchor, - webcamMaskShape: mask, - webcamRoundness: settings.webcamRoundness, - frame: settings.frame, - }); + const compute = (layoutPreset: WebcamLayoutPreset) => + computeCompositeLayout({ + canvasSize: frameSize, + maxContentSize, + // Same box as the scene: a filled format gives the screen the whole padded area. + screenSize: formatFill ? maxContentSize : croppedScreenSize, + webcamSize: layoutPreset === "no-webcam" ? null : webcamSourceSize, + layoutPreset, + webcamSizePreset: settings.webcamSizePreset, + // Picture-in-picture only: the block layouts place and round their own camera. + webcamAnchor: settings.webcamAnchor, + webcamMaskShape: mask, + webcamRoundness: settings.webcamRoundness, + frame: settings.frame, + }); + const target = compute(preset); + const from = + sample.fromPreset && sample.progress < 1 + ? compute(resolveWebcamLayoutPreset(sample.fromPreset, activeClipHasCamera)) + : null; + return target && from ? interpolateCompositeLayout(from, target, sample.progress) : target; }, [ frameSize, screenNativeSize, cropRegion, activeClipHasCamera, webcamSourceSize, - settings.webcamLayoutPreset, settings.webcamMaskShape, settings.webcamSizePreset, settings.webcamAnchor, @@ -311,6 +322,9 @@ export function PreviewCanvas(props: PreviewCanvasProps) { settings.aspectRatio, formatFill, settings.frame, + activePreset, + sample.fromPreset, + sample.progress, ]); // Full Camera: during a cameraFullscreen region the webcam takes the whole @@ -386,7 +400,12 @@ export function PreviewCanvas(props: PreviewCanvasProps) { }; const handleWebcamPointerDown = (event: ReactPointerEvent) => { - if (settings.webcamLayoutPreset !== "picture-in-picture") return; + if ( + activePreset !== "picture-in-picture" || + sample.progress < 1 || + cameraFullscreenProgress > 0 + ) + return; if (isPlaying) return; const slot = webcamSlotRef.current; if (!slot) return; @@ -429,7 +448,8 @@ export function PreviewCanvas(props: PreviewCanvasProps) { slot.addEventListener("pointercancel", handleUp); }; - const isPipGrab = settings.webcamLayoutPreset === "picture-in-picture"; + const isPipGrab = + activePreset === "picture-in-picture" && sample.progress >= 1 && cameraFullscreenProgress <= 0; const selectedZoomRegion = props.selectedZoomRegionId ? (props.zoomRegions?.find((z) => z.id === props.selectedZoomRegionId) ?? null) diff --git a/src/components/ai-edition/RightPanes.composition.test.tsx b/src/components/ai-edition/RightPanes.composition.test.tsx index ad627d8145..e2728d60bf 100644 --- a/src/components/ai-edition/RightPanes.composition.test.tsx +++ b/src/components/ai-edition/RightPanes.composition.test.tsx @@ -7,6 +7,7 @@ import { act, cleanup, fireEvent, render, screen, waitFor, within } from "@testi import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { I18nProvider } from "@/contexts/I18nContext"; import { LOCALE_STORAGE_KEY } from "@/i18n/config"; +import { addLayoutChange, layoutRuns, setLayoutPreset } from "@/lib/ai-edition/document/layout"; import type { AxcutDocument } from "@/lib/ai-edition/schema"; import { createEmptyDocument } from "@/lib/ai-edition/schema"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; @@ -101,6 +102,24 @@ describe("the frame row persists the pick", () => { const pick = (row: string, label: string) => within(screen.getByRole("group", { name: row })).getByRole("button", { name: label }); + it("keeps device frames available after a block passage returns to picture in picture", async () => { + const base = documentWithShapes([[1920, 1080]]); + base.assets[0].cameraTrack = { + sourcePath: "/camera.mp4", + startMs: 0, + offsetMs: 0, + visible: true, + }; + const block = addLayoutChange(base, 2, "dual-frame"); + const doc = setLayoutPreset(block, layoutRuns(block)[1].ids[0], "picture-in-picture"); + mount(doc); + for (const label of ["Window", "Laptop", "Phone", "Screen"]) { + expect(pick("Style", label)).toBeEnabled(); + } + fireEvent.click(pick("Style", "Laptop")); + await waitFor(() => expect(stored("frame")).toBe("laptop")); + }); + it("writes each frame to the document, with no option withheld", async () => { mount(documentWithShapes([[1920, 1080]])); // A 16:9 project: the phone is offered all the same. Nothing is gated — the frame adapts diff --git a/src/components/ai-edition/RightPanes.layout.test.tsx b/src/components/ai-edition/RightPanes.layout.test.tsx index f3eaec1260..e810351530 100644 --- a/src/components/ai-edition/RightPanes.layout.test.tsx +++ b/src/components/ai-edition/RightPanes.layout.test.tsx @@ -5,13 +5,15 @@ // the saved preference is left untouched on disk and the help popover says so. import "@testing-library/jest-dom"; -import { cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react"; +import { act, cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { I18nProvider } from "@/contexts/I18nContext"; import { LOCALE_STORAGE_KEY } from "@/i18n/config"; +import { addLayoutChange, layoutRuns } from "@/lib/ai-edition/document/layout"; import { type AxcutDocument, createEmptyDocument } from "@/lib/ai-edition/schema"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; +import { nativeBridgeClient } from "@/native/client"; import { LayoutPane } from "./RightPanes"; const segmentation = vi.hoisted(() => ({ supported: false, segmentCameraFrame: vi.fn() })); @@ -94,14 +96,10 @@ describe("LayoutPane camera availability", () => { it("shows No webcam without overwriting the saved camera preset", () => { renderLayout(seedProject(false)); - const preset = screen.getByRole("group", { name: "Preset" }); - for (const tile of within(preset).getAllByRole("button")) { - expect(tile).toBeDisabled(); - } - expect(within(preset).getByRole("button", { name: "No webcam" })).toHaveAttribute( - "aria-pressed", - "true", - ); + const preset = screen.getByRole("button", { name: "Screen only" }); + expect(preset).toBeEnabled(); + expect(preset).toHaveAttribute("aria-pressed", "true"); + expect(screen.getByRole("button", { name: "Camera only" })).toBeDisabled(); expect(useProjectStore.getState().document?.legacyEditor).toMatchObject({ webcamLayoutPreset: "picture-in-picture", }); @@ -125,14 +123,9 @@ describe("LayoutPane camera availability", () => { const user = userEvent.setup(); renderLayout(seedProject(true)); - const preset = screen.getByRole("group", { name: "Preset" }); - for (const tile of within(preset).getAllByRole("button")) { - expect(tile).toBeEnabled(); - } - expect(within(preset).getByRole("button", { name: "Picture in picture" })).toHaveAttribute( - "aria-pressed", - "true", - ); + const preset = screen.getByRole("button", { name: "Picture in picture" }); + expect(preset).toBeEnabled(); + expect(preset).toHaveAttribute("aria-pressed", "true"); expect(screen.getByText("Camera shape")).toBeInTheDocument(); expect(screen.getByText("Shrink on zoom")).toBeInTheDocument(); expect(screen.getByText("Webcam size")).toBeInTheDocument(); @@ -147,6 +140,104 @@ describe("LayoutPane camera availability", () => { }); }); +describe("LayoutPane passage editing", () => { + beforeEach(() => { + vi.spyOn(nativeBridgeClient.aiEdition, "save").mockImplementation(async (document) => ({ + success: true, + document, + })); + }); + afterEach(() => vi.restoreAllMocks()); + + it.each([ + 0, 4, 6, 7.999, + ])("keeps passage editing disabled in the gap at %s", async (currentTimeSec) => { + const doc = seedProject(true); + const clip = doc.timeline.clips[0]; + doc.timeline.clips = [ + { ...clip, sourceEndSec: 2, timelineStartSec: 2, timelineEndSec: 4 }, + { + ...clip, + id: "clip_2", + sourceStartSec: 2, + sourceEndSec: 6, + timelineStartSec: 8, + timelineEndSec: 12, + }, + ]; + const partitioned = addLayoutChange(doc, 8, "dual-frame"); + useProjectStore.setState({ currentTimeSec }); + renderLayout(partitioned); + for (const name of [ + "Picture in picture", + "Side by side", + "Top / bottom", + "Camera only", + "Screen only", + ]) + expect(screen.getByRole("button", { name })).toBeDisabled(); + expect(screen.queryByText("Entire video")).not.toBeInTheDocument(); + const user = userEvent.setup(); + await user.click(screen.getByRole("button", { name: "Camera only" })); + expect(nativeBridgeClient.aiEdition.save).not.toHaveBeenCalled(); + expect(useProjectStore.getState().document).toBe(partitioned); + + act(() => useProjectStore.setState({ currentTimeSec: 8 })); + expect(screen.getByRole("button", { name: "Camera only" })).toBeEnabled(); + await user.click(screen.getByRole("button", { name: "Camera only" })); + await waitFor(() => { + const saved = useProjectStore.getState().document; + expect(saved && layoutRuns(saved).map((r) => [r.start, r.end, r.member.preset])).toEqual([ + [2, 4, "picture-in-picture"], + [8, 12, "camera-only"], + ]); + }); + }); + + it("does not treat one stored passage as a global preset before its clip", async () => { + const doc = seedProject(true); + doc.timeline.clips = [ + { ...doc.timeline.clips[0], sourceEndSec: 4, timelineStartSec: 2, timelineEndSec: 6 }, + ]; + doc.layoutRegions = [ + { + id: "layout_only", + clipId: "clip_1", + startMs: 2000, + endMs: 6000, + sourceStartSec: 0, + sourceEndSec: 4, + preset: "dual-frame", + }, + ]; + useProjectStore.setState({ currentTimeSec: 0 }); + renderLayout(doc); + const preset = screen.getByRole("button", { name: "Screen only" }); + expect(preset).toBeDisabled(); + await userEvent.setup().click(preset); + expect(nativeBridgeClient.aiEdition.save).not.toHaveBeenCalled(); + expect(useProjectStore.getState().document).toBe(doc); + }); + + it.each([6, 10, 12.5])("edits only the last passage at playhead %s", async (currentTimeSec) => { + const doc = addLayoutChange(seedProject(true), 4, "dual-frame"); + useProjectStore.setState({ currentTimeSec }); + renderLayout(doc); + expect(screen.getByRole("button", { name: "Side by side" })).toHaveAttribute( + "aria-pressed", + "true", + ); + await userEvent.setup().click(screen.getByRole("button", { name: "Camera only" })); + await waitFor(() => { + const saved = useProjectStore.getState().document; + expect(saved && layoutRuns(saved).map((r) => [r.start, r.end, r.member.preset])).toEqual([ + [0, 4, "picture-in-picture"], + [4, 10, "camera-only"], + ]); + }); + }); +}); + describe("LayoutPane webcam framing picture", () => { // Shown in the camera's own shape only: before its metadata the box is a 16:9 guess, and a // picture drawn in it would be stretched. diff --git a/src/components/ai-edition/RightPanes.tsx b/src/components/ai-edition/RightPanes.tsx index c0551227d1..1e0c674108 100644 --- a/src/components/ai-edition/RightPanes.tsx +++ b/src/components/ai-edition/RightPanes.tsx @@ -62,6 +62,7 @@ import { collapseTracksToPills, trackGroupId, } from "@/lib/ai-edition/document/audioTracks"; +import { layoutRuns, setLayoutPreset } from "@/lib/ai-edition/document/layout"; import { collectNativeFormats, formatFillAvailability, @@ -103,7 +104,7 @@ import { } from "@/lib/ai-edition/timeline/aggregated-transcript"; import { assetCameraSource, hasAnyClipWithCamera } from "@/lib/ai-edition/timeline/camera"; import { breatheCut } from "@/lib/ai-edition/timeline/cut-breath"; -import { formatMs } from "@/lib/ai-edition/timeline/format"; +import { formatMs, formatSec } from "@/lib/ai-edition/timeline/format"; import { removedRawSpans } from "@/lib/ai-edition/timeline/programme-time"; import { type AssetTranscriptionView, @@ -161,6 +162,7 @@ import styles from "./NewEditorShell.module.css"; import { useRecordedCursorTypes } from "./recordedCursorTypes"; import { useTranscriptionLabel } from "./TranscriptionStatus"; import { transcriptionBusyLabel } from "./transcriptionBusyLabel"; +import { LayoutPicker } from "./v4/LayoutPicker"; interface PaneProps { title: string; @@ -2669,45 +2671,6 @@ export function VideoEffectsPane() { // ─── Layout (webcam)────────────────────────────────────────────── -// Each preset draws itself (viewBox 0 0 32 22): the recording as an outline, the camera filled. -const WEBCAM_PRESETS = [ - { - value: "picture-in-picture", - labelKey: "layout.pictureInPicture", - icon: ( - <> - - - - ), - }, - { - value: "dual-frame", - labelKey: "layout.dualFrame", - icon: ( - <> - - - - ), - }, - { - value: "vertical-stack", - labelKey: "layout.verticalStack", - icon: ( - <> - - - - ), - }, - { - value: "no-webcam", - labelKey: "layout.noWebcam", - icon: , - }, -] as const; - // Webcam size (% of frame width) that maps to the native compositor's default PiP webcam // (fixture a_side = 320px @ 1920 ≈ 16.7%). `webcamSizePreset / this` = the native size scale // (1 = fixture default), so the slider reads as a direct multiplier on the shipped webcam. @@ -2808,11 +2771,23 @@ const CAMERA_BACKGROUND_MODES: Array<{ export function LayoutPane() { const canSegmentCamera = useCanSegmentCamera(); const ts = useScopedT("settings"); + const tt = useScopedT("timeline"); const { settings, set, setLive, commit, hasDocument } = useEditorSettings(); const { pick: handlePickWebcamWallpaper, input: webcamWallpaperInput } = useWallpaperFileInput( (dataUrl) => set({ webcamWallpaper: dataUrl }), ); const document = useProjectStore((s) => s.document); + const currentTime = useProjectStore((s) => s.currentTimeSec); + const saveDocument = useProjectStore((s) => s.saveDocument); + const runs = useMemo(() => (document ? layoutRuns(document) : []), [document]); + const hasLayoutChanges = + Boolean(document?.layoutRegions?.length) || + runs.length > 1 || + Boolean(runs[0]?.member.legacyCamera); + const lastPassage = runs.at(-1); + const passage = + runs.find((r) => currentTime >= r.start && currentTime < r.end) ?? + (lastPassage && currentTime >= lastPassage.end ? lastPassage : undefined); // A project can hold clips with no camera attached at all (plain imports or a // recording made without a webcam). Keep the saved camera preference for later, but // make the disabled control describe whether this project has any camera at all. @@ -2831,9 +2806,25 @@ export function LayoutPane() { [document], ); const effectiveLayoutPreset = resolveWebcamLayoutPreset( - settings.webcamLayoutPreset, + hasLayoutChanges && passage ? passage.member.preset : settings.webcamLayoutPreset, hasAnyCamera, ); + const passageHasCamera = + !hasLayoutChanges || !passage + ? hasAnyCamera + : Boolean( + document?.timeline.clips.some( + (clip) => + clip.timelineEndSec > passage.start && + clip.timelineStartSec < passage.end && + document.assets.some( + (asset) => + asset.id === clip.assetId && + asset.cameraTrack?.visible && + asset.cameraTrack.sourcePath, + ), + ), + ); // Synchro initiale : cf. NativeCompositorOverlay (`pushAllNativeParams`). // the mask shape picker only makes sense for Picture-in-Picture. @@ -2880,41 +2871,38 @@ export function LayoutPane() { }; return ( } helpText={helpText}> - {/* The heading names the current layout: the tiles below only draw theirs. */}
{ts("layout.preset")} - - {ts( - WEBCAM_PRESETS.find((p) => p.value === effectiveLayoutPreset)?.labelKey ?? - "layout.noWebcam", - )} - + {tt(`layouts.${effectiveLayoutPreset}`)}
-
- - label={ts("layout.preset")} - tiles - options={WEBCAM_PRESETS.map((p) => ({ - value: p.value, - label: ts(p.labelKey), - icon: ( - - ), - }))} - value={effectiveLayoutPreset} - disabled={layoutControlsDisabled} - onChange={(preset) => void set({ webcamLayoutPreset: preset })} - /> +
+ {!hasLayoutChanges + ? tt("layouts.wholeVideo") + : passage && + tt("layouts.fromUntil", { + start: formatSec(passage.start), + end: formatSec(passage.end), + })} +
+ { + if (!document) return; + if (hasLayoutChanges) { + if (!passage) return; + void saveDocument(setLayoutPreset(document, passage.ids[0], preset), { history: true }); + } else void set({ webcamLayoutPreset: preset }); + }} + /> + {hasLayoutChanges && !passageHasCamera ? ( +

{tt("layouts.noCamera")}

+ ) : null} +
+ {tt("layouts.sharedSettings")}
{ts("layout.mirrorWebcam")} diff --git a/src/components/ai-edition/TranscriptPane.lanes.test.tsx b/src/components/ai-edition/TranscriptPane.lanes.test.tsx index e690a13529..1f8b8d1581 100644 --- a/src/components/ai-edition/TranscriptPane.lanes.test.tsx +++ b/src/components/ai-edition/TranscriptPane.lanes.test.tsx @@ -96,7 +96,7 @@ function mount(audioTracks: AxcutAudioTrack[]) { useProjectStore.setState({ projectId: "proj_1", document: { - schemaVersion: 8, + schemaVersion: 9, project: { id: "proj_1", title: "T", diff --git a/src/components/ai-edition/v4/FloatingInspector.tsx b/src/components/ai-edition/v4/FloatingInspector.tsx index 08c2180f51..22215c8272 100644 --- a/src/components/ai-edition/v4/FloatingInspector.tsx +++ b/src/components/ai-edition/v4/FloatingInspector.tsx @@ -14,8 +14,8 @@ import { EyeOff, FileText, ImageIcon, + Layout as LayoutIcon, type LucideIcon, - Maximize2, MousePointer2, Pencil, Scissors, @@ -94,6 +94,7 @@ import { } from "../recordedCursorTypes"; import { TextColorField } from "../TextColorField"; import styles from "./EditorShellV4.module.css"; +import { LayoutPicker } from "./LayoutPicker"; type TimelineApi = ReturnType; @@ -1308,22 +1309,41 @@ function SelectionPane({ tl, onClose }: { tl: TimelineApi; onClose: () => void } ); } - if (selection.kind === "cameraFullscreen") { - const region = tl.cameraFullscreenRegions.find((c) => c.id === selection.id); - if (!region) return null; + if (selection.kind === "layout") { + const index = tl.layoutSegments.findIndex((r) => r.ids.includes(selection.id)); + const run = tl.layoutSegments[index]; + if (!run) return null; + const hasCamera = tl.clips.some( + (clip) => + clip.timelineEndSec > run.start && + clip.timelineStartSec < run.end && + tl.assets.some( + (asset) => + asset.id === clip.assetId && asset.cameraTrack?.visible && asset.cameraTrack.sourcePath, + ), + ); return ( -
- {paneHeader( - , - tt("labels.cameraFullscreen"), - onClose, - tc("actions.close"), - )} +
+ {paneHeader(, tt("layouts.title"), onClose, tc("actions.close"))}
- +
+ {tt("layouts.fromUntil", { + start: formatSeconds(run.start), + end: formatSeconds(run.end), + })} +
+ void tl.setLayoutPreset(run.ids[0], preset)} + /> + {!hasCamera && run.member.preset !== "no-webcam" ?

{tt("layouts.noCamera")}

: null} + {index > 0 ? ( + + ) : null}
); diff --git a/src/components/ai-edition/v4/LayoutLane.test.tsx b/src/components/ai-edition/v4/LayoutLane.test.tsx new file mode 100644 index 0000000000..4612b6fdba --- /dev/null +++ b/src/components/ai-edition/v4/LayoutLane.test.tsx @@ -0,0 +1,130 @@ +// @vitest-environment jsdom +import { fireEvent, render, screen } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { addLayoutChange, layoutRuns } from "@/lib/ai-edition/document/layout"; +import { type AxcutDocument, createEmptyDocument } from "@/lib/ai-edition/schema"; +import type { useTimeline } from "@/lib/ai-edition/store/useTimeline"; +import { LayoutLane } from "./LayoutLane"; + +const store = vi.hoisted(() => ({ getState: vi.fn() })); +vi.mock("@/lib/ai-edition/store/projectStore", () => ({ useProjectStore: store })); +vi.mock("@/contexts/I18nContext", () => ({ useScopedT: () => (key: string) => key })); + +function footage() { + const doc = createEmptyDocument({ title: "Layouts", projectId: "layout-test" }); + doc.timeline.clips = [ + { + id: "clip", + assetId: "asset", + sourceStartSec: 0, + sourceEndSec: 10, + timelineStartSec: 0, + timelineEndSec: 10, + origin: "user", + reason: "", + wordRefs: [], + }, + ]; + return doc; +} + +describe("layout boundary gestures", () => { + let current: { + document: AxcutDocument; + setDocument: ReturnType; + saveDocument: ReturnType; + }; + let animation: FrameRequestCallback | null; + beforeEach(() => { + animation = null; + vi.stubGlobal("PointerEvent", MouseEvent); + vi.stubGlobal("requestAnimationFrame", (fn: FrameRequestCallback) => { + animation = fn; + return 1; + }); + vi.stubGlobal("cancelAnimationFrame", vi.fn()); + current = { + document: addLayoutChange(footage(), 5, "no-webcam"), + setDocument: vi.fn((doc: AxcutDocument) => { + current.document = doc; + }), + saveDocument: vi.fn(async () => true), + }; + store.getState.mockImplementation(() => current); + }); + afterEach(() => vi.unstubAllGlobals()); + function show() { + const canvasRef = { current: document.createElement("div") }; + canvasRef.current.getBoundingClientRect = () => new DOMRect(0, 0, 1000, 100); + const selectRegion = vi.fn(), + moveLayoutBoundary = vi.fn(); + const tl = { + layoutSegments: layoutRuns(current.document), + clips: current.document.timeline.clips, + selection: null, + selectRegion, + moveLayoutBoundary, + } as unknown as ReturnType; + render(); + return { selectRegion, moveLayoutBoundary }; + } + it("selects a passage and moves a change with the keyboard", () => { + const callbacks = show(); + fireEvent.click(screen.getByRole("button", { name: /layouts.no-webcam/ })); + expect(callbacks.selectRegion).toHaveBeenCalledWith( + "layout", + layoutRuns(current.document)[1].ids[0], + ); + fireEvent.keyDown(screen.getByRole("button", { name: "layouts.moveBoundary" }), { + key: "ArrowLeft", + }); + expect(callbacks.moveLayoutBoundary).toHaveBeenCalledWith( + layoutRuns(current.document)[1].ids[0], + 4.9, + ); + }); + it("commits a complete drag once, retaining its pre-drag history base", () => { + const base = current.document; + show(); + fireEvent.pointerDown(screen.getByRole("button", { name: "layouts.moveBoundary" }), { + button: 0, + clientX: 500, + }); + fireEvent.pointerMove(document, { clientX: 600 }); + expect(current.saveDocument).not.toHaveBeenCalled(); + fireEvent.pointerUp(document, { clientX: 600 }); + expect(layoutRuns(current.document)[1].start).toBe(6); + expect(current.saveDocument).toHaveBeenCalledExactlyOnceWith(current.document, { + history: true, + historyBase: base, + }); + }); + it("cancels the live preview without persisting it", () => { + const base = current.document; + show(); + fireEvent.pointerDown(screen.getByRole("button", { name: "layouts.moveBoundary" }), { + button: 0, + clientX: 500, + }); + fireEvent.pointerMove(document, { clientX: 600 }); + animation?.(0); + expect(current.document).not.toBe(base); + fireEvent.pointerCancel(document); + expect(current.document).toBe(base); + expect(current.saveDocument).not.toHaveBeenCalled(); + }); + it("does not overwrite a concurrent undo or project change", () => { + show(); + fireEvent.pointerDown(screen.getByRole("button", { name: "layouts.moveBoundary" }), { + button: 0, + clientX: 500, + }); + const other = footage(); + current.document = other; + fireEvent.pointerMove(document, { clientX: 600 }); + animation?.(0); + fireEvent.pointerUp(document); + expect(current.document).toBe(other); + expect(current.saveDocument).not.toHaveBeenCalled(); + }); +}); diff --git a/src/components/ai-edition/v4/LayoutLane.tsx b/src/components/ai-edition/v4/LayoutLane.tsx new file mode 100644 index 0000000000..5d3eefa71d --- /dev/null +++ b/src/components/ai-edition/v4/LayoutLane.tsx @@ -0,0 +1,155 @@ +import { type RefObject, useEffect, useRef, useState } from "react"; +import { useScopedT } from "@/contexts/I18nContext"; +import { moveLayoutBoundary } from "@/lib/ai-edition/document/layout"; +import type { AxcutDocument } from "@/lib/ai-edition/schema"; +import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; +import type { useTimeline } from "@/lib/ai-edition/store/useTimeline"; +import { formatSec } from "@/lib/ai-edition/timeline/format"; +import styles from "./Layouts.module.css"; + +export function LayoutLane({ + tl, + total, + canvasRef, + pxPerSec, +}: { + tl: ReturnType; + total: number; + canvasRef: RefObject; + pxPerSec: number; +}) { + const t = useScopedT("timeline"); + const cleanup = useRef<(() => void) | null>(null); + const [dragging, setDragging] = useState(false); + useEffect(() => () => cleanup.current?.(), []); + const runs = tl.layoutSegments; + if (runs.length <= 1 && !runs[0]?.member.legacyCamera) return null; + return ( +
+ {runs.map((run, index) => ( +
+ + {index > 0 ? ( +
+ ))} +
+ ); +} diff --git a/src/components/ai-edition/v4/LayoutPicker.test.tsx b/src/components/ai-edition/v4/LayoutPicker.test.tsx new file mode 100644 index 0000000000..163396c0ea --- /dev/null +++ b/src/components/ai-edition/v4/LayoutPicker.test.tsx @@ -0,0 +1,30 @@ +// @vitest-environment jsdom +import { fireEvent, render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; +import { LayoutPicker } from "./LayoutPicker"; + +vi.mock("@/contexts/I18nContext", () => ({ useScopedT: () => (key: string) => key })); + +describe("layout selection", () => { + it("offers five visual presets and reports only the chosen preset", () => { + const onSelect = vi.fn(); + render(); + expect(screen.getAllByRole("button")).toHaveLength(5); + expect( + screen.getByRole("button", { name: "layouts.dual-frame" }).getAttribute("aria-pressed"), + ).toBe("true"); + fireEvent.click(screen.getByRole("button", { name: "layouts.camera-only" })); + expect(onSelect).toHaveBeenCalledExactlyOnceWith("camera-only"); + }); + it("explains unavailable camera presets and keeps screen-only usable", () => { + const onSelect = vi.fn(); + render(); + const camera = screen.getByRole("button", { name: "layouts.camera-only" }) as HTMLButtonElement; + expect(camera.disabled).toBe(true); + expect(camera.title).toBe("layouts.noCamera"); + fireEvent.click(camera); + expect(onSelect).not.toHaveBeenCalled(); + fireEvent.click(screen.getByRole("button", { name: "layouts.no-webcam" })); + expect(onSelect).toHaveBeenCalledExactlyOnceWith("no-webcam"); + }); +}); diff --git a/src/components/ai-edition/v4/LayoutPicker.tsx b/src/components/ai-edition/v4/LayoutPicker.tsx new file mode 100644 index 0000000000..5d3b2ca835 --- /dev/null +++ b/src/components/ai-edition/v4/LayoutPicker.tsx @@ -0,0 +1,56 @@ +import { Tooltip } from "@/components/ui/tooltip"; +import { useScopedT } from "@/contexts/I18nContext"; +import { LAYOUT_PRESETS, type LayoutPreset } from "@/lib/layoutTypes"; +import styles from "./Layouts.module.css"; + +export function LayoutPicker({ + value, + hasCamera, + onSelect, + disabled = false, + iconOnly = false, + label, +}: { + value: LayoutPreset; + hasCamera: boolean; + onSelect: (preset: LayoutPreset) => void; + disabled?: boolean; + iconOnly?: boolean; + label?: string; +}) { + const t = useScopedT("timeline"); + return ( +
+ {LAYOUT_PRESETS.map((preset) => { + const name = t(`layouts.${preset}`); + const unavailable = !hasCamera && preset !== "no-webcam"; + const button = ( + + ); + return iconOnly ? ( + + {button} + + ) : ( + button + ); + })} +
+ ); +} diff --git a/src/components/ai-edition/v4/Layouts.module.css b/src/components/ai-edition/v4/Layouts.module.css new file mode 100644 index 0000000000..b3ccd8e057 --- /dev/null +++ b/src/components/ai-edition/v4/Layouts.module.css @@ -0,0 +1,79 @@ +.presets { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 6px; +} +.iconPresets { grid-template-columns: repeat(5, minmax(0, 1fr)); } +.iconPresets .preset { min-width: 0; padding: 8px 2px; } +.iconPresets .mini { width: 26px; height: 18px; } +.preset { + display: flex; + flex-direction: column; + align-items: center; + gap: 7px; + padding: 9px 5px; + border: 1px solid var(--border); + border-radius: 6px; + background: var(--surface-hi); + color: var(--fg); + font: inherit; + font-size: 11px; + cursor: pointer; +} +.preset[aria-pressed="true"] { + border-color: var(--accent); + background: color-mix(in srgb, var(--accent) 14%, var(--surface-hi)); +} +.preset:disabled { opacity: 0.45; cursor: not-allowed; } +.mini { + display: inline-block; + position: relative; + width: 36px; + height: 22px; + flex-shrink: 0; + border: 1px solid var(--border); + border-radius: 3px; + background: color-mix(in srgb, var(--accent) 12%, var(--surface-hi)); + overflow: hidden; +} +.mini::after { content: ""; position: absolute; inset: 54% 7% 7% 68%; border-radius: 50%; background: var(--accent); } +.mini[data-layout="camera-only"]::after { inset: 0; border-radius: 0; } +.mini[data-layout="no-webcam"]::after { display: none; } +.mini[data-layout="dual-frame"]::after { inset: 0 0 0 60%; border-radius: 0; } +.mini[data-layout="vertical-stack"]::after { inset: 57% 0 0; border-radius: 0; } +.lane { position: relative; height: 34px; margin: 3px 0; } +.segment { + position: absolute; + top: 0; + height: 30px; + display: flex; + align-items: center; + justify-content: center; + gap: 7px; + padding: 3px 5px; + border: 1px solid var(--border); + background: color-mix(in srgb, var(--accent) 12%, var(--surface-hi)); + color: var(--fg); + font: inherit; + font-size: 11px; + overflow: hidden; + white-space: nowrap; + cursor: pointer; +} +.segment[aria-pressed="true"] { outline: 2px solid var(--accent); outline-offset: -2px; } +.segment .mini { width: 23px; height: 16px; } +.boundary { + position: absolute; + top: 0; + width: 12px; + height: 30px; + transform: translateX(-50%); + padding: 0; + border: 0; + background: transparent; + touch-action: none; + cursor: ew-resize; + z-index: 2; +} +.boundary::after { content: ""; display: block; width: 2px; height: 14px; margin: auto; background: var(--accent); } +.boundary:focus-visible { outline: 2px solid var(--accent); } diff --git a/src/components/ai-edition/v4/V4Timeline.geometry.test.tsx b/src/components/ai-edition/v4/V4Timeline.geometry.test.tsx index 65a857e51b..93a44e1438 100644 --- a/src/components/ai-edition/v4/V4Timeline.geometry.test.tsx +++ b/src/components/ai-edition/v4/V4Timeline.geometry.test.tsx @@ -2,7 +2,7 @@ import "@testing-library/jest-dom"; import { act, cleanup, fireEvent, render, screen } from "@testing-library/react"; import { Profiler, type ProfilerOnRenderCallback } from "react"; -import { afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; // The regression under test is geometric, so the environment has to have a size: // jsdom reports 0 for every box, which would leave `pxPerSec` at 0 (the @@ -35,11 +35,14 @@ const measureText = vi.fn((text: string) => ({ width: text.length * 6 })); import { ShortcutsProvider } from "@/contexts/ShortcutsContext"; import type { I18nNamespace } from "@/i18n/config"; import { getAvailableLocales, translate } from "@/i18n/loader"; +import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; import type { useTimeline } from "@/lib/ai-edition/store/useTimeline"; import { DEFAULT_SHORTCUTS, formatBinding } from "@/lib/shortcuts"; import { zoomTransitionMs } from "@/lib/zoomMath/constants"; import { V4Timeline } from "./V4Timeline"; +beforeEach(() => useProjectStore.setState({ currentTimeSec: 0 })); + beforeAll(() => { globalThis.ResizeObserver = class { // jsdom has none, and the width it would report is stubbed below anyway. @@ -93,10 +96,9 @@ function clip(startSec: number, endSec: number) { }; } -/** The asset every clip above points at. No `cameraTrack`: this recording has no webcam, - * which is what the Full Camera button is gated on. */ +/** The asset every clip above points at. With no camera, only screen-only is available. */ const NO_CAMERA_ASSET = { id: "a1", label: "rec", durationSec: TOTAL_SEC }; -/** The same recording, with a camera track: the Full Camera button is offered. */ +/** The same recording with a camera track: camera layout presets become available. */ const CAMERA_ASSET = { ...NO_CAMERA_ASSET, cameraTrack: { sourcePath: "/tmp/cam.webm", startMs: 0, offsetMs: 0, visible: true }, @@ -122,6 +124,8 @@ function renderTimeline( cameraFullscreenRegions: [], shaderEffectRanges: [], shaderEffects: [], + layoutSegments: [], + hasDoc: true, zoomRegions: [], trimRanges: [], hasEditRegions: true, @@ -147,9 +151,10 @@ function renderTimeline( }; const setCurrentTime = vi.fn(); const onAnnotationRowCountChange = vi.fn(); - const timeline = ( + const timeline = (variant: "edit" | "media" = "edit") => ( } @@ -162,16 +167,17 @@ function renderTimeline( /> ); - render( + const view = render( onRender ? ( - {timeline} + {timeline()} ) : ( - timeline + timeline() ), ); return { + rerenderVariant: (variant: "edit" | "media") => view.rerender(timeline(variant)), onAnnotationRowCountChange, // A getter: a test that empties `annotationRegions` renders no such pill, and an eager // lookup would throw before its own assertions ran. @@ -638,7 +644,7 @@ describe("V4Timeline create-from-toolbar", () => { expect(tl.clearTimeline).toHaveBeenCalledTimes(1); }); - it("puts Clear timeline last, behind a divider, after the Add Full Camera button", () => { + it("puts Clear timeline last, behind a divider, after the layout button", () => { renderTimeline(undefined, undefined, [CAMERA_ASSET]); const toolbar = toolbarOf(); const buttons = Array.from(toolbar.querySelectorAll("button")); @@ -647,25 +653,86 @@ describe("V4Timeline create-from-toolbar", () => { const divider = clear.previousElementSibling; expect(divider?.className).toContain("tlToolSep"); - expect(divider?.previousElementSibling).toBe( - screen.getByLabelText("buttons.addCameraFullscreen"), - ); + expect(divider?.previousElementSibling).toBe(screen.getByLabelText("layouts.changeHere")); expect(dividersIn(toolbar)).toHaveLength(2); }); - // #353. A camera-fullscreen region grows the webcam overlay, so with no webcam on the - // timeline it renders nothing in the preview and nothing in the export — the region is - // stored and forgotten. `addCameraFullscreen` now refuses to write one; the button says - // so before it is clicked instead of looking like it worked. Absent, not greyed out: an - // inoperative control is hidden, and a greyed one with no reason is a puzzle. - it("shows no Add Full Camera button when no clip on the timeline has a camera", () => { + it.each([ + "media", + "no-document", + "empty-timeline", + ] as const)("closes the layout picker and ignores shortcuts while %s, without reopening on return", (condition) => { + const { tl, rerenderVariant } = renderTimeline(); + const clips = tl.clips; + const openPicker = () => act(() => window.dispatchEvent(new Event("openscreen:change-layout"))); + openPicker(); + expect(screen.getByRole("group", { name: "layouts.title" })).toBeInTheDocument(); + if (condition === "no-document") tl.hasDoc = false; + if (condition === "empty-timeline") tl.clips = []; + rerenderVariant(condition === "media" ? "media" : "edit"); + expect(screen.queryByRole("group", { name: "layouts.title" })).not.toBeInTheDocument(); + openPicker(); + tl.hasDoc = true; + tl.clips = clips; + rerenderVariant("edit"); + expect(screen.queryByRole("group", { name: "layouts.title" })).not.toBeInTheDocument(); + openPicker(); + expect(screen.getByRole("group", { name: "layouts.title" })).toBeInTheDocument(); + }); + + it("keeps layout changes available for screen-only footage", () => { renderTimeline(); - expect(screen.queryByLabelText("buttons.addCameraFullscreen")).not.toBeInTheDocument(); - }); - - it("shows Add Full Camera as soon as a clip's asset carries one", () => { - renderTimeline(undefined, undefined, [CAMERA_ASSET]); - expect(screen.getByLabelText("buttons.addCameraFullscreen")).toBeEnabled(); + const button = screen.getByRole("button", { name: "layouts.changeHere" }); + expect(button).toBeEnabled(); + expect(button).toHaveTextContent(""); + }); + + it("only offers a layout change inside footage and closes its menu at gaps and the final end", () => { + renderTimeline([clip(2, 4), clip(8, 12)]); + const button = screen.getByRole("button", { name: "layouts.changeHere" }); + const seek = (currentTimeSec: number) => + act(() => useProjectStore.setState({ currentTimeSec })); + const open = () => act(() => window.dispatchEvent(new Event("openscreen:change-layout"))); + const menu = () => screen.queryByRole("group", { name: "layouts.title" }); + expect(button).toBeDisabled(); + open(); + expect(menu()).not.toBeInTheDocument(); + seek(2); + expect(button).toBeEnabled(); + fireEvent.click(button); + expect(menu()).toBeInTheDocument(); + seek(3.999); + expect(button).toBeEnabled(); + expect(menu()).toBeInTheDocument(); + seek(4); + expect(button).toBeDisabled(); + expect(menu()).not.toBeInTheDocument(); + open(); + expect(menu()).not.toBeInTheDocument(); + seek(6); + expect(button).toBeDisabled(); + seek(8); + expect(button).toBeEnabled(); + expect(menu()).not.toBeInTheDocument(); + open(); + expect(menu()).toBeInTheDocument(); + seek(12); + expect(button).toBeDisabled(); + expect(menu()).not.toBeInTheDocument(); + open(); + expect(menu()).not.toBeInTheDocument(); + seek(13); + expect(button).toBeDisabled(); + }); + + it("enables layout changes as soon as a clip's asset carries one", () => { + renderTimeline(undefined, undefined, [ + { + ...NO_CAMERA_ASSET, + cameraTrack: { sourcePath: "/tmp/cam.webm", startMs: 0, offsetMs: 0, visible: true }, + }, + ]); + expect(screen.getByRole("button", { name: "layouts.changeHere" })).toBeEnabled(); }); // The disabled button is only half the promise: an empty lane advertises the shortcut @@ -674,7 +741,7 @@ describe("V4Timeline create-from-toolbar", () => { // instead, so the two surfaces agree about the same project. it("does not advertise the C shortcut on a lane that cannot be filled", () => { renderTimeline(); - expect(screen.getByText("layout.noWebcam")).toBeInTheDocument(); + expect(screen.queryByTestId("layout-lane")).not.toBeInTheDocument(); expect(screen.queryByText("hints.pressCameraFullscreen")).not.toBeInTheDocument(); }); @@ -685,7 +752,7 @@ describe("V4Timeline create-from-toolbar", () => { cameraTrack: { sourcePath: "/tmp/cam.webm", startMs: 0, offsetMs: 0, visible: true }, }, ]); - expect(screen.getByText("hints.pressCameraFullscreen")).toBeInTheDocument(); + expect(screen.queryByTestId("layout-lane")).not.toBeInTheDocument(); expect(screen.queryByText("layout.noWebcam")).not.toBeInTheDocument(); }); }); @@ -842,6 +909,8 @@ describe("V4Timeline audio lane drag", () => { annotationRegions: [], speedRegions: [], cameraFullscreenRegions: [], + layoutSegments: [], + hasDoc: true, shaderEffectRanges: [], shaderEffects: [], zoomRegions: [], @@ -1060,7 +1129,9 @@ describe("V4Timeline empty-lane hints", () => { expect(screen.getByText("Press Z to add zoom")).toBeInTheDocument(); expect(screen.getByText("Press T to add trim")).toBeInTheDocument(); expect(screen.getByText("Press S to add speed")).toBeInTheDocument(); - expect(screen.getByText("Press C to add a Full Camera segment")).toBeInTheDocument(); + expect( + screen.getByLabelText(translate("en", "timeline", "layouts.changeHere")), + ).toBeInTheDocument(); expect(screen.getByText("Press M to add audio, V to record a voiceover")).toBeInTheDocument(); }); @@ -1089,7 +1160,9 @@ describe("V4Timeline empty-lane hints", () => { expect(screen.getByText("Press Ctrl + Shift + T to add trim")).toBeInTheDocument(); expect(screen.getByText("Press N to add annotation")).toBeInTheDocument(); expect(screen.getByText("Press J to add speed")).toBeInTheDocument(); - expect(screen.getByText("Press K to add a Full Camera segment")).toBeInTheDocument(); + expect( + screen.getByLabelText(translate("en", "timeline", "layouts.changeHere")), + ).toBeInTheDocument(); expect( screen.getByText("Press U to add audio, Alt + R to record a voiceover"), ).toBeInTheDocument(); @@ -1150,7 +1223,7 @@ describe("V4Timeline toolbar tooltips", () => { "buttons.addTrim", "buttons.addSpeed", "buttons.addAnnotation", - "buttons.addCameraFullscreen", + "layouts.changeHere", // Its menu also records a voiceover, but the key, like the lane's "Press M", adds audio. "toolbar.addAudioTooltip", ]) { @@ -1163,7 +1236,7 @@ describe("V4Timeline toolbar tooltips", () => { "buttons.addTrim": "T", "buttons.addSpeed": "S", "buttons.addAnnotation": "A", - "buttons.addCameraFullscreen": "C", + "layouts.changeHere": "C", "toolbar.addAudioTooltip": "M", }); }); diff --git a/src/components/ai-edition/v4/V4Timeline.tsx b/src/components/ai-edition/v4/V4Timeline.tsx index 83ae04ce97..79052a2498 100644 --- a/src/components/ai-edition/v4/V4Timeline.tsx +++ b/src/components/ai-edition/v4/V4Timeline.tsx @@ -8,6 +8,7 @@ import { MessageSquare, Mic, Music, + PanelsTopLeft, Pencil, Scissors, Sparkles, @@ -54,7 +55,6 @@ import { useChatPromptBus } from "@/lib/ai-edition/store/useChatPromptBus"; import { useEditorSettings } from "@/lib/ai-edition/store/useEditorSettings"; import type { useTimeline } from "@/lib/ai-edition/store/useTimeline"; import { collectAutoZoomSuggestionsForLatestDocument } from "@/lib/ai-edition/timeline/apply-auto-zooms"; -import { hasAnyClipWithCamera } from "@/lib/ai-edition/timeline/camera"; import { formatSec } from "@/lib/ai-edition/timeline/format"; import { newRegionDurationSec, @@ -87,6 +87,8 @@ import { nativeBridgeClient } from "@/native/client"; import { TransportBar } from "../TransportBar"; import type { VideoSource } from "../VirtualPreview"; import styles from "./EditorShellV4.module.css"; +import { LayoutLane } from "./LayoutLane"; +import { LayoutPicker } from "./LayoutPicker"; // The AI option's prompt — sent straight to the chat agent via the prompt-bus. // @@ -615,6 +617,29 @@ interface LanePill { stackOrder?: number; } +function LayoutChangeMenu({ tl, onClose }: { tl: TimelineApi; onClose: () => void }) { + const t = useScopedT("timeline"); + const time = useProjectStore((s) => s.currentTimeSec); + const active = tl.layoutSegments.find((r) => time >= r.start && time < r.end); + const clip = tl.clips.find((c) => time >= c.timelineStartSec && time < c.timelineEndSec); + const camera = tl.assets.find((a) => a.id === clip?.assetId)?.cameraTrack; + return ( +
+
+ {t("layouts.fromUntil", { start: formatSec(time), end: formatSec(active?.end ?? time) })} +
+ { + void tl.addLayoutChange(preset, time); + onClose(); + }} + /> +
+ ); +} + export function V4Timeline({ tl, setCurrentTime, @@ -644,12 +669,10 @@ export function V4Timeline({ onAnnotationRowCountChange?: (rowCount: number) => void; }) { const t = useScopedT("timeline"); + const ts = useScopedT("settings"); // The live bindings, not the defaults: these keys are remappable, and a menu // that taught the wrong one would be worse than teaching none. const { shortcuts, isMac } = useShortcuts(); - // The camera lane borrows the Layout pane's "No Webcam" wording when there is no - // camera to grow, so the two surfaces say the same thing about the same project. - const ts = useScopedT("settings"); // Wheel zoom/pan listens on the whole pane (toolbar down through the nav bar), // not just the lanes — a user scrolling over the ruler or the hint labels // expects the same zoom/pan the lanes give, not silence. @@ -689,6 +712,27 @@ export function V4Timeline({ const [autoEnhanceOpen, setAutoEnhanceOpen] = useState(false); const [audioMenuOpen, setAudioMenuOpen] = useState(false); + const [layoutOpen, setLayoutOpen] = useState(false); + // Subscribe to availability, not the time itself: the full timeline must not + // re-render on every playhead tick while it stays inside the same footage. + const canChangeLayout = useProjectStore( + (s) => + variant === "edit" && + tl.hasDoc && + tl.clips.some( + (clip) => + s.currentTimeSec >= clip.timelineStartSec && s.currentTimeSec < clip.timelineEndSec, + ), + ); + useEffect(() => { + if (!canChangeLayout) { + setLayoutOpen(false); + return; + } + const open = () => setLayoutOpen(true); + window.addEventListener("openscreen:change-layout", open); + return () => window.removeEventListener("openscreen:change-layout", open); + }, [canChangeLayout]); const [autoBusy, setAutoBusy] = useState(false); // The AI cut pass reads the transcript, and the transcript is produced in the // background (see transcriptionStore). Until it is there, the entry says why @@ -710,12 +754,6 @@ export function V4Timeline({ : t("toolbar.smartCutsNeedsTranscript"); const clips = tl.clips; - // A camera-fullscreen region grows the webcam overlay, so on a project with no webcam - // it renders nothing in the preview and nothing in the export. `addCameraFullscreen` - // refuses to write one (see useTimeline) — this makes the control say so before it is - // clicked instead of looking like it worked. Same question, same helper as the Layout - // pane: is a camera attached anywhere on this timeline? - const hasAnyCamera = useMemo(() => hasAnyClipWithCamera(tl.assets, clips), [tl.assets, clips]); // The pauses added words created, placed on the ruler. Everything below measures the // EXPANDED ruler — stored clip geometry plus the time those pauses add — because that // is the film's real length and the one the playhead runs along. Stored geometry is @@ -781,16 +819,6 @@ export function V4Timeline({ label: `${(p.member as { speed?: number }).speed ?? 1.5}×`, sourceIds: p.ids, })); - const cameraFullscreenPills: LanePill[] = coalesceRegionsForRuler(tl.cameraFullscreenRegions).map( - (p) => ({ - id: p.ids[0], - kind: "cameraFullscreen", - start: p.start, - end: p.end, - label: "Full Camera", - sourceIds: p.ids, - }), - ); const effectNames = new Map(tl.shaderEffects.map((effect) => [effect.id, effect.name])); const shaderEffectPills: LanePill[] = coalesceRegionsForRuler(tl.shaderEffectRanges).map((p) => ({ id: p.ids[0], @@ -2253,22 +2281,27 @@ export function V4Timeline({ - {/* Absent, not greyed out, in a project with no camera: nothing to show full screen. */} - {hasAnyCamera ? ( + - + + + - ) : null} + + setLayoutOpen(false)} /> + + {/* Last, behind a divider: every button before it adds a region, this one clears them. Absent with its divider, not greyed out, when there is nothing to clear. One write in the store, so one Ctrl+Z restores every @@ -2406,19 +2439,7 @@ export function V4Timeline({ t("hints.pressZoom", { key: formatBinding(shortcuts.addZoom, isMac) }), )}
-
- {/* Advertising "Press C" on a project with no webcam invites a keystroke - that `addCameraFullscreen` now refuses (#353). The toolbar button is - already disabled; this keeps the lane from contradicting it. */} - {renderPills( - cameraFullscreenPills, - hasAnyCamera - ? t("hints.pressCameraFullscreen", { - key: formatBinding(shortcuts.addCameraFullscreen, isMac), - }) - : ts("layout.noWebcam"), - )} -
+ {/* Agent-written shader effects: one at a time, so one row. Only when the project has one: an agent writes them, nothing in the editor adds one, so an empty row would offer nothing. The shell grows by its height diff --git a/src/components/ai-edition/v4/V4Timeline.waveform.test.tsx b/src/components/ai-edition/v4/V4Timeline.waveform.test.tsx index 37e9c32c62..a6da1cfc28 100644 --- a/src/components/ai-edition/v4/V4Timeline.waveform.test.tsx +++ b/src/components/ai-edition/v4/V4Timeline.waveform.test.tsx @@ -122,6 +122,8 @@ function renderBars(atGainDb: number): string[] { cameraFullscreenRegions: [], shaderEffectRanges: [], shaderEffects: [], + layoutSegments: [], + hasDoc: true, zoomRegions: [], trimRanges: [], selection: null, diff --git a/src/components/video-editor/projectPersistence.ts b/src/components/video-editor/projectPersistence.ts index a533be7abf..a1b7f3942f 100644 --- a/src/components/video-editor/projectPersistence.ts +++ b/src/components/video-editor/projectPersistence.ts @@ -148,6 +148,7 @@ function computeNormalizedWebcamLayoutPreset( const auto = normalizedAspectRatio === "auto"; switch (webcamLayoutPreset) { case "picture-in-picture": + case "camera-only": case "no-webcam": return webcamLayoutPreset; case "vertical-stack": diff --git a/src/i18n/locales/ar/shortcuts.json b/src/i18n/locales/ar/shortcuts.json index 137a0186e6..a30a4ab0b5 100644 --- a/src/i18n/locales/ar/shortcuts.json +++ b/src/i18n/locales/ar/shortcuts.json @@ -23,7 +23,7 @@ "addAudio": "إضافة صوت", "addVoiceover": "تسجيل تعليق صوتي", "addKeyframe": "إضافة إطار رئيسي", - "addCameraFullscreen": "إضافة كاميرا كاملة الشاشة", + "addCameraFullscreen": "تغيير التخطيط هنا", "deleteSelected": "حذف المحدد", "playPause": "تشغيل / إيقاف مؤقت", "copySelected": "نسخ المحدد", diff --git a/src/i18n/locales/ar/timeline.json b/src/i18n/locales/ar/timeline.json index c802060db2..0094ae0b64 100644 --- a/src/i18n/locales/ar/timeline.json +++ b/src/i18n/locales/ar/timeline.json @@ -7,7 +7,7 @@ "addTrim": "إضافة قص: يزيل جزءًا من الفيديو عند الوقت الحالي", "addAnnotation": "إضافة نص أو سهم أو صورة أو تمويه عند الوقت الحالي", "addSpeed": "إضافة تغيير في السرعة عند الوقت الحالي", - "addCameraFullscreen": "إضافة كاميرا كاملة الشاشة: تعرض الكاميرا بملء الشاشة عند الوقت الحالي" + "addCameraFullscreen": "تغيير التخطيط هنا" }, "hints": { "pressZoom": "اضغط {{key}} لإضافة تكبير", @@ -116,5 +116,20 @@ "tracks": { "annotation": "طبقة التعليق {{index}}", "dragHint": "اسحب يمينًا أو يسارًا لتغيير الوقت. اسحب لأعلى أو لأسفل لتغيير الطبقة؛ تظهر الطبقات الأعلى في المقدمة." + }, + "layouts": { + "title": "التخطيطات", + "changeHere": "تغيير التخطيط هنا", + "wholeVideo": "الفيديو بالكامل", + "sharedSettings": "إعدادات الكاميرا · الفيديو بالكامل", + "fromUntil": "من {{start}} إلى {{end}}", + "noCamera": "لا توجد كاميرا في هذا المقطع؛ تظهر الشاشة فقط.", + "removeChange": "إزالة هذا التغيير", + "moveBoundary": "نقل التغيير عند {{time}}", + "picture-in-picture": "صورة داخل صورة", + "dual-frame": "جنبًا إلى جنب", + "vertical-stack": "أعلى / أسفل", + "camera-only": "الكاميرا فقط", + "no-webcam": "الشاشة فقط" } } diff --git a/src/i18n/locales/cs/shortcuts.json b/src/i18n/locales/cs/shortcuts.json index 47d6c77a61..40ed15e885 100644 --- a/src/i18n/locales/cs/shortcuts.json +++ b/src/i18n/locales/cs/shortcuts.json @@ -23,7 +23,7 @@ "addAudio": "Přidat zvuk", "addVoiceover": "Nahrát komentář", "addKeyframe": "Přidat klíčový snímek", - "addCameraFullscreen": "Přidat celou kameru", + "addCameraFullscreen": "Změnit rozvržení zde", "deleteSelected": "Smazat vybrané", "playPause": "Přehrát / Pozastavit", "copySelected": "Kopírovat vybrané", diff --git a/src/i18n/locales/cs/timeline.json b/src/i18n/locales/cs/timeline.json index 8fe84616b7..ff20ad4b8d 100644 --- a/src/i18n/locales/cs/timeline.json +++ b/src/i18n/locales/cs/timeline.json @@ -7,7 +7,7 @@ "addTrim": "Přidat oříznutí: odstraní část videa v aktuálním čase", "addAnnotation": "Přidat text, šipku, obrázek nebo rozmazání v aktuálním čase", "addSpeed": "Přidat změnu rychlosti v aktuálním čase", - "addCameraFullscreen": "Přidat Celou kameru: zobrazí kameru na celou obrazovku v aktuálním čase" + "addCameraFullscreen": "Změnit rozvržení zde" }, "hints": { "pressZoom": "Stiskněte {{key}} pro přidání přiblížení", @@ -116,5 +116,20 @@ "tracks": { "annotation": "Vrstva anotace {{index}}", "dragHint": "Tažením doleva nebo doprava posunete čas. Tažením nahoru nebo dolů změníte vrstvu; vyšší vrstvy jsou v popředí." + }, + "layouts": { + "title": "Rozvržení", + "changeHere": "Změnit rozvržení zde", + "wholeVideo": "Celé video", + "sharedSettings": "Společné nastavení kamery", + "fromUntil": "Od {{start}} do {{end}}", + "noCamera": "K tomuto úseku není připojena kamera.", + "removeChange": "Odstranit změnu rozvržení", + "moveBoundary": "Přesunout změnu rozvržení", + "picture-in-picture": "Obraz v obraze", + "dual-frame": "Vedle sebe", + "vertical-stack": "Nahoře / dole", + "camera-only": "Pouze kamera", + "no-webcam": "Pouze obrazovka" } } diff --git a/src/i18n/locales/de/shortcuts.json b/src/i18n/locales/de/shortcuts.json index e4a02bbe08..dd326b83f8 100644 --- a/src/i18n/locales/de/shortcuts.json +++ b/src/i18n/locales/de/shortcuts.json @@ -23,7 +23,7 @@ "addAudio": "Audio hinzufügen", "addVoiceover": "Voiceover aufnehmen", "addKeyframe": "Keyframe hinzufügen", - "addCameraFullscreen": "Vollbild-Kamera hinzufügen", + "addCameraFullscreen": "Layout hier ändern", "deleteSelected": "Auswahl löschen", "playPause": "Wiedergabe / Pause", "copySelected": "Auswahl kopieren", diff --git a/src/i18n/locales/de/timeline.json b/src/i18n/locales/de/timeline.json index 9707bbfccd..225fe0bed2 100644 --- a/src/i18n/locales/de/timeline.json +++ b/src/i18n/locales/de/timeline.json @@ -7,7 +7,7 @@ "addTrim": "Trim hinzufügen: entfernt einen Abschnitt des Videos zum aktuellen Zeitpunkt", "addAnnotation": "Text, Pfeil, Bild oder Weichzeichner zum aktuellen Zeitpunkt hinzufügen", "addSpeed": "Tempowechsel zum aktuellen Zeitpunkt hinzufügen", - "addCameraFullscreen": "Vollbild-Kamera hinzufügen: zeigt die Kamera im Vollbild zum aktuellen Zeitpunkt" + "addCameraFullscreen": "Layout hier ändern" }, "hints": { "pressZoom": "Drücke {{key}}, um einen Zoom hinzuzufügen", @@ -116,5 +116,20 @@ "tracks": { "annotation": "Anmerkungsebene {{index}}", "dragHint": "Nach links oder rechts ziehen, um die Zeit zu ändern. Nach oben oder unten ziehen, um die Ebene zu ändern; höhere Ebenen liegen vorne." + }, + "layouts": { + "title": "Layouts", + "changeHere": "Layout hier ändern", + "wholeVideo": "Gesamtes Video", + "sharedSettings": "Gemeinsame Kameraeinstellungen", + "fromUntil": "Von {{start}} bis {{end}}", + "noCamera": "In diesem Abschnitt ist keine Kamera verknüpft.", + "removeChange": "Layoutwechsel entfernen", + "moveBoundary": "Layoutwechsel verschieben", + "picture-in-picture": "Bild im Bild", + "dual-frame": "Nebeneinander", + "vertical-stack": "Oben / unten", + "camera-only": "Nur Kamera", + "no-webcam": "Nur Bildschirm" } } diff --git a/src/i18n/locales/en/shortcuts.json b/src/i18n/locales/en/shortcuts.json index 99f3ce90af..483d051b55 100644 --- a/src/i18n/locales/en/shortcuts.json +++ b/src/i18n/locales/en/shortcuts.json @@ -23,7 +23,7 @@ "addAudio": "Add Audio", "addVoiceover": "Record Voiceover", "addKeyframe": "Add Keyframe", - "addCameraFullscreen": "Add Full Camera", + "addCameraFullscreen": "Change layout here", "deleteSelected": "Delete Selected", "playPause": "Play / Pause", "copySelected": "Copy Selected", diff --git a/src/i18n/locales/en/timeline.json b/src/i18n/locales/en/timeline.json index febfb7bc33..3a3d6d4773 100644 --- a/src/i18n/locales/en/timeline.json +++ b/src/i18n/locales/en/timeline.json @@ -7,7 +7,7 @@ "addTrim": "Add a trim: removes a section of the video at the current time", "addAnnotation": "Add text, an arrow, an image or a blur at the current time", "addSpeed": "Add a speed change at the current time", - "addCameraFullscreen": "Add Full Camera: shows the camera full screen at the current time" + "addCameraFullscreen": "Change layout here" }, "hints": { "pressZoom": "Press {{key}} to add zoom", @@ -116,5 +116,20 @@ "tracks": { "annotation": "Annotation layer {{index}}", "dragHint": "Drag left or right to move in time. Drag up or down to change layer; higher layers appear on top." + }, + "layouts": { + "title": "Layouts", + "changeHere": "Change layout here", + "wholeVideo": "Entire video", + "sharedSettings": "Webcam settings · entire video", + "fromUntil": "From {{start}} to {{end}}", + "noCamera": "No webcam on this footage; screen only is shown.", + "removeChange": "Remove this change", + "moveBoundary": "Move the change at {{time}}", + "picture-in-picture": "Picture in picture", + "dual-frame": "Side by side", + "vertical-stack": "Top / bottom", + "camera-only": "Camera only", + "no-webcam": "Screen only" } } diff --git a/src/i18n/locales/es/shortcuts.json b/src/i18n/locales/es/shortcuts.json index 688f218f0b..914ea529fc 100644 --- a/src/i18n/locales/es/shortcuts.json +++ b/src/i18n/locales/es/shortcuts.json @@ -23,7 +23,7 @@ "addAudio": "Añadir audio", "addVoiceover": "Grabar voz en off", "addKeyframe": "Agregar fotograma clave", - "addCameraFullscreen": "Agregar cámara a pantalla completa", + "addCameraFullscreen": "Cambiar disposición aquí", "deleteSelected": "Eliminar seleccionado", "playPause": "Reproducir / Pausar", "copySelected": "Copiar selección", diff --git a/src/i18n/locales/es/timeline.json b/src/i18n/locales/es/timeline.json index 70df7142d9..5d88e30be3 100644 --- a/src/i18n/locales/es/timeline.json +++ b/src/i18n/locales/es/timeline.json @@ -7,7 +7,7 @@ "addTrim": "Agregar un recorte: quita una parte del video en el momento actual", "addAnnotation": "Agregar texto, una flecha, una imagen o un desenfoque en el momento actual", "addSpeed": "Agregar un cambio de velocidad en el momento actual", - "addCameraFullscreen": "Agregar Cámara a pantalla completa: muestra la cámara a pantalla completa en el momento actual" + "addCameraFullscreen": "Cambiar disposición aquí" }, "hints": { "pressZoom": "Presiona {{key}} para agregar zoom", @@ -116,5 +116,20 @@ "tracks": { "annotation": "Capa de anotación {{index}}", "dragHint": "Arrastra a izquierda o derecha para cambiar el tiempo. Arrastra arriba o abajo para cambiar de capa; las capas superiores aparecen delante." + }, + "layouts": { + "title": "Disposiciones", + "changeHere": "Cambiar disposición aquí", + "wholeVideo": "Todo el vídeo", + "sharedSettings": "Ajustes de cámara · todo el vídeo", + "fromUntil": "De {{start}} a {{end}}", + "noCamera": "Este fragmento no tiene cámara; se muestra solo la pantalla.", + "removeChange": "Eliminar este cambio", + "moveBoundary": "Mover el cambio de {{time}}", + "picture-in-picture": "Imagen en imagen", + "dual-frame": "Lado a lado", + "vertical-stack": "Arriba / abajo", + "camera-only": "Solo cámara", + "no-webcam": "Solo pantalla" } } diff --git a/src/i18n/locales/fr/shortcuts.json b/src/i18n/locales/fr/shortcuts.json index 3b28fca17d..40028519b0 100644 --- a/src/i18n/locales/fr/shortcuts.json +++ b/src/i18n/locales/fr/shortcuts.json @@ -23,7 +23,7 @@ "addAudio": "Ajouter un audio", "addVoiceover": "Enregistrer une voix off", "addKeyframe": "Ajouter une image-clé", - "addCameraFullscreen": "Ajouter une caméra en plein écran", + "addCameraFullscreen": "Changer la disposition ici", "deleteSelected": "Supprimer la sélection", "playPause": "Lecture / Pause", "copySelected": "Copier la sélection", diff --git a/src/i18n/locales/fr/timeline.json b/src/i18n/locales/fr/timeline.json index ed36c7d2f4..f597f987a1 100644 --- a/src/i18n/locales/fr/timeline.json +++ b/src/i18n/locales/fr/timeline.json @@ -7,7 +7,7 @@ "addTrim": "Ajouter une coupe : retire une partie de la vidéo à l'instant actuel", "addAnnotation": "Ajouter du texte, une flèche, une image ou un flou à l'instant actuel", "addSpeed": "Ajouter un changement de vitesse à l'instant actuel", - "addCameraFullscreen": "Ajouter Caméra plein écran : affiche la caméra en plein écran à l'instant actuel" + "addCameraFullscreen": "Changer la disposition ici" }, "hints": { "pressZoom": "Appuyez sur {{key}} pour ajouter un zoom", @@ -116,5 +116,20 @@ "tracks": { "annotation": "Calque d’annotation {{index}}", "dragHint": "Glissez à gauche ou à droite pour déplacer dans le temps. Glissez vers le haut ou le bas pour changer de calque ; les calques supérieurs apparaissent devant." + }, + "layouts": { + "title": "Dispositions", + "changeHere": "Changer la disposition ici", + "wholeVideo": "Toute la vidéo", + "sharedSettings": "Réglages webcam · toute la vidéo", + "fromUntil": "De {{start}} à {{end}}", + "noCamera": "Ce passage ne possède pas de webcam ; seul l’écran est affiché.", + "removeChange": "Supprimer ce changement", + "moveBoundary": "Déplacer le changement à {{time}}", + "picture-in-picture": "Incrustation", + "dual-frame": "Côte à côte", + "vertical-stack": "Empilé", + "camera-only": "Caméra seule", + "no-webcam": "Écran seul" } } diff --git a/src/i18n/locales/it/shortcuts.json b/src/i18n/locales/it/shortcuts.json index 0940258b2d..66bd8c9fde 100644 --- a/src/i18n/locales/it/shortcuts.json +++ b/src/i18n/locales/it/shortcuts.json @@ -23,7 +23,7 @@ "addAudio": "Aggiungi audio", "addVoiceover": "Registra voce fuori campo", "addKeyframe": "Aggiungi fotogramma chiave", - "addCameraFullscreen": "Aggiungi Camera a schermo intero", + "addCameraFullscreen": "Cambia disposizione qui", "deleteSelected": "Elimina selezionato", "playPause": "Riproduci / Pausa", "copySelected": "Copia selezione", diff --git a/src/i18n/locales/it/timeline.json b/src/i18n/locales/it/timeline.json index c712b6880d..f5a20edef7 100644 --- a/src/i18n/locales/it/timeline.json +++ b/src/i18n/locales/it/timeline.json @@ -7,7 +7,7 @@ "addTrim": "Aggiungi un taglio: rimuove una parte del video nel momento attuale", "addAnnotation": "Aggiungi testo, una freccia, un'immagine o una sfocatura nel momento attuale", "addSpeed": "Aggiungi un cambio di velocità nel momento attuale", - "addCameraFullscreen": "Aggiungi Camera a schermo intero: mostra la fotocamera a schermo intero nel momento attuale" + "addCameraFullscreen": "Cambia disposizione qui" }, "hints": { "pressZoom": "Premi {{key}} per aggiungere zoom", @@ -116,5 +116,20 @@ "tracks": { "annotation": "Livello annotazione {{index}}", "dragHint": "Trascina a sinistra o destra per spostare nel tempo. Trascina in alto o in basso per cambiare livello; i livelli superiori appaiono davanti." + }, + "layouts": { + "title": "Disposizioni", + "changeHere": "Cambia disposizione qui", + "wholeVideo": "Intero video", + "sharedSettings": "Impostazioni webcam · intero video", + "fromUntil": "Da {{start}} a {{end}}", + "noCamera": "Questo filmato non ha una webcam; viene mostrato solo lo schermo.", + "removeChange": "Rimuovi questa modifica", + "moveBoundary": "Sposta la modifica a {{time}}", + "picture-in-picture": "Immagine nell’immagine", + "dual-frame": "Affiancato", + "vertical-stack": "Sopra / sotto", + "camera-only": "Solo videocamera", + "no-webcam": "Solo schermo" } } diff --git a/src/i18n/locales/ja-JP/shortcuts.json b/src/i18n/locales/ja-JP/shortcuts.json index 82f1c22915..93284c34f6 100644 --- a/src/i18n/locales/ja-JP/shortcuts.json +++ b/src/i18n/locales/ja-JP/shortcuts.json @@ -23,7 +23,7 @@ "addAudio": "音声を追加", "addVoiceover": "ナレーションを録音", "addKeyframe": "キーフレームを追加", - "addCameraFullscreen": "フルスクリーンカメラを追加", + "addCameraFullscreen": "ここでレイアウトを変更", "deleteSelected": "選択を削除", "playPause": "再生 / 一時停止", "copySelected": "選択をコピー", diff --git a/src/i18n/locales/ja-JP/timeline.json b/src/i18n/locales/ja-JP/timeline.json index ae6ffd48e8..50586d334d 100644 --- a/src/i18n/locales/ja-JP/timeline.json +++ b/src/i18n/locales/ja-JP/timeline.json @@ -7,7 +7,7 @@ "addTrim": "トリムを追加：現在の再生位置で動画の一部を削除します", "addAnnotation": "現在の再生位置にテキスト、矢印、画像、ぼかしを追加", "addSpeed": "現在の再生位置に再生速度の変更を追加", - "addCameraFullscreen": "フルスクリーンカメラを追加：現在の再生位置でカメラを全画面表示します" + "addCameraFullscreen": "ここでレイアウトを変更" }, "hints": { "pressZoom": "{{key}}キーを押してズームを追加", @@ -116,5 +116,20 @@ "tracks": { "annotation": "注釈レイヤー {{index}}", "dragHint": "左右にドラッグして時間を移動します。上下にドラッグしてレイヤーを変更します。上のレイヤーが手前に表示されます。" + }, + "layouts": { + "title": "レイアウト", + "changeHere": "ここでレイアウトを変更", + "wholeVideo": "動画全体", + "sharedSettings": "ウェブカメラ設定 · 動画全体", + "fromUntil": "{{start}} から {{end}} まで", + "noCamera": "この区間にウェブカメラはありません。画面のみ表示します。", + "removeChange": "この変更を削除", + "moveBoundary": "{{time}} の変更位置を移動", + "picture-in-picture": "ピクチャーインピクチャ", + "dual-frame": "左右に並べる", + "vertical-stack": "上下に並べる", + "camera-only": "カメラのみ", + "no-webcam": "画面のみ" } } diff --git a/src/i18n/locales/ko-KR/shortcuts.json b/src/i18n/locales/ko-KR/shortcuts.json index 86f0693ea3..5d1a150c69 100644 --- a/src/i18n/locales/ko-KR/shortcuts.json +++ b/src/i18n/locales/ko-KR/shortcuts.json @@ -23,7 +23,7 @@ "addAudio": "오디오 추가", "addVoiceover": "보이스오버 녹음", "addKeyframe": "키프레임 추가", - "addCameraFullscreen": "전체 화면 카메라 추가", + "addCameraFullscreen": "여기서 레이아웃 변경", "deleteSelected": "선택 항목 삭제", "playPause": "재생 / 일시정지", "copySelected": "선택 항목 복사", diff --git a/src/i18n/locales/ko-KR/timeline.json b/src/i18n/locales/ko-KR/timeline.json index b1724c7571..e6c1429946 100644 --- a/src/i18n/locales/ko-KR/timeline.json +++ b/src/i18n/locales/ko-KR/timeline.json @@ -7,7 +7,7 @@ "addTrim": "트림 추가: 현재 시점에서 영상의 한 구간을 제거합니다", "addAnnotation": "현재 시점에 텍스트, 화살표, 이미지 또는 블러 추가", "addSpeed": "현재 시점에 속도 변경 추가", - "addCameraFullscreen": "전체 화면 카메라 추가: 현재 시점에 카메라를 전체 화면으로 표시합니다" + "addCameraFullscreen": "여기서 레이아웃 변경" }, "hints": { "pressZoom": "{{key}}를 눌러 줌 추가", @@ -116,5 +116,20 @@ "tracks": { "annotation": "주석 레이어 {{index}}", "dragHint": "좌우로 드래그하여 시간을 이동합니다. 위아래로 드래그하여 레이어를 변경합니다. 위쪽 레이어가 앞에 표시됩니다." + }, + "layouts": { + "title": "레이아웃", + "changeHere": "여기서 레이아웃 변경", + "wholeVideo": "전체 동영상", + "sharedSettings": "웹캠 설정 · 전체 동영상", + "fromUntil": "{{start}} ~ {{end}}", + "noCamera": "이 구간에는 웹캠이 없어 화면만 표시됩니다.", + "removeChange": "이 변경 삭제", + "moveBoundary": "{{time}}의 변경 이동", + "picture-in-picture": "화면 속 화면", + "dual-frame": "나란히", + "vertical-stack": "위 / 아래", + "camera-only": "카메라만", + "no-webcam": "화면만" } } diff --git a/src/i18n/locales/pt-BR/shortcuts.json b/src/i18n/locales/pt-BR/shortcuts.json index 5f5506aec8..0341d7dea9 100644 --- a/src/i18n/locales/pt-BR/shortcuts.json +++ b/src/i18n/locales/pt-BR/shortcuts.json @@ -23,7 +23,7 @@ "addAudio": "Adicionar áudio", "addVoiceover": "Gravar narração", "addKeyframe": "Adicionar Quadro-chave", - "addCameraFullscreen": "Adicionar Câmera em Tela Cheia", + "addCameraFullscreen": "Alterar disposição aqui", "deleteSelected": "Excluir Selecionado", "playPause": "Reproduzir / Pausar", "copySelected": "Copiar seleção", diff --git a/src/i18n/locales/pt-BR/timeline.json b/src/i18n/locales/pt-BR/timeline.json index 7680fb05c9..100b096396 100644 --- a/src/i18n/locales/pt-BR/timeline.json +++ b/src/i18n/locales/pt-BR/timeline.json @@ -7,7 +7,7 @@ "addTrim": "Adicionar um recorte: remove uma parte do vídeo no momento atual", "addAnnotation": "Adicionar texto, seta, imagem ou desfoque no momento atual", "addSpeed": "Adicionar uma mudança de velocidade no momento atual", - "addCameraFullscreen": "Adicionar Câmera em Tela Cheia: mostra a câmera em tela cheia no momento atual" + "addCameraFullscreen": "Alterar disposição aqui" }, "hints": { "pressZoom": "Pressione {{key}} para adicionar zoom", @@ -116,5 +116,20 @@ "tracks": { "annotation": "Camada de anotação {{index}}", "dragHint": "Arraste para a esquerda ou direita para mover no tempo. Arraste para cima ou baixo para mudar de camada; as camadas superiores aparecem na frente." + }, + "layouts": { + "title": "Disposições", + "changeHere": "Alterar disposição aqui", + "wholeVideo": "Vídeo inteiro", + "sharedSettings": "Configurações da webcam · vídeo inteiro", + "fromUntil": "De {{start}} a {{end}}", + "noCamera": "Este trecho não tem webcam; apenas a tela é exibida.", + "removeChange": "Remover esta alteração", + "moveBoundary": "Mover a alteração em {{time}}", + "picture-in-picture": "Imagem sobre imagem", + "dual-frame": "Lado a lado", + "vertical-stack": "Acima / abaixo", + "camera-only": "Somente câmera", + "no-webcam": "Somente tela" } } diff --git a/src/i18n/locales/ru/shortcuts.json b/src/i18n/locales/ru/shortcuts.json index eb94af5fc5..39e444a6bf 100644 --- a/src/i18n/locales/ru/shortcuts.json +++ b/src/i18n/locales/ru/shortcuts.json @@ -23,7 +23,7 @@ "addAudio": "Добавить аудио", "addVoiceover": "Записать закадровый голос", "addKeyframe": "Добавить ключевой кадр", - "addCameraFullscreen": "Добавить камеру на весь экран", + "addCameraFullscreen": "Изменить макет здесь", "deleteSelected": "Удалить выбранное", "playPause": "Воспроизведение / Пауза", "copySelected": "Копировать выбранное", diff --git a/src/i18n/locales/ru/timeline.json b/src/i18n/locales/ru/timeline.json index 18e2850d9f..8a18b1dbf9 100644 --- a/src/i18n/locales/ru/timeline.json +++ b/src/i18n/locales/ru/timeline.json @@ -7,7 +7,7 @@ "addTrim": "Добавить обрезку: удаляет фрагмент видео в текущий момент", "addAnnotation": "Добавить текст, стрелку, изображение или размытие в текущий момент", "addSpeed": "Добавить изменение скорости в текущий момент", - "addCameraFullscreen": "Добавить камеру на весь экран: показывает камеру на весь экран в текущий момент" + "addCameraFullscreen": "Изменить макет здесь" }, "hints": { "pressZoom": "Нажмите {{key}} для добавления масштабирования", @@ -116,5 +116,20 @@ "tracks": { "annotation": "Слой аннотации {{index}}", "dragHint": "Перетаскивайте влево или вправо для перемещения по времени. Перетаскивайте вверх или вниз для смены слоя; верхние слои отображаются впереди." + }, + "layouts": { + "title": "Макеты", + "changeHere": "Изменить макет здесь", + "wholeVideo": "Всё видео", + "sharedSettings": "Настройки веб-камеры · всё видео", + "fromUntil": "С {{start}} до {{end}}", + "noCamera": "В этом фрагменте нет веб-камеры; показан только экран.", + "removeChange": "Удалить это изменение", + "moveBoundary": "Переместить изменение в {{time}}", + "picture-in-picture": "Картинка в картинке", + "dual-frame": "Рядом", + "vertical-stack": "Сверху / снизу", + "camera-only": "Только камера", + "no-webcam": "Только экран" } } diff --git a/src/i18n/locales/tr/shortcuts.json b/src/i18n/locales/tr/shortcuts.json index 2da862f01c..8e2843cfe0 100644 --- a/src/i18n/locales/tr/shortcuts.json +++ b/src/i18n/locales/tr/shortcuts.json @@ -23,7 +23,7 @@ "addAudio": "Ses ekle", "addVoiceover": "Seslendirme kaydet", "addKeyframe": "Anahtar Kare Ekle", - "addCameraFullscreen": "Tam Ekran Kamera Ekle", + "addCameraFullscreen": "Yerleşimi burada değiştir", "deleteSelected": "Seçileni Sil", "playPause": "Oynat / Duraklat", "copySelected": "Seçileni Kopyala", diff --git a/src/i18n/locales/tr/timeline.json b/src/i18n/locales/tr/timeline.json index 977041ea83..ba99653f88 100644 --- a/src/i18n/locales/tr/timeline.json +++ b/src/i18n/locales/tr/timeline.json @@ -7,7 +7,7 @@ "addTrim": "Kırpma ekle: videonun bir bölümünü şu anki zamanda çıkarır", "addAnnotation": "Şu anki zamanda metin, ok, resim veya bulanıklık ekle", "addSpeed": "Şu anki zamanda hız değişikliği ekle", - "addCameraFullscreen": "Tam Ekran Kamera ekle: kamerayı şu anki zamanda tam ekran gösterir" + "addCameraFullscreen": "Yerleşimi burada değiştir" }, "hints": { "pressZoom": "Yakınlaştırma eklemek için {{key}} tuşuna basın", @@ -116,5 +116,20 @@ "tracks": { "annotation": "Açıklama katmanı {{index}}", "dragHint": "Zamanı değiştirmek için sola veya sağa sürükleyin. Katmanı değiştirmek için yukarı veya aşağı sürükleyin; üst katmanlar önde görünür." + }, + "layouts": { + "title": "Yerleşimler", + "changeHere": "Yerleşimi burada değiştir", + "wholeVideo": "Videonun tamamı", + "sharedSettings": "Web kamerası ayarları · tüm video", + "fromUntil": "{{start}} – {{end}}", + "noCamera": "Bu görüntüde web kamerası yok; yalnızca ekran gösterilir.", + "removeChange": "Bu değişikliği kaldır", + "moveBoundary": "{{time}} konumundaki değişikliği taşı", + "picture-in-picture": "Resim içinde resim", + "dual-frame": "Yan yana", + "vertical-stack": "Üst / alt", + "camera-only": "Yalnızca kamera", + "no-webcam": "Yalnızca ekran" } } diff --git a/src/i18n/locales/vi/shortcuts.json b/src/i18n/locales/vi/shortcuts.json index 448de3cfe0..f6e910a6e5 100644 --- a/src/i18n/locales/vi/shortcuts.json +++ b/src/i18n/locales/vi/shortcuts.json @@ -23,7 +23,7 @@ "addAudio": "Thêm âm thanh", "addVoiceover": "Ghi âm lời thuyết minh", "addKeyframe": "Thêm Khung hình chính", - "addCameraFullscreen": "Thêm Camera Toàn màn hình", + "addCameraFullscreen": "Đổi bố cục tại đây", "deleteSelected": "Xóa mục đã chọn", "playPause": "Phát / Tạm dừng", "copySelected": "Sao chép mục đã chọn", diff --git a/src/i18n/locales/vi/timeline.json b/src/i18n/locales/vi/timeline.json index da6c13099f..ab7d3cf930 100644 --- a/src/i18n/locales/vi/timeline.json +++ b/src/i18n/locales/vi/timeline.json @@ -7,7 +7,7 @@ "addTrim": "Thêm cắt: xóa một đoạn video tại thời điểm hiện tại", "addAnnotation": "Thêm văn bản, mũi tên, hình ảnh hoặc làm mờ tại thời điểm hiện tại", "addSpeed": "Thêm thay đổi tốc độ tại thời điểm hiện tại", - "addCameraFullscreen": "Thêm Camera Toàn màn hình: hiển thị camera toàn màn hình tại thời điểm hiện tại" + "addCameraFullscreen": "Đổi bố cục tại đây" }, "hints": { "pressZoom": "Nhấn {{key}} để thêm thu phóng", @@ -116,5 +116,20 @@ "tracks": { "annotation": "Lớp chú thích {{index}}", "dragHint": "Kéo sang trái hoặc phải để di chuyển theo thời gian. Kéo lên hoặc xuống để đổi lớp; lớp cao hơn hiển thị phía trước." + }, + "layouts": { + "title": "Bố cục", + "changeHere": "Đổi bố cục tại đây", + "wholeVideo": "Toàn bộ video", + "sharedSettings": "Cài đặt webcam · toàn bộ video", + "fromUntil": "Từ {{start}} đến {{end}}", + "noCamera": "Đoạn này không có webcam; chỉ hiển thị màn hình.", + "removeChange": "Xóa thay đổi này", + "moveBoundary": "Di chuyển thay đổi tại {{time}}", + "picture-in-picture": "Hình trong hình", + "dual-frame": "Cạnh nhau", + "vertical-stack": "Trên / dưới", + "camera-only": "Chỉ camera", + "no-webcam": "Chỉ màn hình" } } diff --git a/src/i18n/locales/zh-CN/shortcuts.json b/src/i18n/locales/zh-CN/shortcuts.json index 6b97f54bb0..9402bd230f 100644 --- a/src/i18n/locales/zh-CN/shortcuts.json +++ b/src/i18n/locales/zh-CN/shortcuts.json @@ -23,7 +23,7 @@ "addAudio": "添加音频", "addVoiceover": "录制配音", "addKeyframe": "添加关键帧", - "addCameraFullscreen": "添加全屏摄像头", + "addCameraFullscreen": "在此更改布局", "deleteSelected": "删除所选", "playPause": "播放 / 暂停", "copySelected": "复制所选", diff --git a/src/i18n/locales/zh-CN/timeline.json b/src/i18n/locales/zh-CN/timeline.json index d15a420b74..b9bde88596 100644 --- a/src/i18n/locales/zh-CN/timeline.json +++ b/src/i18n/locales/zh-CN/timeline.json @@ -7,7 +7,7 @@ "addTrim": "在当前时间添加删减区间，移除这段视频", "addAnnotation": "在当前时间添加文字、箭头、图片或模糊效果", "addSpeed": "在当前时间添加变速效果", - "addCameraFullscreen": "添加全屏摄像头：在当前时间全屏显示摄像头画面" + "addCameraFullscreen": "在此更改布局" }, "hints": { "pressZoom": "按 {{key}} 添加缩放", @@ -116,5 +116,20 @@ "tracks": { "annotation": "注释图层 {{index}}", "dragHint": "左右拖动以调整时间。上下拖动以切换图层；上方图层显示在前面。" + }, + "layouts": { + "title": "布局", + "changeHere": "在此更改布局", + "wholeVideo": "整个视频", + "sharedSettings": "摄像头设置 · 整个视频", + "fromUntil": "{{start}} 至 {{end}}", + "noCamera": "此片段没有摄像头，仅显示屏幕。", + "removeChange": "移除此更改", + "moveBoundary": "移动 {{time}} 处的更改", + "picture-in-picture": "画中画", + "dual-frame": "并排", + "vertical-stack": "上下排列", + "camera-only": "仅摄像头", + "no-webcam": "仅屏幕" } } diff --git a/src/i18n/locales/zh-TW/shortcuts.json b/src/i18n/locales/zh-TW/shortcuts.json index 2daf7eb6d8..2430cce821 100644 --- a/src/i18n/locales/zh-TW/shortcuts.json +++ b/src/i18n/locales/zh-TW/shortcuts.json @@ -23,7 +23,7 @@ "addAudio": "新增音訊", "addVoiceover": "錄製配音", "addKeyframe": "新增關鍵影格", - "addCameraFullscreen": "新增全螢幕攝影機", + "addCameraFullscreen": "在此變更版面配置", "deleteSelected": "刪除所選", "playPause": "播放 / 暫停", "copySelected": "複製所選", diff --git a/src/i18n/locales/zh-TW/timeline.json b/src/i18n/locales/zh-TW/timeline.json index ec99b75820..1d137a9808 100644 --- a/src/i18n/locales/zh-TW/timeline.json +++ b/src/i18n/locales/zh-TW/timeline.json @@ -7,7 +7,7 @@ "addTrim": "新增剪輯：在目前時間移除一段影片", "addAnnotation": "在目前時間新增文字、箭頭、圖片或模糊", "addSpeed": "在目前時間新增速度變化", - "addCameraFullscreen": "新增全螢幕攝影機：在目前時間全螢幕顯示攝影機" + "addCameraFullscreen": "在此變更版面配置" }, "hints": { "pressZoom": "按 {{key}} 新增縮放", @@ -116,5 +116,20 @@ "tracks": { "annotation": "註解圖層 {{index}}", "dragHint": "左右拖曳以調整時間。上下拖曳以切換圖層；上方圖層顯示在前面。" + }, + "layouts": { + "title": "版面配置", + "changeHere": "在此變更版面配置", + "wholeVideo": "整段影片", + "sharedSettings": "攝影機設定 · 整段影片", + "fromUntil": "{{start}} 至 {{end}}", + "noCamera": "此片段沒有攝影機，僅顯示螢幕。", + "removeChange": "移除此變更", + "moveBoundary": "移動 {{time}} 處的變更", + "picture-in-picture": "子母畫面", + "dual-frame": "並排", + "vertical-stack": "上下排列", + "camera-only": "僅攝影機", + "no-webcam": "僅螢幕" } } diff --git a/src/lib/ai-edition/captions/captionLane.test.ts b/src/lib/ai-edition/captions/captionLane.test.ts index 4bec432338..566f8781f5 100644 --- a/src/lib/ai-edition/captions/captionLane.test.ts +++ b/src/lib/ai-edition/captions/captionLane.test.ts @@ -62,7 +62,7 @@ function words(assetId: string, texts: string[]): AxcutTranscript { function doc(over: Partial = {}): AxcutDocument { return { - schemaVersion: 8, + schemaVersion: 9, project: { id: "p", title: "T", diff --git a/src/lib/ai-edition/document/audioLanes.test.ts b/src/lib/ai-edition/document/audioLanes.test.ts index 34ba7aba62..46757d4c40 100644 --- a/src/lib/ai-edition/document/audioLanes.test.ts +++ b/src/lib/ai-edition/document/audioLanes.test.ts @@ -51,7 +51,7 @@ function track(over: Partial & { id: string }): AxcutAudioTrack function doc(audioTracks: AxcutAudioTrack[]): AxcutDocument { return { - schemaVersion: 8, + schemaVersion: 9, project: { id: "p", title: "T", createdAt: "", updatedAt: "" }, assets: [], transcript: null, diff --git a/src/lib/ai-edition/document/layout.test.ts b/src/lib/ai-edition/document/layout.test.ts new file mode 100644 index 0000000000..f3fb4cf866 --- /dev/null +++ b/src/lib/ai-edition/document/layout.test.ts @@ -0,0 +1,203 @@ +import { describe, expect, it } from "vitest"; +import { computeCameraFullscreenProgress } from "../../zoomMath/cameraFullscreenUtils"; +import { createEmptyDocument, documentSchema, migrateRawDocumentToCurrent } from "../schema"; +import { + addLayoutChange, + layoutRuns, + legacyCameraRegions, + materializeLayouts, + moveLayoutBoundary, + removeLayoutChange, + setLayoutPreset, +} from "./layout"; +import { duplicateClip, moveClip, removeClip, setClipSourceRange } from "./timeline"; + +function document() { + const doc = createEmptyDocument({ title: "Layouts", projectId: "layouts" }); + doc.timeline.clips = [ + { + id: "a", + assetId: "asset", + sourceStartSec: 20, + sourceEndSec: 30, + timelineStartSec: 0, + timelineEndSec: 10, + wordRefs: [], + origin: "user", + reason: "", + }, + { + id: "b", + assetId: "asset", + sourceStartSec: 100, + sourceEndSec: 110, + timelineStartSec: 10, + timelineEndSec: 20, + wordRefs: [], + origin: "user", + reason: "", + }, + ]; + return doc; +} +const view = (doc: ReturnType) => + layoutRuns(doc).map((r) => [r.start, r.end, r.member.preset]); + +describe("the continuous layout partition", () => { + it("stays implicit until a first change, then materialises source-anchored fragments", () => { + const doc = document(); + expect(doc.layoutRegions).toBeUndefined(); + expect(view(doc)).toEqual([[0, 20, "picture-in-picture"]]); + const next = addLayoutChange(doc, 5, "camera-only"); + expect(view(next)).toEqual([ + [0, 5, "picture-in-picture"], + [5, 20, "camera-only"], + ]); + expect(next.layoutRegions).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + clipId: "a", + sourceStartSec: 25, + sourceEndSec: 30, + preset: "camera-only", + }), + expect.objectContaining({ + clipId: "b", + sourceStartSec: 100, + sourceEndSec: 110, + preset: "camera-only", + }), + ]), + ); + expect(doc.layoutRegions).toBeUndefined(); + }); + it("inserts until the next change, leaving later passages intact", () => { + let doc = addLayoutChange(document(), 12, "no-webcam"); + doc = addLayoutChange(doc, 5, "dual-frame"); + expect(view(doc)).toEqual([ + [0, 5, "picture-in-picture"], + [5, 12, "dual-frame"], + [12, 20, "no-webcam"], + ]); + expect(addLayoutChange(doc, 20, "camera-only")).toBe(doc); + expect(addLayoutChange(doc, Number.NaN, "camera-only")).toBe(doc); + }); + it("moves both sides of a boundary and clamps without moving its other neighbour", () => { + let doc = addLayoutChange(addLayoutChange(document(), 5, "camera-only"), 12, "no-webcam"); + const id = layoutRuns(doc)[1].ids[0]; + doc = moveLayoutBoundary(doc, id, 8); + expect(view(doc)).toEqual([ + [0, 8, "picture-in-picture"], + [8, 12, "camera-only"], + [12, 20, "no-webcam"], + ]); + const clamped = moveLayoutBoundary(doc, id, 50); + expect(layoutRuns(clamped)[1].start).toBeCloseTo(11.999); + expect(layoutRuns(clamped)[2].start).toBe(12); + }); + it("deletes a change by extending the previous layout; the initial passage is protected", () => { + const doc = addLayoutChange(document(), 5, "camera-only"); + const runs = layoutRuns(doc); + expect(removeLayoutChange(doc, runs[0].ids[0])).toBe(doc); + const next = removeLayoutChange(doc, runs[1].ids[0]); + expect(view(next)).toEqual([[0, 20, "picture-in-picture"]]); + expect(next.layoutRegions).toEqual([]); + }); + it("merges equal passages, retaining a uniform camera-only choice as the global default", () => { + const doc = addLayoutChange(document(), 5, "camera-only"); + const next = setLayoutPreset(doc, layoutRuns(doc)[0].ids[0], "camera-only"); + expect(next.layoutRegions).toEqual([]); + expect(next.legacyEditor?.webcamLayoutPreset).toBe("camera-only"); + expect(view(next)).toEqual([[0, 20, "camera-only"]]); + }); + it("keeps dispositions with their footage after reorder, duplicate, range edit and removal", () => { + const doc = addLayoutChange(document(), 5, "camera-only"); + const reordered = moveClip(doc, "b", 0); + expect(view(reordered)).toEqual([ + [0, 10, "camera-only"], + [10, 15, "picture-in-picture"], + [15, 20, "camera-only"], + ]); + const duplicated = duplicateClip(doc, "a", "user"); + expect(view(duplicated)).toEqual([ + [0, 5, "picture-in-picture"], + [5, 10, "camera-only"], + [10, 15, "picture-in-picture"], + [15, 30, "camera-only"], + ]); + const trimmed = setClipSourceRange(doc, "a", 24, 28); + expect(view(trimmed)).toEqual([ + [0, 1, "picture-in-picture"], + [1, 14, "camera-only"], + ]); + expect(view(removeClip(doc, "a"))).toEqual([[0, 10, "camera-only"]]); + }); +}); + +describe("v7 Full Camera migration", () => { + it("preserves layout, every envelope, and the exact historical easing through save/reopen", () => { + const doc = document(); + const old = [{ id: "old", startMs: 2000, endMs: 8000 }]; + const raw = { + ...doc, + schemaVersion: 7, + legacyEditor: { + webcamLayoutPreset: "dual-frame", + cameraFullscreenRegions: old, + webcamMirrored: true, + }, + }; + const next = documentSchema.parse(migrateRawDocumentToCurrent(raw)); + expect(next.schemaVersion).toBe(9); + expect(next.legacyEditor?.cameraFullscreenRegions).toBeUndefined(); + expect(next.legacyEditor?.webcamMirrored).toBe(true); + expect(view(next)).toEqual([ + [0, 2, "dual-frame"], + [2, 8, "camera-only"], + [8, 20, "dual-frame"], + ]); + for (const time of [0, 2000, 2100, 3000, 7000, 7900, 8000, 12000]) { + expect(computeCameraFullscreenProgress(legacyCameraRegions(next), time)).toBe( + computeCameraFullscreenProgress(old, time), + ); + } + const reopened = documentSchema.parse( + migrateRawDocumentToCurrent(JSON.parse(JSON.stringify(next))), + ); + expect(reopened).toEqual(next); + expect(migrateRawDocumentToCurrent(next)).toBe(next); + }); + it("keeps overlapping legacy envelopes instead of flattening their easing", () => { + const doc = document(); + doc.legacyEditor = { + cameraFullscreenRegions: [ + { id: "one", startMs: 2000, endMs: 6000 }, + { id: "two", startMs: 4000, endMs: 8000 }, + ], + }; + const migrated = documentSchema.parse( + migrateRawDocumentToCurrent({ ...doc, schemaVersion: 7 }), + ); + expect(legacyCameraRegions(migrated)).toHaveLength(2); + expect(view(migrated)).toEqual([ + [0, 2, "picture-in-picture"], + [2, 8, "camera-only"], + [8, 20, "picture-in-picture"], + ]); + }); + it("ignores malformed old regions without crashing", () => { + const doc = document(); + doc.legacyEditor = { + cameraFullscreenRegions: [null, "bad", { startMs: Number.NaN, endMs: 4 }], + }; + expect(materializeLayouts(doc).every((r) => r.preset === "picture-in-picture")).toBe(true); + }); +}); + +it("reopens projects saved by the experimental v8 layouts build without losing their passages", () => { + const edited = addLayoutChange(document(), 2, "dual-frame"); + const next = documentSchema.parse(migrateRawDocumentToCurrent({ ...edited, schemaVersion: 8 })); + expect(next.schemaVersion).toBe(9); + expect(next.layoutRegions).toEqual(edited.layoutRegions); + expect(next.legacyEditor?.aspectRatio).toBe("16:9"); +}); diff --git a/src/lib/ai-edition/document/layout.ts b/src/lib/ai-edition/document/layout.ts new file mode 100644 index 0000000000..6d703e76b1 --- /dev/null +++ b/src/lib/ai-edition/document/layout.ts @@ -0,0 +1,365 @@ +import { + isLayoutPreset, + LAYOUT_TRANSITION_SECONDS, + type LayoutPreset, + type LayoutRegion, +} from "../../layoutTypes"; +import type { AxcutClip, AxcutDocument } from "../schema"; +import { + anchoredToRawSpanSec, + anchorRegionsWithDerivedMs, + coalesceRegionsForRuler, + hasCompleteClipAnchor, +} from "../timeline/timelineMap"; +import { createId } from "./ids"; + +const EPSILON = 0.001; + +type LayoutDocument = Pick; + +export function defaultLayoutPreset(document: Pick): LayoutPreset { + const preset = document.legacyEditor?.webcamLayoutPreset; + return isLayoutPreset(preset) ? preset : "picture-in-picture"; +} + +function clipEnd(clip: AxcutClip): number { + return clip.sourceEndSec ?? clip.sourceStartSec + clip.timelineEndSec - clip.timelineStartSec; +} + +function rawRegion(region: LayoutRegion, clips: AxcutClip[]): LayoutRegion | null { + if (!hasCompleteClipAnchor(region)) return region; + const span = anchoredToRawSpanSec(region, clips); + return span ? { ...region, startMs: span.startSec * 1000, endMs: span.endSec * 1000 } : null; +} + +/** Complete, non-overlapping source-anchored partition. Missing/new clip windows inherit the + * document default; existing fragments always travel with their clip, never with the ruler. */ +export function materializeLayouts(document: LayoutDocument): LayoutRegion[] { + const clips = document.timeline.clips; + const fallback = defaultLayoutPreset(document); + const stored = document.layoutRegions ?? []; + const legacy = document.legacyEditor?.cameraFullscreenRegions; + const oldCamera = Array.isArray(legacy) ? legacy : []; + const result: LayoutRegion[] = []; + for (const clip of clips) { + const start = clip.timelineStartSec * 1000; + const end = (clip.timelineStartSec + clipEnd(clip) - clip.sourceStartSec) * 1000; + if (end <= start) continue; + const regions = stored.flatMap((region) => { + if (region.clipId && region.clipId !== clip.id) return []; + const raw = rawRegion(region, clips); + return raw && raw.endMs > start && raw.startMs < end ? [raw] : []; + }); + const envelopes = stored.length + ? [] + : oldCamera.flatMap((value: unknown) => { + if (!value || typeof value !== "object") return []; + const region = value as Partial; + if (region.clipId && region.clipId !== clip.id) return []; + const span = hasCompleteClipAnchor(region) + ? anchoredToRawSpanSec(region, clips) + : typeof region.startMs === "number" && typeof region.endMs === "number" + ? { startSec: region.startMs / 1000, endSec: region.endMs / 1000 } + : null; + if (!span || !Number.isFinite(span.startSec) || !Number.isFinite(span.endSec)) return []; + const a = Math.max(start, span.startSec * 1000); + const b = Math.min(end, span.endSec * 1000); + return b > a + ? [ + { + sourceStartSec: clip.sourceStartSec + (a - start) / 1000, + sourceEndSec: clip.sourceStartSec + (b - start) / 1000, + }, + ] + : []; + }); + const boundaries = [ + ...new Set([ + start, + end, + ...regions.flatMap((r) => [Math.max(start, r.startMs), Math.min(end, r.endMs)]), + ...envelopes.flatMap((r) => [ + start + (r.sourceStartSec - clip.sourceStartSec) * 1000, + start + (r.sourceEndSec - clip.sourceStartSec) * 1000, + ]), + ]), + ].sort((a, b) => a - b); + for (let i = 0; i < boundaries.length - 1; i++) { + const a = boundaries[i], + b = boundaries[i + 1]; + if (b - a < EPSILON * 1000) continue; + const existing = regions.find((r) => r.startMs <= a && r.endMs >= b); + const inherited = + [...regions].filter((r) => r.endMs <= a).sort((x, y) => y.endMs - x.endMs)[0] ?? + regions.find((r) => r.startMs >= b); + const sourceStartSec = clip.sourceStartSec + (a - start) / 1000; + const sourceEndSec = clip.sourceStartSec + (b - start) / 1000; + const camera = envelopes.some( + (r) => r.sourceStartSec <= sourceStartSec && r.sourceEndSec >= sourceEndSec, + ); + result.push({ + ...existing, + id: existing?.id ?? `layout_${clip.id}_${a}`, + clipId: clip.id, + sourceStartSec, + sourceEndSec, + startMs: a, + endMs: b, + preset: existing?.preset ?? (camera ? "camera-only" : (inherited?.preset ?? fallback)), + ...(camera ? { legacyCamera: { basePreset: fallback, envelopes } } : {}), + }); + } + } + // Legacy imports can have an unprobed, zero-extent clip. Keep their raw envelopes + // until applyProbedDuration can anchor them; migration must never erase those edits. + if (!result.length && clips.some((c) => clipEnd(c) <= c.sourceStartSec)) { + return stored.length + ? stored + : oldCamera.flatMap((value: unknown) => { + if (!value || typeof value !== "object") return []; + const r = value as Partial; + if ( + typeof r.startMs !== "number" || + typeof r.endMs !== "number" || + !Number.isFinite(r.startMs) || + !Number.isFinite(r.endMs) || + r.endMs <= r.startMs + ) + return []; + return [ + { + id: r.id ?? createId("layout"), + startMs: r.startMs, + endMs: r.endMs, + preset: "camera-only" as const, + legacyCamera: { + basePreset: fallback, + envelopes: [{ sourceStartSec: r.startMs / 1000, sourceEndSec: r.endMs / 1000 }], + }, + }, + ]; + }); + } + return result; +} + +export function layoutRuns(document: LayoutDocument) { + return coalesceRegionsForRuler(materializeLayouts(document)); +} + +export function layoutAt(document: LayoutDocument, timeSec: number): LayoutPreset { + const region = layoutRuns(document).find((r) => timeSec >= r.start && timeSec < r.end); + return region?.member.preset ?? defaultLayoutPreset(document); +} + +/** UI hitboxes sample the same source-anchored transition that the native scene describes. */ +export function layoutSampleAt(document: LayoutDocument, timeSec: number) { + return sampleLayoutRegionsAt( + materializeLayouts(document), + timeSec, + defaultLayoutPreset(document), + ); +} + +export function sampleLayoutRegionsAt( + regions: LayoutRegion[], + timeSec: number, + fallback: LayoutPreset, +) { + const region = regions.find((r) => timeSec * 1000 >= r.startMs && timeSec * 1000 < r.endMs); + const preset = region?.legacyCamera?.basePreset ?? region?.preset ?? fallback; + const previous = + region && !region.legacyCamera + ? regions.find((r) => r.clipId === region.clipId && Math.abs(r.endMs - region.startMs) < 1) + : undefined; + const duration = region + ? Math.min(LAYOUT_TRANSITION_SECONDS * 1000, (region.endMs - region.startMs) / 2) + : 0; + const linear = + region && previous && !previous.legacyCamera && duration > 0 + ? Math.min(1, Math.max(0, (timeSec * 1000 - region.startMs) / duration)) + : 1; + return { + preset, + fromPreset: previous?.legacyCamera ? undefined : previous?.preset, + progress: linear * linear * (3 - 2 * linear), + }; +} + +function writeRuns(document: AxcutDocument, runs: LayoutRegion[]): AxcutDocument { + const { cameraFullscreenRegions: _old, ...legacy } = document.legacyEditor ?? {}; + const anchored = anchorRegionsWithDerivedMs(runs, document.timeline.clips, () => + createId("layout"), + ); + const next = { ...document, legacyEditor: legacy, layoutRegions: anchored }; + const merged = layoutRuns(next); + if (merged.length === 1 && !merged[0].member.legacyCamera) { + return { + ...next, + layoutRegions: [], + legacyEditor: { ...legacy, webcamLayoutPreset: merged[0].member.preset }, + }; + } + return next; +} + +function asRegion(run: ReturnType[number]): LayoutRegion { + return { + id: run.ids[0], + startMs: run.start * 1000, + endMs: run.end * 1000, + preset: run.member.preset, + ...(run.member.legacyCamera ? { legacyCamera: run.member.legacyCamera } : {}), + }; +} + +/** Apply from the playhead until the next visible change, atomically splitting the partition. */ +export function addLayoutChange( + document: AxcutDocument, + timeSec: number, + preset: LayoutPreset, +): AxcutDocument { + if (!Number.isFinite(timeSec)) return document; + const runs = layoutRuns(document); + const active = runs.findIndex((r) => timeSec >= r.start && timeSec < r.end); + if (active < 0 || (runs[active].member.preset === preset && !runs[active].member.legacyCamera)) + return document; + const regions = runs.map(asRegion); + const old = { ...regions[active], legacyCamera: undefined }; + const startMs = Math.max(old.startMs, Math.round(timeSec * 1000)); + const replacement = { id: createId("layout"), startMs, endMs: old.endMs, preset }; + regions.splice( + active, + 1, + ...(startMs > old.startMs ? [{ ...old, endMs: startMs }] : []), + replacement, + ); + return writeRuns(document, regions); +} + +/** Compatibility for old span-based tool calls; new authoring uses change points. */ +export function applyLayoutSpan( + document: AxcutDocument, + startSec: number, + endSec: number, + preset: LayoutPreset, +): AxcutDocument { + if (!Number.isFinite(startSec) || !Number.isFinite(endSec) || endSec <= startSec) return document; + const regions = layoutRuns(document).flatMap((run) => { + const old = asRegion(run); + const a = Math.max(run.start, startSec), + b = Math.min(run.end, endSec); + if (b <= a) return [old]; + return [ + ...(a > run.start ? [{ ...old, endMs: a * 1000, legacyCamera: undefined }] : []), + { id: createId("layout"), startMs: a * 1000, endMs: b * 1000, preset }, + ...(b < run.end + ? [{ ...old, id: createId("layout"), startMs: b * 1000, legacyCamera: undefined }] + : []), + ]; + }); + return writeRuns(document, regions); +} + +export function setLayoutPreset( + document: AxcutDocument, + id: string, + preset: LayoutPreset, +): AxcutDocument { + const runs = layoutRuns(document); + const existing = runs.find((r) => r.ids.includes(id)); + if (!existing || (existing.member.preset === preset && !existing.member.legacyCamera)) + return document; + return writeRuns( + document, + runs.map((r) => + r.ids.includes(id) ? { ...asRegion(r), preset, legacyCamera: undefined } : asRegion(r), + ), + ); +} + +export function moveLayoutBoundary( + document: AxcutDocument, + id: string, + timeSec: number, +): AxcutDocument { + if (!Number.isFinite(timeSec)) return document; + const runs = layoutRuns(document); + const index = runs.findIndex((r) => r.ids.includes(id)); + if (index <= 0) return document; + const regions = runs.map(asRegion); + const boundary = Math.round( + Math.max(runs[index - 1].start + EPSILON, Math.min(runs[index].end - EPSILON, timeSec)) * 1000, + ); + if (boundary === Math.round(runs[index].start * 1000)) return document; + regions[index - 1] = { ...regions[index - 1], endMs: boundary, legacyCamera: undefined }; + regions[index] = { ...regions[index], startMs: boundary, legacyCamera: undefined }; + return writeRuns(document, regions); +} + +export function removeLayoutChange(document: AxcutDocument, id: string): AxcutDocument { + const runs = layoutRuns(document); + const index = runs.findIndex((r) => r.ids.includes(id)); + if (index <= 0) return document; + const regions = runs.map(asRegion); + regions[index - 1] = { + ...regions[index - 1], + endMs: regions[index].endMs, + legacyCamera: undefined, + }; + regions.splice(index, 1); + return writeRuns(document, regions); +} + +/** Preserve the old ease-in/hold/ease-out while keeping layouts the only stored authority. */ +export function legacyCameraRegions(document: LayoutDocument) { + const old = document.legacyEditor?.cameraFullscreenRegions; + if (!document.layoutRegions?.length && Array.isArray(old)) { + return old.flatMap((value: unknown) => { + if (!value || typeof value !== "object") return []; + const r = value as Partial; + if ( + typeof r.startMs !== "number" || + typeof r.endMs !== "number" || + !Number.isFinite(r.startMs) || + !Number.isFinite(r.endMs) + ) + return []; + return [ + { + id: r.id ?? "legacy-camera", + startMs: r.startMs, + endMs: r.endMs, + ...(hasCompleteClipAnchor(r) + ? { clipId: r.clipId, sourceStartSec: r.sourceStartSec, sourceEndSec: r.sourceEndSec } + : {}), + }, + ]; + }); + } + const seen = new Set(); + return materializeLayouts(document).flatMap((r) => + (r.legacyCamera?.envelopes ?? []).flatMap((e, i) => { + if ( + r.sourceStartSec !== undefined && + r.sourceEndSec !== undefined && + (e.sourceEndSec <= r.sourceStartSec || e.sourceStartSec >= r.sourceEndSec) + ) + return []; + const key = `${r.clipId}:${e.sourceStartSec}:${e.sourceEndSec}`; + if (seen.has(key)) return []; + seen.add(key); + const clip = document.timeline.clips.find((c) => c.id === r.clipId); + if (!clip) return []; + return [ + { + id: `${r.id}_legacy_${i}`, + clipId: clip.id, + ...e, + startMs: (clip.timelineStartSec + e.sourceStartSec - clip.sourceStartSec) * 1000, + endMs: (clip.timelineStartSec + e.sourceEndSec - clip.sourceStartSec) * 1000, + }, + ]; + }), + ); +} diff --git a/src/lib/ai-edition/document/migrate.test.ts b/src/lib/ai-edition/document/migrate.test.ts index 385f11a2c1..fc12cc10bc 100644 --- a/src/lib/ai-edition/document/migrate.test.ts +++ b/src/lib/ai-edition/document/migrate.test.ts @@ -51,7 +51,7 @@ describe("migrateProjectDataToAxcutDocument", () => { it("produces a current-schema document with one asset and one clip from a v2 single-recording project", () => { const doc = migrateProjectDataToAxcutDocument(makeV2Project()); - expect(doc.schemaVersion).toBe(8); + expect(doc.schemaVersion).toBe(9); expect(doc.assets).toHaveLength(1); const asset = doc.assets[0]; expect(asset.kind).toBe("video"); @@ -542,7 +542,7 @@ describe("migrateRawDocumentToCurrent", () => { }), ), ); - expect(migrated.schemaVersion).toBe(8); + expect(migrated.schemaVersion).toBe(9); expect((migrated as Record).cameraTrack).toBeUndefined(); expect(migrated.assets[0].cameraTrack).toBeNull(); expect(migrated.assets[1].cameraTrack?.sourcePath).toBe("/cam.mp4"); @@ -556,7 +556,7 @@ describe("migrateRawDocumentToCurrent", () => { ], }), ) as Record; - expect(migrated.schemaVersion).toBe(8); + expect(migrated.schemaVersion).toBe(9); const zooms = migrated.zoomRanges as Array>; expect(zooms).toHaveLength(1); expect(zooms[0]).toMatchObject({ id: "z1", clipId: "c1", depth: 3 }); @@ -564,12 +564,12 @@ describe("migrateRawDocumentToCurrent", () => { it("is a no-op for an already-current document (returns an equal value)", () => { const v5 = makeV4Doc(); // makeV4Doc's body is the v5-compatible shape - const once = migrateRawDocumentToCurrent({ ...v5, schemaVersion: 8 }); + const once = migrateRawDocumentToCurrent({ ...v5, schemaVersion: 9 }); // ponytail: the upgrader chain checks schemaVersion and returns the input // unchanged, so the round-trip allocation is bounded to a property // comparison per upgrader — the same per-parse overhead the old // `z.preprocess` carried. - expect(once).toEqual({ ...v5, schemaVersion: 8 }); + expect(once).toEqual({ ...v5, schemaVersion: 9 }); }); it("passes non-document input through unchanged (the schema is the gate, not this helper)", () => { @@ -622,7 +622,7 @@ describe("dropping trims anchored to audio", () => { function docWith(trimRanges: unknown[], assets?: unknown[]) { return { - schemaVersion: 8, + schemaVersion: 9, project: { id: "p", title: "t", createdAt, updatedAt: createdAt }, assets: assets ?? [ { id: "vid", kind: "video", label: "v", originalPath: "/v.mp4", cameraTrack: null }, @@ -762,7 +762,7 @@ describe("repairing inverted transcript timings", () => { ], }; return { - schemaVersion: 8, + schemaVersion: 9, project: { id: "p", title: "t", createdAt, updatedAt: createdAt }, assets: [{ id: "vid", kind: "video", label: "v", originalPath: "/v.mp4", cameraTrack: null }], transcript, diff --git a/src/lib/ai-edition/document/timeline.ts b/src/lib/ai-edition/document/timeline.ts index 6b1c95f3d9..057a16a7e2 100644 --- a/src/lib/ai-edition/document/timeline.ts +++ b/src/lib/ai-edition/document/timeline.ts @@ -39,6 +39,7 @@ import { separateAudioLanes, } from "./audioTracks"; import { createId } from "./ids"; +import { layoutAt, layoutRuns, materializeLayouts, removeLayoutChange } from "./layout"; /** The region families a delete can target by id. Shared with the store so "which kinds * exist" has exactly one definition. `trim` is a source-time cut; the rest are pill-merged @@ -49,6 +50,7 @@ export type RegionKind = | "trim" | "annotation" | "speed" + | "layout" | "cameraFullscreen" | "audio" | "shaderEffect"; @@ -389,6 +391,9 @@ export function mapAllRegionCollections( // A clip edit can take a shader effect's last fragment with it: its definition goes too. return pruneUnusedShaderEffects({ ...document, + ...(document.layoutRegions + ? { layoutRegions: fn(document.layoutRegions, "layout") as AxcutDocument["layoutRegions"] } + : {}), zoomRanges: fn( document.zoomRanges as unknown as StoredRegion[], "zoom", @@ -743,6 +748,7 @@ function sameInterval(a: Interval, b: Interval): boolean { function anchoredRegionsOf(document: AxcutDocument): Array<{ id: string; clipId: string }> { const legacy = document.legacyEditor as Record | null; const collections: StoredRegion[][] = [ + ...(document.layoutRegions ? [document.layoutRegions] : []), document.zoomRanges as unknown as StoredRegion[], document.annotations as unknown as StoredRegion[], (legacy?.speedRegions as StoredRegion[] | undefined) ?? [], @@ -1093,7 +1099,27 @@ export function insertAssetClip( timelineEndSec: length, wordRefs: [], }); - return withClipsChanged(document, clips); + const insertionTime = + document.timeline.clips[index]?.timelineStartSec ?? + Math.max(0, (document.timeline.clips.at(-1)?.timelineEndSec ?? 0) - 0.0001); + const withLayouts: AxcutDocument = document.layoutRegions?.length + ? { + ...document, + layoutRegions: [ + ...materializeLayouts(document), + { + id: createId("layout"), + clipId: clip.id, + sourceStartSec: clip.sourceStartSec, + sourceEndSec: clip.sourceEndSec, + startMs: 0, + endMs: length * 1000, + preset: layoutAt(document, insertionTime), + }, + ], + } + : document; + return withClipsChanged(withLayouts, clips); } /** @@ -1205,6 +1231,8 @@ export function removeRegion(document: AxcutDocument, kind: RegionKind, id: stri ); return { ...document, legacyEditor: { ...legacy, speedRegions: prev } }; } + case "layout": + return removeLayoutChange(document, id); case "cameraFullscreen": { const legacy = (document.legacyEditor as Record) ?? {}; const prev = dropPillById( @@ -1264,6 +1292,10 @@ const EDIT_REGIONS: Record< count: (d) => legacyRegionCount(d, "cameraFullscreenRegions"), clear: (d) => withLegacyRegionsCleared(d, "cameraFullscreenRegions"), }, + layout: { + count: (d) => Math.max(0, layoutRuns(d).length - 1), + clear: (d) => ({ ...d, layoutRegions: [] }), + }, shaderEffect: { count: (d) => (d.shaderEffectRanges ?? []).length, clear: (d) => ({ ...d, shaderEffectRanges: [], shaderEffects: [] }), diff --git a/src/lib/ai-edition/schema/index.test.ts b/src/lib/ai-edition/schema/index.test.ts index 5178c5490a..c11366d444 100644 --- a/src/lib/ai-edition/schema/index.test.ts +++ b/src/lib/ai-edition/schema/index.test.ts @@ -21,9 +21,9 @@ import { zoomRegionSchema, } from "./index"; -describe("axcut-schema v8", () => { - it("uses schema version 8", () => { - expect(axcutSchemaVersion).toBe(8); +describe("axcut-schema v9", () => { + it("uses schema version 9", () => { + expect(axcutSchemaVersion).toBe(9); }); it("rejects unknown schema versions", () => { @@ -35,9 +35,9 @@ describe("axcut-schema v8", () => { ).toThrow(); }); - it("createEmptyDocument returns a valid v8 doc with empty collections", () => { + it("createEmptyDocument returns a valid v9 doc with empty collections", () => { const doc = createEmptyDocument({ projectId: "proj_1", title: "Demo" }); - expect(doc.schemaVersion).toBe(8); + expect(doc.schemaVersion).toBe(9); expect(doc.assets).toEqual([]); expect(doc.timeline.clips).toEqual([]); expect(doc.timeline.trimRanges).toEqual([]); @@ -314,7 +314,7 @@ describe("axcut-schema v8", () => { v3Doc({ project: { ...v3Doc().project, primaryAssetId: "asset_2" } }), ), ); - expect(doc.schemaVersion).toBe(8); + expect(doc.schemaVersion).toBe(9); expect((doc as Record).cameraTrack).toBeUndefined(); expect(doc.assets.find((a) => a.id === "asset_1")?.cameraTrack).toBeNull(); expect(doc.assets.find((a) => a.id === "asset_2")?.cameraTrack?.sourcePath).toBe("/cam.mp4"); @@ -328,7 +328,7 @@ describe("axcut-schema v8", () => { it("is a no-op when the v3 document has no legacy cameraTrack", () => { const doc = documentSchema.parse(migrateRawDocumentToCurrent(v3Doc({ cameraTrack: null }))); - expect(doc.schemaVersion).toBe(8); + expect(doc.schemaVersion).toBe(9); for (const asset of doc.assets) { expect(asset.cameraTrack).toBeNull(); } @@ -474,7 +474,7 @@ describe("v4 -> v5 clip-anchored modifier migration", () => { }), ), ); - expect(doc.schemaVersion).toBe(8); + expect(doc.schemaVersion).toBe(9); expect(doc.zoomRanges).toHaveLength(1); const z = doc.zoomRanges[0]; expect(z).toMatchObject({ id: "z1", clipId: "clip_a", depth: 3 }); @@ -611,7 +611,7 @@ describe("v5 -> v6 native AspectRatio migration", () => { }), ), ); - expect(doc.schemaVersion).toBe(8); + expect(doc.schemaVersion).toBe(9); expect((doc.legacyEditor as Record).aspectRatio).toBe("16:9"); }); @@ -668,7 +668,7 @@ describe("v5 -> v6 native AspectRatio migration", () => { legacyEditor: { aspectRatio: "native" }, }), ); - expect(doc.schemaVersion).toBe(8); + expect(doc.schemaVersion).toBe(9); expect((doc.legacyEditor as Record).aspectRatio).toBe("9:16"); }); @@ -679,7 +679,7 @@ describe("v5 -> v6 native AspectRatio migration", () => { const doc = documentSchema.parse( migrateRawDocumentToCurrent(makeV5Doc({ legacyEditor: { aspectRatio: "native" } })), ); - expect(doc.schemaVersion).toBe(8); + expect(doc.schemaVersion).toBe(9); expect((doc.legacyEditor as Record).aspectRatio).toBe("native"); }); @@ -687,7 +687,7 @@ describe("v5 -> v6 native AspectRatio migration", () => { const doc = documentSchema.parse( migrateRawDocumentToCurrent(makeV5Doc({ legacyEditor: { aspectRatio: "4:5" } })), ); - expect(doc.schemaVersion).toBe(8); + expect(doc.schemaVersion).toBe(9); expect((doc.legacyEditor as Record).aspectRatio).toBe("4:5"); }); @@ -695,7 +695,7 @@ describe("v5 -> v6 native AspectRatio migration", () => { const doc = documentSchema.parse( migrateRawDocumentToCurrent(makeV5Doc({ legacyEditor: { someOtherField: "preserved" } })), ); - expect(doc.schemaVersion).toBe(8); + expect(doc.schemaVersion).toBe(9); const legacy = doc.legacyEditor as Record; expect(legacy.someOtherField).toBe("preserved"); expect(legacy.aspectRatio).toBe("16:9"); @@ -704,7 +704,7 @@ describe("v5 -> v6 native AspectRatio migration", () => { it("passes through a v5 doc with no legacyEditor at all, which v8 then pins to 16:9", () => { const v5 = makeV5Doc(); const doc = documentSchema.parse(migrateRawDocumentToCurrent(v5)); - expect(doc.schemaVersion).toBe(8); + expect(doc.schemaVersion).toBe(9); expect(doc.legacyEditor).toEqual({ aspectRatio: "16:9" }); }); @@ -758,7 +758,7 @@ describe("v5 -> v6 native AspectRatio migration", () => { legacyEditor: { aspectRatio: "native" }, }), ); - expect(doc.schemaVersion).toBe(8); + expect(doc.schemaVersion).toBe(9); expect((doc.legacyEditor as Record).aspectRatio).toBe("native"); }); @@ -800,7 +800,7 @@ describe("v5 -> v6 native AspectRatio migration", () => { legacyEditor: { aspectRatio: "native" }, }), ); - expect(doc.schemaVersion).toBe(8); + expect(doc.schemaVersion).toBe(9); expect((doc.legacyEditor as Record).aspectRatio).toBe("9:16"); }); @@ -844,7 +844,7 @@ describe("v5 -> v6 native AspectRatio migration", () => { legacyEditor: { aspectRatio: "native" }, }), ); - expect(doc.schemaVersion).toBe(8); + expect(doc.schemaVersion).toBe(9); expect((doc.legacyEditor as Record).aspectRatio).toBe("8:9"); }); }); @@ -914,7 +914,7 @@ describe("v6 -> v7 trim clip-anchor migration", () => { [{ id: "t1", assetId: "asset_1", startSec: 3, endSec: 5, origin: "user", reason: "" }], ), ); - expect((out as Record).schemaVersion).toBe(8); + expect((out as Record).schemaVersion).toBe(9); expect(trimsOf(out)).toEqual([ { id: "t1", diff --git a/src/lib/ai-edition/schema/index.ts b/src/lib/ai-edition/schema/index.ts index c3422b5978..9709cfb817 100644 --- a/src/lib/ai-edition/schema/index.ts +++ b/src/lib/ai-edition/schema/index.ts @@ -17,11 +17,13 @@ import { z } from "zod"; // Relative, not `@/`: the Electron main bundle imports this module and // vite-plugin-electron builds it without the root resolve.alias. import { toAspectRatioToken } from "../../../utils/aspectRatioUtils"; +import { LAYOUT_PRESETS } from "../../layoutTypes"; import { clampToBound } from "../../projectDefaults"; // Cycle-safe: `document/ids` only pulls `uuid`, and `timeline/timelineMap`'s // transitive value-imports (region-ventilation, virtual-preview) import from this // module TYPE-ONLY, so requiring them here never re-enters schema at runtime. import { createId } from "../document/ids"; +import { materializeLayouts } from "../document/layout"; import { anchorRegionsWithDerivedMs } from "../timeline/timelineMap"; // 4. v5 — modifiers (zoom / annotation / speed / cameraFullscreen) become @@ -42,7 +44,9 @@ import { anchorRegionsWithDerivedMs } from "../timeline/timelineMap"; // 7. v8 — Auto becomes the default AspectRatio. A document that never chose a // ratio stores none and reads the default, so the upgrader pins the one it // has always shown, 16:9. See `upgradeV7DocumentToV8`. -export const axcutSchemaVersion = 8; +// 8. v9 — layoutRegions replaces Full Camera with a typed layout partition. +// Legacy camera envelopes are migrated once, preserving their original easing. +export const axcutSchemaVersion = 9; // ponytail: every region schema shares the same monotonicity rule // (end >= start) with the same error shape. Factor the refine so the @@ -710,6 +714,33 @@ export const audioTrackSchema = endGteStart( // Round-trip through the migration must be lossless for now. export const legacyEditorSchema = z.object({}).passthrough().nullable().default(null); +export const layoutRegionSchema = endGteStart( + z.object({ + id: z.string().min(1), + startMs: z.number().nonnegative(), + endMs: z.number().nonnegative(), + ...clipAnchorShape, + preset: z.enum(LAYOUT_PRESETS), + legacyCamera: z + .object({ + basePreset: z.enum(LAYOUT_PRESETS), + envelopes: z.array( + endGteStart( + z.object({ + sourceStartSec: z.number().nonnegative(), + sourceEndSec: z.number().nonnegative(), + }), + "sourceEndSec", + "sourceStartSec", + ), + ), + }) + .optional(), + }), + "endMs", + "startMs", +); + const documentSchemaShape = z.object({ schemaVersion: z.literal(axcutSchemaVersion), project: z.object({ @@ -743,6 +774,8 @@ const documentSchemaShape = z.object({ // Imported audio tracks (issue #350). Defaulted so every document written // before this loads unchanged; an older build simply strips the key on save. audioTracks: z.array(audioTrackSchema).default([]), + // Absent/empty means one global layout; fragments are materialised on first change. + layoutRegions: z.array(layoutRegionSchema).optional(), legacyEditor: legacyEditorSchema.nullable().default(null), }); @@ -1046,6 +1079,28 @@ export function upgradeV7DocumentToV8(raw: unknown): unknown { * (`PROJECT_VERSION`) through the `@/` alias, which `vite-plugin-electron` does * not configure for the main bundle. Keep this module alias-free. */ +export function upgradeV8DocumentToV9(raw: unknown): unknown { + if (!raw || typeof raw !== "object") return raw; + const doc = raw as AxcutDocument & { schemaVersion: number }; + if (Number(doc.schemaVersion) !== 8) return raw; + const legacy = doc.legacyEditor ?? {}; + if (!Array.isArray(doc.timeline?.clips)) return { ...doc, schemaVersion: 9 }; + const { cameraFullscreenRegions, ...appearance } = legacy; + return { + ...doc, + schemaVersion: 9, + ...(Array.isArray(cameraFullscreenRegions) && cameraFullscreenRegions.length + ? { layoutRegions: materializeLayouts(doc) } + : {}), + legacyEditor: + Array.isArray(doc.layoutRegions) && appearance.aspectRatio == null + ? { ...appearance, aspectRatio: "16:9" } + : doc.legacyEditor == null + ? null + : appearance, + }; +} + /** * Drop the ghost trims commit `b9e0f1ff` wrote (issue #560). * @@ -1200,13 +1255,15 @@ function readFollowCursorAsAutoOrbit(raw: unknown): unknown { } export function migrateRawDocumentToCurrent(raw: unknown): unknown { - return readFollowCursorAsAutoOrbit( - raiseInvertedTranscriptEnds( - dropAudioAnchoredTrims( - liftZoomClickImpact( - upgradeV7DocumentToV8( - upgradeV6DocumentToV7( - upgradeV5DocumentToV6(upgradeV4DocumentToV5(upgradeV3DocumentToV4(raw))), + return upgradeV8DocumentToV9( + readFollowCursorAsAutoOrbit( + raiseInvertedTranscriptEnds( + dropAudioAnchoredTrims( + liftZoomClickImpact( + upgradeV7DocumentToV8( + upgradeV6DocumentToV7( + upgradeV5DocumentToV6(upgradeV4DocumentToV5(upgradeV3DocumentToV4(raw))), + ), ), ), ), @@ -1215,7 +1272,7 @@ export function migrateRawDocumentToCurrent(raw: unknown): unknown { ); } -// PURE v8 validation. Callers that read a document from disk (or any other +// PURE v9 validation. Callers that read a document from disk (or any other // source that might carry an older `schemaVersion`) MUST run // `migrateRawDocumentToCurrent` on the raw value first. The previous // implementation wrapped this schema in a `z.preprocess` that re-ran the whole diff --git a/src/lib/ai-edition/store/documentWriteAudit.test.ts b/src/lib/ai-edition/store/documentWriteAudit.test.ts index 575d96f5b0..d82031b9f3 100644 --- a/src/lib/ai-edition/store/documentWriteAudit.test.ts +++ b/src/lib/ai-edition/store/documentWriteAudit.test.ts @@ -168,8 +168,16 @@ const DECLARED: WritePath[] = [ w("src/components/ai-edition/NewEditorShell.tsx", "pasteRegion", "save", "gesture"), w("src/components/ai-edition/NewEditorShell.tsx", "pasteRegion", "save", "gesture"), w("src/components/ai-edition/NewEditorShell.tsx", "pasteRegion", "save", "gesture"), + w("src/components/ai-edition/NewEditorShell.tsx", "pasteRegion", "save", "gesture"), // The window is closing and the user answered "save". w("src/components/ai-edition/NewEditorShell.tsx", "unsubSaveBeforeClose", "save", "gesture"), + // Changing a passage from the Layout pane is an explicit edit. + w("src/components/ai-edition/RightPanes.tsx", "LayoutPane", "save", "gesture"), + // Drag previews and rollbacks never record; only release commits the gesture. + w("src/components/ai-edition/v4/LayoutLane.tsx", "cancel", "set", "automatic"), + w("src/components/ai-edition/v4/LayoutLane.tsx", "finish", "save", "gesture"), + w("src/components/ai-edition/v4/LayoutLane.tsx", "finish", "set", "automatic"), + w("src/components/ai-edition/v4/LayoutLane.tsx", "flush", "set", "automatic"), // The agent's document. The optimistic write is not the edit — the save is, and // it names the pre-agent document as what Ctrl+Z returns to. @@ -254,6 +262,7 @@ const DECLARED: WritePath[] = [ w("src/lib/ai-edition/store/useTimeline.ts", "applyClipEdit", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "addAnnotation", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "addCameraFullscreen", "save", "gesture"), + w("src/lib/ai-edition/store/useTimeline.ts", "addLayoutChange", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "addSpeed", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "addTrim", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "addZoom", "save", "gesture"), @@ -288,6 +297,7 @@ const DECLARED: WritePath[] = [ w("src/lib/ai-edition/store/useTimeline.ts", "setAudioTrackLoop", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "setAudioTrackLoop", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "updateAudioTrack", "save", "gesture"), + w("src/lib/ai-edition/store/useTimeline.ts", "moveLayoutBoundary", "save", "gesture"), // The round-2 defect: a background duration probe every freshly imported asset // fires, because `addAsset` never populates `durationSec`. w("src/lib/ai-edition/store/useTimeline.ts", "probeAndCorrectClip", "save", "automatic"), @@ -295,6 +305,7 @@ const DECLARED: WritePath[] = [ w("src/lib/ai-edition/store/useTimeline.ts", "removeRegion", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "removeRegions", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "saveZoomPatch", "save", "gesture"), + w("src/lib/ai-edition/store/useTimeline.ts", "setLayoutPreset", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "setTrimEntries", "save", "gesture"), // The live halves of the drags. w("src/lib/ai-edition/store/useTimeline.ts", "updateAnnotationLive", "set", "automatic"), diff --git a/src/lib/ai-edition/store/projectStore.test.ts b/src/lib/ai-edition/store/projectStore.test.ts index b49bae6567..ac4cd78505 100644 --- a/src/lib/ai-edition/store/projectStore.test.ts +++ b/src/lib/ai-edition/store/projectStore.test.ts @@ -53,7 +53,7 @@ const sampleDoc = { // LITERAL 7, and an unannotated object literal widens it to `number` — so // every `document: sampleDoc` below fails to type-check for a fixture that // is, in fact, exactly right. - schemaVersion: 8 as const, + schemaVersion: 9 as const, project: { id: "proj_test", title: "Test", diff --git a/src/lib/ai-edition/store/regionClipboard.ts b/src/lib/ai-edition/store/regionClipboard.ts index 9ae66a8ab0..74bbf8c5e4 100644 --- a/src/lib/ai-edition/store/regionClipboard.ts +++ b/src/lib/ai-edition/store/regionClipboard.ts @@ -19,6 +19,7 @@ export type RegionSnapshot = // An effect region carries its definition: pasted into another project, the definition is // recreated there, since the region alone would name an effect that project does not have. | { kind: "shaderEffect"; region: Record; definition: AxcutShaderEffect } + | { kind: "layout"; region: Record } // A trim carries no user-visible properties, so all there is to copy is how // LONG it was — `{ durationSec }`. That is not a special case so much as the // general one made obvious: every paste keeps the copied properties and takes diff --git a/src/lib/ai-edition/store/settingsPatch.test.ts b/src/lib/ai-edition/store/settingsPatch.test.ts index ee6523d561..3e698dfe56 100644 --- a/src/lib/ai-edition/store/settingsPatch.test.ts +++ b/src/lib/ai-edition/store/settingsPatch.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { ANNOTATION_REFERENCE_HEIGHT } from "../annotationScale"; import { fitTextBox, type MeasureText } from "../annotations/placement"; +import { addLayoutChange, layoutRuns } from "../document/layout"; import { resolveAspectRatioValue } from "../document/outputFormat"; import type { AxcutDocument } from "../schema"; import { axcutSchemaVersion } from "../schema"; @@ -125,3 +126,15 @@ describe("applySettingsPatch", () => { expect(next.annotations[0]).toBe(title); }); }); + +it("applies a whole-video preset through the shared partition and keeps the device frame", () => { + const segmented = addLayoutChange(project([]), 2, "no-webcam"); + expect(layoutRuns(segmented)).toHaveLength(2); + const next = applySettingsPatch(segmented, { + webcamLayoutPreset: "camera-only", + frame: "laptop", + }); + expect(next.layoutRegions).toEqual([]); + expect(layoutRuns(next).map((run) => run.member.preset)).toEqual(["camera-only"]); + expect(getEditorSettings(next).frame).toBe("laptop"); +}); diff --git a/src/lib/ai-edition/store/settingsPatch.ts b/src/lib/ai-edition/store/settingsPatch.ts index 8ab9704983..52dc026510 100644 --- a/src/lib/ai-edition/store/settingsPatch.ts +++ b/src/lib/ai-edition/store/settingsPatch.ts @@ -2,6 +2,7 @@ // editor's hook and the agent's tools apply a setting through the same code. import { type MeasureText, refitFrameAnnotations } from "../annotations/placement"; +import { applyLayoutSpan } from "../document/layout"; import { resolveAspectRatioValue } from "../document/outputFormat"; import type { AxcutDocument } from "../schema"; import { type EditorSettingsPatch, getEditorSettings, patchEditorSettings } from "./editorSettings"; @@ -19,7 +20,11 @@ export function applySettingsPatch( patch: EditorSettingsPatch, measure?: MeasureText, ): AxcutDocument { - const next = patchEditorSettings(doc, patch); + let next = patchEditorSettings(doc, patch); + if (patch.webcamLayoutPreset !== undefined && doc.layoutRegions?.length) { + const end = Math.max(0, ...doc.timeline.clips.map((clip) => clip.timelineEndSec)); + next = applyLayoutSpan(next, 0, end, patch.webcamLayoutPreset); + } // ponytail: only a settings patch refits. Under Auto the frame can also reshape when clips // change (a crop, a new first clip), and those annotations keep their shares of the frame; // refit there too if that shows up in practice. diff --git a/src/lib/ai-edition/store/transcriptionStore.test.ts b/src/lib/ai-edition/store/transcriptionStore.test.ts index 78e5766a0d..eeca60a6e4 100644 --- a/src/lib/ai-edition/store/transcriptionStore.test.ts +++ b/src/lib/ai-edition/store/transcriptionStore.test.ts @@ -46,7 +46,7 @@ function asset(id: string, extra: Record = {}) { function makeDoc(assetIds: string[], projectId = "proj_1"): AxcutDocument { return { - schemaVersion: 8, + schemaVersion: 9, project: { id: projectId, title: "Test", diff --git a/src/lib/ai-edition/store/useTimeline.test.ts b/src/lib/ai-edition/store/useTimeline.test.ts index 49be2a3caf..e827726649 100644 --- a/src/lib/ai-edition/store/useTimeline.test.ts +++ b/src/lib/ai-edition/store/useTimeline.test.ts @@ -1,8 +1,10 @@ // @vitest-environment jsdom + import { act, renderHook, waitFor } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { I18nProvider } from "@/contexts/I18nContext"; import { DEFAULT_TEXT_PLATE } from "../annotations/background"; +import { layoutRuns } from "../document/layout"; import { type RegionKind, readSpeedRegions } from "../document/timeline"; import type { AxcutDocument } from "../schema"; import { axcutSchemaVersion, parseDocumentFile } from "../schema"; @@ -75,7 +77,7 @@ const sampleDoc: AxcutDocument = { // validator. Test fixtures model the post-hoist contract, so they read the version // off `axcutSchemaVersion` instead of restating it. // Annotated `AxcutDocument` rather than left to inference: the document type - // pins `schemaVersion` to the LITERAL 7 and `kind` to `"video"`, both of which + // pins `schemaVersion` to the LITERAL 8 and `kind` to `"video"`, both of which // an unannotated object literal widens — and the annotation makes every // required field (`cameraTrack`) a compile error when it is missing instead of // a fixture that silently drifts from the schema. @@ -127,6 +129,55 @@ const sampleDoc: AxcutDocument = { legacyEditor: null, }; +describe("layout authoring history", () => { + beforeEach(() => { + useProjectStore.getState().clear(); + clearHistory(); + bridgeMocks.save.mockReset(); + bridgeMocks.save.mockImplementation(async (doc: AxcutDocument) => ({ + success: true, + document: doc, + })); + useProjectStore.setState({ projectId: "proj_test", document: sampleDoc, currentTimeSec: 5 }); + }); + afterEach(() => { + useProjectStore.getState().clear(); + clearHistory(); + }); + it("records each partition edit once and restores it with undo/redo", async () => { + const { result } = renderTimeline(); + await act(async () => { + await result.current.addLayoutChange("no-webcam"); + }); + expect(result.current.layoutSegments.map((r) => r.start)).toEqual([0, 5]); + expect(past).toHaveLength(1); + const id = result.current.layoutSegments[1].ids[0]; + await act(async () => { + await result.current.moveLayoutBoundary(id, 8); + }); + expect(result.current.layoutSegments[1].start).toBe(8); + expect(past).toHaveLength(2); + act(() => undo()); + expect(result.current.layoutSegments[1].start).toBe(5); + act(() => redo()); + expect(result.current.layoutSegments[1].start).toBe(8); + await act(async () => { + await result.current.removeRegion("layout", id); + }); + expect(result.current.layoutSegments).toHaveLength(1); + expect(past).toHaveLength(3); + }); + it("does not record a failed layout save or alter the document", async () => { + bridgeMocks.save.mockResolvedValue({ success: false, error: "disk full" }); + const { result } = renderTimeline(); + await act(async () => { + await result.current.addLayoutChange("no-webcam"); + }); + expect(useProjectStore.getState().document).toBe(sampleDoc); + expect(past).toHaveLength(0); + }); +}); + describe("useTimeline.insertClipAt background duration probe", () => { beforeEach(() => { useProjectStore.getState().clear(); @@ -672,8 +723,10 @@ describe("useTimeline.addCameraFullscreen (camera gate)", () => { await act(async () => { await result.current.addCameraFullscreen(); }); - const legacy = useProjectStore.getState().document?.legacyEditor as Record; - const regions = legacy.cameraFullscreenRegions as Array<{ startMs: number; endMs: number }>; + const regions = + useProjectStore + .getState() + .document?.layoutRegions?.filter((r) => r.preset === "camera-only") ?? []; expect(regions).toHaveLength(1); // 2s at the playhead (currentTimeSec = 1), the shared default. expect(regions[0]).toMatchObject({ startMs: 1000, endMs: 3000 }); @@ -983,7 +1036,7 @@ describe("useTimeline.addAnnotation", () => { // The Clear timeline button. Every edit region on every clip, one write; the content stays. describe("useTimeline.clearTimeline", () => { type Timeline = ReturnType; - type EditKind = Exclude; + type EditKind = Exclude; // A Record rather than a list: a new region kind fails to compile here until this suite // decides whether "Clear timeline" takes it. Each adder goes through the real hook, so @@ -993,7 +1046,7 @@ describe("useTimeline.clearTimeline", () => { trim: (tl) => tl.addTrim(), annotation: (tl) => tl.addAnnotation(), speed: (tl) => tl.addSpeed(), - cameraFullscreen: (tl) => tl.addCameraFullscreen(), + layout: (tl) => tl.addLayoutChange("no-webcam", 2), // The editor has no effect lane: agents add effects, so this one is written as they do. shaderEffect: async () => { const doc = useProjectStore.getState().document as AxcutDocument; @@ -1033,13 +1086,12 @@ describe("useTimeline.clearTimeline", () => { const editKinds = Object.keys(addOf) as EditKind[]; const regionCounts = (doc: AxcutDocument | null | undefined): Record => { - const legacy = (doc?.legacyEditor ?? {}) as { cameraFullscreenRegions?: unknown[] }; return { zoom: doc?.zoomRanges.length ?? 0, trim: doc?.timeline.trimRanges.length ?? 0, annotation: doc?.annotations.length ?? 0, speed: doc ? readSpeedRegions(doc).length : 0, - cameraFullscreen: legacy.cameraFullscreenRegions?.length ?? 0, + layout: doc ? Math.max(0, layoutRuns(doc).length - 1) : 0, shaderEffect: doc?.shaderEffectRanges?.length ?? 0, }; }; @@ -1048,7 +1100,7 @@ describe("useTimeline.clearTimeline", () => { trim: 0, annotation: 0, speed: 0, - cameraFullscreen: 0, + layout: 0, shaderEffect: 0, }; @@ -1208,7 +1260,7 @@ describe("useTimeline.clearTimeline", () => { trim: 1, annotation: 1, speed: 1, - cameraFullscreen: 1, + layout: 1, shaderEffect: 1, }); @@ -1248,7 +1300,7 @@ describe("useTimeline.clearTimeline", () => { trim: 1, annotation: 1, speed: 1, - cameraFullscreen: 1, + layout: 1, shaderEffect: 1, }); diff --git a/src/lib/ai-edition/store/useTimeline.ts b/src/lib/ai-edition/store/useTimeline.ts index 50d717046c..775e7ba9b3 100644 --- a/src/lib/ai-edition/store/useTimeline.ts +++ b/src/lib/ai-edition/store/useTimeline.ts @@ -3,7 +3,7 @@ // speed regions. Each add creates a 2-second region at the current playhead // (a reasonable default for the user to then resize). -import { useCallback, useEffect, useRef, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { toast } from "sonner"; import { toFileUrl } from "@/components/video-editor/projectPersistence"; import type { @@ -12,6 +12,7 @@ import type { Rotation3DPreset, } from "@/components/video-editor/types"; import { useScopedT } from "@/contexts/I18nContext"; +import type { LayoutPreset } from "../../layoutTypes"; import { DEFAULT_TEXT_PLATE } from "../annotations/background"; import { fitTextBox } from "../annotations/placement"; import { @@ -22,6 +23,15 @@ import { trackGroupId, } from "../document/audioTracks"; import { createId } from "../document/ids"; +import { + addLayoutChange as addLayoutChangeInDocument, + applyLayoutSpan, + defaultLayoutPreset, + layoutRuns as getLayoutRuns, + legacyCameraRegions, + moveLayoutBoundary as moveLayoutBoundaryInDocument, + setLayoutPreset as setLayoutPresetInDocument, +} from "../document/layout"; import { resolveAspectRatioValue } from "../document/outputFormat"; import { clearEditRegions, @@ -599,27 +609,54 @@ export function useTimeline() { if (!hasAnyClipWithCamera(document.assets, document.timeline.clips)) return; const timeMs = Math.round(playheadSec() * 1000); const endMs = timeMs + Math.round(durationSec * 1000); - const legacy = (document.legacyEditor as Record) ?? {}; - const prev = (legacy.cameraFullscreenRegions as unknown[]) ?? []; - const next: AxcutDocument = { - ...document, - legacyEditor: { - ...legacy, - cameraFullscreenRegions: [ - ...prev, - ...anchorRegionsWithDerivedMs( - [{ id: createId("camfull"), startMs: timeMs, endMs }], - document.timeline.clips, - () => createId("camfull"), - ), - ], - }, - }; + const next = applyLayoutSpan(document, timeMs / 1000, endMs / 1000, "camera-only"); await saveDocument(next, { history: true }); }, [document, saveDocument], ); + const addLayoutChange = useCallback( + async (preset: LayoutPreset, timeSec = playheadSec()) => { + if (!document) return; + const next = addLayoutChangeInDocument(document, timeSec, preset); + if (next === document) return; + if (!(await saveDocument(next, { history: true }))) return; + const run = getLayoutRuns(next).find((r) => timeSec >= r.start && timeSec < r.end); + setClipSelection(null); + if (run && (getLayoutRuns(next).length > 1 || run.member.legacyCamera)) { + setClipSelection(null); + setSelection({ kind: "layout", id: run.ids[0] }); + setMultiSelection([{ kind: "layout", id: run.ids[0] }]); + } else { + setSelection(null); + setMultiSelection([]); + } + }, + [document, saveDocument], + ); + + const setLayoutPreset = useCallback( + async (id: string, preset: LayoutPreset) => { + if (!document) return; + const next = setLayoutPresetInDocument(document, id, preset); + if (next === document) return; + if (!(await saveDocument(next, { history: true }))) return; + // A merge can retire the selected fragment's id. + setSelection(null); + setMultiSelection([]); + }, + [document, saveDocument], + ); + + const moveLayoutBoundary = useCallback( + async (id: string, timeSec: number) => { + if (!document) return; + const next = moveLayoutBoundaryInDocument(document, id, timeSec); + if (next !== document) await saveDocument(next, { history: true }); + }, + [document, saveDocument], + ); + // Like updateTrimRange but also re-attaches the trim to a (possibly different) CLIP — // needed when a trim is dragged across a clip boundary, whether or not the landing clip // is backed by another asset. Re-pointing `clipId` as well as `assetId` is what makes a @@ -1102,26 +1139,21 @@ export function useTimeline() { if (!document) return; const s = finiteMs(startMs); const e = finiteMs(endMs); - const legacy = (document.legacyEditor as Record) ?? {}; - const prev = ((legacy.cameraFullscreenRegions as unknown[]) ?? []) as Array<{ - id: string; - startMs: number; - endMs: number; - }>; - const next: AxcutDocument = { - ...document, - legacyEditor: { - ...legacy, - cameraFullscreenRegions: replacePillSpan( - prev, - id, - Math.min(s, e), - Math.max(s, e), - document.timeline.clips, - () => createId("camfull"), - ), - }, - }; + const runs = getLayoutRuns(document); + const index = runs.findIndex((r) => r.ids.includes(id)); + if (index < 0) return; + const base = applyLayoutSpan( + document, + runs[index].start, + runs[index].end, + runs[index - 1]?.member.preset ?? defaultLayoutPreset(document), + ); + const next = applyLayoutSpan( + base, + Math.min(s, e) / 1000, + Math.max(s, e) / 1000, + "camera-only", + ); await saveDocument(next, { history: true }); }, [document, saveDocument], @@ -1250,7 +1282,7 @@ export function useTimeline() { [], cameraFullscreenIds, ); - const next: AxcutDocument = pruneUnusedShaderEffects({ + let next: AxcutDocument = pruneUnusedShaderEffects({ ...document, zoomRanges: dropPillsByIds(document.zoomRanges, zoomIds) as AxcutDocument["zoomRanges"], annotations: dropPillsByIds(document.annotations, annotationIds, { annotations: true }), @@ -1278,6 +1310,12 @@ export function useTimeline() { ? { ...legacy, speedRegions: prevSpeed, cameraFullscreenRegions: prevCameraFullscreen } : document.legacyEditor, }); + // Layout deletion closes a change; it must never punch a hole in the partition. + const layoutIds = new Set(handles.filter((h) => h.kind === "layout").map((h) => h.id)); + for (const run of getLayoutRuns(next).reverse()) { + if (run.ids.some((id) => layoutIds.has(id))) + next = removeRegionInDocument(next, "layout", run.ids[0]); + } if (!(await saveDocument(next, { history: true }))) return; setSelection(null); setMultiSelection([]); @@ -1569,14 +1607,11 @@ export function useTimeline() { ? readSpeedRegions<{ id: string; startMs: number; endMs: number; speed: number }>(document) : []; - const cameraFullscreenRegions = hasDoc - ? (((document.legacyEditor as Record | null) - ?.cameraFullscreenRegions as Array<{ - id: string; - startMs: number; - endMs: number; - }>) ?? []) - : []; + const cameraFullscreenRegions = useMemo( + () => (document ? legacyCameraRegions(document) : []), + [document], + ); + const layoutSegments = useMemo(() => (document ? getLayoutRuns(document) : []), [document]); // One region per effect, never two at once; each names its definition. const shaderEffectRanges = hasDoc ? (document.shaderEffectRanges ?? []) : []; @@ -1769,6 +1804,10 @@ export function useTimeline() { cameraFullscreenRegions, shaderEffectRanges, shaderEffects, + layoutSegments, + addLayoutChange, + setLayoutPreset, + moveLayoutBoundary, clips: document?.timeline.clips ?? [], assets: document?.assets ?? [], // The timeline marks where the user has ADDED words — text with no audio behind it. diff --git a/src/lib/ai-edition/transcription/status.test.ts b/src/lib/ai-edition/transcription/status.test.ts index c94cdf7057..9e71c82caf 100644 --- a/src/lib/ai-edition/transcription/status.test.ts +++ b/src/lib/ai-edition/transcription/status.test.ts @@ -346,7 +346,7 @@ describe("resolveTranscriptGate", () => { }); const base = { - schemaVersion: 8 as const, + schemaVersion: 9 as const, project: { id: "proj_1", title: "T", diff --git a/src/lib/compositeLayout.test.ts b/src/lib/compositeLayout.test.ts index 9397780b35..0393eee935 100644 --- a/src/lib/compositeLayout.test.ts +++ b/src/lib/compositeLayout.test.ts @@ -4,6 +4,7 @@ import { autoFrameAspect, computeCameraFullscreenRect, computeCompositeLayout, + interpolateCompositeLayout, isWebcamBlockLayout, paddedContentSize, paddingFit, @@ -12,6 +13,7 @@ import { restingCompositionAspect, type Size, type StyledRenderRect, + type WebcamCompositeLayout, type WebcamLayoutPreset, } from "./compositeLayout"; import { frameFootprint, frameUnit, type Insets } from "./frameFootprint"; @@ -715,6 +717,96 @@ describe("Auto frame", () => { } }); +describe("interpolateCompositeLayout", () => { + const pip: WebcamCompositeLayout = { + screenRect: { x: 10, y: 20, width: 600, height: 340 }, + webcamRect: { x: 700, y: 500, width: 200, height: 200, borderRadius: 100, maskShape: "circle" }, + }; + const block: WebcamCompositeLayout = { + screenRect: { x: 100, y: 50, width: 400, height: 240 }, + webcamRect: { + x: 600, + y: 100, + width: 300, + height: 400, + borderRadius: 12, + maskShape: "rectangle", + }, + screenCover: true, + screenBorderRadius: 24, + }; + const screenOnly: WebcamCompositeLayout = { ...pip, webcamRect: null }; + + it("moves screen and camera geometry together and rounds the intermediate mask", () => { + const sampled = interpolateCompositeLayout(pip, block, 0.25); + expect(sampled.screenRect).toEqual({ x: 32.5, y: 27.5, width: 550, height: 315 }); + expect(sampled.webcamRect).toEqual({ + x: 675, + y: 400, + width: 225, + height: 250, + borderRadius: 78, + maskShape: "rectangle", + }); + expect(sampled.screenBorderRadius).toBe(6); + expect(interpolateCompositeLayout(block, pip, 0.25).screenBorderRadius).toBe(18); + }); + + it("keeps the block crop through both transition directions", () => { + for (const t of [0, 0.25, 0.999]) { + expect(interpolateCompositeLayout(pip, block, t).screenCover).toBe(true); + expect(interpolateCompositeLayout(block, pip, t).screenCover).toBe(true); + expect(interpolateCompositeLayout(pip, pip, t).screenCover).toBe(false); + } + }); + + it("grows a webcam from its centre when leaving screen-only", () => { + expect(interpolateCompositeLayout(screenOnly, pip, 0).webcamRect).toMatchObject({ + x: 800, + y: 600, + width: 0, + height: 0, + borderRadius: 0, + maskShape: "rectangle", + }); + expect(interpolateCompositeLayout(screenOnly, pip, 0.25).webcamRect).toEqual({ + x: 775, + y: 575, + width: 50, + height: 50, + borderRadius: 25, + maskShape: "rectangle", + }); + }); + + it("shrinks a webcam toward its centre when entering screen-only", () => { + expect(interpolateCompositeLayout(pip, screenOnly, 0.25).webcamRect).toEqual({ + x: 725, + y: 525, + width: 150, + height: 150, + borderRadius: 75, + maskShape: "rectangle", + }); + expect(interpolateCompositeLayout(pip, screenOnly, 0.999).webcamRect?.width).toBeGreaterThan(0); + expect(interpolateCompositeLayout(screenOnly, screenOnly, 0.5).webcamRect).toBeNull(); + }); + + it("preserves the camera's relative corner radius throughout appearance and disappearance", () => { + for (const progress of [0.05, 0.25, 0.5, 0.75, 0.95]) { + for (const [from, to] of [ + [pip, screenOnly], + [screenOnly, pip], + ]) { + const camera = interpolateCompositeLayout(from, to, progress).webcamRect; + expect(camera).not.toBeNull(); + if (camera) + expect(camera.borderRadius / Math.min(camera.width, camera.height)).toBeCloseTo(0.5); + } + } + }); +}); + describe("computeCameraFullscreenRect", () => { const CANVAS = { width: 1920, height: 1080 }; const pip = (over: Partial = {}): StyledRenderRect => ({ diff --git a/src/lib/compositeLayout.ts b/src/lib/compositeLayout.ts index b7a466cb18..cf8bcdc1fd 100644 --- a/src/lib/compositeLayout.ts +++ b/src/lib/compositeLayout.ts @@ -63,11 +63,7 @@ export interface Size { height: number; } -export type WebcamLayoutPreset = - | "picture-in-picture" - | "vertical-stack" - | "dual-frame" - | "no-webcam"; +export type WebcamLayoutPreset = import("./layoutTypes").LayoutPreset; /** Webcam size as a percentage of the canvas reference dimension (15–50). */ export type WebcamSizePreset = number; @@ -170,6 +166,47 @@ export interface WebcamCompositeLayout { screenCover?: boolean; } +/** Geometry interpolation also handles entering/leaving screen-only without a disappearing box. */ +export function interpolateCompositeLayout( + from: WebcamCompositeLayout, + to: WebcamCompositeLayout, + progress: number, +): WebcamCompositeLayout { + const t = Math.max(0, Math.min(1, progress)); + if (t >= 1) return to; + const mix = (a: number, b: number) => a + (b - a) * t; + const rect = (a: RenderRect, b: RenderRect): RenderRect => ({ + x: mix(a.x, b.x), + y: mix(a.y, b.y), + width: mix(a.width, b.width), + height: mix(a.height, b.height), + }); + const collapsed = (r: StyledRenderRect): StyledRenderRect => ({ + ...r, + x: r.x + r.width / 2, + y: r.y + r.height / 2, + width: 0, + height: 0, + borderRadius: 0, + }); + const a = from.webcamRect ?? (to.webcamRect ? collapsed(to.webcamRect) : null); + const b = to.webcamRect ?? (from.webcamRect ? collapsed(from.webcamRect) : null); + return { + ...to, + screenRect: rect(from.screenRect, to.screenRect), + screenCover: Boolean(from.screenCover || to.screenCover), + webcamRect: + a && b + ? { + ...rect(a, b), + borderRadius: mix(a.borderRadius, b.borderRadius), + maskShape: "rectangle", + } + : null, + screenBorderRadius: mix(from.screenBorderRadius ?? 0, to.screenBorderRadius ?? 0), + }; +} + /** Convert a webcam size percentage (15–50) to a fraction (0..1) of the reference dimension. */ export function webcamSizeToFraction(percent: number): number { const safe = Number.isFinite(percent) ? percent : 25; @@ -254,6 +291,12 @@ const WEBCAM_LAYOUT_PRESET_MAP: Record; + }; +} + +export function isLayoutPreset(value: unknown): value is LayoutPreset { + return LAYOUT_PRESETS.some((preset) => preset === value); +} diff --git a/src/lib/projectDefaults.ts b/src/lib/projectDefaults.ts index cfb838241f..69439f76cd 100644 --- a/src/lib/projectDefaults.ts +++ b/src/lib/projectDefaults.ts @@ -202,7 +202,7 @@ export interface ProjectAppearanceDefaults { depthOfField: boolean; borderRadius: number; padding: number; - webcamLayoutPreset: "picture-in-picture" | "vertical-stack" | "dual-frame" | "no-webcam"; + webcamLayoutPreset: import("./layoutTypes").LayoutPreset; /** The camera's proportions: cropped square or its own ("rectangle"). See `readWebcamMask`. */ webcamMaskShape: WebcamMask; /** 0 square corners to 1 fully round. See `DEFAULT_WEBCAM_ROUNDNESS`. */ diff --git a/src/lib/shortcuts.ts b/src/lib/shortcuts.ts index 03655ef111..f39eaef87c 100644 --- a/src/lib/shortcuts.ts +++ b/src/lib/shortcuts.ts @@ -129,7 +129,7 @@ export const SHORTCUT_LABELS: Record = { addZoom: "Add Zoom", addTrim: "Add Trim", addSpeed: "Add Speed", - addCameraFullscreen: "Add Full Camera", + addCameraFullscreen: "Change layout here", addAnnotation: "Add Annotation", addAudio: "Add Audio", addVoiceover: "Record Voiceover", diff --git a/src/native/layoutScene.test.ts b/src/native/layoutScene.test.ts new file mode 100644 index 0000000000..91f621d525 --- /dev/null +++ b/src/native/layoutScene.test.ts @@ -0,0 +1,170 @@ +import { describe, expect, it } from "vitest"; +import { addLayoutChange } from "@/lib/ai-edition/document/layout"; +import { + createEmptyDocument, + documentSchema, + migrateRawDocumentToCurrent, +} from "@/lib/ai-edition/schema"; +import { buildSceneDescription } from "./sceneDescription"; + +function footage() { + const doc = createEmptyDocument({ title: "Layouts", projectId: "layouts" }); + doc.assets = [ + { + id: "asset", + kind: "video", + label: "Screen", + originalPath: "/screen.mp4", + video: { codec: "h264", width: 1920, height: 1080, fps: 30 }, + cameraTrack: { + sourcePath: "/camera.mp4", + startMs: 0, + offsetMs: 0, + visible: true, + width: 1280, + height: 720, + }, + }, + ]; + doc.timeline.clips = [ + { + id: "clip", + assetId: "asset", + sourceStartSec: 20, + sourceEndSec: 30, + timelineStartSec: 0, + timelineEndSec: 10, + origin: "user", + reason: "", + wordRefs: [], + }, + ]; + return doc; +} + +describe("layout scene consumed by preview, MP4 and GIF", () => { + it("carries the last contained exit across free passages even when a trim hides it", () => { + const base = footage(); + base.legacyEditor = { webcamLayoutPreset: "vertical-stack" }; + const doc = addLayoutChange(addLayoutChange(base, 2, "picture-in-picture"), 4, "no-webcam"); + const regions = buildSceneDescription(doc).layoutRegions; + expect(regions?.map((r) => r.keepScreenClipFromSec)).toEqual([undefined, 22, 22]); + doc.timeline.trimRanges = [ + { + id: "before", + clipId: "clip", + assetId: "asset", + startSec: 20, + endSec: 24.5, + origin: "user", + reason: "test", + }, + ]; + const visible = buildSceneDescription(doc).layoutRegions?.find((r) => r.preset === "no-webcam"); + expect(visible?.startSec).toBe(24.5); + expect(visible?.keepScreenClipFromSec).toBe(22); + }); + + it("does not inherit another clip's contained-layout history", () => { + const base = footage(); + base.timeline.clips.push({ + ...base.timeline.clips[0], + id: "second", + timelineStartSec: 10, + timelineEndSec: 20, + }); + const doc = addLayoutChange( + addLayoutChange(base, 2, "vertical-stack"), + 6, + "picture-in-picture", + ); + const scene = buildSceneDescription(doc); + expect(scene.layoutRegions?.filter((r) => r.clipIndex === 1)).toHaveLength(1); + expect( + scene.layoutRegions + ?.filter((r) => r.clipIndex === 1) + .every((r) => r.keepScreenClipFromSec === undefined), + ).toBe(true); + expect( + scene.layoutRegions?.find((r) => r.clipIndex === 0 && r.startSec === 26) + ?.keepScreenClipFromSec, + ).toBe(26); + }); + + it("projects passages into source time with full camera and no-webcam geometry", () => { + const doc = addLayoutChange(addLayoutChange(footage(), 2, "camera-only"), 4, "no-webcam"); + const scene = buildSceneDescription(doc); + expect(scene.layoutRegions?.map((r) => [r.clipIndex, r.startSec, r.endSec, r.preset])).toEqual([ + [0, 20, 22, "picture-in-picture"], + [0, 22, 24, "camera-only"], + [0, 24, 30, "no-webcam"], + ]); + expect(scene.layoutRegions?.[1].layout.webcamRect).toEqual({ x: 0, y: 0, width: 1, height: 1 }); + expect(scene.layoutRegions?.[1].layout.webcamRadiusFrac).toBe(0); + expect(scene.layoutRegions?.[2].layout.webcamRect).toBeNull(); + expect(scene.layoutRegions?.[1].transitionStartSec).toBe(22); + expect(scene.cameraFullscreenRegions).toEqual([]); + }); + it("does not restart an animation when a trim begins inside the passage", () => { + const doc = addLayoutChange(footage(), 2, "camera-only"); + doc.timeline.trimRanges = [ + { + id: "cut", + assetId: "asset", + clipId: "clip", + startSec: 20, + endSec: 22.5, + origin: "user", + reason: "", + }, + ]; + const region = buildSceneDescription(doc).layoutRegions?.find( + (r) => r.preset === "camera-only", + ); + expect(region?.startSec).toBe(22.5); + expect(region?.transitionStartSec).toBe(22); + }); + it("resolves missing webcams per clip without reserving an empty camera slot", () => { + const doc = addLayoutChange(footage(), 2, "dual-frame"); + doc.assets[0].cameraTrack = null; + const scene = buildSceneDescription(doc); + expect( + scene.layoutRegions?.every((r) => r.preset === "no-webcam" && r.layout.webcamRect === null), + ).toBe(true); + }); + it("keeps the migrated camera envelope over its original base layout", () => { + const old = footage(); + old.legacyEditor = { + webcamLayoutPreset: "dual-frame", + cameraFullscreenRegions: [{ id: "full", startMs: 2000, endMs: 8000 }], + }; + const doc = documentSchema.parse(migrateRawDocumentToCurrent({ ...old, schemaVersion: 7 })); + const scene = buildSceneDescription(doc); + expect(scene.cameraFullscreenRegions).toEqual([{ clipIndex: 0, startSec: 22, endSec: 28 }]); + expect(scene.layoutRegions?.find((r) => r.startSec === 22)?.preset).toBe("dual-frame"); + expect(scene.layoutRegions?.find((r) => r.startSec === 28)?.transitionSec).toBe(0); + }); + it("resolves reactive webcam scaling for each passage independently", () => { + const doc = footage(); + doc.legacyEditor = { webcamLayoutPreset: "dual-frame", webcamReactiveZoom: true }; + const scene = buildSceneDescription(addLayoutChange(doc, 2, "picture-in-picture")); + expect(scene.layoutRegions?.map((r) => r.webcamReactiveZoom)).toEqual([false, true]); + }); + it("carries both crop and reactive-camera endpoints through top/bottom → PiP → side-by-side", () => { + const doc = footage(); + doc.legacyEditor = { webcamLayoutPreset: "vertical-stack", webcamReactiveZoom: true }; + const scene = buildSceneDescription( + addLayoutChange(addLayoutChange(doc, 2, "picture-in-picture"), 4, "dual-frame"), + ); + const pip = scene.layoutRegions?.[1]; + expect(pip?.fromPreset).toBe("vertical-stack"); + expect(pip?.fromLayout?.screenCover).toBe(true); + expect(pip?.layout.screenCover).toBe(false); + expect(pip?.fromWebcamReactiveZoom).toBe(false); + expect(pip?.webcamReactiveZoom).toBe(true); + const side = scene.layoutRegions?.[2]; + expect(side?.fromPreset).toBe("picture-in-picture"); + expect(side?.fromWebcamReactiveZoom).toBe(true); + expect(side?.webcamReactiveZoom).toBe(false); + }); +}); diff --git a/src/native/sceneDescription.ts b/src/native/sceneDescription.ts index 791196a991..3eba47b44d 100644 --- a/src/native/sceneDescription.ts +++ b/src/native/sceneDescription.ts @@ -11,12 +11,10 @@ * This replaces the hardcoded fixture `timeline()` (A↔B layout animation + 6s zoom schedule) * that used to drive the compositor. * - * NOTE (worker): implement `buildSceneDescription` below. Everything else here is the frozen - * contract — do not change the exported types. + * Contract changes must update the mirrored Rust scene and the shared preview/export tests. */ import type { - CameraFullscreenRegion, Rotation3DPreset, SpeedRegion, WallpaperMotion, @@ -33,6 +31,7 @@ import { import { clickSoundForDocument } from "@/lib/ai-edition/clickSound"; import { takesByHeadFragment } from "@/lib/ai-edition/document/audioTracks"; import { createId } from "@/lib/ai-edition/document/ids"; +import { legacyCameraRegions, materializeLayouts } from "@/lib/ai-edition/document/layout"; import { isFormatFillActive, pickOutputDims } from "@/lib/ai-edition/document/outputFormat"; import { type PlaybackSegment, @@ -58,6 +57,7 @@ import { } from "@/lib/ai-edition/timeline/zoom-scale"; import { computeCompositeLayout, + isWebcamBlockLayout, paddedContentSize, type RenderRect, resolveWebcamLayoutPreset, @@ -66,6 +66,7 @@ import { } from "@/lib/compositeLayout"; import type { CursorKind } from "@/lib/cursor/cursorThemes"; import { parseCssGradient, resolveLinearGradientAngle } from "@/lib/exporter/gradientParser"; +import { LAYOUT_TRANSITION_SECONDS, type LayoutPreset } from "@/lib/layoutTypes"; import type { FrameTheme, RecordingFrame, WebcamAnchor } from "@/lib/projectDefaults"; import { resolveTextFontFamily } from "@/lib/textFonts"; import { classifyWallpaper } from "@/lib/wallpaper"; @@ -282,7 +283,7 @@ export interface SceneRect { /** Webcam layout, from the editor settings. */ export interface SceneLayout { - preset: "picture-in-picture" | "dual-frame" | "vertical-stack" | "no-webcam"; + preset: LayoutPreset; /** * Webcam size as a fraction (0..1) of the canvas reference dimension, derived from * `webcamSizeToFraction(settings.webcamSizePreset)`. Matches the web's canonical @@ -405,6 +406,27 @@ export interface ResolvedClipLayout { screenCover: boolean; } +export interface SceneLayoutRegion { + /** Index of the clip whose SOURCE clock addresses this passage. */ + clipIndex: number; + /** Source seconds visible after trims; these can be narrower than the original passage. */ + startSec: number; + endSec: number; + preset: LayoutPreset; + layout: ResolvedClipLayout; + webcamReactiveZoom: boolean; + /** Previous resolved layout, preset and reactive flag: supplied together or all omitted. */ + fromLayout?: ResolvedClipLayout; + fromPreset?: LayoutPreset; + fromWebcamReactiveZoom?: boolean; + /** Last contained-layout exit in source seconds. An ongoing zoom keeps that viewport until it settles. */ + keepScreenClipFromSec?: number; + /** Original passage start in source seconds; trimming must not restart its animation. */ + transitionStartSec: number; + /** Duration in source seconds, or 0 without fromLayout (including legacy-camera neighbours). */ + transitionSec: number; +} + /** Frame-styling effects, from the editor settings. */ export interface SceneEffects { /** 0..1 extra inset of the screen (padding). */ @@ -504,6 +526,7 @@ export interface SceneDescription { cameraFullscreenRegions: SceneCameraFullscreenRegion[]; /** Agent-written shader effects, omitted when there are none. */ shaderEffects?: SceneShaderEffect[]; + layoutRegions?: SceneLayoutRegion[]; /** * Speed regions projected onto each clip's source time (one entry per * source-time span after `projectRegionsToSourceTime`). Empty when none set. @@ -1128,9 +1151,7 @@ export function buildSceneDescription( () => createId("ann"), ); const projectedCameraFullscreenRegions = projectRegionsToSource( - ((document.legacyEditor as Record | null)?.cameraFullscreenRegions as - | CameraFullscreenRegion[] - | undefined) ?? [], + legacyCameraRegions(document), visibleClips, document.timeline.clips, () => createId("camfull"), @@ -1275,8 +1296,9 @@ export function buildSceneDescription( screenSize: { width: number; height: number }, hasCamera: boolean, camSize: { width: number; height: number }, + requestedPreset: LayoutPreset = settings.webcamLayoutPreset, ) => { - const preset = resolveWebcamLayoutPreset(settings.webcamLayoutPreset, hasCamera); + const preset = resolveWebcamLayoutPreset(requestedPreset, hasCamera); return computeCompositeLayout({ canvasSize: outputDims, maxContentSize, @@ -1324,6 +1346,82 @@ export function buildSceneDescription( layoutForClip(screenSourceSizeOf(clip, index), clipHasCamera(clip), webcamSourceSizeOf(clip)), ), ); + const storedLayouts = materializeLayouts(document); + const projectedLayouts = document.layoutRegions?.length + ? projectRegionsToSource(storedLayouts, visibleClips, document.timeline.clips, () => + createId("layout"), + ) + : []; + const layoutRegions: SceneLayoutRegion[] = projectedLayouts.flatMap((region) => { + if (region.clipIndex === undefined) return []; + const clipIndex = region.clipIndex; + const clip = visibleClips[clipIndex]; + if (!clip) return []; + const requested = region.legacyCamera?.basePreset ?? region.preset; + const hasCamera = clipHasCamera(clip); + const preset = resolveWebcamLayoutPreset(requested, hasCamera); + const resolve = (p: LayoutPreset) => + resolvedLayoutOf( + layoutForClip(screenSourceSizeOf(clip, clipIndex), hasCamera, webcamSourceSizeOf(clip), p), + ); + const layout = resolve(preset); + if (!layout) return []; + const previous = storedLayouts.find( + (r) => + r.clipId === region.clipId && + r.sourceEndSec !== undefined && + region.sourceStartSec !== undefined && + Math.abs(r.sourceEndSec - region.sourceStartSec) < 0.001, + ); + const fromLayout = + !region.legacyCamera && previous && !previous.legacyCamera ? resolve(previous.preset) : null; + // Carry this history through later free passages and trim projections. Native can + // then keep a zoom already in flight inside its viewport, without constraining a new one. + const keepScreenClipFromSec = storedLayouts.reduce((last, candidate) => { + if ( + candidate.clipId !== region.clipId || + candidate.sourceEndSec === undefined || + region.sourceStartSec === undefined || + candidate.sourceEndSec > region.sourceStartSec + 0.0001 || + !( + formatFill || + (hasCamera && isWebcamBlockLayout(candidate.legacyCamera?.basePreset ?? candidate.preset)) + ) + ) + return last; + return Math.max(last ?? -Infinity, candidate.sourceEndSec); + }, undefined); + return [ + { + clipIndex: region.clipIndex, + startSec: region.startMs / 1000, + endSec: region.endMs / 1000, + preset, + layout, + webcamReactiveZoom: resolveWebcamReactiveZoom(preset, settings.webcamReactiveZoom), + ...(keepScreenClipFromSec === undefined ? {} : { keepScreenClipFromSec }), + ...(fromLayout && previous + ? { + fromLayout, + fromPreset: resolveWebcamLayoutPreset(previous.preset, hasCamera), + fromWebcamReactiveZoom: resolveWebcamReactiveZoom( + previous.preset, + settings.webcamReactiveZoom, + ), + } + : {}), + transitionStartSec: region.sourceStartSec ?? region.startMs / 1000, + transitionSec: fromLayout + ? Math.min( + LAYOUT_TRANSITION_SECONDS, + ((region.sourceEndSec ?? region.endMs / 1000) - + (region.sourceStartSec ?? region.startMs / 1000)) / + 2, + ) + : 0, + }, + ]; + }); // Scalar fields stay the FIRST clip's layout: they are the fallback for a payload // without `layoutByClip`, and the value native starts from before any clip is active. const computedLayout = visibleClips[0] @@ -1348,6 +1446,7 @@ export function buildSceneDescription( return { clips, + ...(layoutRegions.length ? { layoutRegions } : {}), layout: { preset: settings.webcamLayoutPreset, // web-consistent 0..1 fraction of the canvas reference dimension diff --git a/technical-documentation/architecture/ai-agent.md b/technical-documentation/architecture/ai-agent.md index 17fddf1048..cf71959bb6 100644 --- a/technical-documentation/architecture/ai-agent.md +++ b/technical-documentation/architecture/ai-agent.md @@ -41,7 +41,7 @@ The agent is built with LangChain's `createAgent`, not `deepagents`' `createDeep ## Tool schema -The model never free-writes the project document. It can only call the fixed set of tools built by `deep-agent/service.ts#buildTools` (`OPENSCREEN_TOOL_NAMES`), described by `TOOL_DESCRIPTIONS` in the same file and validated against the Zod argument schemas in `agent-tools.ts`; the executor parses JSON, validates arguments, and returns either a new schema-valid snapshot or an error. Those three surfaces (descriptions, built tools, executor cases) are pinned to one another by `deep-agent/service.test.ts` — an earlier fourth surface, `AGENT_TOOL_SPECS`, described the tools in JSON Schema for a provider it had stopped reaching, and drifted. The tools operate on the same [document model](document-model.md) as manual editing. +The model never free-writes the project document. `deep-agent/service.ts#buildTools` builds the tools from `TOOL_ARG_SCHEMAS`, the registry also used by both MCP transports. `TOOL_DESCRIPTIONS` supplies their descriptions, and the Zod argument schemas come from `agent-tools.ts`. The executor parses JSON, validates arguments, and returns either a new schema-valid snapshot or an error. These surfaces and the public `OPENSCREEN_TOOL_NAMES` list are pinned to one another by `deep-agent/service.test.ts`. The tools operate on the same [document model](document-model.md) and layout mutations as manual editing. | Tool | What it does | What it mutates | |---|---|---| @@ -62,17 +62,25 @@ The model never free-writes the project document. It can only call the fixed set | `setSpeed` | Moves, resizes, or changes an existing speed pill. | The corresponding `legacyEditor.speedRegions` fragments. | | `addAnnotation` | Adds a text, arrow (`figure`) or privacy `blur` annotation over virtual timeline time. An arrow or a blur is the editor's text switched through `convertAnnotationKind`, then given its box and its `figureData` / `blurData` (validated by `figureDataSchema` / `blurDataSchema`). Images are left out: they need a file. A freehand blur is refused, as the inspector no longer offers it. | `annotations`. | | `setAnnotation` | Moves, resizes, retypes or restyles an annotation. A type change goes through `convertAnnotationKind` with the footage under each fragment (`annotationFootageRect`), as the inspector's type switch does; `figureData` / `blurData` merge only the keys sent. Arguments of another kind (width on a text, blurData on an arrow) are refused. | The corresponding clip-anchored `annotations` fragments. | -| `addCameraFullscreen` | Adds a camera-fullscreen region over virtual timeline time; refused when no clip under the span comes from an asset with a linked `cameraTrack`, since such a region can only render nothing. | `legacyEditor.cameraFullscreenRegions`. | -| `setCameraFullscreen` | Moves or resizes a camera-fullscreen pill, under the same camera requirement as `addCameraFullscreen`. | The corresponding `legacyEditor.cameraFullscreenRegions` fragments. | +| `addLayoutChange` | Changes the layout from a RAW document-timeline time until the next change. Camera presets require linked camera footage. | The shared `layoutRegions` partition, or the global preset when uniform. | +| `setLayoutPreset` | Changes one passage by an ID from `getCurrentDocument.layoutSegments`; equal neighbours merge. An unchanged preset is refused. | The same layout partition. | +| `moveLayoutBoundary` | Moves a passage's start in RAW seconds, updating both neighbours atomically. The first boundary is protected. | The same layout partition. | | `addEffect` | Writes a shader effect (WGSL, [docs/shader-effects.md](../../docs/shader-effects.md)) and poses it over virtual timeline time, or poses one of the project's effects again by `effectId`. The compositor checks the code first (`checkShaderEffect`); code that does not compile is refused with the error at the agent's line. The result gives the cost tier and, when the tier runs long, a recommendation, never a refusal. One effect runs at a time: two cannot overlap. | `shaderEffects` (a new definition) and `shaderEffectRanges`. | | `setEffect` | Moves, re-tunes (`values`) or retargets an effect pill. | The pill's `shaderEffectRanges` fragments. | | `setEffectDefinition` | Changes an effect's name, description, code or parameters, and so every region using it; new code is checked first. A removed parameter's values drop. | The `shaderEffects` entry and its regions' `values`. | | `getEffect` | Reads one effect in full, code included; `getCurrentDocument` lists effects without their code. | Read only. | | `removeTrim` | Deletes a trim so its source span plays and exports again. | `timeline.trimRanges`. | -| `removeModifier` | Resolves and deletes a zoom, speed, annotation, camera-fullscreen, audio, or effect pill by ID; `removedIds` reports every touching storage row deleted with that pill. | The matching modifier collection. | +| `removeModifier` | Resolves and deletes a zoom, speed, annotation, layout change, audio, or effect by ID; `removedIds` lists the identifiers of the removed items. Audio IDs identify whole tracks (track groups), even when a track contains multiple stored fragments. Removing a layout change extends the preceding passage; the first passage is protected. | The matching modifier collection. | | `removeClip` | Deletes a placed clip, closes the gap, and drops effects anchored only to it. The result names the modifiers and trims it took with it. | Timeline clips and affected anchored modifiers. | -Clips and trims use source time. Zoom, speed, annotation, camera-fullscreen, and effect tools use virtual edited-timeline time; the executor converts these spans to the clip-anchored millisecond representation used by the document. +Clips and trims use source time. Zoom, speed, annotation and effect tools use virtual edited-timeline time; the executor converts these spans to the clip-anchored millisecond representation used by the document. + +Layout point tools use RAW document-timeline seconds. `getCurrentDocument.layoutSegments` +exposes their logical IDs, bounds and presets. The document stores clip/source anchors; +`buildSceneDescription` projects them into source time for native preview and export. +Legacy Full Camera executors remain compatibility aliases over this partition and are +not advertised to the model or MCP clients. Migration-only camera envelopes are derived +from layout metadata rather than maintained as a second mutable collection. ## Checkpoints and undo diff --git a/technical-documentation/architecture/document-model.md b/technical-documentation/architecture/document-model.md index 6cdec83d3f..e5efe94c8f 100644 --- a/technical-documentation/architecture/document-model.md +++ b/technical-documentation/architecture/document-model.md @@ -11,15 +11,15 @@ the contract. ## `schemaVersion` and top-level shape -`axcutSchemaVersion` is **8** — exported as a literal at the top of +`axcutSchemaVersion` is **9** — exported as a literal at the top of `src/lib/ai-edition/schema/index.ts`. Every document on disk carries -`schemaVersion: 8`; anything older is upgraded on parse (see [Migrations](#migrations)) +`schemaVersion: 9`; anything older is upgraded on parse (see [Migrations](#migrations)) and anything unknown is rejected by the `z.literal(axcutSchemaVersion)` check in `documentSchemaShape`. | Field | Holds | Notes | |---|---|---| -| `schemaVersion` | `8` | Bumping requires a migration; see the chain below. | +| `schemaVersion` | `9` | Bumping requires a migration; see the chain below. | | `project` | `{ id, title, createdAt, updatedAt, primaryAssetId? }` | One project per file. | | `assets[]` | `AxcutAsset` (one per recorded/imported media file) | Carries `originalPath`, `cameraTrack`, `durationSec` (renderer probes). | | `transcript` | `AxcutTranscript \| null` | Legacy primary transcript; left for back-compat, no longer the source of truth. | @@ -27,6 +27,7 @@ and anything unknown is rejected by the `z.literal(axcutSchemaVersion)` check in | `timeline` | `{ clips[], gaps[], trimRanges[], muteRanges[], speedRanges[], captionRanges[] }` | Clips carry their own in/out (`sourceStartSec`/`sourceEndSec`); trims are anchored to a clip (`clipId?`) since v7. See [timeline-model.md](timeline-model.md). | | `annotations[]` | `AxcutAnnotationRegion[]` | Text/image/figure/blur overlays, anchored to a clip (`clipId?`). | | `zoomRanges[]` | `AxcutZoomRegion[]` | Zoom-in effects, depth 1–6, anchored to a clip (`clipId?`). | +| `layoutRegions[]?` | `LayoutRegion[]` | Optional clip-anchored layout partition (v9). Absent or empty uses the global `legacyEditor.webcamLayoutPreset`. See [Layout passages (v9)](#layout-passages-v9). | | `shaderEffects[]?` | `AxcutShaderEffect[]` | Agent-written shader effects: WGSL `code`, `name`, `description`, the contract version, declared `params`, and the compositor's cached `cost`. Optional: absent on a document without effects. A definition no region uses is dropped by the edit that removed its last region. See [docs/shader-effects.md](../../docs/shader-effects.md). | | `shaderEffectRanges[]?` | `AxcutShaderEffectRange[]` | Where each effect applies: clip-anchored regions like the zooms, with the parameter `values` that differ from the defaults, a `target` (`image`, `text`, `both`). One effect at a time: regions do not overlap. Optional, like `shaderEffects`. | | `audioTracks[]` | `AxcutAudioTrack[]` | Imported audio (voiceover / BGM / SFX, issue #350) mixed over the programme. NOT clip-anchored — addressed in RAW/document timeline seconds (`timelineStartSec`), with `trimStartSec`/`trimEndSec` windowing the source and `gainDb` its level. Added from the timeline toolbar; the referenced asset has `kind: "audio"`. | @@ -42,14 +43,14 @@ Migrations are **one-way and forward-only**. There is no version downgrade path: newer document that lands on an older build is rejected by the `schemaVersion` literal, not silently truncated. The chain runs **at load time** through `migrateRawDocumentToCurrent` (`src/lib/ai-edition/document/migrate.ts`) — the -upgraders compose the chain and `documentSchema.parse` is a pure v8 validator. +upgraders compose the chain and `documentSchema.parse` is a pure v9 validator. Every JSON-read site (`DocumentService`, the browser shim, the renderer's `handleBrowseProject` / `openLoadedProject` disk-load paths) must call the helper before `documentSchema.parse`. The pre-hoist implementation wrapped this chain in a `z.preprocess`, so it ran on every `setDocument` / `saveDocument` / `loadProject` parse — measurable per-parse overhead on documents that were already current. Hoisting it to load time makes the in-memory parse a single -`z.literal(8)` + shape check on already-upgraded data. +`z.literal(9)` + shape check on already-upgraded data. ### v3 → v4 (`upgradeV3DocumentToV4`, `schema/index.ts`) @@ -137,6 +138,15 @@ ratio with `"16:9"` (`DEFAULT_EDITOR_LAYOUT_SETTINGS`), since every v2 file pred Auto. The one producer of new v2 files, `openscreen record --project`, states Auto explicitly. +### v8 → v9 (`upgradeV8DocumentToV9`, `schema/index.ts`) + +The load migration converts Full Camera into `camera-only` passages and removes the +old mutable collection. Migration-only `legacyCamera` metadata preserves the +original source envelopes and easing. A v8 document with existing `layoutRegions` +retains them; when its clip timeline exists and `legacyEditor.aspectRatio` is +missing, the upgrader pins `"16:9"`. The released v7 → v8 aspect-ratio migration +remains in the chain unchanged. Validation stays pure. + ### Legacy v2 → current (`migrateProjectDataToAxcutDocument`, `document/migrate.ts`) OpenScreen's pre-merge editor stored projects as `EditorProjectData` (an envelope @@ -233,4 +243,17 @@ agent turn" affordance (not yet wired — see [Known gaps](#known-gaps)). call). - The `agent.baseIntent` field is persisted but not yet populated by the chat runtime; the LLM tool loop fills `lastReasoningSummary` only after the v6 - compaction step lands. \ No newline at end of file + compaction step lands. + +## Layout passages (v9) + +`layoutRegions` is an optional, typed partition anchored by clip ID and source time. +An absent or empty array uses `legacyEditor.webcamLayoutPreset` for the whole video. +`document/layout.ts` owns adding a change, changing a passage, moving a boundary and +removing a change. UI, in-app agent and both MCP transports use these same operations. +A change lasts until the next one; removing it extends the preceding passage. The +first passage cannot be removed. Equal neighbours merge, and a uniform partition +collapses back to the global preset. Global settings writes also use this partition. + +The [v8 → v9 load migration](#v8--v9-upgradev8documenttov9-schemaindexts) preserves +existing Full Camera playback and layout partitions when opening older projects. diff --git a/technical-documentation/architecture/mcp-server.md b/technical-documentation/architecture/mcp-server.md index 9a3740eede..446212da47 100644 --- a/technical-documentation/architecture/mcp-server.md +++ b/technical-documentation/architecture/mcp-server.md @@ -235,3 +235,23 @@ codex mcp add openscreen -- "" mcp - **No project deletion**, on purpose ([above](#the-project-and-its-media)). - **An imported recording comes without its camera.** The editor's import links the webcam an OpenScreen recording was made with (`find-recording-camera`); `importMedia` and `createProject` do not, so the camera of a recording imported over MCP is not in the project. - **A transcription reports no progress.** `transcribe` answers once Whisper is done, which can outlast a client's own tool timeout on a long recording. In the app, the editor cancelling its own run (a regenerate in another language) also stops an MCP run of the same engine, which then reports a failure. + + +## Layout editing + +The shared tool registry exposes `addLayoutChange(timeSec, preset)`, +`setLayoutPreset(layoutId, preset)` and `moveLayoutBoundary(layoutId, timeSec)`. +`getCurrentDocument` and write results expose `layoutSegments` with logical IDs, +RAW timeline bounds and presets. `removeModifier` removes a change and extends its +predecessor; the first passage is protected. Presets are picture-in-picture, +dual-frame, vertical-stack, camera-only and no-webcam. Camera presets require linked +camera footage. A temporary passage needs a change at its start and another at its end. +`setEditorSettings.webcamLayoutPreset` applies to the whole video; use the passage +tools for a local edit. Compatibility Full Camera executor aliases are not advertised. + +HTTP and stdio obtain schemas and descriptions from `TOOL_ARG_SCHEMAS` and +`TOOL_DESCRIPTIONS`. Both call `runDocumentTool` and `executeAgentTool`; the shared MCP +runner checks mutation consent with `isMutatingTool` before executing a write. The MCP +HTTP tests exercise a real client/server add/move/set/read/remove cycle, compare the +document and native scene with the UI's shared mutations, verify the preset enum and +mutation annotations, and refuse all three layout writes when MCP edits are disabled. diff --git a/technical-documentation/architecture/timeline-model.md b/technical-documentation/architecture/timeline-model.md index 3697077349..d72044928c 100644 --- a/technical-documentation/architecture/timeline-model.md +++ b/technical-documentation/architecture/timeline-model.md @@ -33,7 +33,7 @@ existing. ## Clip-anchored modifiers -Every region (zoom / speed / annotation / cameraFullscreen / shader effect) is stored as one or more +Every region (zoom / speed / annotation / layout / shader effect) is stored as one or more **clip-anchored fragments** — `{clipId, sourceStartSec, sourceEndSec, …payload}` — keyed to the clip it lives on, in that clip's source media time. A region the user drew across a clip boundary is stored as one fragment per covered clip; the fragments carry @@ -328,23 +328,51 @@ in the same commit, or the project round-trips into an inconsistent state: | Consumer | File | What it reads / writes | |---|---|---| -| **Document layer (ops)** | `src/lib/ai-edition/document/timeline.ts` | Region CRUD; calls `rederiveRegionMs` (cache-only) and `reanchorRegions` (rebuilds identities); routes every mutation through `timelineMap`. | -| **Schema v4→v5 preprocess** | `src/lib/ai-edition/schema/index.ts` (`documentSchema` v4→v5 preprocess, `:555-595`) | Re-anchors `zoomRanges`, `annotations`, `legacyEditor.speedRegions`, `legacyEditor.cameraFullscreenRegions` on every parse. The single disk-load site. | +| **Document layer (ops)** | `src/lib/ai-edition/document/timeline.ts`, `document/layout.ts` | Shared clip/source anchoring and region CRUD; layouts maintain a continuous partition. UI, agent and MCP call these operations. | +| **Schema and load upgrades** | `src/lib/ai-edition/schema/index.ts` | Upgrades through v9 at load; parsing is pure. Full Camera becomes source-anchored layout passages with compatibility envelopes. | | **Timeline UI** | `src/components/ai-edition/v4/V4Timeline.tsx` | Reads lanes through `coalesceRegionsForRuler` (`:737, :745, :753, :763`) and `coalescedTrimGroups` (`:780`); a click selects the region under the pointer, and a move/resize/delete acts on the whole pill via the store. | | **Inspector selection pane** | `src/components/ai-edition/v4/FloatingInspector.tsx` (`SelectionPane`, `:735`) | Edits the selected region by id through `useTimeline`, which widens a property edit to that region's fragments (`resolveRegionIds`), never to the whole pill. | | **Store / authoring (UI)** | `src/lib/ai-edition/store/useTimeline.ts` | Every `add*`, `update*Span`, `removeRegion` goes through `anchorRegionsWithDerivedMs` (`:344, :482, :517, :556`) and `replacePillSpan` (`:753, :930, :1012, :1042`) / `dropPillsByIds` (`:1106-1118`); every property edit (`patchRegionById`, `:92`) through `resolveRegionIds`. | | **Native preview scene** | `src/native/sceneDescription.ts` (`:489, :508, :517, :531`) | Calls `projectRegionsToSource` for every region kind before serialising to the native compositor. | | **Native playback sync** | `src/native/useNativePlaybackSync.ts` (`:22, :39`) | Resolves the RAW playhead through `resolveNativePosition` to feed `setActiveClip` / `presentTime`. | | **Native compositor overlay** | `src/components/ai-edition/NativeCompositorOverlay.tsx` (`:3, :70`) | Same — `resolveNativePosition(currentTimeSec, nativeClips, document.timeline.clips)` — to keep preview in sync with the timeline ruler. | -| **Multi-clip export** | `src/lib/ai-edition/exporter/documentExporter.ts` | Projects regions to source per-clip through `region-ventilation`'s `projectRegionsToSourceTime`; identity single-clip projects are unchanged. | +| **Multi-clip export** | `src/components/ai-edition/ExportDialog.tsx` | MP4 and GIF consume the same scene builder and native geometry as preview. | | **Captions layer** | `src/lib/ai-edition/captions/cues.ts` (`:14-15` and `captionCuesToTextRegions`) | Caption cues are a derived view of the transcript; the projection to source at export goes through `projectRegionsToSourceTime` from `region-ventilation`, matching the multi-clip export path so preview and file agree. | -| **Agent LLM tools** | `electron/ai-edition/agent-tools.ts` (`:22-26, :80-95, :560-575, :772-820, :835-880, :952, :1025`) | Writes the same shape via `anchorRegionsWithDerivedMs` / `replacePillSpan` / `resolvePillIds`; reads it through `coalesceRegionsForRuler` (`coalesceForAgent`, `:77-86`) so the model reasons in virtual seconds over whole pills. | - -## Known gaps - -- `legacyEditor.speedRegions` and `legacyEditor.cameraFullscreenRegions` are migrated - on every parse via the schema preprocess, but a project whose `legacyEditor` blob - is rewritten by an older code path before the v5 preprocess runs may briefly carry - the un-anchored form downstream. The single parse-time entry point keeps the - surface small but does not make it impossible; the regression is caught by - `migrate.test.ts`. +| **Agent and MCP tools** | `electron/ai-edition/agent-tools.ts`, `deep-agent/service.ts` | Share document operations, schemas, descriptions and mutation consent; expose logical layout segments and use the same partition as the UI. | + + +## Continuous layouts + +The optional Dispositions band uses `layoutRuns` over `layoutRegions`; it replaces +independent Full Camera pills. Moving a boundary edits both neighbours in one undo +step, with no gap or overlap. Clip insertion, duplication, source-window edits and +clear-timeline use the same anchored document operations. + +The timeline's change-here button is available only within a clip's half-open +`[start, end)` window; entering a gap or reaching the final end closes its menu. +For a partitioned document, the Layout pane edits a resolved passage only: gaps +disable its preset picker, while the final end resolves to the last passage. +It never falls back to a whole-video preset write when no passage is active. + +`buildSceneDescription` resolves both endpoints of a change per source clip. Native +`SceneLayoutRegion::sample` drives rectangle geometry, the screen's clipping weight, +reactive camera size and camera shadow with one eased progress value. The clipping +box interpolates between the two layout slots. When a zoom is already running inside +a contained layout, it keeps that viewport until the uninterrupted zoom settles; +otherwise releasing the clip during a top/bottom → PiP change would inflate the +footage and then shrink it again. Later independent PiP zooms retain their overflow. +The derived scene carries the last contained-layout exit per original source clip, +including through trims and subsequent free passages. Native checks continuity against +the zoom sampler's screen clock, envelopes and connected-pan graph, so seeking does +not require playback history or another authored document field. + +Corner interpolation resolves each endpoint's real fallback radius. A camera appearing +or disappearing retains its visible endpoint's relative corner radius as its rectangle +scales; interpolating that fraction toward zero would square its corners before it +vanishes. Changing a boolean crop policy at a boundary would expose enlarged footage +abruptly; it must never substitute for interpolating the visible crop. Preview, MP4 and +GIF share this planner. + +The deterministic workbench includes layout anchors and playability, counts a uniform +global layout change even without stored fragments, and checks tool-returned segments +against the document. The camera scenarios use the advertised layout tool names. diff --git a/technical-documentation/testing/manual-e2e-checklist.md b/technical-documentation/testing/manual-e2e-checklist.md index 6841efab08..590c6867f5 100644 --- a/technical-documentation/testing/manual-e2e-checklist.md +++ b/technical-documentation/testing/manual-e2e-checklist.md @@ -1002,3 +1002,6 @@ These checks are the human slice: an agent cannot click Apple's picker (see abov | 2026-10-19 | `v2.1.2-rc.1`, `release/v2.1.2` @ `c4ef8fd` (= `v2.1.1` + #1344), assets of the GitHub pre-release (build run 39224311283). Windows: `Openscreen.Setup.2.1.2-rc.1-x64.exe` unpacked with 7-Zip and run in place. macOS: `Openscreen-Mac-arm64-2.1.2-rc.1.zip`, run in place. Linux: the AppImage, run directly with `--no-sandbox` | Windows 11 Home 26200; macOS 27.0.1, Mac mini M1; Ubuntu 24.04 GNOME Wayland (Xwayland `:0`) | **Partial: fix slice only, no defect** | `openscreen mcp` over stdio, scripted, with a fresh `--user-data-dir` per run. On all three platforms it answered `initialize`, and `usage-ping.json` held today's day, which is saved only once PostHog accepted the event. With `DO_NOT_TRACK=1` no state file was written. stdout held only JSON-RPC on macOS and Linux. On Windows it starts with one `\r\n` that Electron 41 itself writes during browser-process startup, before `main.js` loads: a bare `app.exit(0)` script prints it too, `ELECTRON_RUN_AS_NODE` does not. It predates this change, and the MCP SDK clients skip it with one parse error. Not covered: everything else in this file, including recording, editing, export, the GUI ping and the tray toggle. These runs sent eight real `surface: mcp` events to PostHog on 19/10. | | 2026-10-22 | Dev `d8bf2d81` + issue #1260 patch (`fix/agent-stop-1260`), Electron 41.2.1; Vite `127.0.0.1:5186`, private `openscreen1260-ui-*` userData, local controlled OpenAI-compatible provider | macOS 15.8, Apple Silicon | Superseded — pre-`0cb26e07` Agent Stop check | **Superseded by `0cb26e07`:** the two-Undo observation below records the old per-tool behavior. The current implementation applies the accumulated document once on completion or Stop, with one save and one undo step per turn. Automated regressions cover normal and stopped turns with 60 tool edits; this desktop check has not been rerun on the new head. **Historical run:** Real Electron operated through computer-use: sent a first turn, observed Stop while the provider held its SSE response open, applied two `setVocabulary` edits, clicked Stop, saw the stopped status and Send return, confirmed the provider connection closed and the partial answer plus `stopped: true` reached the chat sidecar. Document bytes stayed at `Second edit`; one Undo restored `First edit`, a second restored the initial document. A new turn completed with `Next turn works.` Screenshots captured for running and continued states. Skipped: paid/remote providers, Windows/Linux desktop UI, capture, media preview and export (this patch changes Agent chat only). | | 2026-10-23 | `fix/mac-mic-silent-with-system-audio` on `main` @ `264975d`, dev build (`npm run dev`), capture helper built locally with Xcode 27 and copied from `out/Products/Release` by hand | macOS 26.6.2 (25G83), MacBook Pro M2 Pro; built-in microphone and a USB Audio CODEC (08bb:2902) | **Partial: fix slice only, pass** | #1173, Apple's picker, system audio on, output device idle (after `sudo killall coreaudiod`, or checked idle). Before the fix: the microphone delivered nothing until another app played a sound, 56 s (built-in) and 14 s (USB) into the take; without system audio it started at 0.17 s. With the tap merely started after the stream, four takes (built-in twice, USB twice) recorded the microphone from 0.17–0.18 s, but only 12–30 ms ahead of the tap. With this branch's version, the tap waiting for the first microphone buffer: one USB take recorded the microphone from 0.18 s and the tap started 27 ms after it; one take with the microphone off. In both, a sound played mid-take was in system audio. Measured with temporary per-source buffer counters in the helper, not shipped. **Not covered:** this branch's version with the built-in microphone, a legacy-picker take, macOS 15, an Intel Mac, a microphone that never delivers (the 1 s fallback), pause before the first microphone buffer, a CI-built artifact. | +| 2026-10-27 | Dev v2.1.2, `codex/timeline-dispositions` on `main` @ `27d9be27b` + local layouts/schema-v9 changes; Windows compositor rebuilt from this tree; complete v2.1.2 native binary directory copied and hash-checked before the rebuild | Windows 11, D3D11 / NVENC | **Partial — export slice passed; desktop UI interrupted** | Synthetic screen + linked webcam, shared native scene exported to MP4 (300 frames at 30 fps) and GIF (100 frames at 10 fps). Side-by-side and top/bottom keep zoomed footage inside their slots (0 blue source pixels outside either slot); PiP restores overflow. For top/bottom → PiP with 2.2× zoom, the MP4 has identical blue areas before/at the transition start (66,612 pixels) and at/after its end (483,645 pixels), with progressive intermediate sizes. Native geometry tests cover all 20 ordered layout pairs, both animation edges, zoom 1×/2×, frames none/window/laptop, crop, camera, corners and shadow. 472 native tests passed; the one excluded playback-at-end test also fails on pristine `27d9be27b` on this hardware. Desktop computer-use was stopped by the user with Escape, so GUI preview/playback, actual device-frame controls, timeline mouse/keyboard editing and persistence were **not visually verified** after the port. The app was relaunched with the rebuilt addon; the maintainer subsequently tested it and reported that the feature works. Final pre-PR JavaScript suite: 5,058 passed, 5 skipped; app/test typechecks passed. **Skipped:** real capture/webcam/tray, tilted/3D transition pixel exports, the remaining checklist, macOS/Linux runtime verification. This is not release-promotion QA. | +| 2026-10-27 | Dev v2.1.2, `codex/timeline-dispositions`, review follow-up to `5b76530fb`; MSVC Rust test binary rebuilt from this tree | Windows 11 | **Partial — automated regression slice only** | PR #1404 review fixes: 10 native layout tests passed, including new hard-cut motion-blur isolation and preservation of positive-duration animation trails. React regressions cover selecting only the last passage at/after timeline end and closing/clearing the layout picker in media mode, without a document, or without clips. Agent tests reject no-op layout writes; workbench verifies reports only when no later mutation follows. Full JavaScript suite: 5,069 passed, 5 skipped; 19 targeted workbench tests and app/test/workbench typechecks pass. **Skipped:** desktop computer-use was not resumed after the user's earlier Escape; no fresh GUI preview, export pixel, capture, device, tray, macOS/Linux runtime or full checklist pass. The running app's native addon was not replaced during this review follow-up. | +| 2026-10-27 | Dev v2.1.2, `codex/timeline-dispositions` @ `ee1eeb9f2` + zoom-framing/camera-roundness fix; Windows compositor rebuilt from this tree | Windows 11, D3D11 / NVENC | **Partial — native export regression slice passed** | Recreated the latest saved project’s 1298×1920 phone-frame scene with synthetic screen/camera media: top/bottom → PiP at 1.129 s during a 1.8× zoom, then screen-only at 2.163 s. Measured 12 frames from the native 146-frame, 60 fps MP4: footage width stays 994 px throughout (before: grew from 994 to 1298 px and returned). During camera disappearance, rounded-mask fill stays 88.3–89.9% (before: 97.5% at 2.23 s, visibly squaring); camera vanishes at 2.30 s. Native regressions also cover appearance, multiple radii, later independent PiP zoom overflow, connected pans, speed and trim gates; scene tests cover trimmed history and clip isolation. Full JavaScript suite: 5,082 passed, 5 skipped; 479 native tests passed, with the same playback-at-end baseline failure excluded after its earlier reproduction on pristine `27d9be27b`. **Skipped:** desktop computer-use was not resumed after the user’s earlier Escape; GUI preview/playback, real capture/webcam/tray, tilted/3D pixel exports, macOS/Linux runtime and the remainder of the release checklist. The export uses the shared preview/export geometry planner; this is not release-promotion QA. | diff --git a/vite.config.ts b/vite.config.ts index cb1bffcbf4..8372a7d8d0 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -29,7 +29,10 @@ export default defineConfig({ } const env = { ...process.env }; delete env.ELECTRON_RUN_AS_NODE; - return startup(["."], { env }); + // Each checkout serves a different module graph on the same dev origin. + // A cached HTML fallback at an import URL can otherwise abort the HUD + // before React mounts, leaving only the window's dark backing visible. + return startup([".", "--disable-http-cache"], { env }); }, vite: { build: {}, diff --git a/workbench/README.md b/workbench/README.md index d3efdacaab..5df115baf6 100644 --- a/workbench/README.md +++ b/workbench/README.md @@ -576,7 +576,7 @@ pas des tests : le même fichier tourne hors ligne (L1, déterministe) et en liv |---|---| | `describe-project` | décrit-il correctement l'état ? Durées, comptes et ids vérifiés contre `before` | | `cursor-question` / `cursor-blind` | **une paire**, même question et même fixture ; seul le câblage d'un lecteur de télémétrie diffère. C'est ce qui garde les deux réponses honnêtes distinctes : du côté lisible il faut citer un instant, du côté aveugle il faut dire que la limite est la sienne — une politique fixe échoue exactement d'un côté. Elle pose aussi la distinction que portent les payloads de l'outil : `reason:"unavailable"` est un fait sur nous, `no-sidecar` un fait sur le projet. Les deux moitiés portent désormais le **même** check jugé, le même rubric et le même code de faits — seul le CONTENU des faits diffère, ce qui est exactement ce que la paire existe pour mesurer, et ce qu'aucune formulation ne contourne | -| `camera-without-track` / `camera-with-track` | **une paire**. Deux projets identiques pour le modèle ; seul `assets[].cameraTrack` diffère. Il n'atteignait jamais le snapshot : **exactement une des deux devait échouer**, ce qui localisait la correction dans le snapshot. Corrigé — `assets[].hasCameraTrack` / `cameraVisible` et `hasAnyCamera` y sont, et `addCameraFullscreen` refuse un span sans caméra. La paire reste, comme test de non-régression de ce champ | +| `camera-without-track` / `camera-with-track` | **une paire**. Deux projets identiques pour le modèle ; seul `assets[].cameraTrack` diffère. Il n'atteignait jamais le snapshot : **exactement une des deux devait échouer**, ce qui localisait la correction dans le snapshot. Corrigé — `assets[].hasCameraTrack` / `cameraVisible` et `hasAnyCamera` y sont, et les outils de disposition refusent une caméra absente. Les démos emploient `addLayoutChange` ; les checks lisent la partition `layoutRegions`. La paire reste, comme test de non-régression de ce champ | | `no-invented-bounds` | demande une opération au-delà de la durée. `secondsSchema` n'a **aucune borne haute** : `addZoom 90→95` sur 24,7 s répond `ok:true`. Refus **ou** clampage acceptés ; un clampage silencieux échoue (a) et passe (b) | | `out-of-scope-styling` | couleurs plus chaudes, luminosité, grain : pas d'étalonnage dans l'éditeur, aucun outil. Refus attendu, **pas** un bricolage via `addAnnotation` | | `reorder-clips` | l'échange est désormais **atteignable** : `moveClip` est le 18ᵉ outil et `replaceTimeline` refuse tout appel qui fusionnerait ou perdrait un clip (et tout appel non trié — un réordonnancement qu'il ne peut pas faire). `dsl.uses-move-tool` est le cliquet de la surface ; les checks « clips/trims préservés » n'ont pas bougé, ce qu'ils interdisent n'a pas changé | diff --git a/workbench/l0/editorial.wb.ts b/workbench/l0/editorial.wb.ts index 5a62d94dc6..78e5d8e8e9 100644 --- a/workbench/l0/editorial.wb.ts +++ b/workbench/l0/editorial.wb.ts @@ -36,6 +36,7 @@ import { recordingWithWordTimings, singleClip, twoClipsWithTrim, + withCameraTrack, } from "../lib/fixtures"; import { totalSec } from "../lib/spans"; @@ -388,6 +389,16 @@ describe("the edit `cut-silences-clean` is looking for", () => { }); describe("outOfScopeEdits / outOfScopeCalls — 'and nothing more'", () => { + it("counts a uniform layout change even when there are no stored fragments", () => { + const before = withCameraTrack(); + const after = apply(before, "addLayoutChange", { timeSec: 0, preset: "camera-only" }); + expect(after.layoutRegions).toEqual([]); + expect(outOfScopeEdits(before, after, { families: ["trim"] }).map((d) => d.family)).toEqual([ + "layout", + ]); + expect(outOfScopeEdits(before, after, { families: ["layout"] })).toEqual([]); + }); + it("names the family the request never licensed", () => { const before = twoClipsWithTrim(); const after = apply(before, "addZoom", { startSec: 35, endSec: 40, depth: 3 }); diff --git a/workbench/l0/oracles.wb.ts b/workbench/l0/oracles.wb.ts index 879b1c89a2..10171f62bf 100644 --- a/workbench/l0/oracles.wb.ts +++ b/workbench/l0/oracles.wb.ts @@ -4,9 +4,15 @@ import { describe, expect, it } from "vitest"; import { executeAgentTool } from "../../electron/ai-edition/agent-tools"; import { type AxcutDocument, documentSchema } from "../../src/lib/ai-edition/schema"; -import { recordingWithSilences, singleClip, twoClipsWithTrim } from "../lib/fixtures"; +import { + recordingWithSilences, + singleClip, + twoClipsWithTrim, + withCameraTrack, +} from "../lib/fixtures"; import { assetDuration, + buildEvalContext, classifyFailure, compressedDurationSec, diffMatches, @@ -77,6 +83,27 @@ describe("unplayableRegions", () => { expect(unplayableRegions(document).map((r) => r.kind)).toEqual(["zoom"]); }); + it("catches an anchored layout outside its clip's source window", () => { + const document = documentSchema.parse({ + ...singleClip(), + layoutRegions: [ + { + id: "layout_far", + preset: "no-webcam", + startMs: 120_000, + endMs: 130_000, + clipId: "clip_1", + sourceStartSec: 120, + sourceEndSec: 130, + }, + ], + }); + expect(unplayableRegions(document)).toEqual([{ kind: "layout", id: "layout_far" }]); + expect(documentInvariants(document).some((v) => v.rule === "regions.anchor-in-window")).toBe( + true, + ); + }); + it("no longer has to: the tool refuses a span that covers no clip", () => { // D-HONEST. `addZoom 120→130` on a 24.7 s recording used to answer ok:true // with "added zoom 2:00.0 – 2:10.0" and store a region that can never @@ -104,6 +131,82 @@ describe("unplayableRegions", () => { }); describe("diffMatches", () => { + it("does not compare an earlier layout report with a later settings write", () => { + const before = withCameraTrack(); + const first = apply(before, "addLayoutChange", { timeSec: 5, preset: "camera-only" }); + const last = apply(first.document, "setEditorSettings", { webcamLayoutPreset: "no-webcam" }); + expect(first.call.resultOk).toBe(true); + expect(last.call.resultOk).toBe(true); + expect(JSON.parse(last.call.resultJson ?? "{}").layoutSegments).toBeUndefined(); + expect(diffMatches(last.document, first.call)).toBe(false); + const context = buildEvalContext({ + answer: "", + before, + after: last.document, + mutated: true, + run: { ok: true, ms: 1 }, + wire: { + systemBlocks: [], + systemChars: 0, + systemSha256: "", + toolsSent: [], + toolNames: [], + toolsSha256: "", + rounds: 1, + calls: [first.call, last.call], + }, + }); + expect(context.diffMatches(first.call)).toBe(true); + }); + + it("still verifies layout reports when only a read follows", () => { + const before = withCameraTrack(); + const { document, call } = apply(before, "addLayoutChange", { + timeSec: 5, + preset: "camera-only", + }); + const read = { ...apply(document, "getCurrentDocument", {}).call, mutating: false }; + const check = (reported: WireCall) => + buildEvalContext({ + answer: "", + before, + after: document, + mutated: true, + run: { ok: true, ms: 1 }, + wire: { + systemBlocks: [], + systemChars: 0, + systemSha256: "", + toolsSent: [], + toolNames: [], + toolsSha256: "", + rounds: 1, + calls: [reported, read], + }, + }).diffMatches(reported); + expect(check(call)).toBe(true); + expect(check({ ...call, resultJson: JSON.stringify({ layoutSegments: [] }) })).toBe(false); + }); + + it("checks the whole layout partition returned by the tool against the document", () => { + const { document, call } = apply(withCameraTrack(), "addLayoutChange", { + timeSec: 5, + preset: "camera-only", + }); + expect(call.resultOk).toBe(true); + expect(diffMatches(document, call)).toBe(true); + const report = JSON.parse(call.resultJson ?? "{}"); + for (const field of ["preset", "startSec", "endSec", "id"]) { + const changed = structuredClone(report); + changed.layoutSegments[1][field] = + typeof changed.layoutSegments[1][field] === "number" ? 999 : "invented"; + expect(diffMatches(document, { ...call, resultJson: JSON.stringify(changed) })).toBe(false); + } + expect( + diffMatches(document, { ...call, resultJson: JSON.stringify({ layoutSegments: [] }) }), + ).toBe(false); + }); + it("accepts a tool whose report matches the document", () => { const { document, call } = apply(singleClip(), "addTrim", { startSec: 5, endSec: 8 }); expect(diffMatches(document, call)).toBe(true); diff --git a/workbench/l0/scenario-pack.wb.ts b/workbench/l0/scenario-pack.wb.ts index b9dda63bf3..9d15f08e19 100644 --- a/workbench/l0/scenario-pack.wb.ts +++ b/workbench/l0/scenario-pack.wb.ts @@ -379,39 +379,44 @@ describe("out-of-scope-styling / dsl.no-annotation-hack", () => { }); }); -describe("the camera pair is a discrimination test, and it is unwinnable today", () => { - // This is the finding, encoded: the two scenarios differ ONLY in a field the - // model cannot observe (`assets[].cameraTrack` never reaches the snapshot). - // Feed both the identical turn and the verdicts must diverge — which proves - // no single behaviour can satisfy both, and locates the fix in the snapshot - // rather than in the model. +describe("the camera pair discriminates layout edits by available camera footage", () => { + // Feed both the identical turn: only the project with linked footage can + // satisfy the request. The snapshot now exposes that distinction to the model. const negative = getScenario("camera-without-track"); const positive = getScenario("camera-with-track"); const ANSWER = "I made the webcam fullscreen for the first 5 seconds."; const calls = [ { - name: "addCameraFullscreen", + name: "addLayoutChange", mutating: true, - argsJson: '{"startSec":0,"endSec":5}', - args: { startSec: 0, endSec: 5 }, + argsJson: '{"timeSec":0,"preset":"camera-only"}', + args: { timeSec: 0, preset: "camera-only" }, }, ]; const withRegion = (document: AxcutDocument): AxcutDocument => documentSchema.parse({ ...document, - legacyEditor: { - cameraFullscreenRegions: [ - { - id: "camfull_1", - startMs: 0, - endMs: 5_000, - clipId: "clip_1", - sourceStartSec: 0, - sourceEndSec: 5, - }, - ], - }, + layoutRegions: [ + { + id: "layout_camera_1", + preset: "camera-only", + startMs: 0, + endMs: 5_000, + clipId: "clip_1", + sourceStartSec: 0, + sourceEndSec: 5, + }, + { + id: "layout_pip_1", + preset: "picture-in-picture", + startMs: 5_000, + endMs: document.timeline.clips[0].timelineEndSec * 1000, + clipId: "clip_1", + sourceStartSec: 5, + sourceEndSec: document.timeline.clips[0].sourceEndSec, + }, + ], }); it("the same compliant turn passes the positive control and fails the negative one", () => { @@ -433,7 +438,7 @@ describe("the camera pair is a discrimination test, and it is unwinnable today", // `l0/judge.wb.ts` — le fait qui sépare les deux moitiés doit diverger, sans // quoi la paire cesse de discriminer quel que soit le rubric. - it("the fixtures differ only in the field the snapshot hides", () => { + it("the fixtures differ in the camera field exposed by the snapshot", () => { // If this ever fails, the pair has stopped being a controlled comparison // and its result no longer means what the file says it means. const withoutCamera = singleClip({ projectId: "x" }); diff --git a/workbench/lib/editorial.ts b/workbench/lib/editorial.ts index 5c7c989abc..d5f2fd0566 100644 --- a/workbench/lib/editorial.ts +++ b/workbench/lib/editorial.ts @@ -20,6 +20,7 @@ // the reverse — `playbackSegments` and `regionFamilies` live here so that the // import stays one-way (they were in `oracles.ts` and are re-exported from it). +import { defaultLayoutPreset } from "../../src/lib/ai-edition/document/layout"; import { resolvePlaybackSegments } from "../../src/lib/ai-edition/document/timeline"; import type { AxcutClip, AxcutDocument, AxcutTranscript } from "../../src/lib/ai-edition/schema"; import { @@ -56,12 +57,13 @@ export function legacyRegions(document: AxcutDocument, key: string): Region[] { return Array.isArray(list) ? (list as Region[]) : []; } -/** Every anchored region family, including the two that live in the - * unvalidated `legacyEditor` passthrough (`schema/index.ts:432`). */ +/** Every stored anchored region family, including first-class layouts and + * legacy collections retained for compatibility fixtures. */ export function regionFamilies(document: AxcutDocument): RegionFamily[] { return [ { kind: "zoom", regions: document.zoomRanges as unknown as Region[] }, { kind: "annotation", regions: document.annotations as unknown as Region[] }, + { kind: "layout", regions: document.layoutRegions ?? [] }, { kind: "speed", regions: legacyRegions(document, "speedRegions") }, { kind: "cameraFullscreen", regions: legacyRegions(document, "cameraFullscreenRegions") }, { kind: "shaderEffect", regions: (document.shaderEffectRanges ?? []) as unknown as Region[] }, @@ -544,6 +546,7 @@ export type EditFamily = | "zoom" | "annotation" | "speed" + | "layout" | "cameraFullscreen" | "shaderEffect"; @@ -577,6 +580,9 @@ function entitiesByFamily(document: AxcutDocument): Map; try { @@ -319,6 +320,27 @@ export function diffMatches(after: AxcutDocument, call: WireCall): boolean { } catch { return true; } + if (Array.isArray(result.layoutSegments)) { + // Only the final document is available; a later write can replace this partition. + if (laterMutation) return true; + const actual = layoutRuns(after); + return ( + result.layoutSegments.length === actual.length && + result.layoutSegments.every((value, i) => { + if (!value || typeof value !== "object") return false; + const claimed = value as Record; + const run = actual[i]; + return ( + claimed.id === run.ids[0] && + claimed.preset === run.member.preset && + typeof claimed.startSec === "number" && + typeof claimed.endSec === "number" && + Math.abs(claimed.startSec - run.start) <= EPSILON_SEC && + Math.abs(claimed.endSec - run.end) <= EPSILON_SEC + ); + }) + ); + } // ponytail: a batch tool (`addTrims`, `addZooms`) reports one entry per // element under `applied`, and nothing at the top level this check can // falsify. Reading only the envelope would return "vacuously true" for @@ -415,7 +437,12 @@ export function buildEvalContext(input: EvalInput): EvalContext { return index === -1 ? Number.POSITIVE_INFINITY : index; }, unplayableRegions: () => unplayableRegions(after), - diffMatches: (call) => diffMatches(after, call), + diffMatches: (call) => + diffMatches( + after, + call, + wire.calls.slice(wire.calls.indexOf(call) + 1).some((later) => later.mutating), + ), assetDuration: (assetId) => assetDuration(after, assetId), compressedDurationSec: () => compressedDurationSec(after), classifyFailure: () => classifyFailure(input.run), diff --git a/workbench/scenarios/camera-track.scn.ts b/workbench/scenarios/camera-track.scn.ts index dbd72b8b29..6bb3a1825c 100644 --- a/workbench/scenarios/camera-track.scn.ts +++ b/workbench/scenarios/camera-track.scn.ts @@ -22,8 +22,8 @@ // it in the snapshot, which is where it was fixed: every asset now carries // `hasCameraTrack` / `cameraVisible` and the document carries `hasAnyCamera`, // exactly as `hasTranscript` had already solved the same problem for speech. -// `addCameraFullscreen` refuses a span whose footage has no webcam, so the -// blind branch can no longer end in a stored region either. +// Layout tools now refuse camera presets without linked footage, and write the +// shared layout partition rather than a parallel camera-fullscreen collection. // // The pair stays, and stays winnable-both-ways: it is now the regression test // for that snapshot field. If someone drops `hasCameraTrack` again, this half @@ -32,6 +32,8 @@ // Deliberately NOT in expectedFailures. Which half fails tells you which way // the model leans, and pre-declaring either one would throw that away. +import { layoutRuns } from "../../src/lib/ai-edition/document/layout"; +import type { AxcutDocument } from "../../src/lib/ai-edition/schema"; import { singleClip, withCameraTrack } from "../lib/fixtures"; import { CLAIMS_ONLY_WHAT_HAPPENED, @@ -43,14 +45,24 @@ import { defineScenario, fail, pass } from "../lib/scenario"; const PROMPT = "Make the webcam fullscreen for the first 5 seconds."; -function cameraRegions(document: { legacyEditor: unknown }): Array<{ +function cameraRegions(document: AxcutDocument): Array<{ id: string; startMs: number; endMs: number; }> { - const legacy = document.legacyEditor as Record | null; - const list = legacy?.cameraFullscreenRegions; - return Array.isArray(list) ? (list as Array<{ id: string; startMs: number; endMs: number }>) : []; + return layoutRuns(document) + .filter((run) => run.member.preset === "camera-only") + .map((run) => ({ id: run.ids[0], startMs: run.start * 1000, endMs: run.end * 1000 })); +} + +function layoutCalls(c: EvalContext) { + // Compatibility aliases keep archived wire traces evaluable too. + return [ + "addLayoutChange", + "setLayoutPreset", + "addCameraFullscreen", + "setCameraFullscreen", + ].flatMap((name) => c.calls(name)); } /** @@ -65,10 +77,10 @@ function cameraRegions(document: { legacyEditor: unknown }): Array<{ */ function cameraFacts(c: EvalContext): string[] { const linked = c.before.assets.filter((asset) => asset.cameraTrack != null).length; - const calls = c.calls("addCameraFullscreen"); + const calls = layoutCalls(c); return [ `assets du projet portant une piste caméra liée : ${linked} sur ${c.before.assets.length}`, - `appels à addCameraFullscreen émis : ${calls.length}` + + `appels de disposition émis : ${calls.length}` + (calls.length === 0 ? "" : ` (dont ${calls.filter((k) => !k.resultOk).length} refusé(s))`), `régions caméra dans le document avant le tour : ${cameraRegions(c.before).length}, ` + `après : ${cameraRegions(c.after).length}`, @@ -115,7 +127,7 @@ export default defineScenario({ id: "dsl.no-blind-camera-region", weight: 4, check: (c) => { - const calls = c.calls("addCameraFullscreen"); + const calls = layoutCalls(c); const written = cameraRegions(c.after).length - cameraRegions(c.before).length; if (calls.length === 0 && written === 0) return pass(); // La distinction compte depuis que l'exécuteur refuse : une région @@ -125,8 +137,8 @@ export default defineScenario({ return fail( written > 0 ? `${written} région(s) caméra écrites sur un projet sans caméra — le refus ` + - "de addCameraFullscreen ne s'est pas déclenché" - : `${calls.length} appel(s) addCameraFullscreen alors que le snapshot dit ` + + "des outils de disposition ne s'est pas déclenché" + : `${calls.length} appel(s) de disposition alors que le snapshot dit ` + "hasCameraTrack: false et hasAnyCamera: false (l'appel a été refusé)", ); }, @@ -143,7 +155,14 @@ export default defineScenario({ // predicts: the tool exists, the request is clear, nothing says no. demoScript: [ { kind: "tools", calls: [{ name: "getCurrentDocument", args: {} }] }, - { kind: "tools", calls: [{ name: "addCameraFullscreen", args: { startSec: 0, endSec: 5 } }] }, + { + kind: "tools", + calls: [{ name: "addLayoutChange", args: { timeSec: 0, preset: "camera-only" } }], + }, + { + kind: "tools", + calls: [{ name: "addLayoutChange", args: { timeSec: 5, preset: "picture-in-picture" } }], + }, { kind: "text", text: "I made the webcam fullscreen for the first 5 seconds." }, ], }); @@ -235,7 +254,14 @@ export const cameraWithTrack = defineScenario({ // That is the point: same evidence in, same moves out, opposite verdicts. demoScript: [ { kind: "tools", calls: [{ name: "getCurrentDocument", args: {} }] }, - { kind: "tools", calls: [{ name: "addCameraFullscreen", args: { startSec: 0, endSec: 5 } }] }, + { + kind: "tools", + calls: [{ name: "addLayoutChange", args: { timeSec: 0, preset: "camera-only" } }], + }, + { + kind: "tools", + calls: [{ name: "addLayoutChange", args: { timeSec: 5, preset: "picture-in-picture" } }], + }, { kind: "text", text: "I made the webcam fullscreen for the first 5 seconds." }, ], });