From cb65c22f836e51333f085f5a9e1e2cac6adafbb1 Mon Sep 17 00:00:00 2001 From: thesammykins <26049947+thesammykins@users.noreply.github.com> Date: Fri, 23 Oct 2026 15:07:25 +1100 Subject: [PATCH 1/2] feat(desktop): remove filler words in the native editor Generate local word captions when needed, then apply reversible source-timed cuts with existing ripple helpers. Preserve media, caption visibility and one-step undo, and skip transition handles and held frames. Closes #2960 --- apps/desktop-gpui/README.md | 21 + apps/desktop-gpui/src/app_windows.rs | 14 + apps/desktop-gpui/src/editor_filler.rs | 622 ++++++++++++++++++ .../src/editor_filler/speech_pauses.rs | 292 ++++++++ apps/desktop-gpui/src/editor_sidebar.rs | 2 + apps/desktop-gpui/src/editor_tabs.rs | 170 ++++- apps/desktop-gpui/src/main.rs | 1 + apps/desktop-gpui/src/transcription.rs | 68 +- 8 files changed, 1176 insertions(+), 14 deletions(-) create mode 100644 apps/desktop-gpui/src/editor_filler.rs create mode 100644 apps/desktop-gpui/src/editor_filler/speech_pauses.rs diff --git a/apps/desktop-gpui/README.md b/apps/desktop-gpui/README.md index e54d9bbb361..2f43832c92f 100644 --- a/apps/desktop-gpui/README.md +++ b/apps/desktop-gpui/README.md @@ -10,6 +10,27 @@ This is the app that ships as **Cap**, from 0.6.1 on. The Tauri app in The main recording window uses a fixed 330×432 layout with device controls and recording and screenshot browsers. The expanded view is no longer available. +## Remove filler words + +Open the editor's Captions tab, select English, then choose **Remove Filler +Words**. If word-timed captions are missing, Cap generates them locally using +the selected downloaded model. Cleanup preserves the caption visibility setting +and original media. All cuts form one undo step. + +The action removes whole `um`, `uh`, `uhm`, `er` and `erm` hesitation tokens, +including drawn-out forms, repeated or retimed clips, and ripples timed overlays. +Audio analysis keeps detected pauses after fillers rather than treating the next +word's onset as the filler end. In noisy audio it retains the existing timings. +Words near transition handles or held frames are skipped and reported. Invalid or +overlapping word timings leave the project unchanged. Review playback after +cleanup: transcript-based detection can miss hesitations or misidentify speech. +The resulting clips remain editable: trim clip handles to recover a boundary, +or split and delete a missed filler. Undo restores the complete cleanup in one step. + +For an isolated verification run, `CAP_GPUI_AUTO_SIDEBAR=captions` and +`CAP_GPUI_AUTO_REMOVE_FILLERS=1` invoke the same action after the editor loads. +Use a separate app-data directory, store and copied project as described below. + ## Running it ```sh diff --git a/apps/desktop-gpui/src/app_windows.rs b/apps/desktop-gpui/src/app_windows.rs index 33a5a3ee08b..68a30d7ee0a 100644 --- a/apps/desktop-gpui/src/app_windows.rs +++ b/apps/desktop-gpui/src/app_windows.rs @@ -5555,6 +5555,8 @@ async fn drive_auto_sidebar(handle: WindowHandle, cx: &mut gpui::A let add_track = std::env::var("CAP_GPUI_AUTO_ADD_TRACK").ok(); let generate_zoom = std::env::var("CAP_GPUI_AUTO_GENERATE_ZOOM").is_ok_and(|value| value == "1"); + let remove_fillers = + std::env::var("CAP_GPUI_AUTO_REMOVE_FILLERS").is_ok_and(|value| value == "1"); if tab.is_none() && select.is_none() && canvas.is_none() @@ -5563,6 +5565,7 @@ async fn drive_auto_sidebar(handle: WindowHandle, cx: &mut gpui::A && camera3d.is_none() && add_track.is_none() && !generate_zoom + && !remove_fillers { return; } @@ -5597,6 +5600,17 @@ async fn drive_auto_sidebar(handle: WindowHandle, cx: &mut gpui::A .ok(); } + if remove_fillers { + cx.background_executor() + .timer(std::time::Duration::from_millis(100)) + .await; + handle + .update(cx, |view, window, cx| { + view.remove_filler_words_clicked(window, cx) + }) + .ok(); + } + if let Some(spec) = select { handle .update(cx, |view, _window, cx| view.auto_select_segments(&spec, cx)) diff --git a/apps/desktop-gpui/src/editor_filler.rs b/apps/desktop-gpui/src/editor_filler.rs new file mode 100644 index 00000000000..75c712fbe0c --- /dev/null +++ b/apps/desktop-gpui/src/editor_filler.rs @@ -0,0 +1,622 @@ +use cap_project::{CaptionSegment, CaptionWord, ProjectConfiguration}; + +use crate::{editor_edits, editor_timeline}; + +mod speech_pauses; + +const EDGE_TOLERANCE: f64 = 0.000_001; + +#[derive(Debug, Default)] +pub struct RemovalSummary { + pub removed_words: usize, + pub skipped_words: usize, + pub removed_seconds: f64, +} + +struct Cut { + clip_index: usize, + start: f64, + end: f64, +} + +pub fn has_word_timings(project: &ProjectConfiguration) -> bool { + project.captions.as_ref().is_some_and(|captions| { + captions.source_timed + && captions + .segments + .iter() + .any(|segment| !segment.words.is_empty()) + }) +} + +fn is_filler(text: &str) -> bool { + let token = text.trim().trim_matches(|c: char| { + matches!( + c, + '.' | ',' + | '!' + | '?' + | ';' + | ':' + | '"' + | '\'' + | '“' + | '”' + | '‘' + | '’' + | '(' + | ')' + | '[' + | ']' + ) + }); + if !token.chars().any(char::is_lowercase) { + return false; + } + let token = token.to_ascii_lowercase(); + if token == "err" { + return false; + } + let mut normalized = String::new(); + let mut previous = None; + for letter in token.chars() { + if previous != Some(letter) { + normalized.push(letter); + } + previous = Some(letter); + } + matches!(normalized.as_str(), "um" | "uh" | "uhm" | "er" | "erm") +} + +pub fn refine_word_ends( + segments: &mut [CaptionSegment], + samples: &[f32], + sample_rate: u32, +) -> Result<(), String> { + if sample_rate == 0 || samples.is_empty() { + return Err("The recording has no valid audio for filler timing.".into()); + } + let mut words: Vec<_> = segments.iter().flat_map(|segment| &segment.words).collect(); + validate_words(&mut words, samples.len() as f64 / f64::from(sample_rate))?; + let Some(pauses) = speech_pauses::SpeechPauses::detect(samples, sample_rate) else { + tracing::info!("Filler timing: speech is not clearly above the noise; keeping word ends"); + return Ok(()); + }; + // Parakeet can stretch a word to the next onset. Keep audible release and + // the following pause instead of cutting away both (see Cap PR #1880). + for segment in segments { + for word in &mut segment.words { + if is_filler(&word.text) { + word.end = pauses.audible_end(word.start, word.end); + } + } + } + Ok(()) +} + +fn validate_words(words: &mut [&CaptionWord], recording_duration: f64) -> Result<(), String> { + for word in words.iter() { + if !word.start.is_finite() + || !word.end.is_finite() + || word.start < 0.0 + || word.end <= word.start + || f64::from(word.end) > recording_duration + 0.005 + { + return Err( + "Caption word timings are invalid. Regenerate captions and try again.".into(), + ); + } + } + words.sort_by(|left, right| left.start.total_cmp(&right.start)); + if words.windows(2).any(|pair| pair[0].end > pair[1].start) { + return Err("Caption words overlap. Review their timings before removing fillers.".into()); + } + Ok(()) +} + +pub fn remove_filler_words( + project: &mut ProjectConfiguration, + recording_durations: &[f64], +) -> Result { + if !has_word_timings(project) { + return Err("Generate word-timed captions before removing filler words.".into()); + } + let mut recording_offsets = Vec::with_capacity(recording_durations.len()); + let mut recording_duration = 0.0; + for duration in recording_durations { + if !duration.is_finite() || *duration <= 0.0 { + return Err("The recording has invalid source durations.".into()); + } + recording_offsets.push(recording_duration); + recording_duration += duration; + } + if recording_offsets.is_empty() || !recording_duration.is_finite() { + return Err("The recording has no valid source durations.".into()); + } + let captions = project + .captions + .as_ref() + .expect("word timings were checked"); + let mut words: Vec<&CaptionWord> = captions + .segments + .iter() + .flat_map(|segment| &segment.words) + .collect(); + validate_words(&mut words, recording_duration)?; + let fillers: Vec<_> = words + .into_iter() + .filter(|word| is_filler(&word.text)) + .collect(); + let mut edited = project.clone(); + if !editor_edits::ensure_timeline(&mut edited, recording_durations) { + return Err("The recording has no editable timeline.".into()); + } + let timeline = edited.timeline.as_ref().expect("timeline was ensured"); + let offsets = editor_timeline::clip_timeline_offsets(timeline); + let mut held = 0.0; + let hold_points: Vec<_> = timeline + .hold_windows() + .into_iter() + .map(|(start, end)| { + let effective = start - held; + held += end - start; + effective + }) + .collect(); + let mut cuts: Vec = Vec::new(); + let mut summary = RemovalSummary::default(); + + for (clip_index, clip) in timeline.segments.iter().enumerate() { + let recording = clip.recording_clip as usize; + let Some(source_offset) = recording_offsets.get(recording) else { + return Err("A timeline clip refers to a missing recording.".into()); + }; + if !clip.start.is_finite() + || !clip.end.is_finite() + || !clip.timescale.is_finite() + || clip.timescale <= 0.0 + || clip.start < 0.0 + || clip.end <= clip.start + || clip.end > recording_durations[recording] + 0.005 + { + return Err("The timeline has invalid clip bounds or speed.".into()); + } + let incoming = timeline + .effective_transition(clip_index) + .map_or(0.0, |transition| transition.duration); + let outgoing = timeline + .effective_transition(clip_index + 1) + .map_or(0.0, |transition| transition.duration); + for word in &fillers { + let start = (f64::from(word.start) - source_offset).max(clip.start); + let end = (f64::from(word.end) - source_offset).min(clip.end); + if end <= start { + continue; + } + let local_start = (start - clip.start) / clip.timescale; + let local_end = (end - clip.start) / clip.timescale; + let effective_start = offsets[clip_index] + local_start; + let effective_end = offsets[clip_index] + local_end; + // Keep transition handles and frozen frames intact rather than shortening them implicitly. + if (incoming > 0.0 && local_start < incoming * 2.0 + EDGE_TOLERANCE) + || (outgoing > 0.0 && clip.duration() - local_end < outgoing * 2.0 + EDGE_TOLERANCE) + || hold_points + .iter() + .any(|point| *point >= effective_start && *point < effective_end) + { + summary.skipped_words += 1; + continue; + } + summary.removed_words += 1; + summary.removed_seconds += (end - start) / clip.timescale; + if let Some(previous) = cuts.last_mut() + && previous.clip_index == clip_index + && start <= previous.end + { + previous.end = previous.end.max(end); + } else { + cuts.push(Cut { + clip_index, + start, + end, + }); + } + } + } + if cuts.is_empty() { + return Ok(summary); + } + if timeline.duration() - summary.removed_seconds < 0.02 { + return Err("Removing these fillers would leave no playable recording.".into()); + } + let timeline = edited.timeline.as_mut().expect("timeline was ensured"); + for cut in cuts.into_iter().rev() { + let clip = timeline.segments[cut.clip_index].clone(); + let offset = editor_timeline::clip_timeline_offsets(timeline)[cut.clip_index]; + let holds = timeline.hold_windows(); + if cut.end < clip.end { + let end = editor_edits::effective_to_output_end( + &holds, + offset + (cut.end - clip.start) / clip.timescale, + ); + if !editor_edits::split_clip_segment(timeline, end, Some(cut.clip_index)) { + return Err( + "A filler cut could not be split safely. The project was not changed.".into(), + ); + } + // Preserve exact source boundaries across the source/output round trip, including nonbinary speeds. + timeline.segments[cut.clip_index].end = cut.end; + timeline.segments[cut.clip_index + 1].start = cut.end; + } + let middle = if cut.start > clip.start { + let start = editor_edits::effective_to_output( + &holds, + offset + (cut.start - clip.start) / clip.timescale, + ); + if !editor_edits::split_clip_segment(timeline, start, Some(cut.clip_index)) { + return Err( + "A filler cut could not be split safely. The project was not changed.".into(), + ); + } + timeline.segments[cut.clip_index].end = cut.start; + timeline.segments[cut.clip_index + 1].start = cut.start; + cut.clip_index + 1 + } else { + cut.clip_index + }; + if !editor_edits::delete_clip_segments(timeline, &[middle]) { + return Err( + "A filler cut could not be removed safely. The project was not changed.".into(), + ); + } + } + *project = edited; + Ok(summary) +} + +#[cfg(test)] +mod tests { + use super::*; + use cap_project::{CaptionSegment, CaptionsData}; + + fn project(clips: serde_json::Value, words: &[(&str, f32, f32)]) -> ProjectConfiguration { + let mut project: ProjectConfiguration = serde_json::from_value(serde_json::json!({ + "timeline": { "segments": clips, "zoomSegments": [{ "start": 6.0, "end": 8.0, "amount": 1.5, "mode": "auto" }] } + })).unwrap(); + project.captions = Some(CaptionsData { + source_timed: true, + segments: vec![CaptionSegment { + words: words + .iter() + .map(|(text, start, end)| CaptionWord { + text: text.to_string(), + start: *start, + end: *end, + }) + .collect(), + ..Default::default() + }], + ..Default::default() + }); + project + } + + fn single(words: &[(&str, f32, f32)]) -> ProjectConfiguration { + project( + serde_json::json!([{ "recordingSegment": 0, "start": 0.0, "end": 10.0, "timescale": 1.0 }]), + words, + ) + } + + #[test] + fn whole_english_tokens_keep_acronyms_and_lexical_words() { + for text in [ + "um", "Um,", "ummm", "uh", "uhhh.", "(Uhh)", "uhm", "uhhmm", "er", "Erm,", "errrm", + ] { + assert!(is_filler(text), "{text}"); + } + for text in [ + "UM", "UH", "ER", "UML", "umbrella", "human", "hmm", "mhm", "ah", "like", "uh-oh", + "err", "error", "you know", "u", "ummh", + ] { + assert!(!is_filler(text), "{text}"); + } + } + + #[test] + fn cuts_ripple_overlays_keep_source_words_and_are_idempotent() { + let mut project = single(&[ + ("Hello", 0.0, 1.0), + ("um", 2.0, 3.0), + ("uh", 3.0, 4.0), + ("ending", 8.0, 9.5), + ]); + let transcript = serde_json::to_value(&project.captions).unwrap(); + let original = project.clone(); + let summary = remove_filler_words(&mut project, &[10.0]).unwrap(); + assert_eq!(summary.removed_words, 2); + assert_eq!(summary.removed_seconds, 2.0); + let timeline = project.timeline.as_ref().unwrap(); + assert_eq!( + timeline + .segments + .iter() + .map(|clip| (clip.start, clip.end)) + .collect::>(), + vec![(0.0, 2.0), (4.0, 10.0)] + ); + assert_eq!( + ( + timeline.zoom_segments[0].start, + timeline.zoom_segments[0].end + ), + (4.0, 6.0) + ); + assert_eq!(serde_json::to_value(&project.captions).unwrap(), transcript); + let derived = cap_project::derive_caption_track_segments( + &project.captions.as_ref().unwrap().segments, + timeline, + &[10.0], + ); + assert!( + derived + .iter() + .flat_map(|caption| &caption.words) + .all(|word| !is_filler(&word.text)) + ); + let mut history = editor_edits::ProjectHistory::new(original.clone()); + history.record(&project); + assert_eq!( + serde_json::to_value(history.undo().unwrap()).unwrap(), + serde_json::to_value(original).unwrap() + ); + assert_eq!( + serde_json::to_value(history.redo().unwrap()).unwrap(), + serde_json::to_value(&project).unwrap() + ); + assert_eq!( + remove_filler_words(&mut project, &[10.0]) + .unwrap() + .removed_words, + 0 + ); + } + + #[test] + fn maps_repeated_reordered_sources_and_speed() { + let mut project = project( + serde_json::json!([ + { "recordingSegment": 1, "start": 0.0, "end": 10.0, "timescale": 2.0 }, + { "recordingSegment": 0, "start": 0.0, "end": 10.0, "timescale": 1.0 }, + { "recordingSegment": 1, "start": 0.0, "end": 10.0, "timescale": 1.0 } + ]), + &[("um", 2.0, 3.0), ("uh", 14.0, 16.0)], + ); + let summary = remove_filler_words(&mut project, &[10.0, 10.0]).unwrap(); + assert_eq!(summary.removed_words, 3); + assert_eq!(summary.removed_seconds, 4.0); + let timeline = project.timeline.unwrap(); + assert_eq!(timeline.duration(), 21.0); + assert_eq!( + timeline + .segments + .iter() + .map(|clip| (clip.recording_clip, clip.start, clip.end)) + .collect::>(), + vec![ + (1, 0.0, 4.0), + (1, 6.0, 10.0), + (0, 0.0, 2.0), + (0, 3.0, 10.0), + (1, 0.0, 4.0), + (1, 6.0, 10.0) + ] + ); + } + + #[test] + fn invalid_or_all_filler_transcripts_do_not_change_project() { + for words in [ + vec![("um", 0.0, 10.0)], + vec![("um", 1.0, 3.0), ("speech", 2.0, 4.0)], + vec![("uh", f32::NAN, 4.0)], + ] { + let mut project = single(&words); + let before = serde_json::to_value(&project).unwrap(); + assert!(remove_filler_words(&mut project, &[10.0]).is_err()); + assert_eq!(serde_json::to_value(project).unwrap(), before); + } + } + + #[test] + fn filler_at_clip_edges_preserves_the_complete_ending() { + let mut project = single(&[("um", 0.0, 1.0), ("ending", 8.0, 9.0), ("uh", 9.5, 10.0)]); + assert_eq!( + remove_filler_words(&mut project, &[10.0]) + .unwrap() + .removed_words, + 2 + ); + let clips = &project.timeline.as_ref().unwrap().segments; + assert_eq!(clips.len(), 1); + assert_eq!((clips[0].start, clips[0].end), (1.0, 9.5)); + } + + #[test] + fn transition_handles_are_skipped_but_middle_cuts_keep_the_transition() { + let mut project = project( + serde_json::json!([ + { "recordingSegment": 0, "start": 0.0, "end": 5.0, "timescale": 1.0 }, + { "recordingSegment": 0, "start": 5.0, "end": 10.0, "timescale": 1.0 } + ]), + &[("um", 2.0, 2.5), ("uh", 4.5, 5.5)], + ); + project.timeline.as_mut().unwrap().transitions = + serde_json::from_value(serde_json::json!([ + { "segmentIndex": 1, "duration": 0.5, "type": "cross-fade" } + ])) + .unwrap(); + let summary = remove_filler_words(&mut project, &[10.0]).unwrap(); + assert_eq!(summary.removed_words, 1); + assert_eq!(summary.skipped_words, 2); + let timeline = project.timeline.unwrap(); + assert_eq!(timeline.transitions.len(), 1); + assert_eq!(timeline.transitions[0].duration, 0.5); + assert_eq!(timeline.transitions[0].segment_index, 2); + assert_eq!(timeline.duration(), 9.0); + } + + #[test] + fn held_frames_survive_and_following_overlays_shift_in_output_time() { + let mut project = single(&[("um", 2.0, 3.0), ("uh", 5.0, 6.0)]); + project.timeline.as_mut().unwrap().text_segments = + serde_json::from_value(serde_json::json!([ + { "start": 2.5, "end": 4.5, "content": "Keep this hold", "layout": "fullscreen" } + ])) + .unwrap(); + let summary = remove_filler_words(&mut project, &[10.0]).unwrap(); + assert_eq!(summary.removed_words, 1); + assert_eq!(summary.skipped_words, 1); + let timeline = project.timeline.unwrap(); + assert_eq!(timeline.hold_windows(), vec![(2.5, 4.5)]); + assert_eq!(timeline.duration(), 11.0); + assert_eq!( + ( + timeline.zoom_segments[0].start, + timeline.zoom_segments[0].end + ), + (6.0, 7.0) + ); + } + + #[test] + fn absent_word_timings_and_no_candidates_leave_the_project_unchanged() { + let mut project = single(&[("Hello", 1.0, 2.0)]); + let before = serde_json::to_value(&project).unwrap(); + assert_eq!( + remove_filler_words(&mut project, &[10.0]) + .unwrap() + .removed_words, + 0 + ); + assert_eq!(serde_json::to_value(&project).unwrap(), before); + project.captions.as_mut().unwrap().source_timed = false; + let before = serde_json::to_value(&project).unwrap(); + assert!(remove_filler_words(&mut project, &[10.0]).is_err()); + assert_eq!(serde_json::to_value(project).unwrap(), before); + } + + #[test] + fn fractional_speeds_do_not_leave_filler_slivers_on_repeat() { + for (speed, source_start, filler_end) in [ + (0.7, 1.31, 2.89), + (1.1, 1.31, 2.89), + (1.7, 1.31, 2.89), + (2.3, 1.31, 2.89), + (3.1, 1.31, 2.89), + (3.0681731581839884, 0.9100351728607, 4.141_660_7), + ] { + let mut project = project( + serde_json::json!([ + { "recordingSegment": 0, "start": source_start, "end": 9.87, "timescale": speed } + ]), + &[("um", 2.37, filler_end), ("uh", 5.12, 5.77)], + ); + assert_eq!( + remove_filler_words(&mut project, &[10.0]) + .unwrap() + .removed_words, + 2 + ); + let after = serde_json::to_value(&project).unwrap(); + assert_eq!( + remove_filler_words(&mut project, &[10.0]) + .unwrap() + .removed_words, + 0, + "speed {speed}" + ); + assert_eq!(serde_json::to_value(project).unwrap(), after); + } + } + + #[test] + fn cleanup_keeps_the_audible_release_and_pause_before_the_next_word() { + let mut project = single(&[("Hello", 0.0, 1.0), ("um", 1.0, 3.0), ("next", 3.0, 3.5)]); + let mut samples = vec![0.0; 16_000 * 10]; + for (index, sample) in samples.iter_mut().enumerate() { + let time = index as f32 / 16_000.0; + if (0.1..0.8).contains(&time) + || (1.0..1.5).contains(&time) + || (3.0..3.5).contains(&time) + { + *sample = 0.1 * (std::f32::consts::TAU * 220.0 * time).sin(); + } + } + refine_word_ends( + &mut project.captions.as_mut().unwrap().segments, + &samples, + 16_000, + ) + .unwrap(); + let words = &project.captions.as_ref().unwrap().segments[0].words; + assert!((1.59..1.63).contains(&words[1].end), "{}", words[1].end); + assert_eq!((words[0].start, words[0].end), (0.0, 1.0)); + assert_eq!((words[2].start, words[2].end), (3.0, 3.5)); + let summary = remove_filler_words(&mut project, &[10.0]).unwrap(); + assert_eq!(summary.removed_words, 1); + let clips = &project.timeline.as_ref().unwrap().segments; + assert!((1.59..1.63).contains(&clips[1].start)); + assert!(3.0 - clips[1].start > 1.3); + assert_eq!( + remove_filler_words(&mut project, &[10.0]) + .unwrap() + .removed_words, + 0 + ); + } + + #[test] + fn refinement_does_not_hide_invalid_or_overlapping_input() { + let mut project = single(&[("um", 1.0, 3.0), ("speech", 2.0, 4.0)]); + let segments = &mut project.captions.as_mut().unwrap().segments; + let before = serde_json::to_value(&segments).unwrap(); + assert!(refine_word_ends(segments, &vec![0.0; 16_000 * 10], 16_000).is_err()); + assert_eq!(serde_json::to_value(segments).unwrap(), before); + } + + #[test] + fn cleaned_clips_allow_manual_recovery_split_delete_and_undo() { + let mut project = single(&[("um", 2.0, 3.0), ("hmm", 5.0, 6.0), ("ending", 8.0, 9.0)]); + let original = serde_json::to_value(&project).unwrap(); + let mut history = editor_edits::ProjectHistory::new(project.clone()); + remove_filler_words(&mut project, &[10.0]).unwrap(); + history.record(&project); + let cleaned = serde_json::to_value(&project).unwrap(); + let timeline = project.timeline.as_mut().unwrap(); + assert_eq!( + editor_edits::clip_trim_end(timeline, 0, 3.0, 0.001, &[10.0], 10.0), + Some(3.0) + ); + let recovered_end = + editor_edits::clip_trim_end(timeline, 0, 2.1, 0.001, &[10.0], 10.0).unwrap(); + assert_eq!(recovered_end, 2.1); + timeline.segments[0].end = recovered_end; + assert!(editor_edits::split_clip_segment(timeline, 5.1, Some(1))); + assert!(editor_edits::split_clip_segment(timeline, 4.1, Some(1))); + assert!(editor_edits::delete_clip_segments(timeline, &[2])); + assert_eq!(timeline.segments.len(), 3); + assert_eq!(timeline.segments[1].start, 3.0); + assert!((timeline.segments[1].end - 5.0).abs() < 0.000_001); + assert!((timeline.segments[2].start - 6.0).abs() < 0.000_001); + history.record(&project); + assert_eq!( + serde_json::to_value(history.undo().unwrap()).unwrap(), + cleaned + ); + assert_eq!( + serde_json::to_value(history.undo().unwrap()).unwrap(), + original + ); + } +} diff --git a/apps/desktop-gpui/src/editor_filler/speech_pauses.rs b/apps/desktop-gpui/src/editor_filler/speech_pauses.rs new file mode 100644 index 00000000000..7c96aea66bc --- /dev/null +++ b/apps/desktop-gpui/src/editor_filler/speech_pauses.rs @@ -0,0 +1,292 @@ +use std::ops::Range; + +const FRAMES_PER_SECOND: u32 = 100; +const LEVEL_FLOOR_DB: f32 = -90.0; +const NOISE_PERCENTILE: f32 = 0.1; +const SPEECH_PERCENTILE: f32 = 0.9; +const MIN_SPEECH_ABOVE_NOISE_DB: f32 = 20.0; +const QUIET_ABOVE_NOISE_DB: f32 = 6.0; +const QUIET_BELOW_SPEECH_DB: f32 = 40.0; +const MAX_QUIET_DB: f32 = -45.0; +const SPEECH_BELOW_PEAK_DB: f32 = 20.0; +const MIN_PAUSE_SECS: f32 = 0.2; +const MIN_SPEECH_SECS: f32 = 0.05; +const SPEECH_RELEASE_SECS: f32 = 0.1; +const MIN_WORD_SECS: f32 = 0.1; +const NEXT_ONSET_SECS: f32 = 0.25; + +#[derive(Debug, Clone, PartialEq)] +pub struct SpeechPauses { + frame_secs: f32, + pauses: Vec>, + speech_frames: Vec, +} + +impl SpeechPauses { + pub fn detect(samples: &[f32], sample_rate: u32) -> Option { + let hop = (sample_rate / FRAMES_PER_SECOND) as usize; + if hop == 0 || samples.is_empty() { + return None; + } + + let energies = samples + .chunks(hop) + .map(|frame| { + frame.iter().map(|sample| sample * sample).sum::() / frame.len() as f32 + }) + .collect::>(); + let levels = (0..energies.len()) + .map(|index| { + let window = &energies[index.saturating_sub(1)..(index + 2).min(energies.len())]; + let energy = window.iter().sum::() / window.len() as f32; + (10.0 * energy.log10()).max(LEVEL_FLOOR_DB) + }) + .collect::>(); + + let mut sorted = levels.clone(); + sorted.sort_by(f32::total_cmp); + let audible = &sorted[sorted.partition_point(|level| *level <= LEVEL_FLOOR_DB)..]; + let noise = percentile(&sorted, NOISE_PERCENTILE)?; + let speech = percentile(audible, SPEECH_PERCENTILE)?; + if speech - noise < MIN_SPEECH_ABOVE_NOISE_DB { + return None; + } + + // Noise-gated mics leave digital silence in pauses, which pins the + // noise floor at the bottom of the scale while the gate's tail after + // each word still sits far above it; following the speech level too + // keeps that tail out of the word. + let quiet = (noise + QUIET_ABOVE_NOISE_DB) + .max(speech - QUIET_BELOW_SPEECH_DB) + .min(MAX_QUIET_DB); + let loud = speech - SPEECH_BELOW_PEAK_DB; + + let frame_secs = hop as f32 / sample_rate as f32; + let min_pause_frames = (MIN_PAUSE_SECS / frame_secs).round() as usize; + let mut pauses = Vec::new(); + let mut quiet_from = None; + for (index, level) in levels.iter().chain([&f32::INFINITY]).enumerate() { + match (*level < quiet, quiet_from) { + (true, None) => quiet_from = Some(index), + (false, Some(from)) => { + if index - from >= min_pause_frames { + pauses.push(from as f32 * frame_secs..index as f32 * frame_secs); + } + quiet_from = None; + } + _ => {} + } + } + + let speech_frames = std::iter::once(0) + .chain(levels.iter().scan(0, |count, level| { + *count += u32::from(*level > loud); + Some(*count) + })) + .collect(); + + Some(Self { + frame_secs, + pauses, + speech_frames, + }) + } + + // parakeet-rs ends each word where the next one starts, so a word's end + // runs through the pause after it; only the audio shows where speech + // stopped. Its word starts drift ~0.2s either way (sometimes more), so the + // next word may already be audible in the last moments before the stored + // end, and a word's own speech may only begin after a pause inside its + // span. Breaths and clicks in a pause stay below speech level. + pub fn audible_end(&self, start: f32, end: f32) -> f32 { + let first = self.pauses.partition_point(|pause| pause.start <= start); + for pause in self.pauses[first..] + .iter() + .take_while(|pause| pause.start < end) + { + let next_onset = (end - NEXT_ONSET_SECS).max(pause.end); + if self.speech_secs(pause.end..next_onset) >= MIN_SPEECH_SECS { + continue; + } + let late = self.speech_secs(next_onset..end); + if late >= MIN_SPEECH_SECS && late > self.speech_secs(start..pause.start) { + continue; + } + return end.min((pause.start + SPEECH_RELEASE_SECS).max(start + MIN_WORD_SECS)); + } + + if first + .checked_sub(1) + .is_some_and(|index| self.pauses[index].end >= end) + { + return end.min(start + MIN_WORD_SECS); + } + + end + } + + fn speech_secs(&self, range: Range) -> f32 { + let last_frame = self.speech_frames.len() - 1; + let frame = + |time: f32| ((time / self.frame_secs).round().max(0.0) as usize).min(last_frame); + let (first, last) = (frame(range.start), frame(range.end)); + if last <= first { + return 0.0; + } + (self.speech_frames[last] - self.speech_frames[first]) as f32 * self.frame_secs + } +} + +fn percentile(sorted: &[f32], fraction: f32) -> Option { + let last = sorted.len().checked_sub(1)?; + Some(sorted[((sorted.len() as f32 * fraction) as usize).min(last)]) +} + +#[cfg(test)] +mod tests { + use super::SpeechPauses; + use std::ops::Range; + + const RATE: u32 = 16_000; + + fn silence(secs: f32) -> Vec { + vec![0.0; (secs * RATE as f32) as usize] + } + + fn sample_range(samples: &[f32], range: Range) -> Range { + let index = |time: f32| ((time * RATE as f32) as usize).min(samples.len()); + index(range.start)..index(range.end) + } + + fn hiss(samples: &mut [f32], range: Range, db: f32) { + let amplitude = 10f32.powf(db / 20.0) * 3f32.sqrt(); + let mut state = 0x2545_f491_u32 ^ range.start.to_bits(); + for index in sample_range(samples, range) { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + samples[index] += (state as f32 / u32::MAX as f32 * 2.0 - 1.0) * amplitude; + } + } + + fn tone(samples: &mut [f32], range: Range, db: f32) { + let amplitude = 10f32.powf(db / 20.0) * 2f32.sqrt(); + for index in sample_range(samples, range) { + let time = index as f32 / RATE as f32; + samples[index] += amplitude * (std::f32::consts::TAU * 220.0 * time).sin(); + } + } + + fn speech(noise_db: f32, speech_db: f32, words: &[Range]) -> Vec { + let mut samples = silence(5.0); + hiss(&mut samples, 0.0..5.0, noise_db); + for word in words { + tone(&mut samples, word.clone(), speech_db); + } + samples + } + + fn pauses(samples: &[f32]) -> SpeechPauses { + SpeechPauses::detect(samples, RATE).expect("speech stands above the noise") + } + + fn assert_close(actual: f32, expected: f32) { + assert!( + (actual - expected).abs() <= 0.02, + "{actual} is not within 20ms of {expected}" + ); + } + + #[test] + fn stretched_word_ends_where_its_speech_stops() { + let pauses = pauses(&speech(-60.0, -20.0, &[1.0..1.6, 3.0..3.5])); + + assert_close(pauses.audible_end(1.0, 3.0), 1.7); + assert_eq!(pauses.audible_end(3.0, 3.5), 3.5); + } + + #[test] + fn word_followed_immediately_by_speech_keeps_its_end() { + let pauses = pauses(&speech(-60.0, -20.0, &[1.0..1.47, 1.53..2.0, 3.5..4.0])); + + assert_eq!(pauses.audible_end(1.0, 1.5), 1.5); + assert_eq!(pauses.audible_end(1.5, 2.0), 2.0); + } + + #[test] + fn next_word_heard_just_before_its_start_still_ends_at_the_pause() { + let pauses = pauses(&speech(-60.0, -20.0, &[1.0..1.6, 2.8..3.3])); + + assert_close(pauses.audible_end(1.0, 3.0), 1.7); + assert_eq!(pauses.audible_end(1.0, 3.2), 3.2); + } + + #[test] + fn speech_after_a_pause_inside_the_word_keeps_its_end() { + let pauses = pauses(&speech(-60.0, -20.0, &[1.0..1.3, 1.6..2.4])); + + assert_eq!(pauses.audible_end(1.0, 2.0), 2.0); + } + + #[test] + fn short_word_heard_after_a_pause_keeps_its_end() { + let pauses = pauses(&speech(-60.0, -20.0, &[1.0..2.05, 2.3..2.45, 2.5..3.0])); + + assert_eq!(pauses.audible_end(2.0, 2.5), 2.5); + } + + #[test] + fn clicks_and_breaths_in_a_pause_do_not_hold_the_word() { + let mut samples = speech(-60.0, -20.0, &[1.0..1.5, 4.0..4.5]); + tone(&mut samples, 2.0..2.02, -30.0); + hiss(&mut samples, 2.5..2.8, -50.0); + let pauses = pauses(&samples); + + assert_close(pauses.audible_end(1.0, 3.5), 1.6); + } + + #[test] + fn noise_gate_tail_after_a_word_counts_as_quiet() { + let mut samples = silence(5.0); + tone(&mut samples, 1.0..1.6, -20.0); + hiss(&mut samples, 1.6..1.9, -65.0); + tone(&mut samples, 3.0..3.5, -20.0); + let pauses = pauses(&samples); + + assert_close(pauses.audible_end(1.0, 3.0), 1.7); + } + + #[test] + fn word_inside_a_pause_keeps_a_sliver_from_its_start() { + let pauses = pauses(&speech(-60.0, -20.0, &[1.0..1.5, 4.0..4.5])); + + assert_close(pauses.audible_end(2.0, 3.0), 2.1); + } + + #[test] + fn pause_before_the_words_own_speech_keeps_its_end() { + let pauses = pauses(&speech(-60.0, -20.0, &[1.0..1.5, 2.2..2.6])); + + assert_eq!(pauses.audible_end(2.0, 2.4), 2.4); + } + + #[test] + fn threshold_follows_the_clips_noise_floor() { + let quiet = pauses(&speech(-85.0, -45.0, &[1.0..1.6, 3.0..3.5])); + let noisy = pauses(&speech(-50.0, -15.0, &[1.0..1.6, 3.0..3.5])); + + assert_close(quiet.audible_end(1.0, 3.0), 1.7); + assert_close(noisy.audible_end(1.0, 3.0), 1.7); + } + + #[test] + fn noisy_or_quiet_clips_are_left_alone() { + let words = [1.0..1.6, 3.0..3.5]; + + assert!(SpeechPauses::detect(&speech(-30.0, -20.0, &words), RATE).is_none()); + assert!(SpeechPauses::detect(&speech(-40.0, -30.0, &words), RATE).is_none()); + assert!(SpeechPauses::detect(&speech(-70.0, -70.0, &[]), RATE).is_none()); + assert!(SpeechPauses::detect(&silence(5.0), RATE).is_none()); + assert!(SpeechPauses::detect(&[], RATE).is_none()); + } +} diff --git a/apps/desktop-gpui/src/editor_sidebar.rs b/apps/desktop-gpui/src/editor_sidebar.rs index d11bb7172e3..f1b3dd9c26a 100644 --- a/apps/desktop-gpui/src/editor_sidebar.rs +++ b/apps/desktop-gpui/src/editor_sidebar.rs @@ -827,6 +827,7 @@ pub struct SidebarState { /// state in the source too, not project config. pub caption_model: &'static str, pub caption_language: &'static str, + pub filler_removal_status: Option>, /// The text panel's Style chip row: 0 is "All", `n` is /// `TEXT_PRESET_GROUPS[n - 1]`. Local UI state, like the source's own /// signal, so it survives selecting another text segment. @@ -911,6 +912,7 @@ impl SidebarState { menu: None, caption_model: "best", caption_language: "auto", + filler_removal_status: None, text_style_group: 0, text_anim_edge: false, editing_end_pose: false, diff --git a/apps/desktop-gpui/src/editor_tabs.rs b/apps/desktop-gpui/src/editor_tabs.rs index a793647488b..9d50f2c7505 100644 --- a/apps/desktop-gpui/src/editor_tabs.rs +++ b/apps/desktop-gpui/src/editor_tabs.rs @@ -36,6 +36,7 @@ use serde_json::Value; use crate::{ editor_color::GradeTarget, + editor_filler, editor_sidebar::{SliderKey, collapsible, dashed_divider}, editor_window::EditorWindow, store, @@ -2893,6 +2894,55 @@ impl EditorWindow { ui::Field::section(&theme, "Captions") .badge("Beta") .child(transcription) + .child( + ui::Field::section(&theme, "Filler Words").child( + div() + .flex() + .flex_col() + .gap(px(8.)) + .child( + div() + .text_size(px(12.)) + .text_color(Hsla::from(theme.gray_11)) + .child("Remove English hesitation words such as ‘um’, ‘uh’ and ‘erm’. Captions are generated locally when needed. Review or adjust cuts on the timeline; Undo restores them."), + ) + .child( + ui::Button::plain( + &theme, + "caption-remove-fillers", + ui::ButtonVariant::Gray, + ui::ButtonSize::Md, + ) + .label("Remove Filler Words") + .full_width() + .disabled( + !self.project_ready() + || generating + || any_deleting + || !has_audio + || self.sidebar.caption_language != "en" + || (!editor_filler::has_word_timings(&self.project) + && !model_downloaded), + ) + .on_click(cx.listener(|this, _, window, cx| { + this.remove_filler_words_clicked(window, cx); + })), + ) + .children((self.sidebar.caption_language != "en").then(|| { + div() + .text_size(px(12.)) + .text_color(Hsla::from(theme.gray_11)) + .child("Select English above to use filler removal.") + })) + .children(self.sidebar.filler_removal_status.as_ref().map(|status| { + let (message, color) = match status { + Ok(message) => (message.clone(), Hsla::from(theme.gray_11)), + Err(message) => (message.clone(), error_color), + }; + div().text_size(px(12.)).text_color(color).child(message) + })), + ), + ) .child(style.mt(px(8.))) .into_any_element() } @@ -3171,6 +3221,95 @@ impl EditorWindow { /// tokio blocking pool, then apply the result through one `edit_project` /// write so a regenerate is a single undo entry. fn generate_captions_clicked(&mut self, window: &mut Window, cx: &mut Context) { + self.start_caption_generation(false, window, cx); + } + + pub(crate) fn remove_filler_words_clicked( + &mut self, + window: &mut Window, + cx: &mut Context, + ) { + if !self.project_ready() + || self.sidebar.caption_language != "en" + || transcription::ui_snapshot(&self.project_path).generating + { + return; + } + self.sidebar.filler_removal_status = None; + self.start_caption_generation(true, window, cx); + } + + fn apply_filler_removal( + &mut self, + segments: Vec, + window: &mut Window, + cx: &mut Context, + ) { + let Some(summary) = self.summary() else { + return; + }; + let recording_durations = summary.clip_display_durations.clone(); + let recording_duration = summary.recording_duration; + let mut edited = self.project.clone(); + let captions_enabled = edited + .captions + .as_ref() + .is_some_and(|captions| captions.settings.enabled); + transcription::apply_caption_result( + &mut edited, + segments, + &recording_durations, + recording_duration, + ); + if let Some(captions) = edited.captions.as_mut() { + captions.settings.enabled = captions_enabled; + } + match editor_filler::remove_filler_words(&mut edited, &recording_durations) { + Ok(summary) => { + if serde_json::to_value(&edited).ok() != serde_json::to_value(&self.project).ok() { + self.stop_playback(cx); + self.selection = None; + self.canvas_selection = None; + self.edit_project("remove-filler-words", window, cx, move |project| { + *project = edited; + true + }); + } + let mut message = if summary.removed_words == 0 { + "No removable filler words found.".to_string() + } else { + format!( + "Removed {} filler {} ({:.1}s).", + summary.removed_words, + if summary.removed_words == 1 { + "word" + } else { + "words" + }, + summary.removed_seconds + ) + }; + if summary.skipped_words > 0 { + message.push_str(&format!( + " Skipped {} near transitions or held frames.", + summary.skipped_words + )); + } + self.sidebar.filler_removal_status = Some(Ok(message)); + } + Err(error) => { + self.sidebar.filler_removal_status = Some(Err(error)); + } + } + cx.notify(); + } + + fn start_caption_generation( + &mut self, + remove_fillers: bool, + window: &mut Window, + cx: &mut Context, + ) { let Some(summary) = self.summary() else { return; }; @@ -3179,6 +3318,22 @@ impl EditorWindow { let project_path = self.project_path.clone(); let model = self.selected_caption_model().name; let language = self.sidebar.caption_language; + let existing = + (remove_fillers && editor_filler::has_word_timings(&self.project)).then(|| { + self.project + .captions + .as_ref() + .expect("word timings were checked") + .segments + .clone() + }); + let before = match serde_json::to_value(&self.project) { + Ok(value) => value, + Err(error) => { + tracing::error!("could not snapshot project before transcription: {error}"); + return; + } + }; if !transcription::begin_generation(&project_path) { return; @@ -3189,7 +3344,11 @@ impl EditorWindow { let task_path = project_path.clone(); let task = gpui_tokio::Tokio::spawn(cx, async move { tokio::task::spawn_blocking(move || { - transcription::transcribe_blocking(&task_path, model, language) + if remove_fillers { + transcription::prepare_filler_captions_blocking(&task_path, model, language, existing) + } else { + transcription::transcribe_blocking(&task_path, model, language) + } }) .await .map_err(|error| format!("Transcription task panicked: {error}"))? @@ -3203,6 +3362,15 @@ impl EditorWindow { Ok(segments) => { transcription::finish_generation(&project_path, None); let _ = this.update_in(cx, |this, window, cx| { + if remove_fillers { + if !this.project_ready() || serde_json::to_value(&this.project).ok().as_ref() != Some(&before) { + this.sidebar.filler_removal_status = Some(Err("The project changed while captions were being generated. Try filler removal again.".into())); + cx.notify(); + return; + } + this.apply_filler_removal(segments, window, cx); + return; + } this.apply_generated_captions( segments, &recording_durations, diff --git a/apps/desktop-gpui/src/main.rs b/apps/desktop-gpui/src/main.rs index 9e886003506..ecffd4b74c5 100644 --- a/apps/desktop-gpui/src/main.rs +++ b/apps/desktop-gpui/src/main.rs @@ -30,6 +30,7 @@ mod editor_color; mod editor_crop; mod editor_edits; mod editor_export; +mod editor_filler; mod editor_hover_bench; #[cfg(target_os = "linux")] mod editor_modal; diff --git a/apps/desktop-gpui/src/transcription.rs b/apps/desktop-gpui/src/transcription.rs index 638b535cf42..27f4965da2c 100644 --- a/apps/desktop-gpui/src/transcription.rs +++ b/apps/desktop-gpui/src/transcription.rs @@ -1753,20 +1753,62 @@ pub fn transcribe_blocking( } transcribe_project_with(project_path, |audio_path| { - let _guard = lock_transcription_worker_slot(); - if is_parakeet_model(model) { - tracing::info!("Using Parakeet TDT engine"); - let result = process_with_parakeet(audio_path, model_file.to_string_lossy().as_ref()); - schedule_parakeet_release(); - result - } else { - tracing::info!("Using Whisper engine"); - let hints = transcription_hints(); - let context = get_whisper_context(model_file.to_string_lossy().as_ref())?; - let result = process_with_whisper(audio_path, context, language, &hints); - release_whisper_context_after_transcription(); - result + transcribe_audio_with_model(audio_path, model, language) + }) +} + +fn transcribe_audio_with_model( + audio_path: &Path, + model: &str, + language: &str, +) -> Result, String> { + let model_file = model_path(model); + let _guard = lock_transcription_worker_slot(); + if is_parakeet_model(model) { + tracing::info!("Using Parakeet TDT engine"); + let result = process_with_parakeet(audio_path, model_file.to_string_lossy().as_ref()); + schedule_parakeet_release(); + result + } else { + tracing::info!("Using Whisper engine"); + let hints = transcription_hints(); + let context = get_whisper_context(model_file.to_string_lossy().as_ref())?; + let result = process_with_whisper(audio_path, context, language, &hints); + release_whisper_context_after_transcription(); + result + } +} + +pub fn prepare_filler_captions_blocking( + project_path: &Path, + model: &str, + language: &str, + existing: Option>, +) -> Result, String> { + if existing.is_none() && !model_files_present(model) { + return Err("Download a transcription model before removing filler words.".into()); + } + transcribe_project_with(project_path, |audio_path| { + let mut segments = match existing { + Some(segments) => segments, + None => transcribe_audio_with_model(audio_path, model, language)?, + }; + let mut reader = hound::WavReader::open(audio_path) + .map_err(|error| format!("Failed to open cleanup audio: {error}"))?; + let spec = reader.spec(); + if spec.channels != 1 + || spec.sample_rate != WHISPER_SAMPLE_RATE + || spec.bits_per_sample != 16 + { + return Err("Expected 16 kHz mono PCM audio for filler timing.".into()); } + let samples = reader + .samples::() + .map(|sample| sample.map(|sample| f32::from(sample) / 32768.0)) + .collect::, _>>() + .map_err(|error| format!("Failed to read cleanup audio: {error}"))?; + crate::editor_filler::refine_word_ends(&mut segments, &samples, spec.sample_rate)?; + Ok(segments) }) } From c80cb817ae4ff95622642f18053eb9d934fee7ce Mon Sep 17 00:00:00 2001 From: thesammykins <26049947+thesammykins@users.noreply.github.com> Date: Fri, 23 Oct 2026 15:55:46 +1100 Subject: [PATCH 2/2] docs(desktop): remove filler cleanup section from readme --- apps/desktop-gpui/README.md | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/apps/desktop-gpui/README.md b/apps/desktop-gpui/README.md index 2f43832c92f..e54d9bbb361 100644 --- a/apps/desktop-gpui/README.md +++ b/apps/desktop-gpui/README.md @@ -10,27 +10,6 @@ This is the app that ships as **Cap**, from 0.6.1 on. The Tauri app in The main recording window uses a fixed 330×432 layout with device controls and recording and screenshot browsers. The expanded view is no longer available. -## Remove filler words - -Open the editor's Captions tab, select English, then choose **Remove Filler -Words**. If word-timed captions are missing, Cap generates them locally using -the selected downloaded model. Cleanup preserves the caption visibility setting -and original media. All cuts form one undo step. - -The action removes whole `um`, `uh`, `uhm`, `er` and `erm` hesitation tokens, -including drawn-out forms, repeated or retimed clips, and ripples timed overlays. -Audio analysis keeps detected pauses after fillers rather than treating the next -word's onset as the filler end. In noisy audio it retains the existing timings. -Words near transition handles or held frames are skipped and reported. Invalid or -overlapping word timings leave the project unchanged. Review playback after -cleanup: transcript-based detection can miss hesitations or misidentify speech. -The resulting clips remain editable: trim clip handles to recover a boundary, -or split and delete a missed filler. Undo restores the complete cleanup in one step. - -For an isolated verification run, `CAP_GPUI_AUTO_SIDEBAR=captions` and -`CAP_GPUI_AUTO_REMOVE_FILLERS=1` invoke the same action after the editor loads. -Use a separate app-data directory, store and copied project as described below. - ## Running it ```sh