From 65d938d408ff642cd9b1d7f830588206d119a5f6 Mon Sep 17 00:00:00 2001 From: Vagarth Date: Thu, 15 Oct 2026 18:45:29 +0530 Subject: [PATCH 1/2] perf(export): encode MP4 export audio with AudioToolbox on macOS FFmpeg's built-in AAC encoder is the busiest CPU thread in an MP4 export. On macOS, exports now encode audio with AudioToolbox (`aac_at`, already in the bundled FFmpeg) in average-bitrate mode and fall back to FFmpeg's encoder if it fails to open. Recording paths keep FFmpeg's encoder: their fragmented MP4s keep encoder priming in the timeline, and AudioToolbox primes 2112 samples instead of 1024. libavcodec pads AudioToolbox's final frame to a full frame and its packet durations count the padding, so the audio track is trimmed back to the end of the input. Encoders that accept a short final frame (FFmpeg `aac`, `libopus`) are not affected. Adds an export benchmark example to the GPUI workspace that exports the way the export page does. --- .../desktop-gpui/examples/export-benchmark.rs | 134 ++++++++ crates/enc-ffmpeg/src/audio/aac.rs | 303 +++++++++++++++++- crates/enc-ffmpeg/src/audio/base.rs | 67 +++- crates/export/src/mp4.rs | 2 +- 4 files changed, 488 insertions(+), 18 deletions(-) create mode 100644 apps/desktop-gpui/examples/export-benchmark.rs diff --git a/apps/desktop-gpui/examples/export-benchmark.rs b/apps/desktop-gpui/examples/export-benchmark.rs new file mode 100644 index 00000000000..4c10debc300 --- /dev/null +++ b/apps/desktop-gpui/examples/export-benchmark.rs @@ -0,0 +1,134 @@ +//! Exports a project to MP4 the way the GPUI export page does (`run_export` in +//! `src/editor_export.rs`) and reports wall time, frames per second and process +//! CPU time. +//! +//! Run: cargo run --release --example export-benchmark -- [WIDTH] [HEIGHT] [FPS] [RUNS] + +use std::{ + path::PathBuf, + sync::{Arc, atomic::AtomicBool}, + time::Instant, +}; + +use cap_export::{ + ExporterBase, + mp4::{ExportCompression, Mp4ExportSettings}, +}; +use cap_project::XY; + +#[cfg(unix)] +fn cpu_secs() -> f64 { + let mut usage = std::mem::MaybeUninit::::zeroed(); + unsafe { libc::getrusage(libc::RUSAGE_SELF, usage.as_mut_ptr()) }; + let usage = unsafe { usage.assume_init() }; + let secs = |time: libc::timeval| time.tv_sec as f64 + time.tv_usec as f64 / 1_000_000.0; + secs(usage.ru_utime) + secs(usage.ru_stime) +} + +#[cfg(windows)] +fn cpu_secs() -> f64 { + use windows_sys::Win32::{ + Foundation::FILETIME, + System::Threading::{GetCurrentProcess, GetProcessTimes}, + }; + + let empty = FILETIME { + dwLowDateTime: 0, + dwHighDateTime: 0, + }; + let (mut creation, mut exit, mut kernel, mut user) = (empty, empty, empty, empty); + unsafe { + GetProcessTimes( + GetCurrentProcess(), + &mut creation, + &mut exit, + &mut kernel, + &mut user, + ) + }; + let secs = |time: FILETIME| { + ((u64::from(time.dwHighDateTime) << 32) | u64::from(time.dwLowDateTime)) as f64 + / 10_000_000.0 + }; + secs(kernel) + secs(user) +} + +async fn run(project_path: PathBuf, size: XY, fps: u32, runs: u32) -> anyhow::Result<()> { + let output_dir = std::env::temp_dir().join("cap-export-benchmark"); + std::fs::create_dir_all(&output_dir)?; + for run in 0..runs { + let output_path = output_dir.join(format!("run-{run}.mp4")); + let _ = std::fs::remove_file(&output_path); + let cancel = Arc::new(AtomicBool::new(false)); + let started = Instant::now(); + let cpu_started = cpu_secs(); + let base = ExporterBase::builder(project_path.clone()) + .with_output_path(output_path.clone()) + .build_for_mp4(cancel) + .await + .map_err(|error| anyhow::anyhow!("{error}"))?; + let total = base.total_frames(fps); + let prepared_ms = started.elapsed().as_secs_f64() * 1000.0; + let path = Mp4ExportSettings { + fps, + resolution_base: size, + compression: ExportCompression::Maximum, + custom_bpp: None, + force_ffmpeg_decoder: false, + optimize_filesize: false, + } + .export_prepared(base, |_| true) + .await + .map_err(|error| anyhow::anyhow!("{error}"))?; + let wall = started.elapsed().as_secs_f64(); + let cpu = cpu_secs() - cpu_started; + let bytes = std::fs::metadata(&path).map(|meta| meta.len()).unwrap_or(0); + println!( + "{}", + serde_json::json!({ + "run": run, + "frames": total, + "width": size.x, + "height": size.y, + "fps": fps, + "prepareMs": prepared_ms, + "wallSecs": wall, + "exportFps": f64::from(total) / wall, + "cpuSecs": cpu, + "bytes": bytes, + }) + ); + } + Ok(()) +} + +fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "warn".into()), + ) + .with_writer(std::io::stderr) + .init(); + + let mut arguments = std::env::args().skip(1); + let project_path = arguments + .next() + .map(PathBuf::from) + .ok_or_else(|| anyhow::anyhow!("Pass a studio .cap project path"))?; + let mut number = |default: u32| { + arguments + .next() + .and_then(|value| value.parse().ok()) + .unwrap_or(default) + }; + let size = XY::new(number(1920), number(1080)); + let fps = number(60); + let runs = number(3).max(1); + + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .thread_stack_size(16 * 1024 * 1024) + .enable_all() + .build()? + .block_on(run(project_path, size, fps, runs)) +} diff --git a/crates/enc-ffmpeg/src/audio/aac.rs b/crates/enc-ffmpeg/src/audio/aac.rs index 89ce84fcd3a..77a9aef7463 100644 --- a/crates/enc-ffmpeg/src/audio/aac.rs +++ b/crates/enc-ffmpeg/src/audio/aac.rs @@ -44,7 +44,41 @@ impl AACEncoder { input_config: AudioInfo, output: &mut format::context::Output, ) -> Result { - let codec = encoder::find_by_name("aac").ok_or(AACEncoderError::CodecNotFound)?; + Self::init_with_codec("aac", ffmpeg::Dictionary::new(), input_config, output) + } + + // A plain MP4's edit list trims AudioToolbox's 2112-sample encoder priming. + // Fragmented recording outputs keep priming in the timeline, so they stay on + // FFmpeg's encoder: AudioToolbox would start their audio ~23 ms later. + pub fn init_for_export( + input_config: AudioInfo, + output: &mut format::context::Output, + ) -> Result { + #[cfg(target_os = "macos")] + { + // AudioToolbox defaults to constant bitrate, which fills silence up to + // the full 320 kbps; average bitrate keeps silent stretches near-free. + let mut options = ffmpeg::Dictionary::new(); + options.set("aac_at_mode", "abr"); + match Self::init_with_codec("aac_at", options, input_config, output) { + Ok(encoder) => return Ok(encoder), + Err(error) => { + tracing::warn!(%error, "AudioToolbox AAC encoder unavailable, using FFmpeg's"); + } + } + } + + Self::init(input_config, output) + } + + fn init_with_codec( + name: &str, + options: ffmpeg::Dictionary, + input_config: AudioInfo, + output: &mut format::context::Output, + ) -> Result { + let codec = encoder::find_by_name(name).ok_or(AACEncoderError::CodecNotFound)?; + let audio_codec = codec.audio()?; let mut encoder_ctx = context::Context::new_with_codec(codec); let thread_count = thread::available_parallelism() .map(|v| v.get()) @@ -53,27 +87,41 @@ impl AACEncoder { let mut encoder = encoder_ctx.encoder().audio()?; let rate = { - let mut rates = codec - .audio() - .unwrap() + let mut rates = audio_codec .rates() .into_iter() .flatten() .collect::>(); rates.sort(); - let Some(&rate) = rates - .iter() - .find(|r| **r >= input_config.rate()) - .or(rates.first()) - else { - return Err(AACEncoderError::RateNotSupported(input_config.rate())); - }; - rate + if rates.is_empty() { + input_config.rate() + } else { + let Some(&rate) = rates + .iter() + .find(|r| **r >= input_config.rate()) + .or(rates.first()) + else { + return Err(AACEncoderError::RateNotSupported(input_config.rate())); + }; + rate + } + }; + + let sample_format = { + let formats = audio_codec + .formats() + .into_iter() + .flatten() + .collect::>(); + match formats.first() { + Some(&first) if !formats.contains(&Self::SAMPLE_FORMAT) => first, + _ => Self::SAMPLE_FORMAT, + } }; let mut output_config = input_config; - output_config.sample_format = Self::SAMPLE_FORMAT; + output_config.sample_format = sample_format; output_config.sample_rate = rate as u32; let resampler = BufferedResampler::new(input_config, output_config) @@ -85,7 +133,7 @@ impl AACEncoder { encoder.set_channel_layout(output_config.channel_layout()); encoder.set_time_base(FFRational(1, output_config.rate())); - let encoder = encoder.open()?; + let encoder = encoder.open_with(options)?; let mut output_stream = output.add_stream(codec)?; output_stream.set_time_base(FFRational(1, output_config.rate())); @@ -287,6 +335,233 @@ mod tests { assert_eq!(encode_audio(true), encode_audio_through_queue()); } + const CLICK_AT: usize = 24_000; + + fn click_frame(start: usize, samples: usize) -> frame::Audio { + let mut audio = + frame::Audio::new(Sample::F32(Type::Packed), samples, ChannelLayout::STEREO); + audio.set_rate(48_000); + audio.set_pts(Some(start as i64)); + for (index, sample) in audio.data_mut(0)[..samples * 2 * size_of::()] + .chunks_exact_mut(2 * size_of::()) + .enumerate() + { + let position = start + index; + let value = if (CLICK_AT..CLICK_AT + 48).contains(&position) { + 0.9_f32 + } else { + 0.0 + }; + sample[..4].copy_from_slice(&value.to_le_bytes()); + sample[4..].copy_from_slice(&value.to_le_bytes()); + } + audio + } + + fn encode_click_for_export(path: &std::path::Path) -> i32 { + let mut output = format::output(path).unwrap(); + let mut encoder = AACEncoder::init_for_export( + AudioInfo::new_raw(Sample::F32(Type::Packed), 48_000, 2), + &mut output, + ) + .unwrap(); + let initial_padding = unsafe { + (*output + .stream(encoder.stream_index()) + .unwrap() + .parameters() + .as_ptr()) + .initial_padding + }; + output.write_header().unwrap(); + + let mut position = 0; + while position < 48_000 { + let samples = 960.min(48_000 - position); + encoder + .send_frame(click_frame(position, samples), Duration::MAX, &mut output) + .unwrap(); + position += samples; + } + + encoder.flush(&mut output).unwrap(); + output.write_trailer().unwrap(); + initial_padding + } + + fn decode_first_channel(path: &std::path::Path) -> (Vec, i64) { + use ffmpeg::Rescale; + + let mut input = format::input(path).unwrap(); + let stream = input.streams().best(ffmpeg::media::Type::Audio).unwrap(); + let stream_index = stream.index(); + let time_base = stream.time_base(); + let mut decoder = context::Context::from_parameters(stream.parameters()) + .unwrap() + .decoder() + .audio() + .unwrap(); + let mut decoded = frame::Audio::empty(); + let mut samples = Vec::new(); + let mut track_end = i64::MIN; + + for (stream, packet) in input.packets() { + if stream.index() != stream_index { + continue; + } + if let Some(pts) = packet.pts() { + track_end = + track_end.max((pts + packet.duration()).rescale(time_base, (1, 48_000))); + } + decoder.send_packet(&packet).unwrap(); + while decoder.receive_frame(&mut decoded).is_ok() { + samples.extend_from_slice(decoded.plane::(0)); + } + } + decoder.send_eof().unwrap(); + while decoder.receive_frame(&mut decoded).is_ok() { + samples.extend_from_slice(decoded.plane::(0)); + } + (samples, track_end) + } + + #[test] + fn export_encoder_keeps_audio_aligned_in_mp4() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("click.mp4"); + encode_click_for_export(&path); + + let (samples, track_end) = decode_first_channel(&path); + let click = samples + .iter() + .position(|sample| sample.abs() > 0.3) + .expect("the encoded click was not found"); + assert!( + click.abs_diff(CLICK_AT) < 64, + "click decoded at sample {click}, encoded at {CLICK_AT}" + ); + assert_eq!(track_end, 48_000); + } + + #[test] + fn export_encoder_spends_almost_nothing_on_silence() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("silence.mp4"); + let mut output = format::output(&path).unwrap(); + let mut encoder = AACEncoder::init_for_export( + AudioInfo::new_raw(Sample::F32(Type::Packed), 48_000, 2), + &mut output, + ) + .unwrap(); + output.write_header().unwrap(); + for start in (0..480_000).step_by(960) { + let mut audio = + frame::Audio::new(Sample::F32(Type::Packed), 960, ChannelLayout::STEREO); + audio.set_rate(48_000); + audio.set_pts(Some(start)); + audio.data_mut(0)[..960 * 2 * size_of::()].fill(0); + encoder + .send_frame(audio, Duration::MAX, &mut output) + .unwrap(); + } + encoder.flush(&mut output).unwrap(); + output.write_trailer().unwrap(); + drop(encoder); + drop(output); + + let mut input = format::input(&path).unwrap(); + let stream_index = input + .streams() + .best(ffmpeg::media::Type::Audio) + .unwrap() + .index(); + let bytes = input + .packets() + .filter(|(stream, _)| stream.index() == stream_index) + .map(|(_, packet)| packet.size()) + .sum::(); + assert!(bytes < 40_000, "10 s of silence took {bytes} bytes"); + } + + #[cfg(target_os = "macos")] + #[test] + fn export_encoder_uses_audiotoolbox_on_macos() { + let directory = tempfile::tempdir().unwrap(); + let initial_padding = encode_click_for_export(&directory.path().join("click.mp4")); + assert_eq!(initial_padding, 2112); + } + + #[test] + #[ignore] + fn benchmark_export_aac_encoder() { + type Init = + fn(AudioInfo, &mut format::context::Output) -> Result; + + fn voice_like_frame(start: usize, samples: usize, noise: &mut u32) -> frame::Audio { + let mut audio = + frame::Audio::new(Sample::F32(Type::Packed), samples, ChannelLayout::STEREO); + audio.set_rate(48_000); + audio.set_pts(Some(start as i64)); + for (index, sample) in audio.data_mut(0)[..samples * 2 * size_of::()] + .chunks_exact_mut(size_of::()) + .enumerate() + { + *noise ^= *noise << 13; + *noise ^= *noise >> 17; + *noise ^= *noise << 5; + let time = (start + index / 2) as f32 / 48_000.0; + let value = 0.3 * (std::f32::consts::TAU * 220.0 * time).sin() + + 0.15 * (std::f32::consts::TAU * 1_330.0 * time).sin() + + 0.1 * (*noise as f32 / u32::MAX as f32 - 0.5); + sample.copy_from_slice(&value.to_le_bytes()); + } + audio + } + + let seconds = std::env::var("CAP_AAC_BENCH_SECONDS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(60usize); + let samples = seconds * 48_000; + let directory = tempfile::tempdir().unwrap(); + let encoders: [(&str, Init); 2] = [ + ("init (FFmpeg aac)", AACEncoder::init), + ("init_for_export", AACEncoder::init_for_export), + ]; + for round in 0..3 { + for (label, init) in encoders { + let path = directory.path().join(format!("{round}.mp4")); + let mut output = format::output(&path).unwrap(); + let mut encoder = init( + AudioInfo::new_raw(Sample::F32(Type::Packed), 48_000, 2), + &mut output, + ) + .unwrap(); + output.write_header().unwrap(); + let mut noise = 0x9e37_79b9_u32; + let frames = (0..samples) + .step_by(960) + .map(|position| { + voice_like_frame(position, 960.min(samples - position), &mut noise) + }) + .collect::>(); + let started = std::time::Instant::now(); + for frame in frames { + encoder + .send_frame(frame, Duration::MAX, &mut output) + .unwrap(); + } + encoder.flush(&mut output).unwrap(); + output.write_trailer().unwrap(); + let wall = started.elapsed().as_secs_f64(); + println!( + "round {round} {label}: {seconds}s of 48 kHz stereo in {wall:.3}s ({:.0}x realtime)", + seconds as f64 / wall + ); + } + } + } + #[test] fn checked_audio_submission_reports_a_closed_encoder() { let directory = tempfile::tempdir().unwrap(); diff --git a/crates/enc-ffmpeg/src/audio/base.rs b/crates/enc-ffmpeg/src/audio/base.rs index cd5197a7c71..932649ccf42 100644 --- a/crates/enc-ffmpeg/src/audio/base.rs +++ b/crates/enc-ffmpeg/src/audio/base.rs @@ -1,12 +1,17 @@ use super::buffered_resampler::BufferedResampler; use crate::base::{EncoderBase, PacketSink}; -use ffmpeg::{codec::encoder, frame}; +use ffmpeg::{ + Packet, Rational, Rescale, + codec::{Capabilities, encoder}, + frame, +}; use std::time::Duration; pub struct AudioEncoderBase { inner: EncoderBase, encoder: encoder::Audio, resampler: BufferedResampler, + input_end: Option, } impl AudioEncoderBase { @@ -15,6 +20,7 @@ impl AudioEncoderBase { inner: EncoderBase::new(stream_index), encoder, resampler, + input_end: None, } } @@ -33,6 +39,7 @@ impl AudioEncoderBase { self.resampler.add_frame(frame); while let Some(frame) = self.resampler.get_frame(self.encoder.frame_size() as usize) { + self.track_input_end(&frame); self.inner.send_frame(&frame, output, &mut self.encoder)?; } @@ -40,10 +47,64 @@ impl AudioEncoderBase { } pub fn flush(&mut self, output: &mut S) -> Result<(), ffmpeg::Error> { + let mut frames = Vec::new(); while let Some(frame) = self.resampler.flush(self.encoder.frame_size() as usize) { - self.inner.send_frame(&frame, output, &mut self.encoder)?; + self.track_input_end(&frame); + frames.push(frame); + } + + // Encoders that only take full frames (AudioToolbox) get the last one + // padded by libavcodec, and their packet durations count that padding, + // so the track would end up to a frame after the input did. + let pads_final_frame = self.encoder.codec().is_some_and(|codec| { + !codec + .capabilities() + .intersects(Capabilities::SMALL_LAST_FRAME | Capabilities::VARIABLE_FRAME_SIZE) + }); + let mut output = EndTrimmedSink { + output, + end: self.input_end.filter(|_| pads_final_frame), + time_base: self.encoder.time_base(), + }; + for frame in &frames { + self.inner + .send_frame(frame, &mut output, &mut self.encoder)?; } + self.inner.process_eof(&mut output, &mut self.encoder) + } - self.inner.process_eof(output, &mut self.encoder) + fn track_input_end(&mut self, frame: &frame::Audio) { + if let Some(pts) = frame.pts() { + let end = pts + frame.samples() as i64; + self.input_end = Some(self.input_end.map_or(end, |current| current.max(end))); + } + } +} + +struct EndTrimmedSink<'a, S: PacketSink + ?Sized> { + output: &'a mut S, + end: Option, + time_base: Rational, +} + +impl PacketSink for EndTrimmedSink<'_, S> { + fn stream_time_base(&self, stream_index: usize) -> Rational { + self.output.stream_time_base(stream_index) + } + + fn write_packet(&mut self, mut packet: Packet) -> Result<(), ffmpeg::Error> { + if let (Some(end), Some(pts)) = (self.end, packet.pts()) { + let end = end.rescale( + self.time_base, + self.output.stream_time_base(packet.stream()), + ); + if pts >= end { + return Ok(()); + } + if pts + packet.duration() > end { + packet.set_duration(end - pts); + } + } + self.output.write_packet(packet) } } diff --git a/crates/export/src/mp4.rs b/crates/export/src/mp4.rs index f120f6cff10..7d070bd0570 100644 --- a/crates/export/src/mp4.rs +++ b/crates/export/src/mp4.rs @@ -451,7 +451,7 @@ impl Mp4ExportSettings { if !has_audio { return None; } - match AACEncoder::init(AudioRenderer::info(), o) { + match AACEncoder::init_for_export(AudioRenderer::info(), o) { Ok(audio) => { detached_audio = Some(audio); None From 3bc89760340e842e735dea17a6ef7aa06655b10d Mon Sep 17 00:00:00 2001 From: Vagarth Date: Mon, 19 Oct 2026 18:10:05 +0530 Subject: [PATCH 2/2] chore(export-benchmark): write runs to a per-process temp dir, drop narrating comment --- apps/desktop-gpui/Cargo.lock | 1 + apps/desktop-gpui/Cargo.toml | 1 + apps/desktop-gpui/examples/export-benchmark.rs | 10 ++-------- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/apps/desktop-gpui/Cargo.lock b/apps/desktop-gpui/Cargo.lock index 4d2ac0436be..8a41257d6ec 100644 --- a/apps/desktop-gpui/Cargo.lock +++ b/apps/desktop-gpui/Cargo.lock @@ -1584,6 +1584,7 @@ dependencies = [ "sha2", "smallvec", "tauri-winres", + "tempfile", "tiny-skia", "tokio", "tracing", diff --git a/apps/desktop-gpui/Cargo.toml b/apps/desktop-gpui/Cargo.toml index c803d72db27..f3a193603a3 100644 --- a/apps/desktop-gpui/Cargo.toml +++ b/apps/desktop-gpui/Cargo.toml @@ -170,6 +170,7 @@ tauri-winres = "0.3.5" # runtime would deadlock the moment the test thread blocks on a frame. Kept in # dev-dependencies so the shipped binary's tokio features are unchanged. tokio = { version = "1", features = ["rt-multi-thread", "time", "test-util"] } +tempfile = "3" [target.'cfg(unix)'.dependencies] libc = "0.2" diff --git a/apps/desktop-gpui/examples/export-benchmark.rs b/apps/desktop-gpui/examples/export-benchmark.rs index 4c10debc300..58b842ff2a5 100644 --- a/apps/desktop-gpui/examples/export-benchmark.rs +++ b/apps/desktop-gpui/examples/export-benchmark.rs @@ -1,7 +1,3 @@ -//! Exports a project to MP4 the way the GPUI export page does (`run_export` in -//! `src/editor_export.rs`) and reports wall time, frames per second and process -//! CPU time. -//! //! Run: cargo run --release --example export-benchmark -- [WIDTH] [HEIGHT] [FPS] [RUNS] use std::{ @@ -54,11 +50,9 @@ fn cpu_secs() -> f64 { } async fn run(project_path: PathBuf, size: XY, fps: u32, runs: u32) -> anyhow::Result<()> { - let output_dir = std::env::temp_dir().join("cap-export-benchmark"); - std::fs::create_dir_all(&output_dir)?; + let output_dir = tempfile::tempdir()?; for run in 0..runs { - let output_path = output_dir.join(format!("run-{run}.mp4")); - let _ = std::fs::remove_file(&output_path); + let output_path = output_dir.path().join(format!("run-{run}.mp4")); let cancel = Arc::new(AtomicBool::new(false)); let started = Instant::now(); let cpu_started = cpu_secs();