diff --git a/apps/desktop-gpui/examples/export-benchmark.rs b/apps/desktop-gpui/examples/export-benchmark.rs new file mode 100644 index 00000000000..58b842ff2a5 --- /dev/null +++ b/apps/desktop-gpui/examples/export-benchmark.rs @@ -0,0 +1,128 @@ +//! Run: cargo run --release --example export-benchmark -- [WIDTH] [HEIGHT] [FPS] [RUNS] + +use std::{ + path::PathBuf, + sync::{Arc, atomic::AtomicBool}, + time::Instant, +}; + +use cap_export::{ + ExporterBase, + mp4::{ExportCompression, Mp4ExportSettings}, +}; +use cap_project::XY; + +#[cfg(unix)] +fn cpu_secs() -> f64 { + let mut usage = std::mem::MaybeUninit::::zeroed(); + unsafe { libc::getrusage(libc::RUSAGE_SELF, usage.as_mut_ptr()) }; + let usage = unsafe { usage.assume_init() }; + let secs = |time: libc::timeval| time.tv_sec as f64 + time.tv_usec as f64 / 1_000_000.0; + secs(usage.ru_utime) + secs(usage.ru_stime) +} + +#[cfg(windows)] +fn cpu_secs() -> f64 { + use windows_sys::Win32::{ + Foundation::FILETIME, + System::Threading::{GetCurrentProcess, GetProcessTimes}, + }; + + let empty = FILETIME { + dwLowDateTime: 0, + dwHighDateTime: 0, + }; + let (mut creation, mut exit, mut kernel, mut user) = (empty, empty, empty, empty); + unsafe { + GetProcessTimes( + GetCurrentProcess(), + &mut creation, + &mut exit, + &mut kernel, + &mut user, + ) + }; + let secs = |time: FILETIME| { + ((u64::from(time.dwHighDateTime) << 32) | u64::from(time.dwLowDateTime)) as f64 + / 10_000_000.0 + }; + secs(kernel) + secs(user) +} + +async fn run(project_path: PathBuf, size: XY, fps: u32, runs: u32) -> anyhow::Result<()> { + let output_dir = tempfile::tempdir()?; + for run in 0..runs { + let output_path = output_dir.path().join(format!("run-{run}.mp4")); + let cancel = Arc::new(AtomicBool::new(false)); + let started = Instant::now(); + let cpu_started = cpu_secs(); + let base = ExporterBase::builder(project_path.clone()) + .with_output_path(output_path.clone()) + .build_for_mp4(cancel) + .await + .map_err(|error| anyhow::anyhow!("{error}"))?; + let total = base.total_frames(fps); + let prepared_ms = started.elapsed().as_secs_f64() * 1000.0; + let path = Mp4ExportSettings { + fps, + resolution_base: size, + compression: ExportCompression::Maximum, + custom_bpp: None, + force_ffmpeg_decoder: false, + optimize_filesize: false, + } + .export_prepared(base, |_| true) + .await + .map_err(|error| anyhow::anyhow!("{error}"))?; + let wall = started.elapsed().as_secs_f64(); + let cpu = cpu_secs() - cpu_started; + let bytes = std::fs::metadata(&path).map(|meta| meta.len()).unwrap_or(0); + println!( + "{}", + serde_json::json!({ + "run": run, + "frames": total, + "width": size.x, + "height": size.y, + "fps": fps, + "prepareMs": prepared_ms, + "wallSecs": wall, + "exportFps": f64::from(total) / wall, + "cpuSecs": cpu, + "bytes": bytes, + }) + ); + } + Ok(()) +} + +fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "warn".into()), + ) + .with_writer(std::io::stderr) + .init(); + + let mut arguments = std::env::args().skip(1); + let project_path = arguments + .next() + .map(PathBuf::from) + .ok_or_else(|| anyhow::anyhow!("Pass a studio .cap project path"))?; + let mut number = |default: u32| { + arguments + .next() + .and_then(|value| value.parse().ok()) + .unwrap_or(default) + }; + let size = XY::new(number(1920), number(1080)); + let fps = number(60); + let runs = number(3).max(1); + + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .thread_stack_size(16 * 1024 * 1024) + .enable_all() + .build()? + .block_on(run(project_path, size, fps, runs)) +} diff --git a/crates/enc-ffmpeg/src/audio/aac.rs b/crates/enc-ffmpeg/src/audio/aac.rs index ca58b26641e..e6291de13b7 100644 --- a/crates/enc-ffmpeg/src/audio/aac.rs +++ b/crates/enc-ffmpeg/src/audio/aac.rs @@ -25,6 +25,12 @@ pub enum AACEncoderError { Resampler(ffmpeg::Error), } +#[derive(Clone, Copy)] +enum Priming { + Keep, + TrimToFfmpegAac, +} + pub struct AACEncoder { base: AudioEncoderBase, stream_index: usize, @@ -49,6 +55,7 @@ impl AACEncoder { "aac", ffmpeg::Dictionary::new(), Self::OUTPUT_BITRATE, + Priming::Keep, input_config, output, ) @@ -76,7 +83,44 @@ impl AACEncoder { } else { 256_000 }; - match Self::init_with_codec("aac_at", options, bit_rate, input_config, output) { + match Self::init_with_codec( + "aac_at", + options, + bit_rate, + Priming::TrimToFfmpegAac, + input_config, + output, + ) { + Ok(encoder) => return Ok(encoder), + Err(error) => { + tracing::warn!(%error, "AudioToolbox AAC encoder unavailable, using FFmpeg's"); + } + } + } + + Self::init(input_config, output) + } + + // A plain MP4's edit list skips AudioToolbox's 2112-sample encoder priming, + // so exports keep every input sample instead of trimming like recordings. + pub fn init_for_export( + input_config: AudioInfo, + output: &mut format::context::Output, + ) -> Result { + #[cfg(target_os = "macos")] + { + // AudioToolbox defaults to constant bitrate, which fills silence up to + // the full 320 kbps; average bitrate keeps silent stretches near-free. + let mut options = ffmpeg::Dictionary::new(); + options.set("aac_at_mode", "abr"); + match Self::init_with_codec( + "aac_at", + options, + Self::OUTPUT_BITRATE, + Priming::Keep, + input_config, + output, + ) { Ok(encoder) => return Ok(encoder), Err(error) => { tracing::warn!(%error, "AudioToolbox AAC encoder unavailable, using FFmpeg's"); @@ -91,6 +135,7 @@ impl AACEncoder { name: &str, options: ffmpeg::Dictionary, bit_rate: usize, + priming: Priming, input_config: AudioInfo, output: &mut format::context::Output, ) -> Result { @@ -157,9 +202,12 @@ impl AACEncoder { // dropping the priming a codec adds beyond FFmpeg's aac (e.g. 2112 vs // 1024 samples for aac_at) keeps every content sample where the native // encoder put it. - let leading_trim = unsafe { (*encoder.as_ptr()).initial_padding } - .saturating_sub(Self::FFMPEG_AAC_PRIMING) - .max(0) as usize; + let leading_trim = match priming { + Priming::TrimToFfmpegAac => unsafe { (*encoder.as_ptr()).initial_padding } + .saturating_sub(Self::FFMPEG_AAC_PRIMING) + .max(0) as usize, + Priming::Keep => 0, + }; let mut output_stream = output.add_stream(codec)?; output_stream.set_time_base(FFRational(1, output_config.rate())); @@ -366,6 +414,236 @@ mod tests { assert_eq!(encode_audio(true), encode_audio_through_queue()); } + const CLICKS: [usize; 2] = [240, 24_000]; + + fn click_frame(start: usize, samples: usize) -> frame::Audio { + let mut audio = + frame::Audio::new(Sample::F32(Type::Packed), samples, ChannelLayout::STEREO); + audio.set_rate(48_000); + audio.set_pts(Some(start as i64)); + for (index, sample) in audio.data_mut(0)[..samples * 2 * size_of::()] + .chunks_exact_mut(2 * size_of::()) + .enumerate() + { + let position = start + index; + let value = if CLICKS + .iter() + .any(|click| (*click..click + 48).contains(&position)) + { + 0.9_f32 + } else { + 0.0 + }; + sample[..4].copy_from_slice(&value.to_le_bytes()); + sample[4..].copy_from_slice(&value.to_le_bytes()); + } + audio + } + + fn encode_clicks_for_export(path: &std::path::Path) { + let mut output = format::output(path).unwrap(); + let mut encoder = AACEncoder::init_for_export( + AudioInfo::new_raw(Sample::F32(Type::Packed), 48_000, 2), + &mut output, + ) + .unwrap(); + output.write_header().unwrap(); + + let mut position = 0; + while position < 48_000 { + let samples = 960.min(48_000 - position); + encoder + .send_frame(click_frame(position, samples), Duration::MAX, &mut output) + .unwrap(); + position += samples; + } + + encoder.flush(&mut output).unwrap(); + output.write_trailer().unwrap(); + } + + fn decode_first_channel(path: &std::path::Path) -> (Vec, i64) { + use ffmpeg::Rescale; + + let mut input = format::input(path).unwrap(); + let stream = input.streams().best(ffmpeg::media::Type::Audio).unwrap(); + let stream_index = stream.index(); + let time_base = stream.time_base(); + let mut decoder = context::Context::from_parameters(stream.parameters()) + .unwrap() + .decoder() + .audio() + .unwrap(); + let mut decoded = frame::Audio::empty(); + let mut samples = Vec::new(); + let mut track_end = i64::MIN; + + for (stream, packet) in input.packets() { + if stream.index() != stream_index { + continue; + } + if let Some(pts) = packet.pts() { + track_end = + track_end.max((pts + packet.duration()).rescale(time_base, (1, 48_000))); + } + decoder.send_packet(&packet).unwrap(); + while decoder.receive_frame(&mut decoded).is_ok() { + samples.extend_from_slice(decoded.plane::(0)); + } + } + decoder.send_eof().unwrap(); + while decoder.receive_frame(&mut decoded).is_ok() { + samples.extend_from_slice(decoded.plane::(0)); + } + (samples, track_end) + } + + #[test] + fn export_encoder_keeps_audio_aligned_in_mp4() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("clicks.mp4"); + encode_clicks_for_export(&path); + + let (samples, track_end) = decode_first_channel(&path); + for encoded in CLICKS { + let search_from = encoded.saturating_sub(64); + let decoded = samples[search_from..] + .iter() + .position(|sample| sample.abs() > 0.3) + .map(|offset| search_from + offset) + .expect("the encoded click was not found"); + assert!( + decoded.abs_diff(encoded) < 64, + "click decoded at sample {decoded}, encoded at {encoded}" + ); + } + assert_eq!(track_end, 48_000); + } + + #[test] + fn export_encoder_spends_almost_nothing_on_silence() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("silence.mp4"); + let mut output = format::output(&path).unwrap(); + let mut encoder = AACEncoder::init_for_export( + AudioInfo::new_raw(Sample::F32(Type::Packed), 48_000, 2), + &mut output, + ) + .unwrap(); + output.write_header().unwrap(); + for start in (0..480_000).step_by(960) { + let mut audio = + frame::Audio::new(Sample::F32(Type::Packed), 960, ChannelLayout::STEREO); + audio.set_rate(48_000); + audio.set_pts(Some(start)); + audio.data_mut(0)[..960 * 2 * size_of::()].fill(0); + encoder + .send_frame(audio, Duration::MAX, &mut output) + .unwrap(); + } + encoder.flush(&mut output).unwrap(); + output.write_trailer().unwrap(); + drop(encoder); + drop(output); + + let mut input = format::input(&path).unwrap(); + let stream_index = input + .streams() + .best(ffmpeg::media::Type::Audio) + .unwrap() + .index(); + let bytes = input + .packets() + .filter(|(stream, _)| stream.index() == stream_index) + .map(|(_, packet)| packet.size()) + .sum::(); + assert!(bytes < 40_000, "10 s of silence took {bytes} bytes"); + } + + #[cfg(target_os = "macos")] + #[test] + fn export_encoder_uses_audiotoolbox_on_macos() { + let directory = tempfile::tempdir().unwrap(); + let mut output = format::output(&directory.path().join("audio.mp4")).unwrap(); + let encoder = AACEncoder::init_for_export( + AudioInfo::new_raw(Sample::F32(Type::Packed), 48_000, 2), + &mut output, + ) + .unwrap(); + assert_eq!(encoder.codec_name().as_deref(), Some("aac_at")); + } + + #[test] + #[ignore] + fn benchmark_export_aac_encoder() { + type Init = + fn(AudioInfo, &mut format::context::Output) -> Result; + + fn voice_like_frame(start: usize, samples: usize, noise: &mut u32) -> frame::Audio { + let mut audio = + frame::Audio::new(Sample::F32(Type::Packed), samples, ChannelLayout::STEREO); + audio.set_rate(48_000); + audio.set_pts(Some(start as i64)); + for (index, sample) in audio.data_mut(0)[..samples * 2 * size_of::()] + .chunks_exact_mut(size_of::()) + .enumerate() + { + *noise ^= *noise << 13; + *noise ^= *noise >> 17; + *noise ^= *noise << 5; + let time = (start + index / 2) as f32 / 48_000.0; + let value = 0.3 * (std::f32::consts::TAU * 220.0 * time).sin() + + 0.15 * (std::f32::consts::TAU * 1_330.0 * time).sin() + + 0.1 * (*noise as f32 / u32::MAX as f32 - 0.5); + sample.copy_from_slice(&value.to_le_bytes()); + } + audio + } + + let seconds = std::env::var("CAP_AAC_BENCH_SECONDS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(60usize); + let samples = seconds * 48_000; + let directory = tempfile::tempdir().unwrap(); + let encoders: [(&str, Init); 2] = [ + ("init (FFmpeg aac)", AACEncoder::init), + ("init_for_export", AACEncoder::init_for_export), + ]; + for round in 0..3 { + for (label, init) in encoders { + let path = directory.path().join(format!("{round}.mp4")); + let mut output = format::output(&path).unwrap(); + let mut encoder = init( + AudioInfo::new_raw(Sample::F32(Type::Packed), 48_000, 2), + &mut output, + ) + .unwrap(); + output.write_header().unwrap(); + let mut noise = 0x9e37_79b9_u32; + let frames = (0..samples) + .step_by(960) + .map(|position| { + voice_like_frame(position, 960.min(samples - position), &mut noise) + }) + .collect::>(); + let started = std::time::Instant::now(); + for frame in frames { + encoder + .send_frame(frame, Duration::MAX, &mut output) + .unwrap(); + } + encoder.flush(&mut output).unwrap(); + output.write_trailer().unwrap(); + let wall = started.elapsed().as_secs_f64(); + println!( + "round {round} {label}: {seconds}s of 48 kHz stereo in {wall:.3}s ({:.0}x realtime)", + seconds as f64 / wall + ); + } + } + } + #[test] fn checked_audio_submission_reports_a_closed_encoder() { let directory = tempfile::tempdir().unwrap(); diff --git a/crates/export/src/mp4.rs b/crates/export/src/mp4.rs index 4bb938d1c3f..2ed82ddfb10 100644 --- a/crates/export/src/mp4.rs +++ b/crates/export/src/mp4.rs @@ -451,7 +451,7 @@ impl Mp4ExportSettings { if !has_audio { return None; } - match AACEncoder::init(AudioRenderer::info(), o) { + match AACEncoder::init_for_export(AudioRenderer::info(), o) { Ok(audio) => { detached_audio = Some(audio); None