From 48db1cacc5e053ead8062268f994208f0caf3ebd Mon Sep 17 00:00:00 2001 From: John Tennant Date: Thu, 20 Aug 2026 15:14:18 -0400 Subject: [PATCH 01/31] feat(voice): support AirPods stem input mute --- src-tauri/Cargo.lock | 36 +++--- src-tauri/Cargo.toml | 1 + src-tauri/src/commands/mod.rs | 1 + src-tauri/src/commands/native_input_mute.rs | 79 +++++++++++++ src-tauri/src/commands/native_voice.rs | 124 +++++++++++++++++++- 5 files changed, 220 insertions(+), 21 deletions(-) create mode 100644 src-tauri/src/commands/native_input_mute.rs diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 6e35ac8bb..d48871f52 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -9,9 +9,9 @@ dependencies = [ "anyhow", "audioadapter-buffers", "base64 0.22.1", + "berd-voice", "block2", "builderbot-auth", - "berd-voice", "bytes", "bzip2 0.6.1", "cc", @@ -34,6 +34,7 @@ dependencies = [ "nucleo-matcher", "objc2", "objc2-app-kit", + "objc2-avf-audio", "objc2-foundation", "objc2-user-notifications", "percent-encoding", @@ -580,6 +581,20 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" +[[package]] +name = "berd-voice" +version = "0.1.0" +dependencies = [ + "ort", + "ort-sys", + "rand 0.10.2", + "sentencepiece-model", + "serde", + "serde_json", + "sherpa-onnx", + "tokenizers", +] + [[package]] name = "berdctl" version = "0.6.2" @@ -713,20 +728,6 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" -[[package]] -name = "berd-voice" -version = "0.1.0" -dependencies = [ - "ort", - "ort-sys", - "rand 0.10.2", - "sentencepiece-model", - "serde", - "serde_json", - "sherpa-onnx", - "tokenizers", -] - [[package]] name = "bytemuck" version = "1.25.2" @@ -3932,6 +3933,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08" dependencies = [ "bitflags 2.13.1", + "block2", "libc", "objc2", "objc2-core-audio", @@ -3946,7 +3948,11 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be" dependencies = [ + "bitflags 2.13.1", + "block2", "objc2", + "objc2-audio-toolbox", + "objc2-core-audio-types", "objc2-foundation", ] diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index ccc523bca..feac03007 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -109,6 +109,7 @@ windows-sys = { version = "0.59", features = [ block2 = "0.6" objc2 = "0.6" objc2-app-kit = { version = "0.3.2", features = ["NSApplication", "NSImage", "NSMenu", "NSMenuItem", "NSAlert", "NSButton", "NSControl", "NSCell", "NSResponder", "NSView"] } +objc2-avf-audio = { version = "0.3.2", features = ["AVAudioApplication", "block2"] } objc2-foundation = { version = "0.3.2", features = ["NSDictionary", "NSError", "NSFileManager", "NSObject", "NSProcessInfo", "NSString", "NSURL"] } objc2-user-notifications = "0.3.2" rodio = { version = "0.22", default-features = false, features = ["playback"] } diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index 465a05e20..485664fe5 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -31,6 +31,7 @@ pub mod local_mcp_inventory; pub mod message_queues; pub mod migration; pub mod model_setup; +mod native_input_mute; pub mod native_voice; pub mod notifications; #[cfg(feature = "block-voice-dictation")] diff --git a/src-tauri/src/commands/native_input_mute.rs b/src-tauri/src/commands/native_input_mute.rs new file mode 100644 index 000000000..8659da2b9 --- /dev/null +++ b/src-tauri/src/commands/native_input_mute.rs @@ -0,0 +1,79 @@ +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; + +pub fn start(input_muted: &Arc) { + clear(input_muted); + + #[cfg(target_os = "macos")] + if let Err(error) = macos::install(Arc::clone(input_muted)) { + log::info!("AirPods input mute listener is unavailable: {error}"); + } +} + +pub fn stop(input_muted: &Arc) { + clear(input_muted); + + #[cfg(target_os = "macos")] + if let Err(error) = macos::uninstall() { + log::info!("Could not stop the AirPods input mute listener: {error}"); + } +} + +fn clear(input_muted: &AtomicBool) { + input_muted.store(false, Ordering::Release); +} + +#[cfg(target_os = "macos")] +mod macos { + use super::*; + use block2::RcBlock; + use objc2::runtime::Bool; + use objc2_avf_audio::AVAudioApplication; + + pub fn install(input_muted: Arc) -> Result<(), String> { + // SAFETY: Berd's minimum macOS version is 14.0, where + // AVAudioApplication and these selectors are public API. + let application = unsafe { AVAudioApplication::sharedInstance() }; + unsafe { application.setInputMuted_error(false) } + .map_err(|error| error.localizedDescription().to_string())?; + + let handler = RcBlock::new(move |muted: Bool| { + let muted = muted.as_bool(); + input_muted.store(muted, Ordering::Release); + log::info!("AirPods input mute changed muted={muted}"); + Bool::YES + }); + // SAFETY: The block has the generated AVFAudio signature. The API + // copies and retains it until a later registration or cancellation. + unsafe { application.setInputMuteStateChangeHandler_error(Some(&handler)) } + .map_err(|error| error.localizedDescription().to_string())?; + log::info!("AirPods input mute listener started"); + Ok(()) + } + + pub fn uninstall() -> Result<(), String> { + // SAFETY: Berd targets macOS 14+, and nil is the documented way to + // cancel the process-wide handler at the end of a call lifecycle. + let application = unsafe { AVAudioApplication::sharedInstance() }; + unsafe { application.setInputMuteStateChangeHandler_error(None) } + .map_err(|error| error.localizedDescription().to_string())?; + // Do not leave another Berd microphone feature inheriting the voice + // conversation's last input-mute state after its handler is gone. + unsafe { application.setInputMuted_error(false) } + .map_err(|error| error.localizedDescription().to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn lifecycle_boundary_clears_mute() { + let input_muted = Arc::new(AtomicBool::new(true)); + clear(&input_muted); + assert!(!input_muted.load(Ordering::Acquire)); + } +} diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index eaff54ede..58551aabd 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -16,7 +16,9 @@ use serde::Serialize; use tauri::{AppHandle, Emitter, Manager, State, WebviewWindow}; use tokio::sync::mpsc as tokio_mpsc; -use super::{pocket_voice::parakeet_model_dir, voice_capture::VoiceCaptureState}; +use super::{ + native_input_mute, pocket_voice::parakeet_model_dir, voice_capture::VoiceCaptureState, +}; const EVENT_NAME: &str = "voice-conversation:event"; const MAX_AUDIO_BATCH_BYTES: usize = 100 * 1024; @@ -124,6 +126,7 @@ pub struct NativeVoiceState { runtime: Arc>, pending: Arc>>, capture_suppressions: Arc, + input_muted: Arc, } #[must_use = "capture suppression ends when the guard is dropped"] @@ -172,24 +175,38 @@ struct SttPipeline { audio_tx: SyncSender>, audio_seen: AtomicBool, shutdown: Arc, + input_muted: Arc, thread: Option>, } impl SttPipeline { - fn new(model_dir: PathBuf) -> Result<(Self, tokio_mpsc::Receiver), String> { + fn new( + model_dir: PathBuf, + input_muted: Arc, + ) -> Result<(Self, tokio_mpsc::Receiver), String> { let (audio_tx, audio_rx) = mpsc::sync_channel(AUDIO_QUEUE_DEPTH); let (event_tx, event_rx) = tokio_mpsc::channel(64); let shutdown = Arc::new(AtomicBool::new(false)); let worker_shutdown = Arc::clone(&shutdown); + let worker_input_muted = Arc::clone(&input_muted); let thread = thread::Builder::new() .name("berd-native-stt".into()) - .spawn(move || stt_worker(model_dir, audio_rx, event_tx, worker_shutdown)) + .spawn(move || { + stt_worker( + model_dir, + audio_rx, + event_tx, + worker_shutdown, + worker_input_muted, + ) + }) .map_err(|error| format!("start native transcription: {error}"))?; Ok(( Self { audio_tx, audio_seen: AtomicBool::new(false), shutdown, + input_muted, thread: Some(thread), }, event_rx, @@ -206,6 +223,9 @@ impl SttPipeline { if !bytes.len().is_multiple_of(4) { return Err("audio batch must contain complete f32 samples".to_string()); } + if self.input_muted.load(Ordering::Acquire) { + return Ok(()); + } if !self.audio_seen.swap(true, Ordering::AcqRel) { log::info!( "Native Parakeet received its first audio batch ({} bytes)", @@ -389,7 +409,7 @@ pub async fn start_native_voice_conversation( return Err(error); } }; - let (pipeline, mut events) = match SttPipeline::new(model_dir) { + let (pipeline, mut events) = match SttPipeline::new(model_dir, Arc::clone(&state.input_muted)) { Ok(result) => result, Err(error) => { if microphone_claimed { @@ -421,6 +441,7 @@ pub async fn start_native_voice_conversation( runtime.lifecycle_id.clone().unwrap_or_default(), ) }; + native_input_mute::start(&state.input_muted); let _ = webview_window.emit( EVENT_NAME, NativeVoiceEvent::Startup { @@ -516,6 +537,7 @@ pub async fn start_native_voice_conversation( if let Some(pipeline) = pipeline { shutdown_pipeline(pipeline).await; } + native_input_mute::stop(&event_app.state::().input_muted); event_app .state::() .release_owner(&window_label, &owner_id); @@ -583,6 +605,9 @@ pub async fn stop_native_voice_conversation( if let Some((owner, owner_id)) = owner.as_ref() { capture.release_owner(&owner.window_label, owner_id); } + if session_id.is_some() { + native_input_mute::stop(&state.input_muted); + } if let Some(session_id) = session_id { let target = owner .as_ref() @@ -637,6 +662,7 @@ impl NativeVoiceState { } runtime.revision }; + let had_session = session_id.is_some(); if let (Some(owner), Some(session_id)) = (owner, session_id) { capture.release_owner(&owner.window_label, &native_owner_id(&session_id)); if let Some(window) = app.get_webview_window(&owner.window_label) { @@ -649,6 +675,9 @@ impl NativeVoiceState { ); } } + if had_session { + native_input_mute::stop(&self.input_muted); + } Ok(()) } @@ -679,8 +708,10 @@ impl NativeVoiceState { runtime.revision = runtime.revision.wrapping_add(1); } } + native_input_mute::stop(&self.input_muted); return true; } + native_input_mute::stop(&self.input_muted); let runtime = Arc::clone(&self.runtime); tauri::async_runtime::spawn(async move { shutdown_pipeline(pipeline.expect("pipeline checked above")).await; @@ -708,6 +739,9 @@ impl NativeVoiceState { ) }; drop(pipeline); + if session_id.is_some() { + native_input_mute::stop(&self.input_muted); + } if let Ok(mut runtime) = self.runtime.lock() { if runtime.revision == revision && runtime.session_id == session_id { runtime.session_id = None; @@ -772,6 +806,7 @@ fn stt_worker( audio_rx: Receiver>, event_tx: tokio_mpsc::Sender, shutdown: Arc, + input_muted: Arc, ) { use rubato::{Fft, FixedSync, Resampler}; use sherpa_onnx::{OfflineRecognizer, OfflineRecognizerConfig}; @@ -812,10 +847,25 @@ fn stt_worker( let mut in_speech = false; while !shutdown.load(Ordering::Acquire) { let bytes = match audio_rx.recv_timeout(Duration::from_millis(50)) { - Ok(bytes) => bytes, - Err(mpsc::RecvTimeoutError::Timeout) => continue, + Ok(bytes) => Some(bytes), + Err(mpsc::RecvTimeoutError::Timeout) => None, Err(mpsc::RecvTimeoutError::Disconnected) => break, }; + if input_muted.load(Ordering::Acquire) { + if clear_buffered_audio( + &mut input_48k, + &mut leftover_16k, + &mut speech, + &mut silence_frames, + &mut in_speech, + ) { + let _ = event_tx.blocking_send(SttMessage::Speaking(false)); + } + continue; + } + let Some(bytes) = bytes else { + continue; + }; input_48k.extend( bytes .chunks_exact(4) @@ -865,6 +915,20 @@ fn stt_worker( } } +fn clear_buffered_audio( + input_48k: &mut Vec, + leftover_16k: &mut Vec, + speech: &mut Vec, + silence_frames: &mut usize, + in_speech: &mut bool, +) -> bool { + input_48k.clear(); + leftover_16k.clear(); + speech.clear(); + *silence_frames = 0; + std::mem::take(in_speech) +} + fn resample(resampler: &mut rubato::Fft, samples: &[f32]) -> Vec { use audioadapter_buffers::direct::InterleavedSlice; use rubato::Resampler; @@ -972,6 +1036,7 @@ mod tests { let pipeline = SttPipeline { audio_tx: sender, shutdown: Arc::new(AtomicBool::new(false)), + input_muted: Arc::new(AtomicBool::new(false)), audio_seen: AtomicBool::new(false), thread: None, }; @@ -985,6 +1050,7 @@ mod tests { let pipeline = SttPipeline { audio_tx: sender, shutdown: Arc::new(AtomicBool::new(false)), + input_muted: Arc::new(AtomicBool::new(false)), audio_seen: AtomicBool::new(false), thread: None, }; @@ -996,6 +1062,50 @@ mod tests { .contains("overrun")); } + #[test] + fn input_mute_discards_audio_and_unmute_resumes_queueing() { + let (sender, receiver) = mpsc::sync_channel(1); + let input_muted = Arc::new(AtomicBool::new(true)); + let pipeline = SttPipeline { + audio_tx: sender, + shutdown: Arc::new(AtomicBool::new(false)), + input_muted: Arc::clone(&input_muted), + audio_seen: AtomicBool::new(false), + thread: None, + }; + + pipeline + .push(vec![0; 4]) + .expect("muted microphone input is accepted and discarded"); + assert!(receiver.try_recv().is_err()); + + input_muted.store(false, Ordering::Release); + pipeline.push(vec![0; 4]).expect("unmuted audio is queued"); + assert_eq!(receiver.try_recv().expect("unmuted audio"), vec![0; 4]); + } + + #[test] + fn input_mute_clears_partial_utterance_even_without_a_new_batch() { + let mut input_48k = vec![0.1]; + let mut leftover_16k = vec![0.2]; + let mut speech = vec![0.3]; + let mut silence_frames = 4; + let mut in_speech = true; + + assert!(clear_buffered_audio( + &mut input_48k, + &mut leftover_16k, + &mut speech, + &mut silence_frames, + &mut in_speech, + )); + assert!(input_48k.is_empty()); + assert!(leftover_16k.is_empty()); + assert!(speech.is_empty()); + assert_eq!(silence_frames, 0); + assert!(!in_speech); + } + #[test] fn only_owning_window_can_inject_audio() { let state = NativeVoiceState::default(); @@ -1008,6 +1118,7 @@ mod tests { runtime.pipeline = Some(SttPipeline { audio_tx: sender, shutdown: Arc::new(AtomicBool::new(false)), + input_muted: Arc::new(AtomicBool::new(false)), audio_seen: AtomicBool::new(false), thread: None, }); @@ -1033,6 +1144,7 @@ mod tests { runtime.pipeline = Some(SttPipeline { audio_tx: sender, shutdown: Arc::new(AtomicBool::new(false)), + input_muted: Arc::new(AtomicBool::new(false)), audio_seen: AtomicBool::new(false), thread: Some(worker), }); From f72e3471712ff18184fac48d991e7483f7e5278f Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 10:41:30 -0400 Subject: [PATCH 02/31] fix(voice): serialize mute handler teardown --- src-tauri/src/commands/native_voice.rs | 69 ++++++++++++++++++++------ 1 file changed, 54 insertions(+), 15 deletions(-) diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index 58551aabd..e712e7aed 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -175,6 +175,7 @@ struct SttPipeline { audio_tx: SyncSender>, audio_seen: AtomicBool, shutdown: Arc, + discard_on_shutdown: Arc, input_muted: Arc, thread: Option>, } @@ -187,7 +188,9 @@ impl SttPipeline { let (audio_tx, audio_rx) = mpsc::sync_channel(AUDIO_QUEUE_DEPTH); let (event_tx, event_rx) = tokio_mpsc::channel(64); let shutdown = Arc::new(AtomicBool::new(false)); + let discard_on_shutdown = Arc::new(AtomicBool::new(false)); let worker_shutdown = Arc::clone(&shutdown); + let worker_discard_on_shutdown = Arc::clone(&discard_on_shutdown); let worker_input_muted = Arc::clone(&input_muted); let thread = thread::Builder::new() .name("berd-native-stt".into()) @@ -197,6 +200,7 @@ impl SttPipeline { audio_rx, event_tx, worker_shutdown, + worker_discard_on_shutdown, worker_input_muted, ) }) @@ -206,6 +210,7 @@ impl SttPipeline { audio_tx, audio_seen: AtomicBool::new(false), shutdown, + discard_on_shutdown, input_muted, thread: Some(thread), }, @@ -245,9 +250,16 @@ impl SttPipeline { } fn begin_shutdown(&mut self) -> Option> { + self.latch_muted_shutdown(); self.shutdown.store(true, Ordering::Release); self.thread.take() } + + fn latch_muted_shutdown(&self) { + if self.input_muted.load(Ordering::Acquire) { + self.discard_on_shutdown.store(true, Ordering::Release); + } + } } impl Drop for SttPipeline { @@ -436,12 +448,12 @@ pub async fn start_native_voice_conversation( window_label: window_label.clone(), }); runtime.pipeline = Some(pipeline); + native_input_mute::start(&state.input_muted); ( runtime.revision, runtime.lifecycle_id.clone().unwrap_or_default(), ) }; - native_input_mute::start(&state.input_muted); let _ = webview_window.emit( EVENT_NAME, NativeVoiceEvent::Startup { @@ -456,6 +468,7 @@ pub async fn start_native_voice_conversation( let event_window = webview_window.clone(); let runtime = Arc::clone(&state.runtime); let pending = Arc::clone(&state.pending); + let input_muted = Arc::clone(&state.input_muted); tauri::async_runtime::spawn(async move { while let Some(event) = events.recv().await { let active = runtime.lock().ok().is_some_and(|current| { @@ -528,6 +541,7 @@ pub async fn start_native_voice_conversation( { break; } + native_input_mute::stop(&input_muted); current.session_id = None; current.lifecycle_id = None; current.owner = None; @@ -537,7 +551,6 @@ pub async fn start_native_voice_conversation( if let Some(pipeline) = pipeline { shutdown_pipeline(pipeline).await; } - native_input_mute::stop(&event_app.state::().input_muted); event_app .state::() .release_owner(&window_label, &owner_id); @@ -595,6 +608,7 @@ pub async fn stop_native_voice_conversation( .lock() .map_err(|_| "native voice state lock was poisoned".to_string())?; if runtime.revision == revision && runtime.session_id == session_id { + native_input_mute::stop(&state.input_muted); runtime.session_id = None; runtime.lifecycle_id = None; runtime.owner = None; @@ -605,9 +619,6 @@ pub async fn stop_native_voice_conversation( if let Some((owner, owner_id)) = owner.as_ref() { capture.release_owner(&owner.window_label, owner_id); } - if session_id.is_some() { - native_input_mute::stop(&state.input_muted); - } if let Some(session_id) = session_id { let target = owner .as_ref() @@ -655,6 +666,7 @@ impl NativeVoiceState { .lock() .map_err(|_| "native voice state lock was poisoned".to_string())?; if runtime.revision == revision && runtime.session_id == session_id { + native_input_mute::stop(&self.input_muted); runtime.session_id = None; runtime.lifecycle_id = None; runtime.owner = None; @@ -662,7 +674,6 @@ impl NativeVoiceState { } runtime.revision }; - let had_session = session_id.is_some(); if let (Some(owner), Some(session_id)) = (owner, session_id) { capture.release_owner(&owner.window_label, &native_owner_id(&session_id)); if let Some(window) = app.get_webview_window(&owner.window_label) { @@ -675,9 +686,6 @@ impl NativeVoiceState { ); } } - if had_session { - native_input_mute::stop(&self.input_muted); - } Ok(()) } @@ -702,21 +710,22 @@ impl NativeVoiceState { if pipeline.is_none() { if let Ok(mut runtime) = self.runtime.lock() { if runtime.revision == revision && runtime.session_id == session_id { + native_input_mute::stop(&self.input_muted); runtime.session_id = None; runtime.lifecycle_id = None; runtime.owner = None; runtime.revision = runtime.revision.wrapping_add(1); } } - native_input_mute::stop(&self.input_muted); return true; } - native_input_mute::stop(&self.input_muted); let runtime = Arc::clone(&self.runtime); + let input_muted = Arc::clone(&self.input_muted); tauri::async_runtime::spawn(async move { shutdown_pipeline(pipeline.expect("pipeline checked above")).await; if let Ok(mut runtime) = runtime.lock() { if runtime.revision == revision && runtime.session_id == session_id { + native_input_mute::stop(&input_muted); runtime.session_id = None; runtime.lifecycle_id = None; runtime.owner = None; @@ -732,6 +741,10 @@ impl NativeVoiceState { let Ok(mut runtime) = self.runtime.lock() else { return; }; + if let Some(pipeline) = runtime.pipeline.as_ref() { + pipeline.latch_muted_shutdown(); + } + native_input_mute::stop(&self.input_muted); ( runtime.session_id.clone(), runtime.revision, @@ -739,9 +752,6 @@ impl NativeVoiceState { ) }; drop(pipeline); - if session_id.is_some() { - native_input_mute::stop(&self.input_muted); - } if let Ok(mut runtime) = self.runtime.lock() { if runtime.revision == revision && runtime.session_id == session_id { runtime.session_id = None; @@ -806,6 +816,7 @@ fn stt_worker( audio_rx: Receiver>, event_tx: tokio_mpsc::Sender, shutdown: Arc, + discard_on_shutdown: Arc, input_muted: Arc, ) { use rubato::{Fft, FixedSync, Resampler}; @@ -908,7 +919,10 @@ fn stt_worker( } } } - if !speech.is_empty() { + if !speech.is_empty() + && !input_muted.load(Ordering::Acquire) + && !discard_on_shutdown.load(Ordering::Acquire) + { let (delivered_tx, delivered_rx) = mpsc::sync_channel(0); flush_speech(&speech, &recognizer, &event_tx, Some(delivered_tx)); let _ = delivered_rx.recv_timeout(Duration::from_secs(5)); @@ -1036,6 +1050,7 @@ mod tests { let pipeline = SttPipeline { audio_tx: sender, shutdown: Arc::new(AtomicBool::new(false)), + discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::new(AtomicBool::new(false)), audio_seen: AtomicBool::new(false), thread: None, @@ -1050,6 +1065,7 @@ mod tests { let pipeline = SttPipeline { audio_tx: sender, shutdown: Arc::new(AtomicBool::new(false)), + discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::new(AtomicBool::new(false)), audio_seen: AtomicBool::new(false), thread: None, @@ -1069,6 +1085,7 @@ mod tests { let pipeline = SttPipeline { audio_tx: sender, shutdown: Arc::new(AtomicBool::new(false)), + discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::clone(&input_muted), audio_seen: AtomicBool::new(false), thread: None, @@ -1106,6 +1123,26 @@ mod tests { assert!(!in_speech); } + #[test] + fn muted_shutdown_keeps_final_utterance_discarded_after_handler_reset() { + let (sender, _receiver) = mpsc::sync_channel(1); + let input_muted = Arc::new(AtomicBool::new(true)); + let discard_on_shutdown = Arc::new(AtomicBool::new(false)); + let mut pipeline = SttPipeline { + audio_tx: sender, + shutdown: Arc::new(AtomicBool::new(false)), + discard_on_shutdown: Arc::clone(&discard_on_shutdown), + input_muted: Arc::clone(&input_muted), + audio_seen: AtomicBool::new(false), + thread: None, + }; + + pipeline.begin_shutdown(); + input_muted.store(false, Ordering::Release); + + assert!(discard_on_shutdown.load(Ordering::Acquire)); + } + #[test] fn only_owning_window_can_inject_audio() { let state = NativeVoiceState::default(); @@ -1118,6 +1155,7 @@ mod tests { runtime.pipeline = Some(SttPipeline { audio_tx: sender, shutdown: Arc::new(AtomicBool::new(false)), + discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::new(AtomicBool::new(false)), audio_seen: AtomicBool::new(false), thread: None, @@ -1144,6 +1182,7 @@ mod tests { runtime.pipeline = Some(SttPipeline { audio_tx: sender, shutdown: Arc::new(AtomicBool::new(false)), + discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::new(AtomicBool::new(false)), audio_seen: AtomicBool::new(false), thread: Some(worker), From 488531285680732ee1e525619ab8b9dc8b764705 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 10:49:52 -0400 Subject: [PATCH 03/31] fix(voice): snapshot mute state during shutdown --- src-tauri/src/commands/native_voice.rs | 39 ++++++++++++++++++-------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index e712e7aed..1d43fdd2f 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -701,16 +701,16 @@ impl NativeVoiceState { { return false; } - ( - runtime.session_id.clone(), - runtime.revision, - runtime.pipeline.take(), - ) + let pipeline = runtime.pipeline.take(); + if let Some(pipeline) = pipeline.as_ref() { + pipeline.latch_muted_shutdown(); + } + native_input_mute::stop(&self.input_muted); + (runtime.session_id.clone(), runtime.revision, pipeline) }; if pipeline.is_none() { if let Ok(mut runtime) = self.runtime.lock() { if runtime.revision == revision && runtime.session_id == session_id { - native_input_mute::stop(&self.input_muted); runtime.session_id = None; runtime.lifecycle_id = None; runtime.owner = None; @@ -720,12 +720,10 @@ impl NativeVoiceState { return true; } let runtime = Arc::clone(&self.runtime); - let input_muted = Arc::clone(&self.input_muted); tauri::async_runtime::spawn(async move { shutdown_pipeline(pipeline.expect("pipeline checked above")).await; if let Ok(mut runtime) = runtime.lock() { if runtime.revision == revision && runtime.session_id == session_id { - native_input_mute::stop(&input_muted); runtime.session_id = None; runtime.lifecycle_id = None; runtime.owner = None; @@ -919,10 +917,7 @@ fn stt_worker( } } } - if !speech.is_empty() - && !input_muted.load(Ordering::Acquire) - && !discard_on_shutdown.load(Ordering::Acquire) - { + if !speech.is_empty() && !discard_on_shutdown.load(Ordering::Acquire) { let (delivered_tx, delivered_rx) = mpsc::sync_channel(0); flush_speech(&speech, &recognizer, &event_tx, Some(delivered_tx)); let _ = delivered_rx.recv_timeout(Duration::from_secs(5)); @@ -1143,6 +1138,26 @@ mod tests { assert!(discard_on_shutdown.load(Ordering::Acquire)); } + #[test] + fn unmuted_shutdown_keeps_final_utterance_after_later_mute_event() { + let (sender, _receiver) = mpsc::sync_channel(1); + let input_muted = Arc::new(AtomicBool::new(false)); + let discard_on_shutdown = Arc::new(AtomicBool::new(false)); + let mut pipeline = SttPipeline { + audio_tx: sender, + shutdown: Arc::new(AtomicBool::new(false)), + discard_on_shutdown: Arc::clone(&discard_on_shutdown), + input_muted: Arc::clone(&input_muted), + audio_seen: AtomicBool::new(false), + thread: None, + }; + + pipeline.begin_shutdown(); + input_muted.store(true, Ordering::Release); + + assert!(!discard_on_shutdown.load(Ordering::Acquire)); + } + #[test] fn only_owning_window_can_inject_audio() { let state = NativeVoiceState::default(); From 9414becdeedc5b55423d5436c378a7f2371c917b Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 10:55:05 -0400 Subject: [PATCH 04/31] fix(voice): signal shutdown before mute reset --- src-tauri/src/commands/native_voice.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index 1d43fdd2f..bd229d7a7 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -250,9 +250,13 @@ impl SttPipeline { } fn begin_shutdown(&mut self) -> Option> { + self.signal_shutdown(); + self.thread.take() + } + + fn signal_shutdown(&self) { self.latch_muted_shutdown(); self.shutdown.store(true, Ordering::Release); - self.thread.take() } fn latch_muted_shutdown(&self) { @@ -703,7 +707,7 @@ impl NativeVoiceState { } let pipeline = runtime.pipeline.take(); if let Some(pipeline) = pipeline.as_ref() { - pipeline.latch_muted_shutdown(); + pipeline.signal_shutdown(); } native_input_mute::stop(&self.input_muted); (runtime.session_id.clone(), runtime.revision, pipeline) @@ -860,6 +864,9 @@ fn stt_worker( Err(mpsc::RecvTimeoutError::Timeout) => None, Err(mpsc::RecvTimeoutError::Disconnected) => break, }; + if shutdown.load(Ordering::Acquire) { + break; + } if input_muted.load(Ordering::Acquire) { if clear_buffered_audio( &mut input_48k, @@ -1187,6 +1194,7 @@ mod tests { async fn window_destroy_schedules_blocked_worker_join_off_callback() { let state = NativeVoiceState::default(); let (sender, _receiver) = mpsc::sync_channel(1); + let shutdown = Arc::new(AtomicBool::new(false)); let worker = thread::spawn(|| thread::sleep(Duration::from_millis(250))); { let mut runtime = state.runtime.lock().expect("lock native runtime"); @@ -1196,7 +1204,7 @@ mod tests { }); runtime.pipeline = Some(SttPipeline { audio_tx: sender, - shutdown: Arc::new(AtomicBool::new(false)), + shutdown: Arc::clone(&shutdown), discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::new(AtomicBool::new(false)), audio_seen: AtomicBool::new(false), @@ -1207,6 +1215,7 @@ mod tests { let started = std::time::Instant::now(); assert!(state.stop_for_window_destroyed("owner-window")); assert!(started.elapsed() < Duration::from_millis(50)); + assert!(shutdown.load(Ordering::Acquire)); tokio::time::sleep(Duration::from_millis(300)).await; assert!(state .runtime From 78cc70c19628472103e7acd6d085c9ed790a6fe1 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 10:59:53 -0400 Subject: [PATCH 05/31] fix(voice): preserve accepted shutdown audio --- src-tauri/src/commands/native_voice.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index bd229d7a7..79114007d 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -864,7 +864,7 @@ fn stt_worker( Err(mpsc::RecvTimeoutError::Timeout) => None, Err(mpsc::RecvTimeoutError::Disconnected) => break, }; - if shutdown.load(Ordering::Acquire) { + if shutdown.load(Ordering::Acquire) && discard_on_shutdown.load(Ordering::Acquire) { break; } if input_muted.load(Ordering::Acquire) { From 4e1896c91274bffe6579ed58913c03c171fa8b86 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 11:06:31 -0400 Subject: [PATCH 06/31] fix(voice): honor latched mute during worker shutdown --- src-tauri/src/commands/native_voice.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index 79114007d..2c7d72857 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -864,10 +864,11 @@ fn stt_worker( Err(mpsc::RecvTimeoutError::Timeout) => None, Err(mpsc::RecvTimeoutError::Disconnected) => break, }; - if shutdown.load(Ordering::Acquire) && discard_on_shutdown.load(Ordering::Acquire) { + let shutting_down = shutdown.load(Ordering::Acquire); + if shutting_down && (discard_on_shutdown.load(Ordering::Acquire) || bytes.is_none()) { break; } - if input_muted.load(Ordering::Acquire) { + if !shutting_down && input_muted.load(Ordering::Acquire) { if clear_buffered_audio( &mut input_48k, &mut leftover_16k, From 43b8161c96a59799dd9ade9a391d9b7d115f2792 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 13:27:56 -0400 Subject: [PATCH 07/31] fix(voice): route AirPods mute through native capture --- src-tauri/Cargo.lock | 7 +- src-tauri/Cargo.toml | 2 +- src-tauri/build.rs | 7 +- src-tauri/src/commands/native_input_mute.rs | 195 +++++++++++++++--- src-tauri/src/commands/native_voice.rs | 115 ++++++++++- src-tauri/src/lib.rs | 1 + .../swift/BerdAirPodsBridge/Package.swift | 20 ++ .../BerdAirPodsBridge/BerdAirPodsBridge.swift | 122 +++++++++++ .../api/voiceConversation.test.ts | 54 +++++ .../api/voiceConversation.ts | 21 ++ .../useVoiceConversationController.test.ts | 18 ++ .../hooks/useVoiceConversationController.ts | 14 ++ .../stores/voiceConversationStore.test.ts | 22 ++ .../stores/voiceConversationStore.ts | 18 ++ 14 files changed, 571 insertions(+), 45 deletions(-) create mode 100644 src-tauri/swift/BerdAirPodsBridge/Package.swift create mode 100644 src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index d48871f52..63f1ac2e6 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -34,7 +34,6 @@ dependencies = [ "nucleo-matcher", "objc2", "objc2-app-kit", - "objc2-avf-audio", "objc2-foundation", "objc2-user-notifications", "percent-encoding", @@ -50,6 +49,7 @@ dependencies = [ "sherpa-onnx", "sqlx", "ssstretch", + "swift-rs", "sysinfo", "tar", "tauri", @@ -3933,7 +3933,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08" dependencies = [ "bitflags 2.13.1", - "block2", "libc", "objc2", "objc2-core-audio", @@ -3948,11 +3947,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be" dependencies = [ - "bitflags 2.13.1", - "block2", "objc2", - "objc2-audio-toolbox", - "objc2-core-audio-types", "objc2-foundation", ] diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index feac03007..c9e7f584e 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -18,6 +18,7 @@ members = ["crates/berd-voice", "crates/berdctl", "plugins/berdctl"] exclude = ["plugins/app-test-driver"] [build-dependencies] +swift-rs = { version = "1.0.7", features = ["build"] } tauri-build = { version = "2", features = [] } cc = "1" @@ -109,7 +110,6 @@ windows-sys = { version = "0.59", features = [ block2 = "0.6" objc2 = "0.6" objc2-app-kit = { version = "0.3.2", features = ["NSApplication", "NSImage", "NSMenu", "NSMenuItem", "NSAlert", "NSButton", "NSControl", "NSCell", "NSResponder", "NSView"] } -objc2-avf-audio = { version = "0.3.2", features = ["AVAudioApplication", "block2"] } objc2-foundation = { version = "0.3.2", features = ["NSDictionary", "NSError", "NSFileManager", "NSObject", "NSProcessInfo", "NSString", "NSURL"] } objc2-user-notifications = "0.3.2" rodio = { version = "0.22", default-features = false, features = ["playback"] } diff --git a/src-tauri/build.rs b/src-tauri/build.rs index 69ecf7ee2..ab8a5f53f 100644 --- a/src-tauri/build.rs +++ b/src-tauri/build.rs @@ -18,7 +18,12 @@ fn main() { for framework in ["Foundation", "AVFoundation", "AudioToolbox", "CoreAudio"] { println!("cargo:rustc-link-lib=framework={framework}"); } - } + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("macos") { + swift_rs::SwiftLinker::new("14.0") + .with_package("BerdAirPodsBridge", "swift/BerdAirPodsBridge") + .link(); + } + } tauri_build::build() } diff --git a/src-tauri/src/commands/native_input_mute.rs b/src-tauri/src/commands/native_input_mute.rs index 8659da2b9..b8c33f0d7 100644 --- a/src-tauri/src/commands/native_input_mute.rs +++ b/src-tauri/src/commands/native_input_mute.rs @@ -3,13 +3,33 @@ use std::sync::{ Arc, }; -pub fn start(input_muted: &Arc) { +pub fn start(input_muted: &Arc, on_change: F, on_audio: A) -> bool +where + F: Fn(bool) + Send + Sync + 'static, + A: Fn(&[f32]) + Send + Sync + 'static, +{ clear(input_muted); #[cfg(target_os = "macos")] - if let Err(error) = macos::install(Arc::clone(input_muted)) { - log::info!("AirPods input mute listener is unavailable: {error}"); - } + let started = match macos::install( + Arc::clone(input_muted), + Arc::new(on_change), + Arc::new(on_audio), + ) { + Ok(()) => true, + Err(error) => { + log::info!("AirPods input mute listener is unavailable: {error}"); + false + } + }; + + #[cfg(not(target_os = "macos"))] + let started = { + let _ = (on_change, on_audio); + false + }; + + started } pub fn stop(input_muted: &Arc) { @@ -21,48 +41,138 @@ pub fn stop(input_muted: &Arc) { } } +pub fn set_muted(input_muted: &AtomicBool, muted: bool) -> Result<(), String> { + #[cfg(target_os = "macos")] + { + macos::set_muted(muted)?; + input_muted.store(muted, Ordering::Release); + Ok(()) + } + + #[cfg(not(target_os = "macos"))] + { + let _ = (input_muted, muted); + Err("native microphone mute is only available on macOS".to_string()) + } +} + fn clear(input_muted: &AtomicBool) { input_muted.store(false, Ordering::Release); } +fn apply_change(input_muted: &AtomicBool, muted: bool, on_change: &dyn Fn(bool)) { + let previous = input_muted.swap(muted, Ordering::AcqRel); + if previous != muted { + on_change(muted); + } +} + #[cfg(target_os = "macos")] mod macos { use super::*; - use block2::RcBlock; - use objc2::runtime::Bool; - use objc2_avf_audio::AVAudioApplication; - - pub fn install(input_muted: Arc) -> Result<(), String> { - // SAFETY: Berd's minimum macOS version is 14.0, where - // AVAudioApplication and these selectors are public API. - let application = unsafe { AVAudioApplication::sharedInstance() }; - unsafe { application.setInputMuted_error(false) } - .map_err(|error| error.localizedDescription().to_string())?; - - let handler = RcBlock::new(move |muted: Bool| { - let muted = muted.as_bool(); - input_muted.store(muted, Ordering::Release); + use std::sync::{Mutex, OnceLock}; + + type MuteChangeHandler = Arc; + type AudioInputHandler = Arc; + + struct CallbackState { + input_muted: Arc, + on_change: MuteChangeHandler, + on_audio: AudioInputHandler, + } + + static CALLBACK_STATE: OnceLock>> = OnceLock::new(); + + fn callback_state() -> &'static Mutex> { + CALLBACK_STATE.get_or_init(|| Mutex::new(None)) + } + + extern "C" { + fn berd_airpods_mute_start( + callback: extern "C" fn(bool), + audio_callback: extern "C" fn(*const f32, usize), + ) -> bool; + fn berd_airpods_mute_stop() -> bool; + fn berd_airpods_mute_set_muted(muted: bool) -> bool; + } + + extern "C" fn handle_input_mute_change(muted: bool) { + let Ok(state) = callback_state().lock() else { + return; + }; + let Some(state) = state.as_ref() else { + return; + }; + apply_change(&state.input_muted, muted, &|muted| { log::info!("AirPods input mute changed muted={muted}"); - Bool::YES + (state.on_change)(muted); }); - // SAFETY: The block has the generated AVFAudio signature. The API - // copies and retains it until a later registration or cancellation. - unsafe { application.setInputMuteStateChangeHandler_error(Some(&handler)) } - .map_err(|error| error.localizedDescription().to_string())?; + } + + extern "C" fn handle_audio_input(samples: *const f32, sample_count: usize) { + if samples.is_null() || sample_count == 0 || sample_count > 48_000 { + return; + } + let callback = { + let Ok(state) = callback_state().lock() else { + return; + }; + let Some(state) = state.as_ref() else { + return; + }; + Arc::clone(&state.on_audio) + }; + // SAFETY: AVAudioEngine owns this non-interleaved Float32 channel for + // the duration of the synchronous tap callback. We do not retain it. + let samples = unsafe { std::slice::from_raw_parts(samples, sample_count) }; + callback(samples); + } + + pub fn install( + input_muted: Arc, + on_change: MuteChangeHandler, + on_audio: AudioInputHandler, + ) -> Result<(), String> { + *callback_state() + .lock() + .map_err(|_| "input mute callback lock was poisoned".to_string())? = + Some(CallbackState { + input_muted, + on_change, + on_audio, + }); + // SAFETY: The Swift shim retains the callback and AVAudioEngine for its + // process-wide lifecycle and invokes it with a C-compatible boolean. + if !unsafe { berd_airpods_mute_start(handle_input_mute_change, handle_audio_input) } { + callback_state() + .lock() + .map_err(|_| "input mute callback lock was poisoned".to_string())? + .take(); + return Err("the Swift AVAudioApplication listener could not start".to_string()); + } log::info!("AirPods input mute listener started"); Ok(()) } pub fn uninstall() -> Result<(), String> { - // SAFETY: Berd targets macOS 14+, and nil is the documented way to - // cancel the process-wide handler at the end of a call lifecycle. - let application = unsafe { AVAudioApplication::sharedInstance() }; - unsafe { application.setInputMuteStateChangeHandler_error(None) } - .map_err(|error| error.localizedDescription().to_string())?; - // Do not leave another Berd microphone feature inheriting the voice - // conversation's last input-mute state after its handler is gone. - unsafe { application.setInputMuted_error(false) } - .map_err(|error| error.localizedDescription().to_string()) + // SAFETY: This mirrors the successful start call and clears the Swift + // process-global before Rust releases its callback state. + let stopped = unsafe { berd_airpods_mute_stop() }; + callback_state() + .lock() + .map_err(|_| "input mute callback lock was poisoned".to_string())? + .take(); + stopped + .then_some(()) + .ok_or_else(|| "the Swift AVAudioApplication listener could not stop".to_string()) + } + + pub fn set_muted(muted: bool) -> Result<(), String> { + // SAFETY: The Swift bridge accepts a C-compatible boolean and remains + // alive for the active native microphone lifecycle. + unsafe { berd_airpods_mute_set_muted(muted) } + .then_some(()) + .ok_or_else(|| "macOS rejected the native microphone mute change".to_string()) } } @@ -76,4 +186,25 @@ mod tests { clear(&input_muted); assert!(!input_muted.load(Ordering::Acquire)); } + + #[test] + fn unchanged_initial_state_is_not_reported_as_a_gesture() { + let input_muted = AtomicBool::new(false); + let changes = std::sync::Mutex::new(Vec::new()); + + apply_change(&input_muted, false, &|muted| { + changes.lock().expect("changes lock").push(muted); + }); + apply_change(&input_muted, true, &|muted| { + changes.lock().expect("changes lock").push(muted); + }); + apply_change(&input_muted, true, &|muted| { + changes.lock().expect("changes lock").push(muted); + }); + apply_change(&input_muted, false, &|muted| { + changes.lock().expect("changes lock").push(muted); + }); + + assert_eq!(*changes.lock().expect("changes lock"), vec![true, false]); + } } diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index 2c7d72857..684372bd0 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -49,6 +49,7 @@ pub struct NativeVoiceStatus { session_id: Option, owner_window_label: Option, revision: u64, + native_microphone_capture: bool, } #[derive(Clone, Debug, Serialize)] @@ -95,6 +96,11 @@ enum NativeVoiceEvent { activity: &'static str, revision: u64, }, + InputMute { + session_id: String, + muted: bool, + revision: u64, + }, CleanShutdown { session_id: String, revision: u64, @@ -127,6 +133,7 @@ pub struct NativeVoiceState { pending: Arc>>, capture_suppressions: Arc, input_muted: Arc, + native_microphone_capture: Arc, } #[must_use = "capture suppression ends when the guard is dropped"] @@ -160,6 +167,12 @@ impl NativeVoiceState { fn capture_is_suppressed(&self) -> bool { self.capture_suppressions.load(Ordering::SeqCst) > 0 } + + fn stop_native_microphone(&self) { + native_input_mute::stop(&self.input_muted); + self.native_microphone_capture + .store(false, Ordering::Release); + } } enum SttMessage { @@ -308,6 +321,7 @@ fn status(app: &AppHandle, state: &NativeVoiceState) -> NativeVoiceStatus { .as_ref() .map(|owner| owner.window_label.clone()), revision: runtime.revision, + native_microphone_capture: state.native_microphone_capture.load(Ordering::Acquire), } } @@ -452,7 +466,6 @@ pub async fn start_native_voice_conversation( window_label: window_label.clone(), }); runtime.pipeline = Some(pipeline); - native_input_mute::start(&state.input_muted); ( runtime.revision, runtime.lifecycle_id.clone().unwrap_or_default(), @@ -467,12 +480,40 @@ pub async fn start_native_voice_conversation( revision, }, ); + let mute_window = webview_window.clone(); + let mute_session_id = session_id.clone(); + let audio_state = state.inner().clone(); + let audio_session_id = session_id.clone(); + let native_capture_started = native_input_mute::start( + &state.input_muted, + move |muted| { + let _ = mute_window.emit( + EVENT_NAME, + NativeVoiceEvent::InputMute { + session_id: mute_session_id.clone(), + muted, + revision, + }, + ); + }, + move |samples| { + if let Err(error) = + push_audio_for_session(&audio_state, &audio_session_id, revision, samples) + { + log::warn!("Native microphone audio was not accepted: {error}"); + } + }, + ); + state + .native_microphone_capture + .store(native_capture_started, Ordering::Release); let event_app = app.clone(); let event_window = webview_window.clone(); let runtime = Arc::clone(&state.runtime); let pending = Arc::clone(&state.pending); let input_muted = Arc::clone(&state.input_muted); + let native_microphone_capture = Arc::clone(&state.native_microphone_capture); tauri::async_runtime::spawn(async move { while let Some(event) = events.recv().await { let active = runtime.lock().ok().is_some_and(|current| { @@ -546,6 +587,7 @@ pub async fn start_native_voice_conversation( break; } native_input_mute::stop(&input_muted); + native_microphone_capture.store(false, Ordering::Release); current.session_id = None; current.lifecycle_id = None; current.owner = None; @@ -612,7 +654,7 @@ pub async fn stop_native_voice_conversation( .lock() .map_err(|_| "native voice state lock was poisoned".to_string())?; if runtime.revision == revision && runtime.session_id == session_id { - native_input_mute::stop(&state.input_muted); + state.stop_native_microphone(); runtime.session_id = None; runtime.lifecycle_id = None; runtime.owner = None; @@ -670,7 +712,7 @@ impl NativeVoiceState { .lock() .map_err(|_| "native voice state lock was poisoned".to_string())?; if runtime.revision == revision && runtime.session_id == session_id { - native_input_mute::stop(&self.input_muted); + self.stop_native_microphone(); runtime.session_id = None; runtime.lifecycle_id = None; runtime.owner = None; @@ -709,7 +751,7 @@ impl NativeVoiceState { if let Some(pipeline) = pipeline.as_ref() { pipeline.signal_shutdown(); } - native_input_mute::stop(&self.input_muted); + self.stop_native_microphone(); (runtime.session_id.clone(), runtime.revision, pipeline) }; if pipeline.is_none() { @@ -746,7 +788,7 @@ impl NativeVoiceState { if let Some(pipeline) = runtime.pipeline.as_ref() { pipeline.latch_muted_shutdown(); } - native_input_mute::stop(&self.input_muted); + self.stop_native_microphone(); ( runtime.session_id.clone(), runtime.revision, @@ -777,6 +819,28 @@ pub fn push_native_voice_audio( push_audio_for_window(&state, webview_window.label(), bytes.to_vec()) } +#[tauri::command] +pub fn set_native_voice_input_muted( + state: State<'_, NativeVoiceState>, + webview_window: WebviewWindow, + muted: bool, +) -> Result<(), String> { + let runtime = state + .runtime + .lock() + .map_err(|_| "native voice state lock was poisoned".to_string())?; + if runtime.session_id.is_none() + || runtime + .owner + .as_ref() + .is_none_or(|owner| owner.window_label != webview_window.label()) + { + return Err("Only the active voice owner may mute its microphone.".to_string()); + } + drop(runtime); + native_input_mute::set_muted(&state.input_muted, muted) +} + fn push_audio_for_window( state: &NativeVoiceState, window_label: &str, @@ -802,6 +866,32 @@ fn push_audio_for_window( Ok(()) } +fn push_audio_for_session( + state: &NativeVoiceState, + session_id: &str, + revision: u64, + samples: &[f32], +) -> Result<(), String> { + if state.capture_is_suppressed() { + return Ok(()); + } + let runtime = state + .runtime + .lock() + .map_err(|_| "native voice state lock was poisoned".to_string())?; + if runtime.session_id.as_deref() != Some(session_id) || runtime.revision != revision { + return Ok(()); + } + let Some(pipeline) = runtime.pipeline.as_ref() else { + return Ok(()); + }; + let mut bytes = Vec::with_capacity(samples.len() * size_of::()); + for sample in samples { + bytes.extend_from_slice(&sample.to_ne_bytes()); + } + pipeline.push(bytes) +} + fn enqueue_pending_transcript( queue: &mut VecDeque, transcript: PendingTranscript, @@ -1289,5 +1379,20 @@ mod tests { "deliveryAttempts": 0, }), ); + + let event = NativeVoiceEvent::InputMute { + session_id: "session-1".to_string(), + muted: true, + revision: 3, + }; + assert_eq!( + serde_json::to_value(event).expect("serialize input mute event"), + serde_json::json!({ + "type": "inputMute", + "sessionId": "session-1", + "muted": true, + "revision": 3, + }), + ); } } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index f90b7b950..8672de987 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -646,6 +646,7 @@ pub fn run() { commands::native_voice::start_native_voice_conversation, commands::native_voice::stop_native_voice_conversation, commands::native_voice::push_native_voice_audio, + commands::native_voice::set_native_voice_input_muted, commands::voice_capture::register_voice_renderer_instance, commands::window_session::get_session_window_support, commands::window_session::open_session_window, diff --git a/src-tauri/swift/BerdAirPodsBridge/Package.swift b/src-tauri/swift/BerdAirPodsBridge/Package.swift new file mode 100644 index 000000000..51862d334 --- /dev/null +++ b/src-tauri/swift/BerdAirPodsBridge/Package.swift @@ -0,0 +1,20 @@ +// swift-tools-version: 5.9 +import PackageDescription + +let package = Package( + name: "BerdAirPodsBridge", + platforms: [.macOS(.v14)], + products: [ + .library( + name: "BerdAirPodsBridge", + type: .static, + targets: ["BerdAirPodsBridge"] + ) + ], + targets: [ + .target( + name: "BerdAirPodsBridge", + linkerSettings: [.linkedFramework("AVFAudio")] + ) + ] +) diff --git a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift new file mode 100644 index 000000000..cf1cf9566 --- /dev/null +++ b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift @@ -0,0 +1,122 @@ +import AVFoundation +import Foundation + +public typealias InputMuteCallback = @convention(c) (Bool) -> Void +public typealias AudioInputCallback = @convention(c) (UnsafePointer, Int) -> Void + +@available(macOS 14.0, *) +private final class AirPodsMuteBridge: @unchecked Sendable { + private let engine: AVAudioEngine + private let inputNode: AVAudioInputNode + private let callback: InputMuteCallback + private let audioCallback: AudioInputCallback + private var inputMuteObserver: NSObjectProtocol? + + init( + callback: @escaping InputMuteCallback, + audioCallback: @escaping AudioInputCallback + ) throws { + self.callback = callback + self.audioCallback = audioCallback + + let engine = AVAudioEngine() + let inputNode = engine.inputNode + let inputFormat = inputNode.outputFormat(forBus: 0) + guard inputFormat.sampleRate > 0, inputFormat.channelCount > 0 else { + throw BridgeError.noInputFormat + } + self.engine = engine + self.inputNode = inputNode + + inputNode.installTap( + onBus: 0, + bufferSize: 4096, + format: inputFormat + ) { [weak self] buffer, _ in + guard + let self, + let channel = buffer.floatChannelData?.pointee, + buffer.frameLength > 0 + else { return } + self.audioCallback(channel, Int(buffer.frameLength)) + } + engine.prepare() + try engine.start() + + // Match voice-conversation-cli exactly: seed before registration and + // ignore the expected error when no prior process handler exists. + try? AVAudioApplication.shared.setInputMuted(false) + try AVAudioApplication.shared.setInputMuteStateChangeHandler { [weak self] muted in + self?.callback(muted) + return true + } + inputMuteObserver = NotificationCenter.default.addObserver( + forName: AVAudioApplication.inputMuteStateChangeNotification, + object: nil, + queue: .main + ) { [weak self] _ in + guard let self else { return } + self.callback(AVAudioApplication.shared.isInputMuted) + } + } + + func stop() { + // Reset while the handler exists, then cancel the process-wide callback. + try? AVAudioApplication.shared.setInputMuted(false) + try? AVAudioApplication.shared.setInputMuteStateChangeHandler(nil) + if let inputMuteObserver { + NotificationCenter.default.removeObserver(inputMuteObserver) + self.inputMuteObserver = nil + } + inputNode.removeTap(onBus: 0) + engine.stop() + } + + private enum BridgeError: Error { + case noInputFormat + } +} + +@available(macOS 14.0, *) +private var activeBridge: AirPodsMuteBridge? + +@_cdecl("berd_airpods_mute_start") +public func berdAirPodsMuteStart( + callback: @escaping InputMuteCallback, + audioCallback: @escaping AudioInputCallback +) -> Bool { + guard #available(macOS 14.0, *) else { return false } + do { + activeBridge?.stop() + activeBridge = try AirPodsMuteBridge( + callback: callback, + audioCallback: audioCallback + ) + return true + } catch { + FileHandle.standardError.write( + Data("Berd AirPods mute bridge failed to start: \(error)\n".utf8) + ) + activeBridge = nil + return false + } +} + +@_cdecl("berd_airpods_mute_stop") +public func berdAirPodsMuteStop() -> Bool { + guard #available(macOS 14.0, *) else { return false } + activeBridge?.stop() + activeBridge = nil + return true +} + +@_cdecl("berd_airpods_mute_set_muted") +public func berdAirPodsMuteSetMuted(_ muted: Bool) -> Bool { + guard #available(macOS 14.0, *), activeBridge != nil else { return false } + do { + try AVAudioApplication.shared.setInputMuted(muted) + return true + } catch { + return false + } +} diff --git a/src/features/voice-conversation/api/voiceConversation.test.ts b/src/features/voice-conversation/api/voiceConversation.test.ts index 4bd905ec8..a66071cde 100644 --- a/src/features/voice-conversation/api/voiceConversation.test.ts +++ b/src/features/voice-conversation/api/voiceConversation.test.ts @@ -162,6 +162,27 @@ describe("voice conversation API", () => { expect(mocks.startMicrophone).not.toHaveBeenCalled(); }); + it("uses backend capture and mute when the native process owns microphone audio", async () => { + const status = { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + revision: 3, + nativeMicrophoneCapture: true, + } as const; + mocks.invoke.mockResolvedValue(undefined); + + await reconcileVoiceConversationMicrophone(status); + await setVoiceConversationMicrophoneMuted(true, status); + + expect(mocks.startMicrophone).not.toHaveBeenCalled(); + expect(mocks.invoke).toHaveBeenCalledWith("set_native_voice_input_muted", { + muted: true, + }); + }); + it("mutes and unmutes without reopening browser capture", async () => { const status = { available: true, @@ -266,6 +287,39 @@ describe("voice conversation API", () => { }); }); + it("applies AirPods input mute events to browser capture", async () => { + const callback = vi.fn(); + mocks.listen.mockImplementation(async (_name, handler) => { + handler({ + payload: { + type: "inputMute", + sessionId: "session-1", + muted: true, + revision: 6, + }, + }); + return vi.fn(); + }); + + await reconcileVoiceConversationMicrophone({ + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + revision: 5, + }); + await listenToVoiceConversation(callback); + + expect(mocks.setMicrophoneMuted).toHaveBeenLastCalledWith(true); + expect(callback).toHaveBeenCalledWith({ + type: "inputMute", + sessionId: "session-1", + muted: true, + revision: 6, + }); + }); + it("stops browser capture when native voice shuts down elsewhere", async () => { mocks.invoke.mockResolvedValue({ available: true, diff --git a/src/features/voice-conversation/api/voiceConversation.ts b/src/features/voice-conversation/api/voiceConversation.ts index e213f5486..f98c4a28a 100644 --- a/src/features/voice-conversation/api/voiceConversation.ts +++ b/src/features/voice-conversation/api/voiceConversation.ts @@ -49,6 +49,10 @@ async function ensureActiveMicrophone(): Promise { export async function reconcileVoiceConversationMicrophone( status: VoiceConversationStatus, ): Promise { + if (status.nativeMicrophoneCapture) { + stopActiveMicrophone(); + return; + } if ( status.lifecycle === "running" && status.ownerWindowLabel === getCurrentWindow().label @@ -66,6 +70,11 @@ export async function setVoiceConversationMicrophoneMuted( const previous = microphoneMuted; microphoneMuted = muted; try { + if (status.nativeMicrophoneCapture) { + await invoke("set_native_voice_input_muted", { muted }); + stopActiveMicrophone(); + return; + } await reconcileVoiceConversationMicrophone(status); } catch (error) { microphoneMuted = previous; @@ -109,6 +118,8 @@ export interface VoiceConversationStatus { ownerWindowLabel: string | null; /** Monotonic native lifecycle revision used to reject stale responses/events. */ revision: number; + /** The backend owns the PCM stream so device gestures reach that process. */ + nativeMicrophoneCapture?: boolean; } export type VoiceConversationEvent = @@ -138,6 +149,12 @@ export type VoiceConversationEvent = | "assistant-idle"; revision: number; } + | { + type: "inputMute"; + sessionId: string; + muted: boolean; + revision: number; + } | { type: "cleanShutdown"; sessionId: string; @@ -244,6 +261,10 @@ export function listenToVoiceConversation( onEvent: (event: VoiceConversationEvent) => void, ): Promise { return listen(VOICE_CONVERSATION_EVENT, (event) => { + if (event.payload.type === "inputMute") { + microphoneMuted = event.payload.muted; + activeMicrophone?.setMuted(event.payload.muted); + } if ( event.payload.type === "cleanShutdown" || (event.payload.type === "error" && event.payload.terminal) diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts index 68e06b5c6..07446a4a8 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts @@ -3,12 +3,15 @@ import { act, renderHook, waitFor } from "@testing-library/react"; import { useChatStore } from "@/features/chat/stores/chatStore"; import { useVoiceConversationStore } from "../stores/voiceConversationStore"; +const toastMocks = vi.hoisted(() => ({ message: vi.fn() })); const nativeAssistantSpeechMocks = vi.hoisted(() => ({ start: vi.fn(), stop: vi.fn(), takeNotices: vi.fn<() => string | null>(() => null), })); +vi.mock("sonner", () => ({ toast: toastMocks })); + vi.mock("../lib/nativeAssistantSpeech", () => ({ startNativeAssistantSpeech: nativeAssistantSpeechMocks.start, stopNativeAssistantSpeech: nativeAssistantSpeechMocks.stop, @@ -21,6 +24,7 @@ import { createVoiceRouteMountRegistry, createVoiceTranscriptDeliveryQueue, hasDeliveredVoiceTranscript, + notifyAirPodsInputMuteGesture, resetVoiceUiWhenRunSettles, resolveVoiceRouteMount, resolveVoiceToggleAction, @@ -32,6 +36,20 @@ import { } from "./useVoiceConversationController"; describe("voice transcript delivery coordination", () => { + it("shows an unmistakable AirPods gesture confirmation", () => { + notifyAirPodsInputMuteGesture(true); + expect(toastMocks.message).toHaveBeenCalledWith( + "AirPods gesture detected — microphone muted", + { id: "airpods-input-mute" }, + ); + + notifyAirPodsInputMuteGesture(false); + expect(toastMocks.message).toHaveBeenLastCalledWith( + "AirPods gesture detected — microphone unmuted", + { id: "airpods-input-mute" }, + ); + }); + it("recognizes a replayed transcript that was already delivered", () => { useChatStore.setState({ messagesBySession: { diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index df364dd87..1e15eb54d 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -1,4 +1,5 @@ import { useCallback, useEffect, useMemo, useRef } from "react"; +import { toast } from "sonner"; import type { ChatInputSendHandler, @@ -187,6 +188,15 @@ function addErrorNotification(sessionId: string | null, message: string) { .addMessage(sessionId, createSystemNotificationMessage(message, "error")); } +export function notifyAirPodsInputMuteGesture(muted: boolean) { + toast.message( + muted + ? "AirPods gesture detected — microphone muted" + : "AirPods gesture detected — microphone unmuted", + { id: "airpods-input-mute" }, + ); +} + export function hasDeliveredVoiceTranscript( sessionId: string, lifecycleId: string, @@ -314,6 +324,10 @@ function ensureVoiceEventDeliveryInitialized() { addErrorNotification(sessionId ?? null, event.message); return; } + if (event.type === "inputMute") { + notifyAirPodsInputMuteGesture(event.muted); + return; + } if (event.type === "activity") return; if (event.type !== "user" || !event.text.trim()) return; if ( diff --git a/src/features/voice-conversation/stores/voiceConversationStore.test.ts b/src/features/voice-conversation/stores/voiceConversationStore.test.ts index af1510a15..7c992fa52 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.test.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.test.ts @@ -389,6 +389,28 @@ describe("voice conversation store lifecycle ordering", () => { }); }); + it("reflects AirPods input mute events in the microphone control", async () => { + const store = await loadStore(); + store.setState({ + status: status("running", 2, "session-1"), + uiState: "user-speaking", + userSpeaking: true, + }); + + emit({ + type: "inputMute", + sessionId: "session-1", + muted: true, + revision: 3, + }); + + expect(store.getState()).toMatchObject({ + microphoneMuted: true, + userSpeaking: false, + uiState: "listening", + }); + }); + it("surfaces an unmute failure without losing muted state", async () => { const store = await loadStore(); store.setState({ diff --git a/src/features/voice-conversation/stores/voiceConversationStore.ts b/src/features/voice-conversation/stores/voiceConversationStore.ts index efff65a0c..56a420d05 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.ts @@ -287,6 +287,24 @@ export const useVoiceConversationStore = create( uiState: activityUiState(nextState), }; } + case "inputMute": { + const nextState = { + ...state, + microphoneMuted: event.muted, + userSpeaking: event.muted ? false : state.userSpeaking, + status: { + ...state.status, + lifecycle: "running" as const, + sessionId: event.sessionId, + revision: event.revision, + }, + error: null, + }; + return { + ...nextState, + uiState: activityUiState(nextState), + }; + } case "cleanShutdown": return { ...state, From 44989e82112e89af414c81ad3462cce164e6ef73 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 13:47:56 -0400 Subject: [PATCH 08/31] fix(voice): preserve native mute routing --- src-tauri/src/commands/native_voice.rs | 20 +++--- .../BerdAirPodsBridge/BerdAirPodsBridge.swift | 62 +++++++++++++++++-- .../api/voiceConversation.test.ts | 4 +- .../api/voiceConversation.ts | 5 +- .../stores/voiceConversationStore.test.ts | 6 +- .../stores/voiceConversationStore.ts | 1 + 6 files changed, 76 insertions(+), 22 deletions(-) diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index 684372bd0..6b3b19fd4 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -82,6 +82,7 @@ enum NativeVoiceEvent { owner_window_label: String, line: String, revision: u64, + native_microphone_capture: bool, }, User { session_id: String, @@ -471,15 +472,6 @@ pub async fn start_native_voice_conversation( runtime.lifecycle_id.clone().unwrap_or_default(), ) }; - let _ = webview_window.emit( - EVENT_NAME, - NativeVoiceEvent::Startup { - session_id: session_id.clone(), - owner_window_label: window_label.clone(), - line: "Native Parakeet voice conversation is on".to_string(), - revision, - }, - ); let mute_window = webview_window.clone(); let mute_session_id = session_id.clone(); let audio_state = state.inner().clone(); @@ -507,6 +499,16 @@ pub async fn start_native_voice_conversation( state .native_microphone_capture .store(native_capture_started, Ordering::Release); + let _ = webview_window.emit( + EVENT_NAME, + NativeVoiceEvent::Startup { + session_id: session_id.clone(), + owner_window_label: window_label.clone(), + line: "Native Parakeet voice conversation is on".to_string(), + revision, + native_microphone_capture: native_capture_started, + }, + ); let event_app = app.clone(); let event_window = webview_window.clone(); diff --git a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift index cf1cf9566..ba6f04b55 100644 --- a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift +++ b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift @@ -10,6 +10,8 @@ private final class AirPodsMuteBridge: @unchecked Sendable { private let inputNode: AVAudioInputNode private let callback: InputMuteCallback private let audioCallback: AudioInputCallback + private let targetFormat: AVAudioFormat + private let converter: AVAudioConverter? private var inputMuteObserver: NSObjectProtocol? init( @@ -25,20 +27,34 @@ private final class AirPodsMuteBridge: @unchecked Sendable { guard inputFormat.sampleRate > 0, inputFormat.channelCount > 0 else { throw BridgeError.noInputFormat } + guard let targetFormat = AVAudioFormat( + commonFormat: .pcmFormatFloat32, + sampleRate: 48_000, + channels: 1, + interleaved: false + ) else { + throw BridgeError.noOutputFormat + } + let converter: AVAudioConverter? + if inputFormat == targetFormat { + converter = nil + } else { + guard let created = AVAudioConverter(from: inputFormat, to: targetFormat) else { + throw BridgeError.noConverter + } + converter = created + } self.engine = engine self.inputNode = inputNode + self.targetFormat = targetFormat + self.converter = converter inputNode.installTap( onBus: 0, bufferSize: 4096, format: inputFormat ) { [weak self] buffer, _ in - guard - let self, - let channel = buffer.floatChannelData?.pointee, - buffer.frameLength > 0 - else { return } - self.audioCallback(channel, Int(buffer.frameLength)) + self?.forward(buffer) } engine.prepare() try engine.start() @@ -72,8 +88,42 @@ private final class AirPodsMuteBridge: @unchecked Sendable { engine.stop() } + private func forward(_ source: AVAudioPCMBuffer) { + let output: AVAudioPCMBuffer + if let converter { + let capacity = AVAudioFrameCount(ceil( + Double(source.frameLength) * targetFormat.sampleRate / source.format.sampleRate + )) + guard capacity > 0, + let converted = AVAudioPCMBuffer( + pcmFormat: targetFormat, + frameCapacity: capacity + ) else { return } + var error: NSError? + nonisolated(unsafe) var consumed = false + converter.convert(to: converted, error: &error) { _, status in + if !consumed { + consumed = true + status.pointee = .haveData + return source + } + status.pointee = .noDataNow + return nil + } + guard error == nil, converted.frameLength > 0 else { return } + output = converted + } else { + output = source + } + guard let channel = output.floatChannelData?.pointee, + output.frameLength > 0 else { return } + audioCallback(channel, Int(output.frameLength)) + } + private enum BridgeError: Error { case noInputFormat + case noOutputFormat + case noConverter } } diff --git a/src/features/voice-conversation/api/voiceConversation.test.ts b/src/features/voice-conversation/api/voiceConversation.test.ts index a66071cde..cc5061191 100644 --- a/src/features/voice-conversation/api/voiceConversation.test.ts +++ b/src/features/voice-conversation/api/voiceConversation.test.ts @@ -287,7 +287,7 @@ describe("voice conversation API", () => { }); }); - it("applies AirPods input mute events to browser capture", async () => { + it("forwards input mute events without mutating browser capture", async () => { const callback = vi.fn(); mocks.listen.mockImplementation(async (_name, handler) => { handler({ @@ -311,7 +311,7 @@ describe("voice conversation API", () => { }); await listenToVoiceConversation(callback); - expect(mocks.setMicrophoneMuted).toHaveBeenLastCalledWith(true); + expect(mocks.setMicrophoneMuted).not.toHaveBeenCalledWith(true); expect(callback).toHaveBeenCalledWith({ type: "inputMute", sessionId: "session-1", diff --git a/src/features/voice-conversation/api/voiceConversation.ts b/src/features/voice-conversation/api/voiceConversation.ts index f98c4a28a..417fe738a 100644 --- a/src/features/voice-conversation/api/voiceConversation.ts +++ b/src/features/voice-conversation/api/voiceConversation.ts @@ -129,6 +129,7 @@ export type VoiceConversationEvent = ownerWindowLabel: string; line: string; revision: number; + nativeMicrophoneCapture: boolean; } | { type: "user"; @@ -261,10 +262,6 @@ export function listenToVoiceConversation( onEvent: (event: VoiceConversationEvent) => void, ): Promise { return listen(VOICE_CONVERSATION_EVENT, (event) => { - if (event.payload.type === "inputMute") { - microphoneMuted = event.payload.muted; - activeMicrophone?.setMuted(event.payload.muted); - } if ( event.payload.type === "cleanShutdown" || (event.payload.type === "error" && event.payload.terminal) diff --git a/src/features/voice-conversation/stores/voiceConversationStore.test.ts b/src/features/voice-conversation/stores/voiceConversationStore.test.ts index 7c992fa52..19aec7cea 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.test.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.test.ts @@ -265,12 +265,16 @@ describe("voice conversation store lifecycle ordering", () => { ownerWindowLabel: "main", line: "type\tid\ttext", revision: 2, + nativeMicrophoneCapture: true, }); response.resolve(status("starting", 1, "session-1")); await starting; expect(store.getState()).toMatchObject({ - status: status("running", 2, "session-1"), + status: { + ...status("running", 2, "session-1"), + nativeMicrophoneCapture: true, + }, uiState: "listening", error: null, }); diff --git a/src/features/voice-conversation/stores/voiceConversationStore.ts b/src/features/voice-conversation/stores/voiceConversationStore.ts index 56a420d05..8c26864ad 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.ts @@ -240,6 +240,7 @@ export const useVoiceConversationStore = create( sessionId: event.sessionId, ownerWindowLabel: event.ownerWindowLabel, revision: event.revision, + nativeMicrophoneCapture: event.nativeMicrophoneCapture, }, uiState: "listening", microphoneMuted: false, From c065964b1ec4d9729d58650221e380cb063035ce Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 13:59:53 -0400 Subject: [PATCH 09/31] fix(voice): latch mute transitions --- src-tauri/src/commands/native_input_mute.rs | 58 ++++++++++---- src-tauri/src/commands/native_voice.rs | 78 +++++++++++++++---- .../BerdAirPodsBridge/BerdAirPodsBridge.swift | 4 +- 3 files changed, 106 insertions(+), 34 deletions(-) diff --git a/src-tauri/src/commands/native_input_mute.rs b/src-tauri/src/commands/native_input_mute.rs index b8c33f0d7..9282e5a7a 100644 --- a/src-tauri/src/commands/native_input_mute.rs +++ b/src-tauri/src/commands/native_input_mute.rs @@ -1,18 +1,24 @@ use std::sync::{ - atomic::{AtomicBool, Ordering}, + atomic::{AtomicBool, AtomicU64, Ordering}, Arc, }; -pub fn start(input_muted: &Arc, on_change: F, on_audio: A) -> bool +pub fn start( + input_muted: &Arc, + mute_epoch: &Arc, + on_change: F, + on_audio: A, +) -> bool where F: Fn(bool) + Send + Sync + 'static, A: Fn(&[f32]) + Send + Sync + 'static, { - clear(input_muted); + clear(input_muted, mute_epoch); #[cfg(target_os = "macos")] let started = match macos::install( Arc::clone(input_muted), + Arc::clone(mute_epoch), Arc::new(on_change), Arc::new(on_audio), ) { @@ -32,8 +38,8 @@ where started } -pub fn stop(input_muted: &Arc) { - clear(input_muted); +pub fn stop(input_muted: &Arc, mute_epoch: &Arc) { + clear(input_muted, mute_epoch); #[cfg(target_os = "macos")] if let Err(error) = macos::uninstall() { @@ -41,28 +47,41 @@ pub fn stop(input_muted: &Arc) { } } -pub fn set_muted(input_muted: &AtomicBool, muted: bool) -> Result<(), String> { +pub fn set_muted( + input_muted: &AtomicBool, + mute_epoch: &AtomicU64, + muted: bool, +) -> Result<(), String> { #[cfg(target_os = "macos")] { macos::set_muted(muted)?; - input_muted.store(muted, Ordering::Release); + apply_change(input_muted, mute_epoch, muted, &|_| {}); Ok(()) } #[cfg(not(target_os = "macos"))] { - let _ = (input_muted, muted); + let _ = (input_muted, mute_epoch, muted); Err("native microphone mute is only available on macOS".to_string()) } } -fn clear(input_muted: &AtomicBool) { +fn clear(input_muted: &AtomicBool, mute_epoch: &AtomicU64) { input_muted.store(false, Ordering::Release); + mute_epoch.store(0, Ordering::Release); } -fn apply_change(input_muted: &AtomicBool, muted: bool, on_change: &dyn Fn(bool)) { +fn apply_change( + input_muted: &AtomicBool, + mute_epoch: &AtomicU64, + muted: bool, + on_change: &dyn Fn(bool), +) { let previous = input_muted.swap(muted, Ordering::AcqRel); if previous != muted { + if muted { + mute_epoch.fetch_add(1, Ordering::AcqRel); + } on_change(muted); } } @@ -77,6 +96,7 @@ mod macos { struct CallbackState { input_muted: Arc, + mute_epoch: Arc, on_change: MuteChangeHandler, on_audio: AudioInputHandler, } @@ -103,7 +123,7 @@ mod macos { let Some(state) = state.as_ref() else { return; }; - apply_change(&state.input_muted, muted, &|muted| { + apply_change(&state.input_muted, &state.mute_epoch, muted, &|muted| { log::info!("AirPods input mute changed muted={muted}"); (state.on_change)(muted); }); @@ -130,6 +150,7 @@ mod macos { pub fn install( input_muted: Arc, + mute_epoch: Arc, on_change: MuteChangeHandler, on_audio: AudioInputHandler, ) -> Result<(), String> { @@ -138,6 +159,7 @@ mod macos { .map_err(|_| "input mute callback lock was poisoned".to_string())? = Some(CallbackState { input_muted, + mute_epoch, on_change, on_audio, }); @@ -183,28 +205,32 @@ mod tests { #[test] fn lifecycle_boundary_clears_mute() { let input_muted = Arc::new(AtomicBool::new(true)); - clear(&input_muted); + let mute_epoch = Arc::new(AtomicU64::new(3)); + clear(&input_muted, &mute_epoch); assert!(!input_muted.load(Ordering::Acquire)); + assert_eq!(mute_epoch.load(Ordering::Acquire), 0); } #[test] fn unchanged_initial_state_is_not_reported_as_a_gesture() { let input_muted = AtomicBool::new(false); + let mute_epoch = AtomicU64::new(0); let changes = std::sync::Mutex::new(Vec::new()); - apply_change(&input_muted, false, &|muted| { + apply_change(&input_muted, &mute_epoch, false, &|muted| { changes.lock().expect("changes lock").push(muted); }); - apply_change(&input_muted, true, &|muted| { + apply_change(&input_muted, &mute_epoch, true, &|muted| { changes.lock().expect("changes lock").push(muted); }); - apply_change(&input_muted, true, &|muted| { + apply_change(&input_muted, &mute_epoch, true, &|muted| { changes.lock().expect("changes lock").push(muted); }); - apply_change(&input_muted, false, &|muted| { + apply_change(&input_muted, &mute_epoch, false, &|muted| { changes.lock().expect("changes lock").push(muted); }); assert_eq!(*changes.lock().expect("changes lock"), vec![true, false]); + assert_eq!(mute_epoch.load(Ordering::Acquire), 1); } } diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index 6b3b19fd4..ee3d93134 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -4,7 +4,7 @@ use std::{ collections::VecDeque, path::PathBuf, sync::{ - atomic::{AtomicBool, AtomicUsize, Ordering}, + atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, mpsc::{self, Receiver, SyncSender, TrySendError}, Arc, Mutex, }, @@ -134,6 +134,7 @@ pub struct NativeVoiceState { pending: Arc>>, capture_suppressions: Arc, input_muted: Arc, + input_mute_epoch: Arc, native_microphone_capture: Arc, } @@ -170,7 +171,7 @@ impl NativeVoiceState { } fn stop_native_microphone(&self) { - native_input_mute::stop(&self.input_muted); + native_input_mute::stop(&self.input_muted, &self.input_mute_epoch); self.native_microphone_capture .store(false, Ordering::Release); } @@ -186,18 +187,25 @@ enum SttMessage { } struct SttPipeline { - audio_tx: SyncSender>, + audio_tx: SyncSender, audio_seen: AtomicBool, shutdown: Arc, discard_on_shutdown: Arc, input_muted: Arc, + input_mute_epoch: Arc, thread: Option>, } +struct AudioBatch { + bytes: Vec, + mute_epoch: u64, +} + impl SttPipeline { fn new( model_dir: PathBuf, input_muted: Arc, + input_mute_epoch: Arc, ) -> Result<(Self, tokio_mpsc::Receiver), String> { let (audio_tx, audio_rx) = mpsc::sync_channel(AUDIO_QUEUE_DEPTH); let (event_tx, event_rx) = tokio_mpsc::channel(64); @@ -206,6 +214,7 @@ impl SttPipeline { let worker_shutdown = Arc::clone(&shutdown); let worker_discard_on_shutdown = Arc::clone(&discard_on_shutdown); let worker_input_muted = Arc::clone(&input_muted); + let worker_input_mute_epoch = Arc::clone(&input_mute_epoch); let thread = thread::Builder::new() .name("berd-native-stt".into()) .spawn(move || { @@ -216,6 +225,7 @@ impl SttPipeline { worker_shutdown, worker_discard_on_shutdown, worker_input_muted, + worker_input_mute_epoch, ) }) .map_err(|error| format!("start native transcription: {error}"))?; @@ -226,6 +236,7 @@ impl SttPipeline { shutdown, discard_on_shutdown, input_muted, + input_mute_epoch, thread: Some(thread), }, event_rx, @@ -245,13 +256,19 @@ impl SttPipeline { if self.input_muted.load(Ordering::Acquire) { return Ok(()); } + let mute_epoch = self.input_mute_epoch.load(Ordering::Acquire); + if self.input_muted.load(Ordering::Acquire) + || mute_epoch != self.input_mute_epoch.load(Ordering::Acquire) + { + return Ok(()); + } if !self.audio_seen.swap(true, Ordering::AcqRel) { log::info!( "Native Parakeet received its first audio batch ({} bytes)", bytes.len() ); } - match self.audio_tx.try_send(bytes) { + match self.audio_tx.try_send(AudioBatch { bytes, mute_epoch }) { Ok(()) => Ok(()), Err(TrySendError::Full(_)) => Err( "Native voice audio overrun: transcription could not keep up with microphone input." @@ -440,7 +457,11 @@ pub async fn start_native_voice_conversation( return Err(error); } }; - let (pipeline, mut events) = match SttPipeline::new(model_dir, Arc::clone(&state.input_muted)) { + let (pipeline, mut events) = match SttPipeline::new( + model_dir, + Arc::clone(&state.input_muted), + Arc::clone(&state.input_mute_epoch), + ) { Ok(result) => result, Err(error) => { if microphone_claimed { @@ -478,6 +499,7 @@ pub async fn start_native_voice_conversation( let audio_session_id = session_id.clone(); let native_capture_started = native_input_mute::start( &state.input_muted, + &state.input_mute_epoch, move |muted| { let _ = mute_window.emit( EVENT_NAME, @@ -515,6 +537,7 @@ pub async fn start_native_voice_conversation( let runtime = Arc::clone(&state.runtime); let pending = Arc::clone(&state.pending); let input_muted = Arc::clone(&state.input_muted); + let input_mute_epoch = Arc::clone(&state.input_mute_epoch); let native_microphone_capture = Arc::clone(&state.native_microphone_capture); tauri::async_runtime::spawn(async move { while let Some(event) = events.recv().await { @@ -588,7 +611,7 @@ pub async fn start_native_voice_conversation( { break; } - native_input_mute::stop(&input_muted); + native_input_mute::stop(&input_muted, &input_mute_epoch); native_microphone_capture.store(false, Ordering::Release); current.session_id = None; current.lifecycle_id = None; @@ -840,7 +863,7 @@ pub fn set_native_voice_input_muted( return Err("Only the active voice owner may mute its microphone.".to_string()); } drop(runtime); - native_input_mute::set_muted(&state.input_muted, muted) + native_input_mute::set_muted(&state.input_muted, &state.input_mute_epoch, muted) } fn push_audio_for_window( @@ -907,11 +930,12 @@ fn enqueue_pending_transcript( fn stt_worker( model_dir: PathBuf, - audio_rx: Receiver>, + audio_rx: Receiver, event_tx: tokio_mpsc::Sender, shutdown: Arc, discard_on_shutdown: Arc, input_muted: Arc, + input_mute_epoch: Arc, ) { use rubato::{Fft, FixedSync, Resampler}; use sherpa_onnx::{OfflineRecognizer, OfflineRecognizerConfig}; @@ -950,17 +974,20 @@ fn stt_worker( let mut speech = Vec::new(); let mut silence_frames = 0_usize; let mut in_speech = false; + let mut observed_mute_epoch = input_mute_epoch.load(Ordering::Acquire); while !shutdown.load(Ordering::Acquire) { - let bytes = match audio_rx.recv_timeout(Duration::from_millis(50)) { - Ok(bytes) => Some(bytes), + let batch = match audio_rx.recv_timeout(Duration::from_millis(50)) { + Ok(batch) => Some(batch), Err(mpsc::RecvTimeoutError::Timeout) => None, Err(mpsc::RecvTimeoutError::Disconnected) => break, }; let shutting_down = shutdown.load(Ordering::Acquire); - if shutting_down && (discard_on_shutdown.load(Ordering::Acquire) || bytes.is_none()) { + if shutting_down && (discard_on_shutdown.load(Ordering::Acquire) || batch.is_none()) { break; } - if !shutting_down && input_muted.load(Ordering::Acquire) { + let current_mute_epoch = input_mute_epoch.load(Ordering::Acquire); + if current_mute_epoch != observed_mute_epoch { + observed_mute_epoch = current_mute_epoch; if clear_buffered_audio( &mut input_48k, &mut leftover_16k, @@ -970,13 +997,19 @@ fn stt_worker( ) { let _ = event_tx.blocking_send(SttMessage::Speaking(false)); } + } + if !shutting_down && input_muted.load(Ordering::Acquire) { continue; } - let Some(bytes) = bytes else { + let Some(batch) = batch else { continue; }; + if batch.mute_epoch != observed_mute_epoch { + continue; + } input_48k.extend( - bytes + batch + .bytes .chunks_exact(4) .map(|sample| f32::from_le_bytes([sample[0], sample[1], sample[2], sample[3]])), ); @@ -1147,6 +1180,7 @@ mod tests { shutdown: Arc::new(AtomicBool::new(false)), discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::new(AtomicBool::new(false)), + input_mute_epoch: Arc::new(AtomicU64::new(0)), audio_seen: AtomicBool::new(false), thread: None, }; @@ -1162,6 +1196,7 @@ mod tests { shutdown: Arc::new(AtomicBool::new(false)), discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::new(AtomicBool::new(false)), + input_mute_epoch: Arc::new(AtomicU64::new(0)), audio_seen: AtomicBool::new(false), thread: None, }; @@ -1177,11 +1212,13 @@ mod tests { fn input_mute_discards_audio_and_unmute_resumes_queueing() { let (sender, receiver) = mpsc::sync_channel(1); let input_muted = Arc::new(AtomicBool::new(true)); + let input_mute_epoch = Arc::new(AtomicU64::new(1)); let pipeline = SttPipeline { audio_tx: sender, shutdown: Arc::new(AtomicBool::new(false)), discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::clone(&input_muted), + input_mute_epoch: Arc::clone(&input_mute_epoch), audio_seen: AtomicBool::new(false), thread: None, }; @@ -1193,7 +1230,9 @@ mod tests { input_muted.store(false, Ordering::Release); pipeline.push(vec![0; 4]).expect("unmuted audio is queued"); - assert_eq!(receiver.try_recv().expect("unmuted audio"), vec![0; 4]); + let batch = receiver.try_recv().expect("unmuted audio"); + assert_eq!(batch.bytes, vec![0; 4]); + assert_eq!(batch.mute_epoch, 1); } #[test] @@ -1228,6 +1267,7 @@ mod tests { shutdown: Arc::new(AtomicBool::new(false)), discard_on_shutdown: Arc::clone(&discard_on_shutdown), input_muted: Arc::clone(&input_muted), + input_mute_epoch: Arc::new(AtomicU64::new(1)), audio_seen: AtomicBool::new(false), thread: None, }; @@ -1248,6 +1288,7 @@ mod tests { shutdown: Arc::new(AtomicBool::new(false)), discard_on_shutdown: Arc::clone(&discard_on_shutdown), input_muted: Arc::clone(&input_muted), + input_mute_epoch: Arc::new(AtomicU64::new(0)), audio_seen: AtomicBool::new(false), thread: None, }; @@ -1272,6 +1313,7 @@ mod tests { shutdown: Arc::new(AtomicBool::new(false)), discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::new(AtomicBool::new(false)), + input_mute_epoch: Arc::new(AtomicU64::new(0)), audio_seen: AtomicBool::new(false), thread: None, }); @@ -1280,7 +1322,10 @@ mod tests { assert!(push_audio_for_window(&state, "other-window", vec![0; 4]).is_err()); assert!(receiver.try_recv().is_err()); push_audio_for_window(&state, "owner-window", vec![0; 4]).expect("owner can send audio"); - assert_eq!(receiver.try_recv().expect("owner audio queued"), vec![0; 4]); + assert_eq!( + receiver.try_recv().expect("owner audio queued").bytes, + vec![0; 4] + ); } #[tokio::test] @@ -1300,6 +1345,7 @@ mod tests { shutdown: Arc::clone(&shutdown), discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::new(AtomicBool::new(false)), + input_mute_epoch: Arc::new(AtomicU64::new(0)), audio_seen: AtomicBool::new(false), thread: Some(worker), }); diff --git a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift index ba6f04b55..a37c05116 100644 --- a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift +++ b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift @@ -59,8 +59,8 @@ private final class AirPodsMuteBridge: @unchecked Sendable { engine.prepare() try engine.start() - // Match voice-conversation-cli exactly: seed before registration and - // ignore the expected error when no prior process handler exists. + // Clear stale process mute before registering this lifecycle's handler. + // The reset can fail when no earlier handler exists, which is harmless. try? AVAudioApplication.shared.setInputMuted(false) try AVAudioApplication.shared.setInputMuteStateChangeHandler { [weak self] muted in self?.callback(muted) From 635a081c779ee0d224c73a91eba38b1d5cb56ae1 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 15:57:33 -0400 Subject: [PATCH 10/31] fix(voice): satisfy cross-platform clippy --- src-tauri/src/commands/native_input_mute.rs | 1 + src-tauri/src/commands/native_voice.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src-tauri/src/commands/native_input_mute.rs b/src-tauri/src/commands/native_input_mute.rs index 9282e5a7a..19a4008a2 100644 --- a/src-tauri/src/commands/native_input_mute.rs +++ b/src-tauri/src/commands/native_input_mute.rs @@ -71,6 +71,7 @@ fn clear(input_muted: &AtomicBool, mute_epoch: &AtomicU64) { mute_epoch.store(0, Ordering::Release); } +#[cfg(any(target_os = "macos", test))] fn apply_change( input_muted: &AtomicBool, mute_epoch: &AtomicU64, diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index ee3d93134..b2cb45dd1 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -910,7 +910,7 @@ fn push_audio_for_session( let Some(pipeline) = runtime.pipeline.as_ref() else { return Ok(()); }; - let mut bytes = Vec::with_capacity(samples.len() * size_of::()); + let mut bytes = Vec::with_capacity(size_of_val(samples)); for sample in samples { bytes.extend_from_slice(&sample.to_ne_bytes()); } From 4dbfaddfbfed96e525a06a4018526da4f4482be7 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 16:26:03 -0400 Subject: [PATCH 11/31] fix(voice): harden native microphone lifecycle --- src-tauri/src/commands/native_input_mute.rs | 33 ++- src-tauri/src/commands/native_voice.rs | 164 +++++++++-- .../swift/BerdAirPodsBridge/Package.swift | 5 + .../BerdAirPodsBridge/BerdAirPodsBridge.swift | 268 ++++++++++++++---- .../BerdObjCExceptionCatch.m | 20 ++ .../include/BerdObjCExceptionCatch.h | 6 + .../api/voiceConversation.test.ts | 57 ++++ .../api/voiceConversation.ts | 46 ++- .../useVoiceConversationController.test.ts | 22 +- .../hooks/useVoiceConversationController.ts | 13 +- .../stores/voiceConversationStore.test.ts | 41 +++ .../stores/voiceConversationStore.ts | 29 +- 12 files changed, 598 insertions(+), 106 deletions(-) create mode 100644 src-tauri/swift/BerdAirPodsBridge/Sources/BerdObjCExceptionCatch/BerdObjCExceptionCatch.m create mode 100644 src-tauri/swift/BerdAirPodsBridge/Sources/BerdObjCExceptionCatch/include/BerdObjCExceptionCatch.h diff --git a/src-tauri/src/commands/native_input_mute.rs b/src-tauri/src/commands/native_input_mute.rs index 19a4008a2..7691ab119 100644 --- a/src-tauri/src/commands/native_input_mute.rs +++ b/src-tauri/src/commands/native_input_mute.rs @@ -3,15 +3,17 @@ use std::sync::{ Arc, }; -pub fn start( +pub fn start( input_muted: &Arc, mute_epoch: &Arc, on_change: F, on_audio: A, + on_capture_state: C, ) -> bool where F: Fn(bool) + Send + Sync + 'static, A: Fn(&[f32]) + Send + Sync + 'static, + C: Fn(bool) + Send + Sync + 'static, { clear(input_muted, mute_epoch); @@ -21,6 +23,7 @@ where Arc::clone(mute_epoch), Arc::new(on_change), Arc::new(on_audio), + Arc::new(on_capture_state), ) { Ok(()) => true, Err(error) => { @@ -31,7 +34,7 @@ where #[cfg(not(target_os = "macos"))] let started = { - let _ = (on_change, on_audio); + let _ = (on_change, on_audio, on_capture_state); false }; @@ -94,12 +97,14 @@ mod macos { type MuteChangeHandler = Arc; type AudioInputHandler = Arc; + type CaptureStateHandler = Arc; struct CallbackState { input_muted: Arc, mute_epoch: Arc, on_change: MuteChangeHandler, on_audio: AudioInputHandler, + on_capture_state: CaptureStateHandler, } static CALLBACK_STATE: OnceLock>> = OnceLock::new(); @@ -112,6 +117,7 @@ mod macos { fn berd_airpods_mute_start( callback: extern "C" fn(bool), audio_callback: extern "C" fn(*const f32, usize), + capture_state_callback: extern "C" fn(bool), ) -> bool; fn berd_airpods_mute_stop() -> bool; fn berd_airpods_mute_set_muted(muted: bool) -> bool; @@ -149,11 +155,25 @@ mod macos { callback(samples); } + extern "C" fn handle_capture_state_change(available: bool) { + let callback = { + let Ok(state) = callback_state().lock() else { + return; + }; + let Some(state) = state.as_ref() else { + return; + }; + Arc::clone(&state.on_capture_state) + }; + callback(available); + } + pub fn install( input_muted: Arc, mute_epoch: Arc, on_change: MuteChangeHandler, on_audio: AudioInputHandler, + on_capture_state: CaptureStateHandler, ) -> Result<(), String> { *callback_state() .lock() @@ -163,10 +183,17 @@ mod macos { mute_epoch, on_change, on_audio, + on_capture_state, }); // SAFETY: The Swift shim retains the callback and AVAudioEngine for its // process-wide lifecycle and invokes it with a C-compatible boolean. - if !unsafe { berd_airpods_mute_start(handle_input_mute_change, handle_audio_input) } { + if !unsafe { + berd_airpods_mute_start( + handle_input_mute_change, + handle_audio_input, + handle_capture_state_change, + ) + } { callback_state() .lock() .map_err(|_| "input mute callback lock was poisoned".to_string())? diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index b2cb45dd1..6f97757b6 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -50,6 +50,7 @@ pub struct NativeVoiceStatus { owner_window_label: Option, revision: u64, native_microphone_capture: bool, + native_microphone_mute_control: bool, } #[derive(Clone, Debug, Serialize)] @@ -83,6 +84,7 @@ enum NativeVoiceEvent { line: String, revision: u64, native_microphone_capture: bool, + native_microphone_mute_control: bool, }, User { session_id: String, @@ -102,6 +104,11 @@ enum NativeVoiceEvent { muted: bool, revision: u64, }, + NativeMicrophoneCapture { + session_id: String, + available: bool, + revision: u64, + }, CleanShutdown { session_id: String, revision: u64, @@ -136,6 +143,8 @@ pub struct NativeVoiceState { input_muted: Arc, input_mute_epoch: Arc, native_microphone_capture: Arc, + native_microphone_mute_control: Arc, + native_microphone_lifecycle: Arc>, } #[must_use = "capture suppression ends when the guard is dropped"] @@ -174,6 +183,8 @@ impl NativeVoiceState { native_input_mute::stop(&self.input_muted, &self.input_mute_epoch); self.native_microphone_capture .store(false, Ordering::Release); + self.native_microphone_mute_control + .store(false, Ordering::Release); } } @@ -340,6 +351,9 @@ fn status(app: &AppHandle, state: &NativeVoiceState) -> NativeVoiceStatus { .map(|owner| owner.window_label.clone()), revision: runtime.revision, native_microphone_capture: state.native_microphone_capture.load(Ordering::Acquire), + native_microphone_mute_control: state + .native_microphone_mute_control + .load(Ordering::Acquire), } } @@ -470,6 +484,10 @@ pub async fn start_native_voice_conversation( return Err(error); } }; + let native_microphone_lifecycle = Arc::clone(&state.native_microphone_lifecycle); + let _native_microphone_guard = native_microphone_lifecycle + .lock() + .map_err(|_| "native microphone lifecycle lock was poisoned".to_string())?; let (revision, lifecycle_id) = { let mut runtime = state .runtime @@ -497,6 +515,9 @@ pub async fn start_native_voice_conversation( let mute_session_id = session_id.clone(); let audio_state = state.inner().clone(); let audio_session_id = session_id.clone(); + let capture_window = webview_window.clone(); + let capture_session_id = session_id.clone(); + let native_microphone_capture_state = Arc::clone(&state.native_microphone_capture); let native_capture_started = native_input_mute::start( &state.input_muted, &state.input_mute_epoch, @@ -517,10 +538,26 @@ pub async fn start_native_voice_conversation( log::warn!("Native microphone audio was not accepted: {error}"); } }, + move |available| { + let previous = native_microphone_capture_state.swap(available, Ordering::AcqRel); + if previous != available { + let _ = capture_window.emit( + EVENT_NAME, + NativeVoiceEvent::NativeMicrophoneCapture { + session_id: capture_session_id.clone(), + available, + revision, + }, + ); + } + }, ); state .native_microphone_capture .store(native_capture_started, Ordering::Release); + state + .native_microphone_mute_control + .store(native_capture_started, Ordering::Release); let _ = webview_window.emit( EVENT_NAME, NativeVoiceEvent::Startup { @@ -529,6 +566,7 @@ pub async fn start_native_voice_conversation( line: "Native Parakeet voice conversation is on".to_string(), revision, native_microphone_capture: native_capture_started, + native_microphone_mute_control: native_capture_started, }, ); @@ -539,6 +577,8 @@ pub async fn start_native_voice_conversation( let input_muted = Arc::clone(&state.input_muted); let input_mute_epoch = Arc::clone(&state.input_mute_epoch); let native_microphone_capture = Arc::clone(&state.native_microphone_capture); + let native_microphone_mute_control = Arc::clone(&state.native_microphone_mute_control); + let native_microphone_lifecycle = Arc::clone(&state.native_microphone_lifecycle); tauri::async_runtime::spawn(async move { while let Some(event) = events.recv().await { let active = runtime.lock().ok().is_some_and(|current| { @@ -603,6 +643,10 @@ pub async fn start_native_voice_conversation( } SttMessage::Failed(message) => { let pipeline = { + let Ok(_native_microphone_guard) = native_microphone_lifecycle.lock() + else { + break; + }; let Ok(mut current) = runtime.lock() else { break; }; @@ -611,13 +655,16 @@ pub async fn start_native_voice_conversation( { break; } - native_input_mute::stop(&input_muted, &input_mute_epoch); - native_microphone_capture.store(false, Ordering::Release); current.session_id = None; current.lifecycle_id = None; current.owner = None; current.revision = current.revision.wrapping_add(1); - current.pipeline.take() + let pipeline = current.pipeline.take(); + drop(current); + native_input_mute::stop(&input_muted, &input_mute_epoch); + native_microphone_capture.store(false, Ordering::Release); + native_microphone_mute_control.store(false, Ordering::Release); + pipeline }; if let Some(pipeline) = pipeline { shutdown_pipeline(pipeline).await; @@ -674,18 +721,29 @@ pub async fn stop_native_voice_conversation( shutdown_pipeline(pipeline).await; } let revision = { + let _native_microphone_guard = state + .native_microphone_lifecycle + .lock() + .map_err(|_| "native microphone lifecycle lock was poisoned".to_string())?; let mut runtime = state .runtime .lock() .map_err(|_| "native voice state lock was poisoned".to_string())?; if runtime.revision == revision && runtime.session_id == session_id { - state.stop_native_microphone(); runtime.session_id = None; runtime.lifecycle_id = None; runtime.owner = None; runtime.revision = runtime.revision.wrapping_add(1); + drop(runtime); + state.stop_native_microphone(); + state + .runtime + .lock() + .map_err(|_| "native voice state lock was poisoned".to_string())? + .revision + } else { + runtime.revision } - runtime.revision }; if let Some((owner, owner_id)) = owner.as_ref() { capture.release_owner(&owner.window_label, owner_id); @@ -732,18 +790,28 @@ impl NativeVoiceState { shutdown_pipeline(pipeline).await; } let next_revision = { + let _native_microphone_guard = self + .native_microphone_lifecycle + .lock() + .map_err(|_| "native microphone lifecycle lock was poisoned".to_string())?; let mut runtime = self .runtime .lock() .map_err(|_| "native voice state lock was poisoned".to_string())?; if runtime.revision == revision && runtime.session_id == session_id { - self.stop_native_microphone(); runtime.session_id = None; runtime.lifecycle_id = None; runtime.owner = None; runtime.revision = runtime.revision.wrapping_add(1); + drop(runtime); + self.stop_native_microphone(); + self.runtime + .lock() + .map_err(|_| "native voice state lock was poisoned".to_string())? + .revision + } else { + runtime.revision } - runtime.revision }; if let (Some(owner), Some(session_id)) = (owner, session_id) { capture.release_owner(&owner.window_label, &native_owner_id(&session_id)); @@ -762,6 +830,9 @@ impl NativeVoiceState { pub fn stop_for_window_destroyed(&self, window_label: &str) -> bool { let (session_id, revision, pipeline) = { + let Ok(_native_microphone_guard) = self.native_microphone_lifecycle.lock() else { + return false; + }; let Ok(mut runtime) = self.runtime.lock() else { return false; }; @@ -776,8 +847,10 @@ impl NativeVoiceState { if let Some(pipeline) = pipeline.as_ref() { pipeline.signal_shutdown(); } + let result = (runtime.session_id.clone(), runtime.revision, pipeline); + drop(runtime); self.stop_native_microphone(); - (runtime.session_id.clone(), runtime.revision, pipeline) + result }; if pipeline.is_none() { if let Ok(mut runtime) = self.runtime.lock() { @@ -807,18 +880,23 @@ impl NativeVoiceState { pub fn stop_for_app_exit(&self) { let (session_id, revision, pipeline) = { + let Ok(_native_microphone_guard) = self.native_microphone_lifecycle.lock() else { + return; + }; let Ok(mut runtime) = self.runtime.lock() else { return; }; if let Some(pipeline) = runtime.pipeline.as_ref() { pipeline.latch_muted_shutdown(); } - self.stop_native_microphone(); - ( + let result = ( runtime.session_id.clone(), runtime.revision, runtime.pipeline.take(), - ) + ); + drop(runtime); + self.stop_native_microphone(); + result }; drop(pipeline); if let Ok(mut runtime) = self.runtime.lock() { @@ -848,24 +926,39 @@ pub fn push_native_voice_audio( pub fn set_native_voice_input_muted( state: State<'_, NativeVoiceState>, webview_window: WebviewWindow, + session_id: String, + revision: u64, muted: bool, ) -> Result<(), String> { + let _native_microphone_guard = state + .native_microphone_lifecycle + .lock() + .map_err(|_| "native microphone lifecycle lock was poisoned".to_string())?; let runtime = state .runtime .lock() .map_err(|_| "native voice state lock was poisoned".to_string())?; - if runtime.session_id.is_none() - || runtime - .owner - .as_ref() - .is_none_or(|owner| owner.window_label != webview_window.label()) - { + if !owns_active_voice_lifecycle(&runtime, webview_window.label(), &session_id, revision) { return Err("Only the active voice owner may mute its microphone.".to_string()); } drop(runtime); native_input_mute::set_muted(&state.input_muted, &state.input_mute_epoch, muted) } +fn owns_active_voice_lifecycle( + runtime: &Runtime, + window_label: &str, + session_id: &str, + revision: u64, +) -> bool { + runtime.session_id.as_deref() == Some(session_id) + && runtime.revision == revision + && runtime + .owner + .as_ref() + .is_some_and(|owner| owner.window_label == window_label) +} + fn push_audio_for_window( state: &NativeVoiceState, window_label: &str, @@ -1125,6 +1218,43 @@ fn deliver_recognition_result( mod tests { use super::*; + #[test] + fn stale_lifecycle_cannot_change_native_input_mute() { + let runtime = Runtime { + session_id: Some("new-session".to_string()), + revision: 8, + owner: Some(RuntimeOwner { + window_label: "main".to_string(), + }), + ..Runtime::default() + }; + + assert!(owns_active_voice_lifecycle( + &runtime, + "main", + "new-session", + 8, + )); + assert!(!owns_active_voice_lifecycle( + &runtime, + "main", + "old-session", + 8, + )); + assert!(!owns_active_voice_lifecycle( + &runtime, + "main", + "new-session", + 7, + )); + assert!(!owns_active_voice_lifecycle( + &runtime, + "other-window", + "new-session", + 8, + )); + } + #[test] fn speaker_playback_blocks_vad_ingestion_until_all_guards_finish() { let state = NativeVoiceState::default(); diff --git a/src-tauri/swift/BerdAirPodsBridge/Package.swift b/src-tauri/swift/BerdAirPodsBridge/Package.swift index 51862d334..5e216c9a1 100644 --- a/src-tauri/swift/BerdAirPodsBridge/Package.swift +++ b/src-tauri/swift/BerdAirPodsBridge/Package.swift @@ -12,8 +12,13 @@ let package = Package( ) ], targets: [ + .target( + name: "BerdObjCExceptionCatch", + publicHeadersPath: "include" + ), .target( name: "BerdAirPodsBridge", + dependencies: ["BerdObjCExceptionCatch"], linkerSettings: [.linkedFramework("AVFAudio")] ) ] diff --git a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift index a37c05116..c7b64aa87 100644 --- a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift +++ b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift @@ -1,28 +1,99 @@ import AVFoundation +import BerdObjCExceptionCatch import Foundation public typealias InputMuteCallback = @convention(c) (Bool) -> Void public typealias AudioInputCallback = @convention(c) (UnsafePointer, Int) -> Void +public typealias CaptureStateCallback = @convention(c) (Bool) -> Void + +private let bridgeQueueKey = DispatchSpecificKey() +private let bridgeQueue: DispatchQueue = { + let queue = DispatchQueue(label: "com.berd.airpods-mute-bridge") + queue.setSpecific(key: bridgeQueueKey, value: ()) + return queue +}() + +private func onBridgeQueue(_ body: () -> T) -> T { + if DispatchQueue.getSpecific(key: bridgeQueueKey) != nil { + return body() + } + return bridgeQueue.sync(execute: body) +} @available(macOS 14.0, *) private final class AirPodsMuteBridge: @unchecked Sendable { - private let engine: AVAudioEngine - private let inputNode: AVAudioInputNode private let callback: InputMuteCallback private let audioCallback: AudioInputCallback - private let targetFormat: AVAudioFormat - private let converter: AVAudioConverter? + private let captureStateCallback: CaptureStateCallback + private var engine: AVAudioEngine? + private var inputNode: AVAudioInputNode? + private var configurationObserver: NSObjectProtocol? private var inputMuteObserver: NSObjectProtocol? + private var restartWorkItem: DispatchWorkItem? + private var restartGeneration: UInt64 = 0 + private var isStopped = false init( callback: @escaping InputMuteCallback, - audioCallback: @escaping AudioInputCallback + audioCallback: @escaping AudioInputCallback, + captureStateCallback: @escaping CaptureStateCallback ) throws { self.callback = callback self.audioCallback = audioCallback + self.captureStateCallback = captureStateCallback + + do { + try startCapture() + + // Clear stale process mute before registering this lifecycle's handler. + // The reset can fail when no earlier handler exists, which is harmless. + try? AVAudioApplication.shared.setInputMuted(false) + try AVAudioApplication.shared.setInputMuteStateChangeHandler { [weak self] muted in + self?.callback(muted) + return true + } + inputMuteObserver = NotificationCenter.default.addObserver( + forName: AVAudioApplication.inputMuteStateChangeNotification, + object: nil, + queue: nil + ) { [weak self] _ in + guard let self else { return } + self.callback(AVAudioApplication.shared.isInputMuted) + } + } catch { + teardownCapture() + throw error + } + } + + func stop() { + guard !isStopped else { return } + isStopped = true + restartGeneration &+= 1 + restartWorkItem?.cancel() + restartWorkItem = nil + // Reset while the handler exists, then cancel the process-wide callback. + try? AVAudioApplication.shared.setInputMuted(false) + try? AVAudioApplication.shared.setInputMuteStateChangeHandler(nil) + if let inputMuteObserver { + NotificationCenter.default.removeObserver(inputMuteObserver) + self.inputMuteObserver = nil + } + teardownCapture() + } + + private func startCapture() throws { let engine = AVAudioEngine() - let inputNode = engine.inputNode + var caughtInputNode: AVAudioInputNode? + var inputNodeError: NSError? + BerdTryObjCBlock({ + caughtInputNode = engine.inputNode + }, &inputNodeError) + guard inputNodeError == nil, let inputNode = caughtInputNode else { + throw BridgeError.objectiveC(inputNodeError?.localizedDescription ?? "no input node") + } + let inputFormat = inputNode.outputFormat(forBus: 0) guard inputFormat.sampleRate > 0, inputFormat.channelCount > 0 else { throw BridgeError.noInputFormat @@ -44,51 +115,114 @@ private final class AirPodsMuteBridge: @unchecked Sendable { } converter = created } + + var tapError: NSError? + BerdTryObjCBlock({ + inputNode.installTap( + onBus: 0, + bufferSize: 4096, + format: inputFormat + ) { [weak self] buffer, _ in + self?.forward(buffer, targetFormat: targetFormat, converter: converter) + } + }, &tapError) + guard tapError == nil else { + throw BridgeError.objectiveC(tapError?.localizedDescription ?? "install tap failed") + } + + do { + engine.prepare() + try engine.start() + } catch { + removeTap(from: inputNode) + engine.stop() + throw error + } + + configurationObserver = NotificationCenter.default.addObserver( + forName: .AVAudioEngineConfigurationChange, + object: engine, + queue: nil + ) { [weak self] _ in + self?.scheduleConfigurationRestart() + } self.engine = engine self.inputNode = inputNode - self.targetFormat = targetFormat - self.converter = converter + } - inputNode.installTap( - onBus: 0, - bufferSize: 4096, - format: inputFormat - ) { [weak self] buffer, _ in - self?.forward(buffer) + private func teardownCapture() { + if let configurationObserver { + NotificationCenter.default.removeObserver(configurationObserver) + self.configurationObserver = nil + } + if let inputNode { + removeTap(from: inputNode) } - engine.prepare() - try engine.start() + engine?.stop() + inputNode = nil + engine = nil + } - // Clear stale process mute before registering this lifecycle's handler. - // The reset can fail when no earlier handler exists, which is harmless. - try? AVAudioApplication.shared.setInputMuted(false) - try AVAudioApplication.shared.setInputMuteStateChangeHandler { [weak self] muted in - self?.callback(muted) - return true + private func removeTap(from inputNode: AVAudioInputNode) { + var removeError: NSError? + BerdTryObjCBlock({ + inputNode.removeTap(onBus: 0) + }, &removeError) + if let removeError { + FileHandle.standardError.write( + Data("Berd AirPods bridge could not remove its microphone tap: \(removeError)\n".utf8) + ) } - inputMuteObserver = NotificationCenter.default.addObserver( - forName: AVAudioApplication.inputMuteStateChangeNotification, - object: nil, - queue: .main - ) { [weak self] _ in - guard let self else { return } - self.callback(AVAudioApplication.shared.isInputMuted) + } + + private func scheduleConfigurationRestart() { + bridgeQueue.async { [weak self] in + guard let self, !self.isStopped else { return } + self.restartGeneration &+= 1 + let generation = self.restartGeneration + self.restartWorkItem?.cancel() + let workItem = DispatchWorkItem { [weak self] in + self?.restartCapture(generation: generation, attempt: 0) + } + self.restartWorkItem = workItem + bridgeQueue.asyncAfter(deadline: .now() + 0.15, execute: workItem) } } - func stop() { - // Reset while the handler exists, then cancel the process-wide callback. - try? AVAudioApplication.shared.setInputMuted(false) - try? AVAudioApplication.shared.setInputMuteStateChangeHandler(nil) - if let inputMuteObserver { - NotificationCenter.default.removeObserver(inputMuteObserver) - self.inputMuteObserver = nil + private func restartCapture(generation: UInt64, attempt: Int) { + guard !isStopped, generation == restartGeneration else { return } + restartWorkItem = nil + captureStateCallback(false) + teardownCapture() + do { + try startCapture() + captureStateCallback(true) + } catch { + let delays: [TimeInterval] = [0.3, 0.6, 1.2, 2.4, 4.8] + if attempt == delays.count { + FileHandle.standardError.write( + Data("Berd AirPods bridge is still retrying microphone capture: \(error)\n".utf8) + ) + } + let workItem = DispatchWorkItem { [weak self] in + self?.restartCapture( + generation: generation, + attempt: min(attempt + 1, delays.count + 1) + ) + } + restartWorkItem = workItem + bridgeQueue.asyncAfter( + deadline: .now() + delays[min(attempt, delays.count - 1)], + execute: workItem + ) } - inputNode.removeTap(onBus: 0) - engine.stop() } - private func forward(_ source: AVAudioPCMBuffer) { + private func forward( + _ source: AVAudioPCMBuffer, + targetFormat: AVAudioFormat, + converter: AVAudioConverter? + ) { let output: AVAudioPCMBuffer if let converter { let capacity = AVAudioFrameCount(ceil( @@ -124,6 +258,7 @@ private final class AirPodsMuteBridge: @unchecked Sendable { case noInputFormat case noOutputFormat case noConverter + case objectiveC(String) } } @@ -133,40 +268,49 @@ private var activeBridge: AirPodsMuteBridge? @_cdecl("berd_airpods_mute_start") public func berdAirPodsMuteStart( callback: @escaping InputMuteCallback, - audioCallback: @escaping AudioInputCallback + audioCallback: @escaping AudioInputCallback, + captureStateCallback: @escaping CaptureStateCallback ) -> Bool { guard #available(macOS 14.0, *) else { return false } - do { - activeBridge?.stop() - activeBridge = try AirPodsMuteBridge( - callback: callback, - audioCallback: audioCallback - ) - return true - } catch { - FileHandle.standardError.write( - Data("Berd AirPods mute bridge failed to start: \(error)\n".utf8) - ) - activeBridge = nil - return false + return onBridgeQueue { + do { + activeBridge?.stop() + activeBridge = try AirPodsMuteBridge( + callback: callback, + audioCallback: audioCallback, + captureStateCallback: captureStateCallback + ) + return true + } catch { + FileHandle.standardError.write( + Data("Berd AirPods mute bridge failed to start: \(error)\n".utf8) + ) + activeBridge = nil + return false + } } } @_cdecl("berd_airpods_mute_stop") public func berdAirPodsMuteStop() -> Bool { guard #available(macOS 14.0, *) else { return false } - activeBridge?.stop() - activeBridge = nil - return true + return onBridgeQueue { + activeBridge?.stop() + activeBridge = nil + return true + } } @_cdecl("berd_airpods_mute_set_muted") public func berdAirPodsMuteSetMuted(_ muted: Bool) -> Bool { - guard #available(macOS 14.0, *), activeBridge != nil else { return false } - do { - try AVAudioApplication.shared.setInputMuted(muted) - return true - } catch { - return false + guard #available(macOS 14.0, *) else { return false } + return onBridgeQueue { + guard activeBridge != nil else { return false } + do { + try AVAudioApplication.shared.setInputMuted(muted) + return true + } catch { + return false + } } } diff --git a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdObjCExceptionCatch/BerdObjCExceptionCatch.m b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdObjCExceptionCatch/BerdObjCExceptionCatch.m new file mode 100644 index 000000000..07cd2e8aa --- /dev/null +++ b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdObjCExceptionCatch/BerdObjCExceptionCatch.m @@ -0,0 +1,20 @@ +#import "BerdObjCExceptionCatch.h" + +BOOL BerdTryObjCBlock(void (NS_NOESCAPE ^_Nonnull block)(void), + NSError *_Nullable *_Nullable error) { + @try { + block(); + return YES; + } @catch (NSException *exception) { + if (error) { + NSDictionary *userInfo = @{ + NSLocalizedDescriptionKey: exception.reason ?: exception.name, + @"NSExceptionName": exception.name, + }; + *error = [NSError errorWithDomain:@"com.berd.objc-exception" + code:-1 + userInfo:userInfo]; + } + return NO; + } +} diff --git a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdObjCExceptionCatch/include/BerdObjCExceptionCatch.h b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdObjCExceptionCatch/include/BerdObjCExceptionCatch.h new file mode 100644 index 000000000..a7a99d574 --- /dev/null +++ b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdObjCExceptionCatch/include/BerdObjCExceptionCatch.h @@ -0,0 +1,6 @@ +#import + +/// Runs a block inside an Objective-C @try/@catch so Swift callers can handle +/// AVAudioEngine NSExceptions without terminating the process. +BOOL BerdTryObjCBlock(void (NS_NOESCAPE ^_Nonnull block)(void), + NSError *_Nullable *_Nullable error); diff --git a/src/features/voice-conversation/api/voiceConversation.test.ts b/src/features/voice-conversation/api/voiceConversation.test.ts index cc5061191..54d0b8c3a 100644 --- a/src/features/voice-conversation/api/voiceConversation.test.ts +++ b/src/features/voice-conversation/api/voiceConversation.test.ts @@ -171,6 +171,7 @@ describe("voice conversation API", () => { ownerWindowLabel: "main", revision: 3, nativeMicrophoneCapture: true, + nativeMicrophoneMuteControl: true, } as const; mocks.invoke.mockResolvedValue(undefined); @@ -179,6 +180,8 @@ describe("voice conversation API", () => { expect(mocks.startMicrophone).not.toHaveBeenCalled(); expect(mocks.invoke).toHaveBeenCalledWith("set_native_voice_input_muted", { + sessionId: "session-1", + revision: 3, muted: true, }); }); @@ -209,6 +212,60 @@ describe("voice conversation API", () => { expect(mocks.invoke).not.toHaveBeenCalled(); }); + it("preserves UI mute when native capture recovers from browser fallback", async () => { + const fallbackStatus = { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + revision: 3, + nativeMicrophoneCapture: false, + nativeMicrophoneMuteControl: true, + } as const; + const recoveredStatus = { + ...fallbackStatus, + nativeMicrophoneCapture: true, + } as const; + mocks.invoke.mockResolvedValue(undefined); + + await reconcileVoiceConversationMicrophone(fallbackStatus); + await setVoiceConversationMicrophoneMuted(true, fallbackStatus); + expect(mocks.invoke).toHaveBeenCalledTimes(1); + expect(mocks.invoke).toHaveBeenLastCalledWith( + "set_native_voice_input_muted", + { + sessionId: "session-1", + revision: 3, + muted: true, + }, + ); + await reconcileVoiceConversationMicrophone(recoveredStatus); + + expect(mocks.invoke).toHaveBeenCalledTimes(2); + expect(mocks.stopMicrophone).toHaveBeenCalledOnce(); + }); + + it("does not change native mute when browser fallback cannot start", async () => { + const fallbackStatus = { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + revision: 3, + nativeMicrophoneCapture: false, + nativeMicrophoneMuteControl: true, + } as const; + mocks.startMicrophone.mockRejectedValueOnce(new Error("capture failed")); + + await expect( + setVoiceConversationMicrophoneMuted(true, fallbackStatus), + ).rejects.toThrow("capture failed"); + + expect(mocks.invoke).not.toHaveBeenCalled(); + }); + it("restores the previous mute state when initial capture fails", async () => { const status = { available: true, diff --git a/src/features/voice-conversation/api/voiceConversation.ts b/src/features/voice-conversation/api/voiceConversation.ts index 417fe738a..e476a97ed 100644 --- a/src/features/voice-conversation/api/voiceConversation.ts +++ b/src/features/voice-conversation/api/voiceConversation.ts @@ -50,6 +50,18 @@ export async function reconcileVoiceConversationMicrophone( status: VoiceConversationStatus, ): Promise { if (status.nativeMicrophoneCapture) { + const fallbackWasActive = + activeMicrophone !== null || microphoneStart !== null; + if (fallbackWasActive) { + if (!status.sessionId) { + throw new Error("Native voice microphone has no active session."); + } + await invoke("set_native_voice_input_muted", { + sessionId: status.sessionId, + revision: status.revision, + muted: microphoneMuted, + }); + } stopActiveMicrophone(); return; } @@ -70,9 +82,21 @@ export async function setVoiceConversationMicrophoneMuted( const previous = microphoneMuted; microphoneMuted = muted; try { - if (status.nativeMicrophoneCapture) { - await invoke("set_native_voice_input_muted", { muted }); - stopActiveMicrophone(); + if (status.nativeMicrophoneMuteControl) { + if (!status.nativeMicrophoneCapture) { + await reconcileVoiceConversationMicrophone(status); + } + if (!status.sessionId) { + throw new Error("Native voice microphone has no active session."); + } + await invoke("set_native_voice_input_muted", { + sessionId: status.sessionId, + revision: status.revision, + muted, + }); + if (status.nativeMicrophoneCapture) { + stopActiveMicrophone(); + } return; } await reconcileVoiceConversationMicrophone(status); @@ -83,6 +107,13 @@ export async function setVoiceConversationMicrophoneMuted( } } +export function applyVoiceConversationMicrophoneMuteEvent( + muted: boolean, +): void { + microphoneMuted = muted; + activeMicrophone?.setMuted(muted); +} + export function stopActiveMicrophoneForTest(): void { if (!import.meta.env.DEV) { throw new Error("Native microphone test controls are development-only."); @@ -120,6 +151,8 @@ export interface VoiceConversationStatus { revision: number; /** The backend owns the PCM stream so device gestures reach that process. */ nativeMicrophoneCapture?: boolean; + /** The process-wide native mute bridge remains usable during route recovery. */ + nativeMicrophoneMuteControl?: boolean; } export type VoiceConversationEvent = @@ -130,6 +163,7 @@ export type VoiceConversationEvent = line: string; revision: number; nativeMicrophoneCapture: boolean; + nativeMicrophoneMuteControl: boolean; } | { type: "user"; @@ -156,6 +190,12 @@ export type VoiceConversationEvent = muted: boolean; revision: number; } + | { + type: "nativeMicrophoneCapture"; + sessionId: string; + available: boolean; + revision: number; + } | { type: "cleanShutdown"; sessionId: string; diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts index 07446a4a8..0371b419a 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts @@ -24,7 +24,7 @@ import { createVoiceRouteMountRegistry, createVoiceTranscriptDeliveryQueue, hasDeliveredVoiceTranscript, - notifyAirPodsInputMuteGesture, + notifyInputMuteChanged, resetVoiceUiWhenRunSettles, resolveVoiceRouteMount, resolveVoiceToggleAction, @@ -36,18 +36,16 @@ import { } from "./useVoiceConversationController"; describe("voice transcript delivery coordination", () => { - it("shows an unmistakable AirPods gesture confirmation", () => { - notifyAirPodsInputMuteGesture(true); - expect(toastMocks.message).toHaveBeenCalledWith( - "AirPods gesture detected — microphone muted", - { id: "airpods-input-mute" }, - ); + it("shows an unmistakable input mute confirmation", () => { + notifyInputMuteChanged(true); + expect(toastMocks.message).toHaveBeenCalledWith("Microphone muted", { + id: "voice-input-mute", + }); - notifyAirPodsInputMuteGesture(false); - expect(toastMocks.message).toHaveBeenLastCalledWith( - "AirPods gesture detected — microphone unmuted", - { id: "airpods-input-mute" }, - ); + notifyInputMuteChanged(false); + expect(toastMocks.message).toHaveBeenLastCalledWith("Microphone unmuted", { + id: "voice-input-mute", + }); }); it("recognizes a replayed transcript that was already delivered", () => { diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index 1e15eb54d..8d08e4adf 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -188,13 +188,10 @@ function addErrorNotification(sessionId: string | null, message: string) { .addMessage(sessionId, createSystemNotificationMessage(message, "error")); } -export function notifyAirPodsInputMuteGesture(muted: boolean) { - toast.message( - muted - ? "AirPods gesture detected — microphone muted" - : "AirPods gesture detected — microphone unmuted", - { id: "airpods-input-mute" }, - ); +export function notifyInputMuteChanged(muted: boolean) { + toast.message(muted ? "Microphone muted" : "Microphone unmuted", { + id: "voice-input-mute", + }); } export function hasDeliveredVoiceTranscript( @@ -325,7 +322,7 @@ function ensureVoiceEventDeliveryInitialized() { return; } if (event.type === "inputMute") { - notifyAirPodsInputMuteGesture(event.muted); + notifyInputMuteChanged(event.muted); return; } if (event.type === "activity") return; diff --git a/src/features/voice-conversation/stores/voiceConversationStore.test.ts b/src/features/voice-conversation/stores/voiceConversationStore.test.ts index 19aec7cea..7239b03c7 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.test.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.test.ts @@ -7,6 +7,7 @@ import type { const mocks = vi.hoisted(() => ({ acknowledge: vi.fn(), + applyInputMute: vi.fn(), drain: vi.fn(), getStatus: vi.fn(), listen: vi.fn(), @@ -19,6 +20,7 @@ const mocks = vi.hoisted(() => ({ vi.mock("../api/voiceConversation", () => ({ acknowledgeVoiceConversationTranscript: mocks.acknowledge, + applyVoiceConversationMicrophoneMuteEvent: mocks.applyInputMute, drainVoiceConversationTranscripts: mocks.drain, getVoiceConversationStatus: mocks.getStatus, listenToVoiceConversation: mocks.listen, @@ -58,6 +60,7 @@ describe("voice conversation store lifecycle ordering", () => { beforeEach(() => { vi.resetModules(); mocks.acknowledge.mockReset().mockResolvedValue(undefined); + mocks.applyInputMute.mockReset(); mocks.drain.mockReset().mockResolvedValue([]); mocks.getStatus.mockReset().mockResolvedValue(status("stopped", 0)); mocks.start.mockReset(); @@ -266,6 +269,7 @@ describe("voice conversation store lifecycle ordering", () => { line: "type\tid\ttext", revision: 2, nativeMicrophoneCapture: true, + nativeMicrophoneMuteControl: true, }); response.resolve(status("starting", 1, "session-1")); await starting; @@ -274,6 +278,7 @@ describe("voice conversation store lifecycle ordering", () => { status: { ...status("running", 2, "session-1"), nativeMicrophoneCapture: true, + nativeMicrophoneMuteControl: true, }, uiState: "listening", error: null, @@ -413,6 +418,42 @@ describe("voice conversation store lifecycle ordering", () => { userSpeaking: false, uiState: "listening", }); + expect(mocks.applyInputMute).toHaveBeenCalledWith(true); + }); + + it("falls back while native capture restarts and returns when it recovers", async () => { + const store = await loadStore(); + store.setState({ + status: { + ...status("running", 3, "session-1"), + nativeMicrophoneCapture: true, + }, + }); + mocks.reconcileMicrophone.mockClear(); + + emit({ + type: "nativeMicrophoneCapture", + sessionId: "session-1", + available: false, + revision: 3, + }); + await vi.waitFor(() => { + expect(mocks.reconcileMicrophone).toHaveBeenLastCalledWith( + expect.objectContaining({ nativeMicrophoneCapture: false }), + ); + }); + + emit({ + type: "nativeMicrophoneCapture", + sessionId: "session-1", + available: true, + revision: 3, + }); + await vi.waitFor(() => { + expect(mocks.reconcileMicrophone).toHaveBeenLastCalledWith( + expect.objectContaining({ nativeMicrophoneCapture: true }), + ); + }); }); it("surfaces an unmute failure without losing muted state", async () => { diff --git a/src/features/voice-conversation/stores/voiceConversationStore.ts b/src/features/voice-conversation/stores/voiceConversationStore.ts index 8c26864ad..05254d42c 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.ts @@ -2,6 +2,7 @@ import { create } from "zustand"; import { acknowledgeVoiceConversationTranscript, + applyVoiceConversationMicrophoneMuteEvent, drainVoiceConversationTranscripts, getVoiceConversationStatus, listenToVoiceConversation, @@ -241,6 +242,8 @@ export const useVoiceConversationStore = create( ownerWindowLabel: event.ownerWindowLabel, revision: event.revision, nativeMicrophoneCapture: event.nativeMicrophoneCapture, + nativeMicrophoneMuteControl: + event.nativeMicrophoneMuteControl, }, uiState: "listening", microphoneMuted: false, @@ -306,6 +309,17 @@ export const useVoiceConversationStore = create( uiState: activityUiState(nextState), }; } + case "nativeMicrophoneCapture": + return { + ...state, + status: { + ...state.status, + lifecycle: "running", + sessionId: event.sessionId, + revision: event.revision, + nativeMicrophoneCapture: event.available, + }, + }; case "cleanShutdown": return { ...state, @@ -348,7 +362,20 @@ export const useVoiceConversationStore = create( } }); - if (event.type === "user") { + if (event.type === "inputMute") { + applyVoiceConversationMicrophoneMuteEvent(event.muted); + for (const subscriber of [...eventSubscribers]) + void subscriber(event); + } else if (event.type === "nativeMicrophoneCapture") { + void reconcileVoiceConversationMicrophone(get().status).catch( + (error) => { + set({ + uiState: "error", + error: error instanceof Error ? error.message : String(error), + }); + }, + ); + } else if (event.type === "user") { void deliverTranscriptOnce(event).catch((error) => { const current = get(); if ( From 8f64ee5f08b4edda9767af81f3073a9cd6bc7a16 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 16:29:09 -0400 Subject: [PATCH 12/31] Revert "fix(voice): harden native microphone lifecycle" This reverts commit c4cb4762ee82ef9aac9e50bd511551a506863f2b. --- src-tauri/src/commands/native_input_mute.rs | 33 +-- src-tauri/src/commands/native_voice.rs | 164 ++--------- .../swift/BerdAirPodsBridge/Package.swift | 5 - .../BerdAirPodsBridge/BerdAirPodsBridge.swift | 268 ++++-------------- .../BerdObjCExceptionCatch.m | 20 -- .../include/BerdObjCExceptionCatch.h | 6 - .../api/voiceConversation.test.ts | 57 ---- .../api/voiceConversation.ts | 46 +-- .../useVoiceConversationController.test.ts | 22 +- .../hooks/useVoiceConversationController.ts | 13 +- .../stores/voiceConversationStore.test.ts | 41 --- .../stores/voiceConversationStore.ts | 29 +- 12 files changed, 106 insertions(+), 598 deletions(-) delete mode 100644 src-tauri/swift/BerdAirPodsBridge/Sources/BerdObjCExceptionCatch/BerdObjCExceptionCatch.m delete mode 100644 src-tauri/swift/BerdAirPodsBridge/Sources/BerdObjCExceptionCatch/include/BerdObjCExceptionCatch.h diff --git a/src-tauri/src/commands/native_input_mute.rs b/src-tauri/src/commands/native_input_mute.rs index 7691ab119..19a4008a2 100644 --- a/src-tauri/src/commands/native_input_mute.rs +++ b/src-tauri/src/commands/native_input_mute.rs @@ -3,17 +3,15 @@ use std::sync::{ Arc, }; -pub fn start( +pub fn start( input_muted: &Arc, mute_epoch: &Arc, on_change: F, on_audio: A, - on_capture_state: C, ) -> bool where F: Fn(bool) + Send + Sync + 'static, A: Fn(&[f32]) + Send + Sync + 'static, - C: Fn(bool) + Send + Sync + 'static, { clear(input_muted, mute_epoch); @@ -23,7 +21,6 @@ where Arc::clone(mute_epoch), Arc::new(on_change), Arc::new(on_audio), - Arc::new(on_capture_state), ) { Ok(()) => true, Err(error) => { @@ -34,7 +31,7 @@ where #[cfg(not(target_os = "macos"))] let started = { - let _ = (on_change, on_audio, on_capture_state); + let _ = (on_change, on_audio); false }; @@ -97,14 +94,12 @@ mod macos { type MuteChangeHandler = Arc; type AudioInputHandler = Arc; - type CaptureStateHandler = Arc; struct CallbackState { input_muted: Arc, mute_epoch: Arc, on_change: MuteChangeHandler, on_audio: AudioInputHandler, - on_capture_state: CaptureStateHandler, } static CALLBACK_STATE: OnceLock>> = OnceLock::new(); @@ -117,7 +112,6 @@ mod macos { fn berd_airpods_mute_start( callback: extern "C" fn(bool), audio_callback: extern "C" fn(*const f32, usize), - capture_state_callback: extern "C" fn(bool), ) -> bool; fn berd_airpods_mute_stop() -> bool; fn berd_airpods_mute_set_muted(muted: bool) -> bool; @@ -155,25 +149,11 @@ mod macos { callback(samples); } - extern "C" fn handle_capture_state_change(available: bool) { - let callback = { - let Ok(state) = callback_state().lock() else { - return; - }; - let Some(state) = state.as_ref() else { - return; - }; - Arc::clone(&state.on_capture_state) - }; - callback(available); - } - pub fn install( input_muted: Arc, mute_epoch: Arc, on_change: MuteChangeHandler, on_audio: AudioInputHandler, - on_capture_state: CaptureStateHandler, ) -> Result<(), String> { *callback_state() .lock() @@ -183,17 +163,10 @@ mod macos { mute_epoch, on_change, on_audio, - on_capture_state, }); // SAFETY: The Swift shim retains the callback and AVAudioEngine for its // process-wide lifecycle and invokes it with a C-compatible boolean. - if !unsafe { - berd_airpods_mute_start( - handle_input_mute_change, - handle_audio_input, - handle_capture_state_change, - ) - } { + if !unsafe { berd_airpods_mute_start(handle_input_mute_change, handle_audio_input) } { callback_state() .lock() .map_err(|_| "input mute callback lock was poisoned".to_string())? diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index 6f97757b6..b2cb45dd1 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -50,7 +50,6 @@ pub struct NativeVoiceStatus { owner_window_label: Option, revision: u64, native_microphone_capture: bool, - native_microphone_mute_control: bool, } #[derive(Clone, Debug, Serialize)] @@ -84,7 +83,6 @@ enum NativeVoiceEvent { line: String, revision: u64, native_microphone_capture: bool, - native_microphone_mute_control: bool, }, User { session_id: String, @@ -104,11 +102,6 @@ enum NativeVoiceEvent { muted: bool, revision: u64, }, - NativeMicrophoneCapture { - session_id: String, - available: bool, - revision: u64, - }, CleanShutdown { session_id: String, revision: u64, @@ -143,8 +136,6 @@ pub struct NativeVoiceState { input_muted: Arc, input_mute_epoch: Arc, native_microphone_capture: Arc, - native_microphone_mute_control: Arc, - native_microphone_lifecycle: Arc>, } #[must_use = "capture suppression ends when the guard is dropped"] @@ -183,8 +174,6 @@ impl NativeVoiceState { native_input_mute::stop(&self.input_muted, &self.input_mute_epoch); self.native_microphone_capture .store(false, Ordering::Release); - self.native_microphone_mute_control - .store(false, Ordering::Release); } } @@ -351,9 +340,6 @@ fn status(app: &AppHandle, state: &NativeVoiceState) -> NativeVoiceStatus { .map(|owner| owner.window_label.clone()), revision: runtime.revision, native_microphone_capture: state.native_microphone_capture.load(Ordering::Acquire), - native_microphone_mute_control: state - .native_microphone_mute_control - .load(Ordering::Acquire), } } @@ -484,10 +470,6 @@ pub async fn start_native_voice_conversation( return Err(error); } }; - let native_microphone_lifecycle = Arc::clone(&state.native_microphone_lifecycle); - let _native_microphone_guard = native_microphone_lifecycle - .lock() - .map_err(|_| "native microphone lifecycle lock was poisoned".to_string())?; let (revision, lifecycle_id) = { let mut runtime = state .runtime @@ -515,9 +497,6 @@ pub async fn start_native_voice_conversation( let mute_session_id = session_id.clone(); let audio_state = state.inner().clone(); let audio_session_id = session_id.clone(); - let capture_window = webview_window.clone(); - let capture_session_id = session_id.clone(); - let native_microphone_capture_state = Arc::clone(&state.native_microphone_capture); let native_capture_started = native_input_mute::start( &state.input_muted, &state.input_mute_epoch, @@ -538,26 +517,10 @@ pub async fn start_native_voice_conversation( log::warn!("Native microphone audio was not accepted: {error}"); } }, - move |available| { - let previous = native_microphone_capture_state.swap(available, Ordering::AcqRel); - if previous != available { - let _ = capture_window.emit( - EVENT_NAME, - NativeVoiceEvent::NativeMicrophoneCapture { - session_id: capture_session_id.clone(), - available, - revision, - }, - ); - } - }, ); state .native_microphone_capture .store(native_capture_started, Ordering::Release); - state - .native_microphone_mute_control - .store(native_capture_started, Ordering::Release); let _ = webview_window.emit( EVENT_NAME, NativeVoiceEvent::Startup { @@ -566,7 +529,6 @@ pub async fn start_native_voice_conversation( line: "Native Parakeet voice conversation is on".to_string(), revision, native_microphone_capture: native_capture_started, - native_microphone_mute_control: native_capture_started, }, ); @@ -577,8 +539,6 @@ pub async fn start_native_voice_conversation( let input_muted = Arc::clone(&state.input_muted); let input_mute_epoch = Arc::clone(&state.input_mute_epoch); let native_microphone_capture = Arc::clone(&state.native_microphone_capture); - let native_microphone_mute_control = Arc::clone(&state.native_microphone_mute_control); - let native_microphone_lifecycle = Arc::clone(&state.native_microphone_lifecycle); tauri::async_runtime::spawn(async move { while let Some(event) = events.recv().await { let active = runtime.lock().ok().is_some_and(|current| { @@ -643,10 +603,6 @@ pub async fn start_native_voice_conversation( } SttMessage::Failed(message) => { let pipeline = { - let Ok(_native_microphone_guard) = native_microphone_lifecycle.lock() - else { - break; - }; let Ok(mut current) = runtime.lock() else { break; }; @@ -655,16 +611,13 @@ pub async fn start_native_voice_conversation( { break; } + native_input_mute::stop(&input_muted, &input_mute_epoch); + native_microphone_capture.store(false, Ordering::Release); current.session_id = None; current.lifecycle_id = None; current.owner = None; current.revision = current.revision.wrapping_add(1); - let pipeline = current.pipeline.take(); - drop(current); - native_input_mute::stop(&input_muted, &input_mute_epoch); - native_microphone_capture.store(false, Ordering::Release); - native_microphone_mute_control.store(false, Ordering::Release); - pipeline + current.pipeline.take() }; if let Some(pipeline) = pipeline { shutdown_pipeline(pipeline).await; @@ -721,29 +674,18 @@ pub async fn stop_native_voice_conversation( shutdown_pipeline(pipeline).await; } let revision = { - let _native_microphone_guard = state - .native_microphone_lifecycle - .lock() - .map_err(|_| "native microphone lifecycle lock was poisoned".to_string())?; let mut runtime = state .runtime .lock() .map_err(|_| "native voice state lock was poisoned".to_string())?; if runtime.revision == revision && runtime.session_id == session_id { + state.stop_native_microphone(); runtime.session_id = None; runtime.lifecycle_id = None; runtime.owner = None; runtime.revision = runtime.revision.wrapping_add(1); - drop(runtime); - state.stop_native_microphone(); - state - .runtime - .lock() - .map_err(|_| "native voice state lock was poisoned".to_string())? - .revision - } else { - runtime.revision } + runtime.revision }; if let Some((owner, owner_id)) = owner.as_ref() { capture.release_owner(&owner.window_label, owner_id); @@ -790,28 +732,18 @@ impl NativeVoiceState { shutdown_pipeline(pipeline).await; } let next_revision = { - let _native_microphone_guard = self - .native_microphone_lifecycle - .lock() - .map_err(|_| "native microphone lifecycle lock was poisoned".to_string())?; let mut runtime = self .runtime .lock() .map_err(|_| "native voice state lock was poisoned".to_string())?; if runtime.revision == revision && runtime.session_id == session_id { + self.stop_native_microphone(); runtime.session_id = None; runtime.lifecycle_id = None; runtime.owner = None; runtime.revision = runtime.revision.wrapping_add(1); - drop(runtime); - self.stop_native_microphone(); - self.runtime - .lock() - .map_err(|_| "native voice state lock was poisoned".to_string())? - .revision - } else { - runtime.revision } + runtime.revision }; if let (Some(owner), Some(session_id)) = (owner, session_id) { capture.release_owner(&owner.window_label, &native_owner_id(&session_id)); @@ -830,9 +762,6 @@ impl NativeVoiceState { pub fn stop_for_window_destroyed(&self, window_label: &str) -> bool { let (session_id, revision, pipeline) = { - let Ok(_native_microphone_guard) = self.native_microphone_lifecycle.lock() else { - return false; - }; let Ok(mut runtime) = self.runtime.lock() else { return false; }; @@ -847,10 +776,8 @@ impl NativeVoiceState { if let Some(pipeline) = pipeline.as_ref() { pipeline.signal_shutdown(); } - let result = (runtime.session_id.clone(), runtime.revision, pipeline); - drop(runtime); self.stop_native_microphone(); - result + (runtime.session_id.clone(), runtime.revision, pipeline) }; if pipeline.is_none() { if let Ok(mut runtime) = self.runtime.lock() { @@ -880,23 +807,18 @@ impl NativeVoiceState { pub fn stop_for_app_exit(&self) { let (session_id, revision, pipeline) = { - let Ok(_native_microphone_guard) = self.native_microphone_lifecycle.lock() else { - return; - }; let Ok(mut runtime) = self.runtime.lock() else { return; }; if let Some(pipeline) = runtime.pipeline.as_ref() { pipeline.latch_muted_shutdown(); } - let result = ( + self.stop_native_microphone(); + ( runtime.session_id.clone(), runtime.revision, runtime.pipeline.take(), - ); - drop(runtime); - self.stop_native_microphone(); - result + ) }; drop(pipeline); if let Ok(mut runtime) = self.runtime.lock() { @@ -926,39 +848,24 @@ pub fn push_native_voice_audio( pub fn set_native_voice_input_muted( state: State<'_, NativeVoiceState>, webview_window: WebviewWindow, - session_id: String, - revision: u64, muted: bool, ) -> Result<(), String> { - let _native_microphone_guard = state - .native_microphone_lifecycle - .lock() - .map_err(|_| "native microphone lifecycle lock was poisoned".to_string())?; let runtime = state .runtime .lock() .map_err(|_| "native voice state lock was poisoned".to_string())?; - if !owns_active_voice_lifecycle(&runtime, webview_window.label(), &session_id, revision) { + if runtime.session_id.is_none() + || runtime + .owner + .as_ref() + .is_none_or(|owner| owner.window_label != webview_window.label()) + { return Err("Only the active voice owner may mute its microphone.".to_string()); } drop(runtime); native_input_mute::set_muted(&state.input_muted, &state.input_mute_epoch, muted) } -fn owns_active_voice_lifecycle( - runtime: &Runtime, - window_label: &str, - session_id: &str, - revision: u64, -) -> bool { - runtime.session_id.as_deref() == Some(session_id) - && runtime.revision == revision - && runtime - .owner - .as_ref() - .is_some_and(|owner| owner.window_label == window_label) -} - fn push_audio_for_window( state: &NativeVoiceState, window_label: &str, @@ -1218,43 +1125,6 @@ fn deliver_recognition_result( mod tests { use super::*; - #[test] - fn stale_lifecycle_cannot_change_native_input_mute() { - let runtime = Runtime { - session_id: Some("new-session".to_string()), - revision: 8, - owner: Some(RuntimeOwner { - window_label: "main".to_string(), - }), - ..Runtime::default() - }; - - assert!(owns_active_voice_lifecycle( - &runtime, - "main", - "new-session", - 8, - )); - assert!(!owns_active_voice_lifecycle( - &runtime, - "main", - "old-session", - 8, - )); - assert!(!owns_active_voice_lifecycle( - &runtime, - "main", - "new-session", - 7, - )); - assert!(!owns_active_voice_lifecycle( - &runtime, - "other-window", - "new-session", - 8, - )); - } - #[test] fn speaker_playback_blocks_vad_ingestion_until_all_guards_finish() { let state = NativeVoiceState::default(); diff --git a/src-tauri/swift/BerdAirPodsBridge/Package.swift b/src-tauri/swift/BerdAirPodsBridge/Package.swift index 5e216c9a1..51862d334 100644 --- a/src-tauri/swift/BerdAirPodsBridge/Package.swift +++ b/src-tauri/swift/BerdAirPodsBridge/Package.swift @@ -12,13 +12,8 @@ let package = Package( ) ], targets: [ - .target( - name: "BerdObjCExceptionCatch", - publicHeadersPath: "include" - ), .target( name: "BerdAirPodsBridge", - dependencies: ["BerdObjCExceptionCatch"], linkerSettings: [.linkedFramework("AVFAudio")] ) ] diff --git a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift index c7b64aa87..a37c05116 100644 --- a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift +++ b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift @@ -1,99 +1,28 @@ import AVFoundation -import BerdObjCExceptionCatch import Foundation public typealias InputMuteCallback = @convention(c) (Bool) -> Void public typealias AudioInputCallback = @convention(c) (UnsafePointer, Int) -> Void -public typealias CaptureStateCallback = @convention(c) (Bool) -> Void - -private let bridgeQueueKey = DispatchSpecificKey() -private let bridgeQueue: DispatchQueue = { - let queue = DispatchQueue(label: "com.berd.airpods-mute-bridge") - queue.setSpecific(key: bridgeQueueKey, value: ()) - return queue -}() - -private func onBridgeQueue(_ body: () -> T) -> T { - if DispatchQueue.getSpecific(key: bridgeQueueKey) != nil { - return body() - } - return bridgeQueue.sync(execute: body) -} @available(macOS 14.0, *) private final class AirPodsMuteBridge: @unchecked Sendable { + private let engine: AVAudioEngine + private let inputNode: AVAudioInputNode private let callback: InputMuteCallback private let audioCallback: AudioInputCallback - private let captureStateCallback: CaptureStateCallback - private var engine: AVAudioEngine? - private var inputNode: AVAudioInputNode? - private var configurationObserver: NSObjectProtocol? + private let targetFormat: AVAudioFormat + private let converter: AVAudioConverter? private var inputMuteObserver: NSObjectProtocol? - private var restartWorkItem: DispatchWorkItem? - private var restartGeneration: UInt64 = 0 - private var isStopped = false init( callback: @escaping InputMuteCallback, - audioCallback: @escaping AudioInputCallback, - captureStateCallback: @escaping CaptureStateCallback + audioCallback: @escaping AudioInputCallback ) throws { self.callback = callback self.audioCallback = audioCallback - self.captureStateCallback = captureStateCallback - - do { - try startCapture() - - // Clear stale process mute before registering this lifecycle's handler. - // The reset can fail when no earlier handler exists, which is harmless. - try? AVAudioApplication.shared.setInputMuted(false) - try AVAudioApplication.shared.setInputMuteStateChangeHandler { [weak self] muted in - self?.callback(muted) - return true - } - inputMuteObserver = NotificationCenter.default.addObserver( - forName: AVAudioApplication.inputMuteStateChangeNotification, - object: nil, - queue: nil - ) { [weak self] _ in - guard let self else { return } - self.callback(AVAudioApplication.shared.isInputMuted) - } - } catch { - teardownCapture() - throw error - } - } - - func stop() { - guard !isStopped else { return } - isStopped = true - restartGeneration &+= 1 - restartWorkItem?.cancel() - restartWorkItem = nil - // Reset while the handler exists, then cancel the process-wide callback. - try? AVAudioApplication.shared.setInputMuted(false) - try? AVAudioApplication.shared.setInputMuteStateChangeHandler(nil) - if let inputMuteObserver { - NotificationCenter.default.removeObserver(inputMuteObserver) - self.inputMuteObserver = nil - } - teardownCapture() - } - - private func startCapture() throws { let engine = AVAudioEngine() - var caughtInputNode: AVAudioInputNode? - var inputNodeError: NSError? - BerdTryObjCBlock({ - caughtInputNode = engine.inputNode - }, &inputNodeError) - guard inputNodeError == nil, let inputNode = caughtInputNode else { - throw BridgeError.objectiveC(inputNodeError?.localizedDescription ?? "no input node") - } - + let inputNode = engine.inputNode let inputFormat = inputNode.outputFormat(forBus: 0) guard inputFormat.sampleRate > 0, inputFormat.channelCount > 0 else { throw BridgeError.noInputFormat @@ -115,114 +44,51 @@ private final class AirPodsMuteBridge: @unchecked Sendable { } converter = created } - - var tapError: NSError? - BerdTryObjCBlock({ - inputNode.installTap( - onBus: 0, - bufferSize: 4096, - format: inputFormat - ) { [weak self] buffer, _ in - self?.forward(buffer, targetFormat: targetFormat, converter: converter) - } - }, &tapError) - guard tapError == nil else { - throw BridgeError.objectiveC(tapError?.localizedDescription ?? "install tap failed") - } - - do { - engine.prepare() - try engine.start() - } catch { - removeTap(from: inputNode) - engine.stop() - throw error - } - - configurationObserver = NotificationCenter.default.addObserver( - forName: .AVAudioEngineConfigurationChange, - object: engine, - queue: nil - ) { [weak self] _ in - self?.scheduleConfigurationRestart() - } self.engine = engine self.inputNode = inputNode - } + self.targetFormat = targetFormat + self.converter = converter - private func teardownCapture() { - if let configurationObserver { - NotificationCenter.default.removeObserver(configurationObserver) - self.configurationObserver = nil - } - if let inputNode { - removeTap(from: inputNode) + inputNode.installTap( + onBus: 0, + bufferSize: 4096, + format: inputFormat + ) { [weak self] buffer, _ in + self?.forward(buffer) } - engine?.stop() - inputNode = nil - engine = nil - } + engine.prepare() + try engine.start() - private func removeTap(from inputNode: AVAudioInputNode) { - var removeError: NSError? - BerdTryObjCBlock({ - inputNode.removeTap(onBus: 0) - }, &removeError) - if let removeError { - FileHandle.standardError.write( - Data("Berd AirPods bridge could not remove its microphone tap: \(removeError)\n".utf8) - ) + // Clear stale process mute before registering this lifecycle's handler. + // The reset can fail when no earlier handler exists, which is harmless. + try? AVAudioApplication.shared.setInputMuted(false) + try AVAudioApplication.shared.setInputMuteStateChangeHandler { [weak self] muted in + self?.callback(muted) + return true } - } - - private func scheduleConfigurationRestart() { - bridgeQueue.async { [weak self] in - guard let self, !self.isStopped else { return } - self.restartGeneration &+= 1 - let generation = self.restartGeneration - self.restartWorkItem?.cancel() - let workItem = DispatchWorkItem { [weak self] in - self?.restartCapture(generation: generation, attempt: 0) - } - self.restartWorkItem = workItem - bridgeQueue.asyncAfter(deadline: .now() + 0.15, execute: workItem) + inputMuteObserver = NotificationCenter.default.addObserver( + forName: AVAudioApplication.inputMuteStateChangeNotification, + object: nil, + queue: .main + ) { [weak self] _ in + guard let self else { return } + self.callback(AVAudioApplication.shared.isInputMuted) } } - private func restartCapture(generation: UInt64, attempt: Int) { - guard !isStopped, generation == restartGeneration else { return } - restartWorkItem = nil - captureStateCallback(false) - teardownCapture() - do { - try startCapture() - captureStateCallback(true) - } catch { - let delays: [TimeInterval] = [0.3, 0.6, 1.2, 2.4, 4.8] - if attempt == delays.count { - FileHandle.standardError.write( - Data("Berd AirPods bridge is still retrying microphone capture: \(error)\n".utf8) - ) - } - let workItem = DispatchWorkItem { [weak self] in - self?.restartCapture( - generation: generation, - attempt: min(attempt + 1, delays.count + 1) - ) - } - restartWorkItem = workItem - bridgeQueue.asyncAfter( - deadline: .now() + delays[min(attempt, delays.count - 1)], - execute: workItem - ) + func stop() { + // Reset while the handler exists, then cancel the process-wide callback. + try? AVAudioApplication.shared.setInputMuted(false) + try? AVAudioApplication.shared.setInputMuteStateChangeHandler(nil) + if let inputMuteObserver { + NotificationCenter.default.removeObserver(inputMuteObserver) + self.inputMuteObserver = nil } + inputNode.removeTap(onBus: 0) + engine.stop() } - private func forward( - _ source: AVAudioPCMBuffer, - targetFormat: AVAudioFormat, - converter: AVAudioConverter? - ) { + private func forward(_ source: AVAudioPCMBuffer) { let output: AVAudioPCMBuffer if let converter { let capacity = AVAudioFrameCount(ceil( @@ -258,7 +124,6 @@ private final class AirPodsMuteBridge: @unchecked Sendable { case noInputFormat case noOutputFormat case noConverter - case objectiveC(String) } } @@ -268,49 +133,40 @@ private var activeBridge: AirPodsMuteBridge? @_cdecl("berd_airpods_mute_start") public func berdAirPodsMuteStart( callback: @escaping InputMuteCallback, - audioCallback: @escaping AudioInputCallback, - captureStateCallback: @escaping CaptureStateCallback + audioCallback: @escaping AudioInputCallback ) -> Bool { guard #available(macOS 14.0, *) else { return false } - return onBridgeQueue { - do { - activeBridge?.stop() - activeBridge = try AirPodsMuteBridge( - callback: callback, - audioCallback: audioCallback, - captureStateCallback: captureStateCallback - ) - return true - } catch { - FileHandle.standardError.write( - Data("Berd AirPods mute bridge failed to start: \(error)\n".utf8) - ) - activeBridge = nil - return false - } + do { + activeBridge?.stop() + activeBridge = try AirPodsMuteBridge( + callback: callback, + audioCallback: audioCallback + ) + return true + } catch { + FileHandle.standardError.write( + Data("Berd AirPods mute bridge failed to start: \(error)\n".utf8) + ) + activeBridge = nil + return false } } @_cdecl("berd_airpods_mute_stop") public func berdAirPodsMuteStop() -> Bool { guard #available(macOS 14.0, *) else { return false } - return onBridgeQueue { - activeBridge?.stop() - activeBridge = nil - return true - } + activeBridge?.stop() + activeBridge = nil + return true } @_cdecl("berd_airpods_mute_set_muted") public func berdAirPodsMuteSetMuted(_ muted: Bool) -> Bool { - guard #available(macOS 14.0, *) else { return false } - return onBridgeQueue { - guard activeBridge != nil else { return false } - do { - try AVAudioApplication.shared.setInputMuted(muted) - return true - } catch { - return false - } + guard #available(macOS 14.0, *), activeBridge != nil else { return false } + do { + try AVAudioApplication.shared.setInputMuted(muted) + return true + } catch { + return false } } diff --git a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdObjCExceptionCatch/BerdObjCExceptionCatch.m b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdObjCExceptionCatch/BerdObjCExceptionCatch.m deleted file mode 100644 index 07cd2e8aa..000000000 --- a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdObjCExceptionCatch/BerdObjCExceptionCatch.m +++ /dev/null @@ -1,20 +0,0 @@ -#import "BerdObjCExceptionCatch.h" - -BOOL BerdTryObjCBlock(void (NS_NOESCAPE ^_Nonnull block)(void), - NSError *_Nullable *_Nullable error) { - @try { - block(); - return YES; - } @catch (NSException *exception) { - if (error) { - NSDictionary *userInfo = @{ - NSLocalizedDescriptionKey: exception.reason ?: exception.name, - @"NSExceptionName": exception.name, - }; - *error = [NSError errorWithDomain:@"com.berd.objc-exception" - code:-1 - userInfo:userInfo]; - } - return NO; - } -} diff --git a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdObjCExceptionCatch/include/BerdObjCExceptionCatch.h b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdObjCExceptionCatch/include/BerdObjCExceptionCatch.h deleted file mode 100644 index a7a99d574..000000000 --- a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdObjCExceptionCatch/include/BerdObjCExceptionCatch.h +++ /dev/null @@ -1,6 +0,0 @@ -#import - -/// Runs a block inside an Objective-C @try/@catch so Swift callers can handle -/// AVAudioEngine NSExceptions without terminating the process. -BOOL BerdTryObjCBlock(void (NS_NOESCAPE ^_Nonnull block)(void), - NSError *_Nullable *_Nullable error); diff --git a/src/features/voice-conversation/api/voiceConversation.test.ts b/src/features/voice-conversation/api/voiceConversation.test.ts index 54d0b8c3a..cc5061191 100644 --- a/src/features/voice-conversation/api/voiceConversation.test.ts +++ b/src/features/voice-conversation/api/voiceConversation.test.ts @@ -171,7 +171,6 @@ describe("voice conversation API", () => { ownerWindowLabel: "main", revision: 3, nativeMicrophoneCapture: true, - nativeMicrophoneMuteControl: true, } as const; mocks.invoke.mockResolvedValue(undefined); @@ -180,8 +179,6 @@ describe("voice conversation API", () => { expect(mocks.startMicrophone).not.toHaveBeenCalled(); expect(mocks.invoke).toHaveBeenCalledWith("set_native_voice_input_muted", { - sessionId: "session-1", - revision: 3, muted: true, }); }); @@ -212,60 +209,6 @@ describe("voice conversation API", () => { expect(mocks.invoke).not.toHaveBeenCalled(); }); - it("preserves UI mute when native capture recovers from browser fallback", async () => { - const fallbackStatus = { - available: true, - unavailableReason: null, - lifecycle: "running", - sessionId: "session-1", - ownerWindowLabel: "main", - revision: 3, - nativeMicrophoneCapture: false, - nativeMicrophoneMuteControl: true, - } as const; - const recoveredStatus = { - ...fallbackStatus, - nativeMicrophoneCapture: true, - } as const; - mocks.invoke.mockResolvedValue(undefined); - - await reconcileVoiceConversationMicrophone(fallbackStatus); - await setVoiceConversationMicrophoneMuted(true, fallbackStatus); - expect(mocks.invoke).toHaveBeenCalledTimes(1); - expect(mocks.invoke).toHaveBeenLastCalledWith( - "set_native_voice_input_muted", - { - sessionId: "session-1", - revision: 3, - muted: true, - }, - ); - await reconcileVoiceConversationMicrophone(recoveredStatus); - - expect(mocks.invoke).toHaveBeenCalledTimes(2); - expect(mocks.stopMicrophone).toHaveBeenCalledOnce(); - }); - - it("does not change native mute when browser fallback cannot start", async () => { - const fallbackStatus = { - available: true, - unavailableReason: null, - lifecycle: "running", - sessionId: "session-1", - ownerWindowLabel: "main", - revision: 3, - nativeMicrophoneCapture: false, - nativeMicrophoneMuteControl: true, - } as const; - mocks.startMicrophone.mockRejectedValueOnce(new Error("capture failed")); - - await expect( - setVoiceConversationMicrophoneMuted(true, fallbackStatus), - ).rejects.toThrow("capture failed"); - - expect(mocks.invoke).not.toHaveBeenCalled(); - }); - it("restores the previous mute state when initial capture fails", async () => { const status = { available: true, diff --git a/src/features/voice-conversation/api/voiceConversation.ts b/src/features/voice-conversation/api/voiceConversation.ts index e476a97ed..417fe738a 100644 --- a/src/features/voice-conversation/api/voiceConversation.ts +++ b/src/features/voice-conversation/api/voiceConversation.ts @@ -50,18 +50,6 @@ export async function reconcileVoiceConversationMicrophone( status: VoiceConversationStatus, ): Promise { if (status.nativeMicrophoneCapture) { - const fallbackWasActive = - activeMicrophone !== null || microphoneStart !== null; - if (fallbackWasActive) { - if (!status.sessionId) { - throw new Error("Native voice microphone has no active session."); - } - await invoke("set_native_voice_input_muted", { - sessionId: status.sessionId, - revision: status.revision, - muted: microphoneMuted, - }); - } stopActiveMicrophone(); return; } @@ -82,21 +70,9 @@ export async function setVoiceConversationMicrophoneMuted( const previous = microphoneMuted; microphoneMuted = muted; try { - if (status.nativeMicrophoneMuteControl) { - if (!status.nativeMicrophoneCapture) { - await reconcileVoiceConversationMicrophone(status); - } - if (!status.sessionId) { - throw new Error("Native voice microphone has no active session."); - } - await invoke("set_native_voice_input_muted", { - sessionId: status.sessionId, - revision: status.revision, - muted, - }); - if (status.nativeMicrophoneCapture) { - stopActiveMicrophone(); - } + if (status.nativeMicrophoneCapture) { + await invoke("set_native_voice_input_muted", { muted }); + stopActiveMicrophone(); return; } await reconcileVoiceConversationMicrophone(status); @@ -107,13 +83,6 @@ export async function setVoiceConversationMicrophoneMuted( } } -export function applyVoiceConversationMicrophoneMuteEvent( - muted: boolean, -): void { - microphoneMuted = muted; - activeMicrophone?.setMuted(muted); -} - export function stopActiveMicrophoneForTest(): void { if (!import.meta.env.DEV) { throw new Error("Native microphone test controls are development-only."); @@ -151,8 +120,6 @@ export interface VoiceConversationStatus { revision: number; /** The backend owns the PCM stream so device gestures reach that process. */ nativeMicrophoneCapture?: boolean; - /** The process-wide native mute bridge remains usable during route recovery. */ - nativeMicrophoneMuteControl?: boolean; } export type VoiceConversationEvent = @@ -163,7 +130,6 @@ export type VoiceConversationEvent = line: string; revision: number; nativeMicrophoneCapture: boolean; - nativeMicrophoneMuteControl: boolean; } | { type: "user"; @@ -190,12 +156,6 @@ export type VoiceConversationEvent = muted: boolean; revision: number; } - | { - type: "nativeMicrophoneCapture"; - sessionId: string; - available: boolean; - revision: number; - } | { type: "cleanShutdown"; sessionId: string; diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts index 0371b419a..07446a4a8 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts @@ -24,7 +24,7 @@ import { createVoiceRouteMountRegistry, createVoiceTranscriptDeliveryQueue, hasDeliveredVoiceTranscript, - notifyInputMuteChanged, + notifyAirPodsInputMuteGesture, resetVoiceUiWhenRunSettles, resolveVoiceRouteMount, resolveVoiceToggleAction, @@ -36,16 +36,18 @@ import { } from "./useVoiceConversationController"; describe("voice transcript delivery coordination", () => { - it("shows an unmistakable input mute confirmation", () => { - notifyInputMuteChanged(true); - expect(toastMocks.message).toHaveBeenCalledWith("Microphone muted", { - id: "voice-input-mute", - }); + it("shows an unmistakable AirPods gesture confirmation", () => { + notifyAirPodsInputMuteGesture(true); + expect(toastMocks.message).toHaveBeenCalledWith( + "AirPods gesture detected — microphone muted", + { id: "airpods-input-mute" }, + ); - notifyInputMuteChanged(false); - expect(toastMocks.message).toHaveBeenLastCalledWith("Microphone unmuted", { - id: "voice-input-mute", - }); + notifyAirPodsInputMuteGesture(false); + expect(toastMocks.message).toHaveBeenLastCalledWith( + "AirPods gesture detected — microphone unmuted", + { id: "airpods-input-mute" }, + ); }); it("recognizes a replayed transcript that was already delivered", () => { diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index 8d08e4adf..1e15eb54d 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -188,10 +188,13 @@ function addErrorNotification(sessionId: string | null, message: string) { .addMessage(sessionId, createSystemNotificationMessage(message, "error")); } -export function notifyInputMuteChanged(muted: boolean) { - toast.message(muted ? "Microphone muted" : "Microphone unmuted", { - id: "voice-input-mute", - }); +export function notifyAirPodsInputMuteGesture(muted: boolean) { + toast.message( + muted + ? "AirPods gesture detected — microphone muted" + : "AirPods gesture detected — microphone unmuted", + { id: "airpods-input-mute" }, + ); } export function hasDeliveredVoiceTranscript( @@ -322,7 +325,7 @@ function ensureVoiceEventDeliveryInitialized() { return; } if (event.type === "inputMute") { - notifyInputMuteChanged(event.muted); + notifyAirPodsInputMuteGesture(event.muted); return; } if (event.type === "activity") return; diff --git a/src/features/voice-conversation/stores/voiceConversationStore.test.ts b/src/features/voice-conversation/stores/voiceConversationStore.test.ts index 7239b03c7..19aec7cea 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.test.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.test.ts @@ -7,7 +7,6 @@ import type { const mocks = vi.hoisted(() => ({ acknowledge: vi.fn(), - applyInputMute: vi.fn(), drain: vi.fn(), getStatus: vi.fn(), listen: vi.fn(), @@ -20,7 +19,6 @@ const mocks = vi.hoisted(() => ({ vi.mock("../api/voiceConversation", () => ({ acknowledgeVoiceConversationTranscript: mocks.acknowledge, - applyVoiceConversationMicrophoneMuteEvent: mocks.applyInputMute, drainVoiceConversationTranscripts: mocks.drain, getVoiceConversationStatus: mocks.getStatus, listenToVoiceConversation: mocks.listen, @@ -60,7 +58,6 @@ describe("voice conversation store lifecycle ordering", () => { beforeEach(() => { vi.resetModules(); mocks.acknowledge.mockReset().mockResolvedValue(undefined); - mocks.applyInputMute.mockReset(); mocks.drain.mockReset().mockResolvedValue([]); mocks.getStatus.mockReset().mockResolvedValue(status("stopped", 0)); mocks.start.mockReset(); @@ -269,7 +266,6 @@ describe("voice conversation store lifecycle ordering", () => { line: "type\tid\ttext", revision: 2, nativeMicrophoneCapture: true, - nativeMicrophoneMuteControl: true, }); response.resolve(status("starting", 1, "session-1")); await starting; @@ -278,7 +274,6 @@ describe("voice conversation store lifecycle ordering", () => { status: { ...status("running", 2, "session-1"), nativeMicrophoneCapture: true, - nativeMicrophoneMuteControl: true, }, uiState: "listening", error: null, @@ -418,42 +413,6 @@ describe("voice conversation store lifecycle ordering", () => { userSpeaking: false, uiState: "listening", }); - expect(mocks.applyInputMute).toHaveBeenCalledWith(true); - }); - - it("falls back while native capture restarts and returns when it recovers", async () => { - const store = await loadStore(); - store.setState({ - status: { - ...status("running", 3, "session-1"), - nativeMicrophoneCapture: true, - }, - }); - mocks.reconcileMicrophone.mockClear(); - - emit({ - type: "nativeMicrophoneCapture", - sessionId: "session-1", - available: false, - revision: 3, - }); - await vi.waitFor(() => { - expect(mocks.reconcileMicrophone).toHaveBeenLastCalledWith( - expect.objectContaining({ nativeMicrophoneCapture: false }), - ); - }); - - emit({ - type: "nativeMicrophoneCapture", - sessionId: "session-1", - available: true, - revision: 3, - }); - await vi.waitFor(() => { - expect(mocks.reconcileMicrophone).toHaveBeenLastCalledWith( - expect.objectContaining({ nativeMicrophoneCapture: true }), - ); - }); }); it("surfaces an unmute failure without losing muted state", async () => { diff --git a/src/features/voice-conversation/stores/voiceConversationStore.ts b/src/features/voice-conversation/stores/voiceConversationStore.ts index 05254d42c..8c26864ad 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.ts @@ -2,7 +2,6 @@ import { create } from "zustand"; import { acknowledgeVoiceConversationTranscript, - applyVoiceConversationMicrophoneMuteEvent, drainVoiceConversationTranscripts, getVoiceConversationStatus, listenToVoiceConversation, @@ -242,8 +241,6 @@ export const useVoiceConversationStore = create( ownerWindowLabel: event.ownerWindowLabel, revision: event.revision, nativeMicrophoneCapture: event.nativeMicrophoneCapture, - nativeMicrophoneMuteControl: - event.nativeMicrophoneMuteControl, }, uiState: "listening", microphoneMuted: false, @@ -309,17 +306,6 @@ export const useVoiceConversationStore = create( uiState: activityUiState(nextState), }; } - case "nativeMicrophoneCapture": - return { - ...state, - status: { - ...state.status, - lifecycle: "running", - sessionId: event.sessionId, - revision: event.revision, - nativeMicrophoneCapture: event.available, - }, - }; case "cleanShutdown": return { ...state, @@ -362,20 +348,7 @@ export const useVoiceConversationStore = create( } }); - if (event.type === "inputMute") { - applyVoiceConversationMicrophoneMuteEvent(event.muted); - for (const subscriber of [...eventSubscribers]) - void subscriber(event); - } else if (event.type === "nativeMicrophoneCapture") { - void reconcileVoiceConversationMicrophone(get().status).catch( - (error) => { - set({ - uiState: "error", - error: error instanceof Error ? error.message : String(error), - }); - }, - ); - } else if (event.type === "user") { + if (event.type === "user") { void deliverTranscriptOnce(event).catch((error) => { const current = get(); if ( From f2c1697affe4318f2841b92450672235e8386868 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 16:29:10 -0400 Subject: [PATCH 13/31] Revert "fix(voice): satisfy cross-platform clippy" This reverts commit c807df1af002f3fadba0dee730061cfb2222bbfd. --- src-tauri/src/commands/native_input_mute.rs | 1 - src-tauri/src/commands/native_voice.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src-tauri/src/commands/native_input_mute.rs b/src-tauri/src/commands/native_input_mute.rs index 19a4008a2..9282e5a7a 100644 --- a/src-tauri/src/commands/native_input_mute.rs +++ b/src-tauri/src/commands/native_input_mute.rs @@ -71,7 +71,6 @@ fn clear(input_muted: &AtomicBool, mute_epoch: &AtomicU64) { mute_epoch.store(0, Ordering::Release); } -#[cfg(any(target_os = "macos", test))] fn apply_change( input_muted: &AtomicBool, mute_epoch: &AtomicU64, diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index b2cb45dd1..ee3d93134 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -910,7 +910,7 @@ fn push_audio_for_session( let Some(pipeline) = runtime.pipeline.as_ref() else { return Ok(()); }; - let mut bytes = Vec::with_capacity(size_of_val(samples)); + let mut bytes = Vec::with_capacity(samples.len() * size_of::()); for sample in samples { bytes.extend_from_slice(&sample.to_ne_bytes()); } From 00c9c7bbae8439be5702641ace7bc42d977bda2c Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 16:29:10 -0400 Subject: [PATCH 14/31] Revert "fix(voice): latch mute transitions" This reverts commit 14231209b5ae2fd8d217f43982cf0a6cee2586e3. --- src-tauri/src/commands/native_input_mute.rs | 58 ++++---------- src-tauri/src/commands/native_voice.rs | 78 ++++--------------- .../BerdAirPodsBridge/BerdAirPodsBridge.swift | 4 +- 3 files changed, 34 insertions(+), 106 deletions(-) diff --git a/src-tauri/src/commands/native_input_mute.rs b/src-tauri/src/commands/native_input_mute.rs index 9282e5a7a..b8c33f0d7 100644 --- a/src-tauri/src/commands/native_input_mute.rs +++ b/src-tauri/src/commands/native_input_mute.rs @@ -1,24 +1,18 @@ use std::sync::{ - atomic::{AtomicBool, AtomicU64, Ordering}, + atomic::{AtomicBool, Ordering}, Arc, }; -pub fn start( - input_muted: &Arc, - mute_epoch: &Arc, - on_change: F, - on_audio: A, -) -> bool +pub fn start(input_muted: &Arc, on_change: F, on_audio: A) -> bool where F: Fn(bool) + Send + Sync + 'static, A: Fn(&[f32]) + Send + Sync + 'static, { - clear(input_muted, mute_epoch); + clear(input_muted); #[cfg(target_os = "macos")] let started = match macos::install( Arc::clone(input_muted), - Arc::clone(mute_epoch), Arc::new(on_change), Arc::new(on_audio), ) { @@ -38,8 +32,8 @@ where started } -pub fn stop(input_muted: &Arc, mute_epoch: &Arc) { - clear(input_muted, mute_epoch); +pub fn stop(input_muted: &Arc) { + clear(input_muted); #[cfg(target_os = "macos")] if let Err(error) = macos::uninstall() { @@ -47,41 +41,28 @@ pub fn stop(input_muted: &Arc, mute_epoch: &Arc) { } } -pub fn set_muted( - input_muted: &AtomicBool, - mute_epoch: &AtomicU64, - muted: bool, -) -> Result<(), String> { +pub fn set_muted(input_muted: &AtomicBool, muted: bool) -> Result<(), String> { #[cfg(target_os = "macos")] { macos::set_muted(muted)?; - apply_change(input_muted, mute_epoch, muted, &|_| {}); + input_muted.store(muted, Ordering::Release); Ok(()) } #[cfg(not(target_os = "macos"))] { - let _ = (input_muted, mute_epoch, muted); + let _ = (input_muted, muted); Err("native microphone mute is only available on macOS".to_string()) } } -fn clear(input_muted: &AtomicBool, mute_epoch: &AtomicU64) { +fn clear(input_muted: &AtomicBool) { input_muted.store(false, Ordering::Release); - mute_epoch.store(0, Ordering::Release); } -fn apply_change( - input_muted: &AtomicBool, - mute_epoch: &AtomicU64, - muted: bool, - on_change: &dyn Fn(bool), -) { +fn apply_change(input_muted: &AtomicBool, muted: bool, on_change: &dyn Fn(bool)) { let previous = input_muted.swap(muted, Ordering::AcqRel); if previous != muted { - if muted { - mute_epoch.fetch_add(1, Ordering::AcqRel); - } on_change(muted); } } @@ -96,7 +77,6 @@ mod macos { struct CallbackState { input_muted: Arc, - mute_epoch: Arc, on_change: MuteChangeHandler, on_audio: AudioInputHandler, } @@ -123,7 +103,7 @@ mod macos { let Some(state) = state.as_ref() else { return; }; - apply_change(&state.input_muted, &state.mute_epoch, muted, &|muted| { + apply_change(&state.input_muted, muted, &|muted| { log::info!("AirPods input mute changed muted={muted}"); (state.on_change)(muted); }); @@ -150,7 +130,6 @@ mod macos { pub fn install( input_muted: Arc, - mute_epoch: Arc, on_change: MuteChangeHandler, on_audio: AudioInputHandler, ) -> Result<(), String> { @@ -159,7 +138,6 @@ mod macos { .map_err(|_| "input mute callback lock was poisoned".to_string())? = Some(CallbackState { input_muted, - mute_epoch, on_change, on_audio, }); @@ -205,32 +183,28 @@ mod tests { #[test] fn lifecycle_boundary_clears_mute() { let input_muted = Arc::new(AtomicBool::new(true)); - let mute_epoch = Arc::new(AtomicU64::new(3)); - clear(&input_muted, &mute_epoch); + clear(&input_muted); assert!(!input_muted.load(Ordering::Acquire)); - assert_eq!(mute_epoch.load(Ordering::Acquire), 0); } #[test] fn unchanged_initial_state_is_not_reported_as_a_gesture() { let input_muted = AtomicBool::new(false); - let mute_epoch = AtomicU64::new(0); let changes = std::sync::Mutex::new(Vec::new()); - apply_change(&input_muted, &mute_epoch, false, &|muted| { + apply_change(&input_muted, false, &|muted| { changes.lock().expect("changes lock").push(muted); }); - apply_change(&input_muted, &mute_epoch, true, &|muted| { + apply_change(&input_muted, true, &|muted| { changes.lock().expect("changes lock").push(muted); }); - apply_change(&input_muted, &mute_epoch, true, &|muted| { + apply_change(&input_muted, true, &|muted| { changes.lock().expect("changes lock").push(muted); }); - apply_change(&input_muted, &mute_epoch, false, &|muted| { + apply_change(&input_muted, false, &|muted| { changes.lock().expect("changes lock").push(muted); }); assert_eq!(*changes.lock().expect("changes lock"), vec![true, false]); - assert_eq!(mute_epoch.load(Ordering::Acquire), 1); } } diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index ee3d93134..6b3b19fd4 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -4,7 +4,7 @@ use std::{ collections::VecDeque, path::PathBuf, sync::{ - atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, + atomic::{AtomicBool, AtomicUsize, Ordering}, mpsc::{self, Receiver, SyncSender, TrySendError}, Arc, Mutex, }, @@ -134,7 +134,6 @@ pub struct NativeVoiceState { pending: Arc>>, capture_suppressions: Arc, input_muted: Arc, - input_mute_epoch: Arc, native_microphone_capture: Arc, } @@ -171,7 +170,7 @@ impl NativeVoiceState { } fn stop_native_microphone(&self) { - native_input_mute::stop(&self.input_muted, &self.input_mute_epoch); + native_input_mute::stop(&self.input_muted); self.native_microphone_capture .store(false, Ordering::Release); } @@ -187,25 +186,18 @@ enum SttMessage { } struct SttPipeline { - audio_tx: SyncSender, + audio_tx: SyncSender>, audio_seen: AtomicBool, shutdown: Arc, discard_on_shutdown: Arc, input_muted: Arc, - input_mute_epoch: Arc, thread: Option>, } -struct AudioBatch { - bytes: Vec, - mute_epoch: u64, -} - impl SttPipeline { fn new( model_dir: PathBuf, input_muted: Arc, - input_mute_epoch: Arc, ) -> Result<(Self, tokio_mpsc::Receiver), String> { let (audio_tx, audio_rx) = mpsc::sync_channel(AUDIO_QUEUE_DEPTH); let (event_tx, event_rx) = tokio_mpsc::channel(64); @@ -214,7 +206,6 @@ impl SttPipeline { let worker_shutdown = Arc::clone(&shutdown); let worker_discard_on_shutdown = Arc::clone(&discard_on_shutdown); let worker_input_muted = Arc::clone(&input_muted); - let worker_input_mute_epoch = Arc::clone(&input_mute_epoch); let thread = thread::Builder::new() .name("berd-native-stt".into()) .spawn(move || { @@ -225,7 +216,6 @@ impl SttPipeline { worker_shutdown, worker_discard_on_shutdown, worker_input_muted, - worker_input_mute_epoch, ) }) .map_err(|error| format!("start native transcription: {error}"))?; @@ -236,7 +226,6 @@ impl SttPipeline { shutdown, discard_on_shutdown, input_muted, - input_mute_epoch, thread: Some(thread), }, event_rx, @@ -256,19 +245,13 @@ impl SttPipeline { if self.input_muted.load(Ordering::Acquire) { return Ok(()); } - let mute_epoch = self.input_mute_epoch.load(Ordering::Acquire); - if self.input_muted.load(Ordering::Acquire) - || mute_epoch != self.input_mute_epoch.load(Ordering::Acquire) - { - return Ok(()); - } if !self.audio_seen.swap(true, Ordering::AcqRel) { log::info!( "Native Parakeet received its first audio batch ({} bytes)", bytes.len() ); } - match self.audio_tx.try_send(AudioBatch { bytes, mute_epoch }) { + match self.audio_tx.try_send(bytes) { Ok(()) => Ok(()), Err(TrySendError::Full(_)) => Err( "Native voice audio overrun: transcription could not keep up with microphone input." @@ -457,11 +440,7 @@ pub async fn start_native_voice_conversation( return Err(error); } }; - let (pipeline, mut events) = match SttPipeline::new( - model_dir, - Arc::clone(&state.input_muted), - Arc::clone(&state.input_mute_epoch), - ) { + let (pipeline, mut events) = match SttPipeline::new(model_dir, Arc::clone(&state.input_muted)) { Ok(result) => result, Err(error) => { if microphone_claimed { @@ -499,7 +478,6 @@ pub async fn start_native_voice_conversation( let audio_session_id = session_id.clone(); let native_capture_started = native_input_mute::start( &state.input_muted, - &state.input_mute_epoch, move |muted| { let _ = mute_window.emit( EVENT_NAME, @@ -537,7 +515,6 @@ pub async fn start_native_voice_conversation( let runtime = Arc::clone(&state.runtime); let pending = Arc::clone(&state.pending); let input_muted = Arc::clone(&state.input_muted); - let input_mute_epoch = Arc::clone(&state.input_mute_epoch); let native_microphone_capture = Arc::clone(&state.native_microphone_capture); tauri::async_runtime::spawn(async move { while let Some(event) = events.recv().await { @@ -611,7 +588,7 @@ pub async fn start_native_voice_conversation( { break; } - native_input_mute::stop(&input_muted, &input_mute_epoch); + native_input_mute::stop(&input_muted); native_microphone_capture.store(false, Ordering::Release); current.session_id = None; current.lifecycle_id = None; @@ -863,7 +840,7 @@ pub fn set_native_voice_input_muted( return Err("Only the active voice owner may mute its microphone.".to_string()); } drop(runtime); - native_input_mute::set_muted(&state.input_muted, &state.input_mute_epoch, muted) + native_input_mute::set_muted(&state.input_muted, muted) } fn push_audio_for_window( @@ -930,12 +907,11 @@ fn enqueue_pending_transcript( fn stt_worker( model_dir: PathBuf, - audio_rx: Receiver, + audio_rx: Receiver>, event_tx: tokio_mpsc::Sender, shutdown: Arc, discard_on_shutdown: Arc, input_muted: Arc, - input_mute_epoch: Arc, ) { use rubato::{Fft, FixedSync, Resampler}; use sherpa_onnx::{OfflineRecognizer, OfflineRecognizerConfig}; @@ -974,20 +950,17 @@ fn stt_worker( let mut speech = Vec::new(); let mut silence_frames = 0_usize; let mut in_speech = false; - let mut observed_mute_epoch = input_mute_epoch.load(Ordering::Acquire); while !shutdown.load(Ordering::Acquire) { - let batch = match audio_rx.recv_timeout(Duration::from_millis(50)) { - Ok(batch) => Some(batch), + let bytes = match audio_rx.recv_timeout(Duration::from_millis(50)) { + Ok(bytes) => Some(bytes), Err(mpsc::RecvTimeoutError::Timeout) => None, Err(mpsc::RecvTimeoutError::Disconnected) => break, }; let shutting_down = shutdown.load(Ordering::Acquire); - if shutting_down && (discard_on_shutdown.load(Ordering::Acquire) || batch.is_none()) { + if shutting_down && (discard_on_shutdown.load(Ordering::Acquire) || bytes.is_none()) { break; } - let current_mute_epoch = input_mute_epoch.load(Ordering::Acquire); - if current_mute_epoch != observed_mute_epoch { - observed_mute_epoch = current_mute_epoch; + if !shutting_down && input_muted.load(Ordering::Acquire) { if clear_buffered_audio( &mut input_48k, &mut leftover_16k, @@ -997,19 +970,13 @@ fn stt_worker( ) { let _ = event_tx.blocking_send(SttMessage::Speaking(false)); } - } - if !shutting_down && input_muted.load(Ordering::Acquire) { continue; } - let Some(batch) = batch else { + let Some(bytes) = bytes else { continue; }; - if batch.mute_epoch != observed_mute_epoch { - continue; - } input_48k.extend( - batch - .bytes + bytes .chunks_exact(4) .map(|sample| f32::from_le_bytes([sample[0], sample[1], sample[2], sample[3]])), ); @@ -1180,7 +1147,6 @@ mod tests { shutdown: Arc::new(AtomicBool::new(false)), discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::new(AtomicBool::new(false)), - input_mute_epoch: Arc::new(AtomicU64::new(0)), audio_seen: AtomicBool::new(false), thread: None, }; @@ -1196,7 +1162,6 @@ mod tests { shutdown: Arc::new(AtomicBool::new(false)), discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::new(AtomicBool::new(false)), - input_mute_epoch: Arc::new(AtomicU64::new(0)), audio_seen: AtomicBool::new(false), thread: None, }; @@ -1212,13 +1177,11 @@ mod tests { fn input_mute_discards_audio_and_unmute_resumes_queueing() { let (sender, receiver) = mpsc::sync_channel(1); let input_muted = Arc::new(AtomicBool::new(true)); - let input_mute_epoch = Arc::new(AtomicU64::new(1)); let pipeline = SttPipeline { audio_tx: sender, shutdown: Arc::new(AtomicBool::new(false)), discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::clone(&input_muted), - input_mute_epoch: Arc::clone(&input_mute_epoch), audio_seen: AtomicBool::new(false), thread: None, }; @@ -1230,9 +1193,7 @@ mod tests { input_muted.store(false, Ordering::Release); pipeline.push(vec![0; 4]).expect("unmuted audio is queued"); - let batch = receiver.try_recv().expect("unmuted audio"); - assert_eq!(batch.bytes, vec![0; 4]); - assert_eq!(batch.mute_epoch, 1); + assert_eq!(receiver.try_recv().expect("unmuted audio"), vec![0; 4]); } #[test] @@ -1267,7 +1228,6 @@ mod tests { shutdown: Arc::new(AtomicBool::new(false)), discard_on_shutdown: Arc::clone(&discard_on_shutdown), input_muted: Arc::clone(&input_muted), - input_mute_epoch: Arc::new(AtomicU64::new(1)), audio_seen: AtomicBool::new(false), thread: None, }; @@ -1288,7 +1248,6 @@ mod tests { shutdown: Arc::new(AtomicBool::new(false)), discard_on_shutdown: Arc::clone(&discard_on_shutdown), input_muted: Arc::clone(&input_muted), - input_mute_epoch: Arc::new(AtomicU64::new(0)), audio_seen: AtomicBool::new(false), thread: None, }; @@ -1313,7 +1272,6 @@ mod tests { shutdown: Arc::new(AtomicBool::new(false)), discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::new(AtomicBool::new(false)), - input_mute_epoch: Arc::new(AtomicU64::new(0)), audio_seen: AtomicBool::new(false), thread: None, }); @@ -1322,10 +1280,7 @@ mod tests { assert!(push_audio_for_window(&state, "other-window", vec![0; 4]).is_err()); assert!(receiver.try_recv().is_err()); push_audio_for_window(&state, "owner-window", vec![0; 4]).expect("owner can send audio"); - assert_eq!( - receiver.try_recv().expect("owner audio queued").bytes, - vec![0; 4] - ); + assert_eq!(receiver.try_recv().expect("owner audio queued"), vec![0; 4]); } #[tokio::test] @@ -1345,7 +1300,6 @@ mod tests { shutdown: Arc::clone(&shutdown), discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::new(AtomicBool::new(false)), - input_mute_epoch: Arc::new(AtomicU64::new(0)), audio_seen: AtomicBool::new(false), thread: Some(worker), }); diff --git a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift index a37c05116..ba6f04b55 100644 --- a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift +++ b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift @@ -59,8 +59,8 @@ private final class AirPodsMuteBridge: @unchecked Sendable { engine.prepare() try engine.start() - // Clear stale process mute before registering this lifecycle's handler. - // The reset can fail when no earlier handler exists, which is harmless. + // Match voice-conversation-cli exactly: seed before registration and + // ignore the expected error when no prior process handler exists. try? AVAudioApplication.shared.setInputMuted(false) try AVAudioApplication.shared.setInputMuteStateChangeHandler { [weak self] muted in self?.callback(muted) From 62a6f2f205ee15ccc85a45ed9ee3658b1bbe655d Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 16:29:10 -0400 Subject: [PATCH 15/31] Revert "fix(voice): preserve native mute routing" This reverts commit 0bea2a015696d90c979616c3d56f784c36f81432. --- src-tauri/src/commands/native_voice.rs | 20 +++--- .../BerdAirPodsBridge/BerdAirPodsBridge.swift | 62 ++----------------- .../api/voiceConversation.test.ts | 4 +- .../api/voiceConversation.ts | 5 +- .../stores/voiceConversationStore.test.ts | 6 +- .../stores/voiceConversationStore.ts | 1 - 6 files changed, 22 insertions(+), 76 deletions(-) diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index 6b3b19fd4..684372bd0 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -82,7 +82,6 @@ enum NativeVoiceEvent { owner_window_label: String, line: String, revision: u64, - native_microphone_capture: bool, }, User { session_id: String, @@ -472,6 +471,15 @@ pub async fn start_native_voice_conversation( runtime.lifecycle_id.clone().unwrap_or_default(), ) }; + let _ = webview_window.emit( + EVENT_NAME, + NativeVoiceEvent::Startup { + session_id: session_id.clone(), + owner_window_label: window_label.clone(), + line: "Native Parakeet voice conversation is on".to_string(), + revision, + }, + ); let mute_window = webview_window.clone(); let mute_session_id = session_id.clone(); let audio_state = state.inner().clone(); @@ -499,16 +507,6 @@ pub async fn start_native_voice_conversation( state .native_microphone_capture .store(native_capture_started, Ordering::Release); - let _ = webview_window.emit( - EVENT_NAME, - NativeVoiceEvent::Startup { - session_id: session_id.clone(), - owner_window_label: window_label.clone(), - line: "Native Parakeet voice conversation is on".to_string(), - revision, - native_microphone_capture: native_capture_started, - }, - ); let event_app = app.clone(); let event_window = webview_window.clone(); diff --git a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift index ba6f04b55..cf1cf9566 100644 --- a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift +++ b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift @@ -10,8 +10,6 @@ private final class AirPodsMuteBridge: @unchecked Sendable { private let inputNode: AVAudioInputNode private let callback: InputMuteCallback private let audioCallback: AudioInputCallback - private let targetFormat: AVAudioFormat - private let converter: AVAudioConverter? private var inputMuteObserver: NSObjectProtocol? init( @@ -27,34 +25,20 @@ private final class AirPodsMuteBridge: @unchecked Sendable { guard inputFormat.sampleRate > 0, inputFormat.channelCount > 0 else { throw BridgeError.noInputFormat } - guard let targetFormat = AVAudioFormat( - commonFormat: .pcmFormatFloat32, - sampleRate: 48_000, - channels: 1, - interleaved: false - ) else { - throw BridgeError.noOutputFormat - } - let converter: AVAudioConverter? - if inputFormat == targetFormat { - converter = nil - } else { - guard let created = AVAudioConverter(from: inputFormat, to: targetFormat) else { - throw BridgeError.noConverter - } - converter = created - } self.engine = engine self.inputNode = inputNode - self.targetFormat = targetFormat - self.converter = converter inputNode.installTap( onBus: 0, bufferSize: 4096, format: inputFormat ) { [weak self] buffer, _ in - self?.forward(buffer) + guard + let self, + let channel = buffer.floatChannelData?.pointee, + buffer.frameLength > 0 + else { return } + self.audioCallback(channel, Int(buffer.frameLength)) } engine.prepare() try engine.start() @@ -88,42 +72,8 @@ private final class AirPodsMuteBridge: @unchecked Sendable { engine.stop() } - private func forward(_ source: AVAudioPCMBuffer) { - let output: AVAudioPCMBuffer - if let converter { - let capacity = AVAudioFrameCount(ceil( - Double(source.frameLength) * targetFormat.sampleRate / source.format.sampleRate - )) - guard capacity > 0, - let converted = AVAudioPCMBuffer( - pcmFormat: targetFormat, - frameCapacity: capacity - ) else { return } - var error: NSError? - nonisolated(unsafe) var consumed = false - converter.convert(to: converted, error: &error) { _, status in - if !consumed { - consumed = true - status.pointee = .haveData - return source - } - status.pointee = .noDataNow - return nil - } - guard error == nil, converted.frameLength > 0 else { return } - output = converted - } else { - output = source - } - guard let channel = output.floatChannelData?.pointee, - output.frameLength > 0 else { return } - audioCallback(channel, Int(output.frameLength)) - } - private enum BridgeError: Error { case noInputFormat - case noOutputFormat - case noConverter } } diff --git a/src/features/voice-conversation/api/voiceConversation.test.ts b/src/features/voice-conversation/api/voiceConversation.test.ts index cc5061191..a66071cde 100644 --- a/src/features/voice-conversation/api/voiceConversation.test.ts +++ b/src/features/voice-conversation/api/voiceConversation.test.ts @@ -287,7 +287,7 @@ describe("voice conversation API", () => { }); }); - it("forwards input mute events without mutating browser capture", async () => { + it("applies AirPods input mute events to browser capture", async () => { const callback = vi.fn(); mocks.listen.mockImplementation(async (_name, handler) => { handler({ @@ -311,7 +311,7 @@ describe("voice conversation API", () => { }); await listenToVoiceConversation(callback); - expect(mocks.setMicrophoneMuted).not.toHaveBeenCalledWith(true); + expect(mocks.setMicrophoneMuted).toHaveBeenLastCalledWith(true); expect(callback).toHaveBeenCalledWith({ type: "inputMute", sessionId: "session-1", diff --git a/src/features/voice-conversation/api/voiceConversation.ts b/src/features/voice-conversation/api/voiceConversation.ts index 417fe738a..f98c4a28a 100644 --- a/src/features/voice-conversation/api/voiceConversation.ts +++ b/src/features/voice-conversation/api/voiceConversation.ts @@ -129,7 +129,6 @@ export type VoiceConversationEvent = ownerWindowLabel: string; line: string; revision: number; - nativeMicrophoneCapture: boolean; } | { type: "user"; @@ -262,6 +261,10 @@ export function listenToVoiceConversation( onEvent: (event: VoiceConversationEvent) => void, ): Promise { return listen(VOICE_CONVERSATION_EVENT, (event) => { + if (event.payload.type === "inputMute") { + microphoneMuted = event.payload.muted; + activeMicrophone?.setMuted(event.payload.muted); + } if ( event.payload.type === "cleanShutdown" || (event.payload.type === "error" && event.payload.terminal) diff --git a/src/features/voice-conversation/stores/voiceConversationStore.test.ts b/src/features/voice-conversation/stores/voiceConversationStore.test.ts index 19aec7cea..7c992fa52 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.test.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.test.ts @@ -265,16 +265,12 @@ describe("voice conversation store lifecycle ordering", () => { ownerWindowLabel: "main", line: "type\tid\ttext", revision: 2, - nativeMicrophoneCapture: true, }); response.resolve(status("starting", 1, "session-1")); await starting; expect(store.getState()).toMatchObject({ - status: { - ...status("running", 2, "session-1"), - nativeMicrophoneCapture: true, - }, + status: status("running", 2, "session-1"), uiState: "listening", error: null, }); diff --git a/src/features/voice-conversation/stores/voiceConversationStore.ts b/src/features/voice-conversation/stores/voiceConversationStore.ts index 8c26864ad..56a420d05 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.ts @@ -240,7 +240,6 @@ export const useVoiceConversationStore = create( sessionId: event.sessionId, ownerWindowLabel: event.ownerWindowLabel, revision: event.revision, - nativeMicrophoneCapture: event.nativeMicrophoneCapture, }, uiState: "listening", microphoneMuted: false, From 3a2430d1401a6ec8d9d36307863a6c51a8b838ed Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 16:29:11 -0400 Subject: [PATCH 16/31] Revert "fix(voice): route AirPods mute through native capture" This reverts commit cc92a1f2cc20ce48a7072918344d38d409069495. --- src-tauri/Cargo.lock | 7 +- src-tauri/Cargo.toml | 2 +- src-tauri/build.rs | 5 - src-tauri/src/commands/native_input_mute.rs | 195 +++--------------- src-tauri/src/commands/native_voice.rs | 115 +---------- src-tauri/src/lib.rs | 1 - .../swift/BerdAirPodsBridge/Package.swift | 20 -- .../BerdAirPodsBridge/BerdAirPodsBridge.swift | 122 ----------- .../api/voiceConversation.test.ts | 54 ----- .../api/voiceConversation.ts | 21 -- .../useVoiceConversationController.test.ts | 18 -- .../hooks/useVoiceConversationController.ts | 14 -- .../stores/voiceConversationStore.test.ts | 22 -- .../stores/voiceConversationStore.ts | 18 -- 14 files changed, 44 insertions(+), 570 deletions(-) delete mode 100644 src-tauri/swift/BerdAirPodsBridge/Package.swift delete mode 100644 src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 63f1ac2e6..d48871f52 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -34,6 +34,7 @@ dependencies = [ "nucleo-matcher", "objc2", "objc2-app-kit", + "objc2-avf-audio", "objc2-foundation", "objc2-user-notifications", "percent-encoding", @@ -49,7 +50,6 @@ dependencies = [ "sherpa-onnx", "sqlx", "ssstretch", - "swift-rs", "sysinfo", "tar", "tauri", @@ -3933,6 +3933,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08" dependencies = [ "bitflags 2.13.1", + "block2", "libc", "objc2", "objc2-core-audio", @@ -3947,7 +3948,11 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be" dependencies = [ + "bitflags 2.13.1", + "block2", "objc2", + "objc2-audio-toolbox", + "objc2-core-audio-types", "objc2-foundation", ] diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index c9e7f584e..feac03007 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -18,7 +18,6 @@ members = ["crates/berd-voice", "crates/berdctl", "plugins/berdctl"] exclude = ["plugins/app-test-driver"] [build-dependencies] -swift-rs = { version = "1.0.7", features = ["build"] } tauri-build = { version = "2", features = [] } cc = "1" @@ -110,6 +109,7 @@ windows-sys = { version = "0.59", features = [ block2 = "0.6" objc2 = "0.6" objc2-app-kit = { version = "0.3.2", features = ["NSApplication", "NSImage", "NSMenu", "NSMenuItem", "NSAlert", "NSButton", "NSControl", "NSCell", "NSResponder", "NSView"] } +objc2-avf-audio = { version = "0.3.2", features = ["AVAudioApplication", "block2"] } objc2-foundation = { version = "0.3.2", features = ["NSDictionary", "NSError", "NSFileManager", "NSObject", "NSProcessInfo", "NSString", "NSURL"] } objc2-user-notifications = "0.3.2" rodio = { version = "0.22", default-features = false, features = ["playback"] } diff --git a/src-tauri/build.rs b/src-tauri/build.rs index ab8a5f53f..afa1d236e 100644 --- a/src-tauri/build.rs +++ b/src-tauri/build.rs @@ -19,11 +19,6 @@ fn main() { println!("cargo:rustc-link-lib=framework={framework}"); } - if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("macos") { - swift_rs::SwiftLinker::new("14.0") - .with_package("BerdAirPodsBridge", "swift/BerdAirPodsBridge") - .link(); - } } tauri_build::build() } diff --git a/src-tauri/src/commands/native_input_mute.rs b/src-tauri/src/commands/native_input_mute.rs index b8c33f0d7..8659da2b9 100644 --- a/src-tauri/src/commands/native_input_mute.rs +++ b/src-tauri/src/commands/native_input_mute.rs @@ -3,33 +3,13 @@ use std::sync::{ Arc, }; -pub fn start(input_muted: &Arc, on_change: F, on_audio: A) -> bool -where - F: Fn(bool) + Send + Sync + 'static, - A: Fn(&[f32]) + Send + Sync + 'static, -{ +pub fn start(input_muted: &Arc) { clear(input_muted); #[cfg(target_os = "macos")] - let started = match macos::install( - Arc::clone(input_muted), - Arc::new(on_change), - Arc::new(on_audio), - ) { - Ok(()) => true, - Err(error) => { - log::info!("AirPods input mute listener is unavailable: {error}"); - false - } - }; - - #[cfg(not(target_os = "macos"))] - let started = { - let _ = (on_change, on_audio); - false - }; - - started + if let Err(error) = macos::install(Arc::clone(input_muted)) { + log::info!("AirPods input mute listener is unavailable: {error}"); + } } pub fn stop(input_muted: &Arc) { @@ -41,138 +21,48 @@ pub fn stop(input_muted: &Arc) { } } -pub fn set_muted(input_muted: &AtomicBool, muted: bool) -> Result<(), String> { - #[cfg(target_os = "macos")] - { - macos::set_muted(muted)?; - input_muted.store(muted, Ordering::Release); - Ok(()) - } - - #[cfg(not(target_os = "macos"))] - { - let _ = (input_muted, muted); - Err("native microphone mute is only available on macOS".to_string()) - } -} - fn clear(input_muted: &AtomicBool) { input_muted.store(false, Ordering::Release); } -fn apply_change(input_muted: &AtomicBool, muted: bool, on_change: &dyn Fn(bool)) { - let previous = input_muted.swap(muted, Ordering::AcqRel); - if previous != muted { - on_change(muted); - } -} - #[cfg(target_os = "macos")] mod macos { use super::*; - use std::sync::{Mutex, OnceLock}; - - type MuteChangeHandler = Arc; - type AudioInputHandler = Arc; - - struct CallbackState { - input_muted: Arc, - on_change: MuteChangeHandler, - on_audio: AudioInputHandler, - } - - static CALLBACK_STATE: OnceLock>> = OnceLock::new(); - - fn callback_state() -> &'static Mutex> { - CALLBACK_STATE.get_or_init(|| Mutex::new(None)) - } - - extern "C" { - fn berd_airpods_mute_start( - callback: extern "C" fn(bool), - audio_callback: extern "C" fn(*const f32, usize), - ) -> bool; - fn berd_airpods_mute_stop() -> bool; - fn berd_airpods_mute_set_muted(muted: bool) -> bool; - } - - extern "C" fn handle_input_mute_change(muted: bool) { - let Ok(state) = callback_state().lock() else { - return; - }; - let Some(state) = state.as_ref() else { - return; - }; - apply_change(&state.input_muted, muted, &|muted| { + use block2::RcBlock; + use objc2::runtime::Bool; + use objc2_avf_audio::AVAudioApplication; + + pub fn install(input_muted: Arc) -> Result<(), String> { + // SAFETY: Berd's minimum macOS version is 14.0, where + // AVAudioApplication and these selectors are public API. + let application = unsafe { AVAudioApplication::sharedInstance() }; + unsafe { application.setInputMuted_error(false) } + .map_err(|error| error.localizedDescription().to_string())?; + + let handler = RcBlock::new(move |muted: Bool| { + let muted = muted.as_bool(); + input_muted.store(muted, Ordering::Release); log::info!("AirPods input mute changed muted={muted}"); - (state.on_change)(muted); + Bool::YES }); - } - - extern "C" fn handle_audio_input(samples: *const f32, sample_count: usize) { - if samples.is_null() || sample_count == 0 || sample_count > 48_000 { - return; - } - let callback = { - let Ok(state) = callback_state().lock() else { - return; - }; - let Some(state) = state.as_ref() else { - return; - }; - Arc::clone(&state.on_audio) - }; - // SAFETY: AVAudioEngine owns this non-interleaved Float32 channel for - // the duration of the synchronous tap callback. We do not retain it. - let samples = unsafe { std::slice::from_raw_parts(samples, sample_count) }; - callback(samples); - } - - pub fn install( - input_muted: Arc, - on_change: MuteChangeHandler, - on_audio: AudioInputHandler, - ) -> Result<(), String> { - *callback_state() - .lock() - .map_err(|_| "input mute callback lock was poisoned".to_string())? = - Some(CallbackState { - input_muted, - on_change, - on_audio, - }); - // SAFETY: The Swift shim retains the callback and AVAudioEngine for its - // process-wide lifecycle and invokes it with a C-compatible boolean. - if !unsafe { berd_airpods_mute_start(handle_input_mute_change, handle_audio_input) } { - callback_state() - .lock() - .map_err(|_| "input mute callback lock was poisoned".to_string())? - .take(); - return Err("the Swift AVAudioApplication listener could not start".to_string()); - } + // SAFETY: The block has the generated AVFAudio signature. The API + // copies and retains it until a later registration or cancellation. + unsafe { application.setInputMuteStateChangeHandler_error(Some(&handler)) } + .map_err(|error| error.localizedDescription().to_string())?; log::info!("AirPods input mute listener started"); Ok(()) } pub fn uninstall() -> Result<(), String> { - // SAFETY: This mirrors the successful start call and clears the Swift - // process-global before Rust releases its callback state. - let stopped = unsafe { berd_airpods_mute_stop() }; - callback_state() - .lock() - .map_err(|_| "input mute callback lock was poisoned".to_string())? - .take(); - stopped - .then_some(()) - .ok_or_else(|| "the Swift AVAudioApplication listener could not stop".to_string()) - } - - pub fn set_muted(muted: bool) -> Result<(), String> { - // SAFETY: The Swift bridge accepts a C-compatible boolean and remains - // alive for the active native microphone lifecycle. - unsafe { berd_airpods_mute_set_muted(muted) } - .then_some(()) - .ok_or_else(|| "macOS rejected the native microphone mute change".to_string()) + // SAFETY: Berd targets macOS 14+, and nil is the documented way to + // cancel the process-wide handler at the end of a call lifecycle. + let application = unsafe { AVAudioApplication::sharedInstance() }; + unsafe { application.setInputMuteStateChangeHandler_error(None) } + .map_err(|error| error.localizedDescription().to_string())?; + // Do not leave another Berd microphone feature inheriting the voice + // conversation's last input-mute state after its handler is gone. + unsafe { application.setInputMuted_error(false) } + .map_err(|error| error.localizedDescription().to_string()) } } @@ -186,25 +76,4 @@ mod tests { clear(&input_muted); assert!(!input_muted.load(Ordering::Acquire)); } - - #[test] - fn unchanged_initial_state_is_not_reported_as_a_gesture() { - let input_muted = AtomicBool::new(false); - let changes = std::sync::Mutex::new(Vec::new()); - - apply_change(&input_muted, false, &|muted| { - changes.lock().expect("changes lock").push(muted); - }); - apply_change(&input_muted, true, &|muted| { - changes.lock().expect("changes lock").push(muted); - }); - apply_change(&input_muted, true, &|muted| { - changes.lock().expect("changes lock").push(muted); - }); - apply_change(&input_muted, false, &|muted| { - changes.lock().expect("changes lock").push(muted); - }); - - assert_eq!(*changes.lock().expect("changes lock"), vec![true, false]); - } } diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index 684372bd0..2c7d72857 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -49,7 +49,6 @@ pub struct NativeVoiceStatus { session_id: Option, owner_window_label: Option, revision: u64, - native_microphone_capture: bool, } #[derive(Clone, Debug, Serialize)] @@ -96,11 +95,6 @@ enum NativeVoiceEvent { activity: &'static str, revision: u64, }, - InputMute { - session_id: String, - muted: bool, - revision: u64, - }, CleanShutdown { session_id: String, revision: u64, @@ -133,7 +127,6 @@ pub struct NativeVoiceState { pending: Arc>>, capture_suppressions: Arc, input_muted: Arc, - native_microphone_capture: Arc, } #[must_use = "capture suppression ends when the guard is dropped"] @@ -167,12 +160,6 @@ impl NativeVoiceState { fn capture_is_suppressed(&self) -> bool { self.capture_suppressions.load(Ordering::SeqCst) > 0 } - - fn stop_native_microphone(&self) { - native_input_mute::stop(&self.input_muted); - self.native_microphone_capture - .store(false, Ordering::Release); - } } enum SttMessage { @@ -321,7 +308,6 @@ fn status(app: &AppHandle, state: &NativeVoiceState) -> NativeVoiceStatus { .as_ref() .map(|owner| owner.window_label.clone()), revision: runtime.revision, - native_microphone_capture: state.native_microphone_capture.load(Ordering::Acquire), } } @@ -466,6 +452,7 @@ pub async fn start_native_voice_conversation( window_label: window_label.clone(), }); runtime.pipeline = Some(pipeline); + native_input_mute::start(&state.input_muted); ( runtime.revision, runtime.lifecycle_id.clone().unwrap_or_default(), @@ -480,40 +467,12 @@ pub async fn start_native_voice_conversation( revision, }, ); - let mute_window = webview_window.clone(); - let mute_session_id = session_id.clone(); - let audio_state = state.inner().clone(); - let audio_session_id = session_id.clone(); - let native_capture_started = native_input_mute::start( - &state.input_muted, - move |muted| { - let _ = mute_window.emit( - EVENT_NAME, - NativeVoiceEvent::InputMute { - session_id: mute_session_id.clone(), - muted, - revision, - }, - ); - }, - move |samples| { - if let Err(error) = - push_audio_for_session(&audio_state, &audio_session_id, revision, samples) - { - log::warn!("Native microphone audio was not accepted: {error}"); - } - }, - ); - state - .native_microphone_capture - .store(native_capture_started, Ordering::Release); let event_app = app.clone(); let event_window = webview_window.clone(); let runtime = Arc::clone(&state.runtime); let pending = Arc::clone(&state.pending); let input_muted = Arc::clone(&state.input_muted); - let native_microphone_capture = Arc::clone(&state.native_microphone_capture); tauri::async_runtime::spawn(async move { while let Some(event) = events.recv().await { let active = runtime.lock().ok().is_some_and(|current| { @@ -587,7 +546,6 @@ pub async fn start_native_voice_conversation( break; } native_input_mute::stop(&input_muted); - native_microphone_capture.store(false, Ordering::Release); current.session_id = None; current.lifecycle_id = None; current.owner = None; @@ -654,7 +612,7 @@ pub async fn stop_native_voice_conversation( .lock() .map_err(|_| "native voice state lock was poisoned".to_string())?; if runtime.revision == revision && runtime.session_id == session_id { - state.stop_native_microphone(); + native_input_mute::stop(&state.input_muted); runtime.session_id = None; runtime.lifecycle_id = None; runtime.owner = None; @@ -712,7 +670,7 @@ impl NativeVoiceState { .lock() .map_err(|_| "native voice state lock was poisoned".to_string())?; if runtime.revision == revision && runtime.session_id == session_id { - self.stop_native_microphone(); + native_input_mute::stop(&self.input_muted); runtime.session_id = None; runtime.lifecycle_id = None; runtime.owner = None; @@ -751,7 +709,7 @@ impl NativeVoiceState { if let Some(pipeline) = pipeline.as_ref() { pipeline.signal_shutdown(); } - self.stop_native_microphone(); + native_input_mute::stop(&self.input_muted); (runtime.session_id.clone(), runtime.revision, pipeline) }; if pipeline.is_none() { @@ -788,7 +746,7 @@ impl NativeVoiceState { if let Some(pipeline) = runtime.pipeline.as_ref() { pipeline.latch_muted_shutdown(); } - self.stop_native_microphone(); + native_input_mute::stop(&self.input_muted); ( runtime.session_id.clone(), runtime.revision, @@ -819,28 +777,6 @@ pub fn push_native_voice_audio( push_audio_for_window(&state, webview_window.label(), bytes.to_vec()) } -#[tauri::command] -pub fn set_native_voice_input_muted( - state: State<'_, NativeVoiceState>, - webview_window: WebviewWindow, - muted: bool, -) -> Result<(), String> { - let runtime = state - .runtime - .lock() - .map_err(|_| "native voice state lock was poisoned".to_string())?; - if runtime.session_id.is_none() - || runtime - .owner - .as_ref() - .is_none_or(|owner| owner.window_label != webview_window.label()) - { - return Err("Only the active voice owner may mute its microphone.".to_string()); - } - drop(runtime); - native_input_mute::set_muted(&state.input_muted, muted) -} - fn push_audio_for_window( state: &NativeVoiceState, window_label: &str, @@ -866,32 +802,6 @@ fn push_audio_for_window( Ok(()) } -fn push_audio_for_session( - state: &NativeVoiceState, - session_id: &str, - revision: u64, - samples: &[f32], -) -> Result<(), String> { - if state.capture_is_suppressed() { - return Ok(()); - } - let runtime = state - .runtime - .lock() - .map_err(|_| "native voice state lock was poisoned".to_string())?; - if runtime.session_id.as_deref() != Some(session_id) || runtime.revision != revision { - return Ok(()); - } - let Some(pipeline) = runtime.pipeline.as_ref() else { - return Ok(()); - }; - let mut bytes = Vec::with_capacity(samples.len() * size_of::()); - for sample in samples { - bytes.extend_from_slice(&sample.to_ne_bytes()); - } - pipeline.push(bytes) -} - fn enqueue_pending_transcript( queue: &mut VecDeque, transcript: PendingTranscript, @@ -1379,20 +1289,5 @@ mod tests { "deliveryAttempts": 0, }), ); - - let event = NativeVoiceEvent::InputMute { - session_id: "session-1".to_string(), - muted: true, - revision: 3, - }; - assert_eq!( - serde_json::to_value(event).expect("serialize input mute event"), - serde_json::json!({ - "type": "inputMute", - "sessionId": "session-1", - "muted": true, - "revision": 3, - }), - ); } } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 8672de987..f90b7b950 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -646,7 +646,6 @@ pub fn run() { commands::native_voice::start_native_voice_conversation, commands::native_voice::stop_native_voice_conversation, commands::native_voice::push_native_voice_audio, - commands::native_voice::set_native_voice_input_muted, commands::voice_capture::register_voice_renderer_instance, commands::window_session::get_session_window_support, commands::window_session::open_session_window, diff --git a/src-tauri/swift/BerdAirPodsBridge/Package.swift b/src-tauri/swift/BerdAirPodsBridge/Package.swift deleted file mode 100644 index 51862d334..000000000 --- a/src-tauri/swift/BerdAirPodsBridge/Package.swift +++ /dev/null @@ -1,20 +0,0 @@ -// swift-tools-version: 5.9 -import PackageDescription - -let package = Package( - name: "BerdAirPodsBridge", - platforms: [.macOS(.v14)], - products: [ - .library( - name: "BerdAirPodsBridge", - type: .static, - targets: ["BerdAirPodsBridge"] - ) - ], - targets: [ - .target( - name: "BerdAirPodsBridge", - linkerSettings: [.linkedFramework("AVFAudio")] - ) - ] -) diff --git a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift deleted file mode 100644 index cf1cf9566..000000000 --- a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift +++ /dev/null @@ -1,122 +0,0 @@ -import AVFoundation -import Foundation - -public typealias InputMuteCallback = @convention(c) (Bool) -> Void -public typealias AudioInputCallback = @convention(c) (UnsafePointer, Int) -> Void - -@available(macOS 14.0, *) -private final class AirPodsMuteBridge: @unchecked Sendable { - private let engine: AVAudioEngine - private let inputNode: AVAudioInputNode - private let callback: InputMuteCallback - private let audioCallback: AudioInputCallback - private var inputMuteObserver: NSObjectProtocol? - - init( - callback: @escaping InputMuteCallback, - audioCallback: @escaping AudioInputCallback - ) throws { - self.callback = callback - self.audioCallback = audioCallback - - let engine = AVAudioEngine() - let inputNode = engine.inputNode - let inputFormat = inputNode.outputFormat(forBus: 0) - guard inputFormat.sampleRate > 0, inputFormat.channelCount > 0 else { - throw BridgeError.noInputFormat - } - self.engine = engine - self.inputNode = inputNode - - inputNode.installTap( - onBus: 0, - bufferSize: 4096, - format: inputFormat - ) { [weak self] buffer, _ in - guard - let self, - let channel = buffer.floatChannelData?.pointee, - buffer.frameLength > 0 - else { return } - self.audioCallback(channel, Int(buffer.frameLength)) - } - engine.prepare() - try engine.start() - - // Match voice-conversation-cli exactly: seed before registration and - // ignore the expected error when no prior process handler exists. - try? AVAudioApplication.shared.setInputMuted(false) - try AVAudioApplication.shared.setInputMuteStateChangeHandler { [weak self] muted in - self?.callback(muted) - return true - } - inputMuteObserver = NotificationCenter.default.addObserver( - forName: AVAudioApplication.inputMuteStateChangeNotification, - object: nil, - queue: .main - ) { [weak self] _ in - guard let self else { return } - self.callback(AVAudioApplication.shared.isInputMuted) - } - } - - func stop() { - // Reset while the handler exists, then cancel the process-wide callback. - try? AVAudioApplication.shared.setInputMuted(false) - try? AVAudioApplication.shared.setInputMuteStateChangeHandler(nil) - if let inputMuteObserver { - NotificationCenter.default.removeObserver(inputMuteObserver) - self.inputMuteObserver = nil - } - inputNode.removeTap(onBus: 0) - engine.stop() - } - - private enum BridgeError: Error { - case noInputFormat - } -} - -@available(macOS 14.0, *) -private var activeBridge: AirPodsMuteBridge? - -@_cdecl("berd_airpods_mute_start") -public func berdAirPodsMuteStart( - callback: @escaping InputMuteCallback, - audioCallback: @escaping AudioInputCallback -) -> Bool { - guard #available(macOS 14.0, *) else { return false } - do { - activeBridge?.stop() - activeBridge = try AirPodsMuteBridge( - callback: callback, - audioCallback: audioCallback - ) - return true - } catch { - FileHandle.standardError.write( - Data("Berd AirPods mute bridge failed to start: \(error)\n".utf8) - ) - activeBridge = nil - return false - } -} - -@_cdecl("berd_airpods_mute_stop") -public func berdAirPodsMuteStop() -> Bool { - guard #available(macOS 14.0, *) else { return false } - activeBridge?.stop() - activeBridge = nil - return true -} - -@_cdecl("berd_airpods_mute_set_muted") -public func berdAirPodsMuteSetMuted(_ muted: Bool) -> Bool { - guard #available(macOS 14.0, *), activeBridge != nil else { return false } - do { - try AVAudioApplication.shared.setInputMuted(muted) - return true - } catch { - return false - } -} diff --git a/src/features/voice-conversation/api/voiceConversation.test.ts b/src/features/voice-conversation/api/voiceConversation.test.ts index a66071cde..4bd905ec8 100644 --- a/src/features/voice-conversation/api/voiceConversation.test.ts +++ b/src/features/voice-conversation/api/voiceConversation.test.ts @@ -162,27 +162,6 @@ describe("voice conversation API", () => { expect(mocks.startMicrophone).not.toHaveBeenCalled(); }); - it("uses backend capture and mute when the native process owns microphone audio", async () => { - const status = { - available: true, - unavailableReason: null, - lifecycle: "running", - sessionId: "session-1", - ownerWindowLabel: "main", - revision: 3, - nativeMicrophoneCapture: true, - } as const; - mocks.invoke.mockResolvedValue(undefined); - - await reconcileVoiceConversationMicrophone(status); - await setVoiceConversationMicrophoneMuted(true, status); - - expect(mocks.startMicrophone).not.toHaveBeenCalled(); - expect(mocks.invoke).toHaveBeenCalledWith("set_native_voice_input_muted", { - muted: true, - }); - }); - it("mutes and unmutes without reopening browser capture", async () => { const status = { available: true, @@ -287,39 +266,6 @@ describe("voice conversation API", () => { }); }); - it("applies AirPods input mute events to browser capture", async () => { - const callback = vi.fn(); - mocks.listen.mockImplementation(async (_name, handler) => { - handler({ - payload: { - type: "inputMute", - sessionId: "session-1", - muted: true, - revision: 6, - }, - }); - return vi.fn(); - }); - - await reconcileVoiceConversationMicrophone({ - available: true, - unavailableReason: null, - lifecycle: "running", - sessionId: "session-1", - ownerWindowLabel: "main", - revision: 5, - }); - await listenToVoiceConversation(callback); - - expect(mocks.setMicrophoneMuted).toHaveBeenLastCalledWith(true); - expect(callback).toHaveBeenCalledWith({ - type: "inputMute", - sessionId: "session-1", - muted: true, - revision: 6, - }); - }); - it("stops browser capture when native voice shuts down elsewhere", async () => { mocks.invoke.mockResolvedValue({ available: true, diff --git a/src/features/voice-conversation/api/voiceConversation.ts b/src/features/voice-conversation/api/voiceConversation.ts index f98c4a28a..e213f5486 100644 --- a/src/features/voice-conversation/api/voiceConversation.ts +++ b/src/features/voice-conversation/api/voiceConversation.ts @@ -49,10 +49,6 @@ async function ensureActiveMicrophone(): Promise { export async function reconcileVoiceConversationMicrophone( status: VoiceConversationStatus, ): Promise { - if (status.nativeMicrophoneCapture) { - stopActiveMicrophone(); - return; - } if ( status.lifecycle === "running" && status.ownerWindowLabel === getCurrentWindow().label @@ -70,11 +66,6 @@ export async function setVoiceConversationMicrophoneMuted( const previous = microphoneMuted; microphoneMuted = muted; try { - if (status.nativeMicrophoneCapture) { - await invoke("set_native_voice_input_muted", { muted }); - stopActiveMicrophone(); - return; - } await reconcileVoiceConversationMicrophone(status); } catch (error) { microphoneMuted = previous; @@ -118,8 +109,6 @@ export interface VoiceConversationStatus { ownerWindowLabel: string | null; /** Monotonic native lifecycle revision used to reject stale responses/events. */ revision: number; - /** The backend owns the PCM stream so device gestures reach that process. */ - nativeMicrophoneCapture?: boolean; } export type VoiceConversationEvent = @@ -149,12 +138,6 @@ export type VoiceConversationEvent = | "assistant-idle"; revision: number; } - | { - type: "inputMute"; - sessionId: string; - muted: boolean; - revision: number; - } | { type: "cleanShutdown"; sessionId: string; @@ -261,10 +244,6 @@ export function listenToVoiceConversation( onEvent: (event: VoiceConversationEvent) => void, ): Promise { return listen(VOICE_CONVERSATION_EVENT, (event) => { - if (event.payload.type === "inputMute") { - microphoneMuted = event.payload.muted; - activeMicrophone?.setMuted(event.payload.muted); - } if ( event.payload.type === "cleanShutdown" || (event.payload.type === "error" && event.payload.terminal) diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts index 07446a4a8..68e06b5c6 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts @@ -3,15 +3,12 @@ import { act, renderHook, waitFor } from "@testing-library/react"; import { useChatStore } from "@/features/chat/stores/chatStore"; import { useVoiceConversationStore } from "../stores/voiceConversationStore"; -const toastMocks = vi.hoisted(() => ({ message: vi.fn() })); const nativeAssistantSpeechMocks = vi.hoisted(() => ({ start: vi.fn(), stop: vi.fn(), takeNotices: vi.fn<() => string | null>(() => null), })); -vi.mock("sonner", () => ({ toast: toastMocks })); - vi.mock("../lib/nativeAssistantSpeech", () => ({ startNativeAssistantSpeech: nativeAssistantSpeechMocks.start, stopNativeAssistantSpeech: nativeAssistantSpeechMocks.stop, @@ -24,7 +21,6 @@ import { createVoiceRouteMountRegistry, createVoiceTranscriptDeliveryQueue, hasDeliveredVoiceTranscript, - notifyAirPodsInputMuteGesture, resetVoiceUiWhenRunSettles, resolveVoiceRouteMount, resolveVoiceToggleAction, @@ -36,20 +32,6 @@ import { } from "./useVoiceConversationController"; describe("voice transcript delivery coordination", () => { - it("shows an unmistakable AirPods gesture confirmation", () => { - notifyAirPodsInputMuteGesture(true); - expect(toastMocks.message).toHaveBeenCalledWith( - "AirPods gesture detected — microphone muted", - { id: "airpods-input-mute" }, - ); - - notifyAirPodsInputMuteGesture(false); - expect(toastMocks.message).toHaveBeenLastCalledWith( - "AirPods gesture detected — microphone unmuted", - { id: "airpods-input-mute" }, - ); - }); - it("recognizes a replayed transcript that was already delivered", () => { useChatStore.setState({ messagesBySession: { diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index 1e15eb54d..df364dd87 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -1,5 +1,4 @@ import { useCallback, useEffect, useMemo, useRef } from "react"; -import { toast } from "sonner"; import type { ChatInputSendHandler, @@ -188,15 +187,6 @@ function addErrorNotification(sessionId: string | null, message: string) { .addMessage(sessionId, createSystemNotificationMessage(message, "error")); } -export function notifyAirPodsInputMuteGesture(muted: boolean) { - toast.message( - muted - ? "AirPods gesture detected — microphone muted" - : "AirPods gesture detected — microphone unmuted", - { id: "airpods-input-mute" }, - ); -} - export function hasDeliveredVoiceTranscript( sessionId: string, lifecycleId: string, @@ -324,10 +314,6 @@ function ensureVoiceEventDeliveryInitialized() { addErrorNotification(sessionId ?? null, event.message); return; } - if (event.type === "inputMute") { - notifyAirPodsInputMuteGesture(event.muted); - return; - } if (event.type === "activity") return; if (event.type !== "user" || !event.text.trim()) return; if ( diff --git a/src/features/voice-conversation/stores/voiceConversationStore.test.ts b/src/features/voice-conversation/stores/voiceConversationStore.test.ts index 7c992fa52..af1510a15 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.test.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.test.ts @@ -389,28 +389,6 @@ describe("voice conversation store lifecycle ordering", () => { }); }); - it("reflects AirPods input mute events in the microphone control", async () => { - const store = await loadStore(); - store.setState({ - status: status("running", 2, "session-1"), - uiState: "user-speaking", - userSpeaking: true, - }); - - emit({ - type: "inputMute", - sessionId: "session-1", - muted: true, - revision: 3, - }); - - expect(store.getState()).toMatchObject({ - microphoneMuted: true, - userSpeaking: false, - uiState: "listening", - }); - }); - it("surfaces an unmute failure without losing muted state", async () => { const store = await loadStore(); store.setState({ diff --git a/src/features/voice-conversation/stores/voiceConversationStore.ts b/src/features/voice-conversation/stores/voiceConversationStore.ts index 56a420d05..efff65a0c 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.ts @@ -287,24 +287,6 @@ export const useVoiceConversationStore = create( uiState: activityUiState(nextState), }; } - case "inputMute": { - const nextState = { - ...state, - microphoneMuted: event.muted, - userSpeaking: event.muted ? false : state.userSpeaking, - status: { - ...state.status, - lifecycle: "running" as const, - sessionId: event.sessionId, - revision: event.revision, - }, - error: null, - }; - return { - ...nextState, - uiState: activityUiState(nextState), - }; - } case "cleanShutdown": return { ...state, From a9bc16464cbdfee07af8e89ea275845e528902ac Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 16:39:19 -0400 Subject: [PATCH 17/31] refactor(voice): minimize AirPods mute bridge --- src-tauri/Cargo.lock | 1 + src-tauri/Cargo.toml | 1 + src-tauri/build.rs | 5 + src-tauri/src/commands/native_input_mute.rs | 83 ++++++++-- src-tauri/src/commands/native_voice.rs | 79 +++++++++- src-tauri/src/lib.rs | 1 + .../swift/BerdAirPodsBridge/Package.swift | 25 +++ .../BerdAirPodsBridge/BerdAirPodsBridge.swift | 144 ++++++++++++++++++ .../BerdObjCExceptionCatch.m | 19 +++ .../include/BerdObjCExceptionCatch.h | 4 + .../api/voiceConversation.test.ts | 23 +++ .../api/voiceConversation.ts | 20 +++ .../hooks/useVoiceConversationController.ts | 7 + .../stores/voiceConversationStore.test.ts | 26 ++++ .../stores/voiceConversationStore.ts | 17 +++ 15 files changed, 441 insertions(+), 14 deletions(-) create mode 100644 src-tauri/swift/BerdAirPodsBridge/Package.swift create mode 100644 src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift create mode 100644 src-tauri/swift/BerdAirPodsBridge/Sources/BerdObjCExceptionCatch/BerdObjCExceptionCatch.m create mode 100644 src-tauri/swift/BerdAirPodsBridge/Sources/BerdObjCExceptionCatch/include/BerdObjCExceptionCatch.h diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index d48871f52..7ce8c3c52 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -50,6 +50,7 @@ dependencies = [ "sherpa-onnx", "sqlx", "ssstretch", + "swift-rs", "sysinfo", "tar", "tauri", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index feac03007..39addf449 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -18,6 +18,7 @@ members = ["crates/berd-voice", "crates/berdctl", "plugins/berdctl"] exclude = ["plugins/app-test-driver"] [build-dependencies] +swift-rs = { version = "1.0.7", features = ["build"] } tauri-build = { version = "2", features = [] } cc = "1" diff --git a/src-tauri/build.rs b/src-tauri/build.rs index afa1d236e..ab8a5f53f 100644 --- a/src-tauri/build.rs +++ b/src-tauri/build.rs @@ -19,6 +19,11 @@ fn main() { println!("cargo:rustc-link-lib=framework={framework}"); } + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("macos") { + swift_rs::SwiftLinker::new("14.0") + .with_package("BerdAirPodsBridge", "swift/BerdAirPodsBridge") + .link(); + } } tauri_build::build() } diff --git a/src-tauri/src/commands/native_input_mute.rs b/src-tauri/src/commands/native_input_mute.rs index 8659da2b9..6660c4322 100644 --- a/src-tauri/src/commands/native_input_mute.rs +++ b/src-tauri/src/commands/native_input_mute.rs @@ -3,12 +3,25 @@ use std::sync::{ Arc, }; -pub fn start(input_muted: &Arc) { +pub fn start(input_muted: &Arc, on_change: F) -> bool +where + F: Fn(bool) + Send + Sync + 'static, +{ clear(input_muted); #[cfg(target_os = "macos")] - if let Err(error) = macos::install(Arc::clone(input_muted)) { - log::info!("AirPods input mute listener is unavailable: {error}"); + return match macos::install(Arc::clone(input_muted), on_change) { + Ok(()) => true, + Err(error) => { + log::info!("AirPods input mute listener is unavailable: {error}"); + false + } + }; + + #[cfg(not(target_os = "macos"))] + { + let _ = on_change; + false } } @@ -21,6 +34,21 @@ pub fn stop(input_muted: &Arc) { } } +pub fn set_muted(input_muted: &AtomicBool, muted: bool) -> Result<(), String> { + #[cfg(target_os = "macos")] + { + macos::set_muted(muted)?; + input_muted.store(muted, Ordering::Release); + Ok(()) + } + + #[cfg(not(target_os = "macos"))] + { + let _ = (input_muted, muted); + Err("native microphone mute is only available on macOS".to_string()) + } +} + fn clear(input_muted: &AtomicBool) { input_muted.store(false, Ordering::Release); } @@ -32,23 +60,44 @@ mod macos { use objc2::runtime::Bool; use objc2_avf_audio::AVAudioApplication; - pub fn install(input_muted: Arc) -> Result<(), String> { + extern "C" { + fn berd_airpods_capture_start() -> bool; + fn berd_airpods_capture_stop(); + } + + pub fn install(input_muted: Arc, on_change: F) -> Result<(), String> + where + F: Fn(bool) + Send + Sync + 'static, + { + // SAFETY: The Swift bridge owns one process-global AVAudioEngine and + // exposes a C-compatible lifecycle API. + if !unsafe { berd_airpods_capture_start() } { + return Err("macOS microphone capture could not start".to_string()); + } // SAFETY: Berd's minimum macOS version is 14.0, where // AVAudioApplication and these selectors are public API. let application = unsafe { AVAudioApplication::sharedInstance() }; - unsafe { application.setInputMuted_error(false) } - .map_err(|error| error.localizedDescription().to_string())?; + if let Err(error) = unsafe { application.setInputMuted_error(false) } { + unsafe { berd_airpods_capture_stop() }; + return Err(error.localizedDescription().to_string()); + } let handler = RcBlock::new(move |muted: Bool| { let muted = muted.as_bool(); - input_muted.store(muted, Ordering::Release); - log::info!("AirPods input mute changed muted={muted}"); + if input_muted.swap(muted, Ordering::AcqRel) != muted { + log::info!("AirPods input mute changed muted={muted}"); + on_change(muted); + } Bool::YES }); // SAFETY: The block has the generated AVFAudio signature. The API // copies and retains it until a later registration or cancellation. - unsafe { application.setInputMuteStateChangeHandler_error(Some(&handler)) } - .map_err(|error| error.localizedDescription().to_string())?; + if let Err(error) = + unsafe { application.setInputMuteStateChangeHandler_error(Some(&handler)) } + { + unsafe { berd_airpods_capture_stop() }; + return Err(error.localizedDescription().to_string()); + } log::info!("AirPods input mute listener started"); Ok(()) } @@ -57,11 +106,19 @@ mod macos { // SAFETY: Berd targets macOS 14+, and nil is the documented way to // cancel the process-wide handler at the end of a call lifecycle. let application = unsafe { AVAudioApplication::sharedInstance() }; - unsafe { application.setInputMuteStateChangeHandler_error(None) } - .map_err(|error| error.localizedDescription().to_string())?; + let handler_result = unsafe { application.setInputMuteStateChangeHandler_error(None) } + .map_err(|error| error.localizedDescription().to_string()); // Do not leave another Berd microphone feature inheriting the voice // conversation's last input-mute state after its handler is gone. - unsafe { application.setInputMuted_error(false) } + let reset_result = unsafe { application.setInputMuted_error(false) } + .map_err(|error| error.localizedDescription().to_string()); + unsafe { berd_airpods_capture_stop() }; + handler_result.and(reset_result) + } + + pub fn set_muted(muted: bool) -> Result<(), String> { + let application = unsafe { AVAudioApplication::sharedInstance() }; + unsafe { application.setInputMuted_error(muted) } .map_err(|error| error.localizedDescription().to_string()) } } diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index 2c7d72857..fd94fc0df 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -49,6 +49,7 @@ pub struct NativeVoiceStatus { session_id: Option, owner_window_label: Option, revision: u64, + native_microphone_mute_control: bool, } #[derive(Clone, Debug, Serialize)] @@ -95,6 +96,11 @@ enum NativeVoiceEvent { activity: &'static str, revision: u64, }, + InputMute { + session_id: String, + muted: bool, + revision: u64, + }, CleanShutdown { session_id: String, revision: u64, @@ -114,6 +120,7 @@ struct Runtime { revision: u64, owner: Option, pipeline: Option, + native_microphone_mute_control: bool, } #[derive(Clone)] @@ -308,6 +315,7 @@ fn status(app: &AppHandle, state: &NativeVoiceState) -> NativeVoiceStatus { .as_ref() .map(|owner| owner.window_label.clone()), revision: runtime.revision, + native_microphone_mute_control: runtime.native_microphone_mute_control, } } @@ -452,7 +460,20 @@ pub async fn start_native_voice_conversation( window_label: window_label.clone(), }); runtime.pipeline = Some(pipeline); - native_input_mute::start(&state.input_muted); + let runtime_revision = runtime.revision; + let mute_window = webview_window.clone(); + let mute_session_id = session_id.clone(); + runtime.native_microphone_mute_control = + native_input_mute::start(&state.input_muted, move |muted| { + let _ = mute_window.emit( + EVENT_NAME, + NativeVoiceEvent::InputMute { + session_id: mute_session_id.clone(), + muted, + revision: runtime_revision, + }, + ); + }); ( runtime.revision, runtime.lifecycle_id.clone().unwrap_or_default(), @@ -546,6 +567,7 @@ pub async fn start_native_voice_conversation( break; } native_input_mute::stop(&input_muted); + current.native_microphone_mute_control = false; current.session_id = None; current.lifecycle_id = None; current.owner = None; @@ -613,6 +635,7 @@ pub async fn stop_native_voice_conversation( .map_err(|_| "native voice state lock was poisoned".to_string())?; if runtime.revision == revision && runtime.session_id == session_id { native_input_mute::stop(&state.input_muted); + runtime.native_microphone_mute_control = false; runtime.session_id = None; runtime.lifecycle_id = None; runtime.owner = None; @@ -671,6 +694,7 @@ impl NativeVoiceState { .map_err(|_| "native voice state lock was poisoned".to_string())?; if runtime.revision == revision && runtime.session_id == session_id { native_input_mute::stop(&self.input_muted); + runtime.native_microphone_mute_control = false; runtime.session_id = None; runtime.lifecycle_id = None; runtime.owner = None; @@ -710,6 +734,7 @@ impl NativeVoiceState { pipeline.signal_shutdown(); } native_input_mute::stop(&self.input_muted); + runtime.native_microphone_mute_control = false; (runtime.session_id.clone(), runtime.revision, pipeline) }; if pipeline.is_none() { @@ -747,6 +772,7 @@ impl NativeVoiceState { pipeline.latch_muted_shutdown(); } native_input_mute::stop(&self.input_muted); + runtime.native_microphone_mute_control = false; ( runtime.session_id.clone(), runtime.revision, @@ -765,6 +791,39 @@ impl NativeVoiceState { } } +#[tauri::command] +pub fn set_native_voice_input_muted( + state: State<'_, NativeVoiceState>, + webview_window: WebviewWindow, + session_id: String, + revision: u64, + muted: bool, +) -> Result<(), String> { + let runtime = state + .runtime + .lock() + .map_err(|_| "native voice state lock was poisoned".to_string())?; + if !owns_native_mute_control(&runtime, webview_window.label(), &session_id, revision) { + return Err("Native microphone mute is unavailable for this conversation.".to_string()); + } + native_input_mute::set_muted(&state.input_muted, muted) +} + +fn owns_native_mute_control( + runtime: &Runtime, + window_label: &str, + session_id: &str, + revision: u64, +) -> bool { + runtime.native_microphone_mute_control + && runtime.session_id.as_deref() == Some(session_id) + && runtime.revision == revision + && runtime + .owner + .as_ref() + .is_some_and(|owner| owner.window_label == window_label) +} + #[tauri::command] pub fn push_native_voice_audio( request: tauri::ipc::Request<'_>, @@ -1000,6 +1059,24 @@ fn deliver_recognition_result( mod tests { use super::*; + #[test] + fn native_mute_control_is_bound_to_window_session_and_revision() { + let runtime = Runtime { + session_id: Some("session-1".to_string()), + revision: 4, + owner: Some(RuntimeOwner { + window_label: "main".to_string(), + }), + native_microphone_mute_control: true, + ..Runtime::default() + }; + + assert!(owns_native_mute_control(&runtime, "main", "session-1", 4)); + assert!(!owns_native_mute_control(&runtime, "other", "session-1", 4)); + assert!(!owns_native_mute_control(&runtime, "main", "session-2", 4)); + assert!(!owns_native_mute_control(&runtime, "main", "session-1", 5)); + } + #[test] fn speaker_playback_blocks_vad_ingestion_until_all_guards_finish() { let state = NativeVoiceState::default(); diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index f90b7b950..8672de987 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -646,6 +646,7 @@ pub fn run() { commands::native_voice::start_native_voice_conversation, commands::native_voice::stop_native_voice_conversation, commands::native_voice::push_native_voice_audio, + commands::native_voice::set_native_voice_input_muted, commands::voice_capture::register_voice_renderer_instance, commands::window_session::get_session_window_support, commands::window_session::open_session_window, diff --git a/src-tauri/swift/BerdAirPodsBridge/Package.swift b/src-tauri/swift/BerdAirPodsBridge/Package.swift new file mode 100644 index 000000000..5e216c9a1 --- /dev/null +++ b/src-tauri/swift/BerdAirPodsBridge/Package.swift @@ -0,0 +1,25 @@ +// swift-tools-version: 5.9 +import PackageDescription + +let package = Package( + name: "BerdAirPodsBridge", + platforms: [.macOS(.v14)], + products: [ + .library( + name: "BerdAirPodsBridge", + type: .static, + targets: ["BerdAirPodsBridge"] + ) + ], + targets: [ + .target( + name: "BerdObjCExceptionCatch", + publicHeadersPath: "include" + ), + .target( + name: "BerdAirPodsBridge", + dependencies: ["BerdObjCExceptionCatch"], + linkerSettings: [.linkedFramework("AVFAudio")] + ) + ] +) diff --git a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift new file mode 100644 index 000000000..d4475e47f --- /dev/null +++ b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift @@ -0,0 +1,144 @@ +import AVFAudio +import BerdObjCExceptionCatch +import Foundation + +private let bridgeQueue = DispatchQueue(label: "com.berd.airpods-capture") + +@available(macOS 14.0, *) +private final class AirPodsCapture: @unchecked Sendable { + private var engine: AVAudioEngine? + private var inputNode: AVAudioInputNode? + private var observer: NSObjectProtocol? + private var restart: DispatchWorkItem? + private var generation: UInt64 = 0 + private var stopped = false + + init() throws { + try startEngine() + } + + func stop() { + guard !stopped else { return } + stopped = true + generation &+= 1 + restart?.cancel() + restart = nil + tearDownEngine() + } + + private func startEngine() throws { + let engine = AVAudioEngine() + var caughtInputNode: AVAudioInputNode? + var inputNodeError: NSError? + BerdTryObjCBlock({ caughtInputNode = engine.inputNode }, &inputNodeError) + guard inputNodeError == nil, let inputNode = caughtInputNode else { + throw CaptureError.objectiveC(inputNodeError?.localizedDescription ?? "no input node") + } + + let format = inputNode.outputFormat(forBus: 0) + guard format.sampleRate > 0, format.channelCount > 0 else { + throw CaptureError.noInputFormat + } + var tapError: NSError? + BerdTryObjCBlock({ + inputNode.installTap(onBus: 0, bufferSize: 4096, format: format) { _, _ in } + }, &tapError) + guard tapError == nil else { + throw CaptureError.objectiveC(tapError?.localizedDescription ?? "install tap failed") + } + + do { + engine.prepare() + try engine.start() + } catch { + removeTap(from: inputNode) + throw error + } + observer = NotificationCenter.default.addObserver( + forName: .AVAudioEngineConfigurationChange, + object: engine, + queue: nil + ) { [weak self] _ in + self?.scheduleRestart() + } + self.engine = engine + self.inputNode = inputNode + } + + private func tearDownEngine() { + if let observer { + NotificationCenter.default.removeObserver(observer) + self.observer = nil + } + if let inputNode { removeTap(from: inputNode) } + engine?.stop() + inputNode = nil + engine = nil + } + + private func removeTap(from inputNode: AVAudioInputNode) { + var error: NSError? + BerdTryObjCBlock({ inputNode.removeTap(onBus: 0) }, &error) + } + + private func scheduleRestart() { + bridgeQueue.async { [weak self] in + guard let self, !stopped else { return } + generation &+= 1 + let expectedGeneration = generation + restart?.cancel() + let work = DispatchWorkItem { [weak self] in + self?.restartEngine(expectedGeneration) + } + restart = work + bridgeQueue.asyncAfter(deadline: .now() + 0.15, execute: work) + } + } + + private func restartEngine(_ expectedGeneration: UInt64) { + guard !stopped, generation == expectedGeneration else { return } + restart = nil + tearDownEngine() + do { + try startEngine() + } catch { + let work = DispatchWorkItem { [weak self] in + self?.restartEngine(expectedGeneration) + } + restart = work + bridgeQueue.asyncAfter(deadline: .now() + 1, execute: work) + } + } + + private enum CaptureError: Error { + case noInputFormat + case objectiveC(String) + } +} + +@available(macOS 14.0, *) +private var activeCapture: AirPodsCapture? + +@_cdecl("berd_airpods_capture_start") +public func berdAirPodsCaptureStart() -> Bool { + guard #available(macOS 14.0, *) else { return false } + return bridgeQueue.sync { + do { + activeCapture?.stop() + activeCapture = try AirPodsCapture() + return true + } catch { + activeCapture = nil + return false + } + } +} + +@_cdecl("berd_airpods_capture_stop") +public func berdAirPodsCaptureStop() { + guard #available(macOS 14.0, *) else { return } + bridgeQueue.sync { + activeCapture?.stop() + activeCapture = nil + } +} diff --git a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdObjCExceptionCatch/BerdObjCExceptionCatch.m b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdObjCExceptionCatch/BerdObjCExceptionCatch.m new file mode 100644 index 000000000..350cd0658 --- /dev/null +++ b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdObjCExceptionCatch/BerdObjCExceptionCatch.m @@ -0,0 +1,19 @@ +#import "BerdObjCExceptionCatch.h" + +// Adapted from voice-conversation-cli's VCTryObjCBlock. +BOOL BerdTryObjCBlock(void (NS_NOESCAPE ^_Nonnull block)(void), + NSError *_Nullable *_Nullable error) { + @try { + block(); + return YES; + } @catch (NSException *exception) { + if (error) { + *error = [NSError errorWithDomain:@"com.berd.objc-exception" + code:-1 + userInfo:@{ + NSLocalizedDescriptionKey: exception.reason ?: exception.name, + }]; + } + return NO; + } +} diff --git a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdObjCExceptionCatch/include/BerdObjCExceptionCatch.h b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdObjCExceptionCatch/include/BerdObjCExceptionCatch.h new file mode 100644 index 000000000..f23c87a37 --- /dev/null +++ b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdObjCExceptionCatch/include/BerdObjCExceptionCatch.h @@ -0,0 +1,4 @@ +#import + +BOOL BerdTryObjCBlock(void (NS_NOESCAPE ^_Nonnull block)(void), + NSError *_Nullable *_Nullable error); diff --git a/src/features/voice-conversation/api/voiceConversation.test.ts b/src/features/voice-conversation/api/voiceConversation.test.ts index 4bd905ec8..7f2596357 100644 --- a/src/features/voice-conversation/api/voiceConversation.test.ts +++ b/src/features/voice-conversation/api/voiceConversation.test.ts @@ -188,6 +188,29 @@ describe("voice conversation API", () => { expect(mocks.invoke).not.toHaveBeenCalled(); }); + it("routes UI mute through macOS while keeping browser capture in sync", async () => { + const status = { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + revision: 3, + nativeMicrophoneMuteControl: true, + } as const; + mocks.invoke.mockResolvedValue(undefined); + + await reconcileVoiceConversationMicrophone(status); + await setVoiceConversationMicrophoneMuted(true, status); + + expect(mocks.invoke).toHaveBeenCalledWith("set_native_voice_input_muted", { + sessionId: "session-1", + revision: 3, + muted: true, + }); + expect(mocks.setMicrophoneMuted).toHaveBeenLastCalledWith(true); + }); + it("restores the previous mute state when initial capture fails", async () => { const status = { available: true, diff --git a/src/features/voice-conversation/api/voiceConversation.ts b/src/features/voice-conversation/api/voiceConversation.ts index e213f5486..3f4735a59 100644 --- a/src/features/voice-conversation/api/voiceConversation.ts +++ b/src/features/voice-conversation/api/voiceConversation.ts @@ -67,6 +67,13 @@ export async function setVoiceConversationMicrophoneMuted( microphoneMuted = muted; try { await reconcileVoiceConversationMicrophone(status); + if (status.nativeMicrophoneMuteControl) { + await invoke("set_native_voice_input_muted", { + sessionId: status.sessionId, + revision: status.revision, + muted, + }); + } } catch (error) { microphoneMuted = previous; activeMicrophone?.setMuted(previous); @@ -74,6 +81,11 @@ export async function setVoiceConversationMicrophoneMuted( } } +export function applyVoiceConversationMicrophoneMuteEvent(muted: boolean) { + microphoneMuted = muted; + activeMicrophone?.setMuted(muted); +} + export function stopActiveMicrophoneForTest(): void { if (!import.meta.env.DEV) { throw new Error("Native microphone test controls are development-only."); @@ -109,6 +121,8 @@ export interface VoiceConversationStatus { ownerWindowLabel: string | null; /** Monotonic native lifecycle revision used to reject stale responses/events. */ revision: number; + /** macOS owns an input session capable of receiving headset mute controls. */ + nativeMicrophoneMuteControl?: boolean; } export type VoiceConversationEvent = @@ -138,6 +152,12 @@ export type VoiceConversationEvent = | "assistant-idle"; revision: number; } + | { + type: "inputMute"; + sessionId: string; + muted: boolean; + revision: number; + } | { type: "cleanShutdown"; sessionId: string; diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index df364dd87..4d94878a6 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -1,4 +1,5 @@ import { useCallback, useEffect, useMemo, useRef } from "react"; +import { toast } from "sonner"; import type { ChatInputSendHandler, @@ -315,6 +316,12 @@ function ensureVoiceEventDeliveryInitialized() { return; } if (event.type === "activity") return; + if (event.type === "inputMute") { + toast.message(`Microphone ${event.muted ? "muted" : "unmuted"}`, { + id: "voice-input-mute", + }); + return; + } if (event.type !== "user" || !event.text.trim()) return; if ( hasDeliveredVoiceTranscript( diff --git a/src/features/voice-conversation/stores/voiceConversationStore.test.ts b/src/features/voice-conversation/stores/voiceConversationStore.test.ts index af1510a15..84798a75e 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.test.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.test.ts @@ -6,6 +6,7 @@ import type { } from "../api/voiceConversation"; const mocks = vi.hoisted(() => ({ + applyMicrophoneMuteEvent: vi.fn(), acknowledge: vi.fn(), drain: vi.fn(), getStatus: vi.fn(), @@ -18,6 +19,7 @@ const mocks = vi.hoisted(() => ({ })); vi.mock("../api/voiceConversation", () => ({ + applyVoiceConversationMicrophoneMuteEvent: mocks.applyMicrophoneMuteEvent, acknowledgeVoiceConversationTranscript: mocks.acknowledge, drainVoiceConversationTranscripts: mocks.drain, getVoiceConversationStatus: mocks.getStatus, @@ -58,6 +60,7 @@ describe("voice conversation store lifecycle ordering", () => { beforeEach(() => { vi.resetModules(); mocks.acknowledge.mockReset().mockResolvedValue(undefined); + mocks.applyMicrophoneMuteEvent.mockReset(); mocks.drain.mockReset().mockResolvedValue([]); mocks.getStatus.mockReset().mockResolvedValue(status("stopped", 0)); mocks.start.mockReset(); @@ -367,6 +370,29 @@ describe("voice conversation store lifecycle ordering", () => { }); }); + it("applies a current stem mute event to capture and UI", async () => { + const store = await loadStore(); + store.setState({ + status: status("running", 2, "session-1"), + uiState: "user-speaking", + userSpeaking: true, + }); + + emit({ + type: "inputMute", + sessionId: "session-1", + muted: true, + revision: 3, + }); + + expect(mocks.applyMicrophoneMuteEvent).toHaveBeenCalledWith(true); + expect(store.getState()).toMatchObject({ + microphoneMuted: true, + userSpeaking: false, + uiState: "listening", + }); + }); + it("ignores stale user-speaking activity while the microphone is muted", async () => { const store = await loadStore(); store.setState({ diff --git a/src/features/voice-conversation/stores/voiceConversationStore.ts b/src/features/voice-conversation/stores/voiceConversationStore.ts index efff65a0c..5663556f0 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.ts @@ -1,6 +1,7 @@ import { create } from "zustand"; import { + applyVoiceConversationMicrophoneMuteEvent, acknowledgeVoiceConversationTranscript, drainVoiceConversationTranscripts, getVoiceConversationStatus, @@ -229,6 +230,10 @@ export const useVoiceConversationStore = create( await listenToVoiceConversation((event) => { if (!shouldApplyEventRevision(get().status, event.revision)) return; + if (event.type === "inputMute") { + applyVoiceConversationMicrophoneMuteEvent(event.muted); + } + set((state) => { switch (event.type) { case "startup": @@ -287,6 +292,18 @@ export const useVoiceConversationStore = create( uiState: activityUiState(nextState), }; } + case "inputMute": { + const nextState = { + ...state, + microphoneMuted: event.muted, + userSpeaking: event.muted ? false : state.userSpeaking, + }; + return { + microphoneMuted: event.muted, + userSpeaking: nextState.userSpeaking, + uiState: activityUiState(nextState), + }; + } case "cleanShutdown": return { ...state, From 8c862702b0a16b2859e16286ff69363bceacc0ba Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 17:11:31 -0400 Subject: [PATCH 18/31] fix(voice): preserve mute lifecycle edges --- src-tauri/src/commands/native_input_mute.rs | 54 +++++-- src-tauri/src/commands/native_voice.rs | 136 +++++++++++++++--- .../api/voiceConversation.ts | 1 + .../stores/voiceConversationStore.test.ts | 9 +- .../stores/voiceConversationStore.ts | 4 + 5 files changed, 173 insertions(+), 31 deletions(-) diff --git a/src-tauri/src/commands/native_input_mute.rs b/src-tauri/src/commands/native_input_mute.rs index 6660c4322..68c69a244 100644 --- a/src-tauri/src/commands/native_input_mute.rs +++ b/src-tauri/src/commands/native_input_mute.rs @@ -1,16 +1,16 @@ use std::sync::{ - atomic::{AtomicBool, Ordering}, + atomic::{AtomicBool, AtomicU64, Ordering}, Arc, }; -pub fn start(input_muted: &Arc, on_change: F) -> bool +pub fn start(input_muted: &Arc, mute_epoch: &Arc, on_change: F) -> bool where F: Fn(bool) + Send + Sync + 'static, { clear(input_muted); #[cfg(target_os = "macos")] - return match macos::install(Arc::clone(input_muted), on_change) { + return match macos::install(Arc::clone(input_muted), Arc::clone(mute_epoch), on_change) { Ok(()) => true, Err(error) => { log::info!("AirPods input mute listener is unavailable: {error}"); @@ -20,7 +20,7 @@ where #[cfg(not(target_os = "macos"))] { - let _ = on_change; + let _ = (mute_epoch, on_change); false } } @@ -34,17 +34,21 @@ pub fn stop(input_muted: &Arc) { } } -pub fn set_muted(input_muted: &AtomicBool, muted: bool) -> Result<(), String> { +pub fn set_muted( + input_muted: &AtomicBool, + mute_epoch: &AtomicU64, + muted: bool, +) -> Result<(), String> { #[cfg(target_os = "macos")] { macos::set_muted(muted)?; - input_muted.store(muted, Ordering::Release); + apply_change(input_muted, mute_epoch, muted, &|_| {}); Ok(()) } #[cfg(not(target_os = "macos"))] { - let _ = (input_muted, muted); + let _ = (input_muted, mute_epoch, muted); Err("native microphone mute is only available on macOS".to_string()) } } @@ -53,6 +57,20 @@ fn clear(input_muted: &AtomicBool) { input_muted.store(false, Ordering::Release); } +fn apply_change( + input_muted: &AtomicBool, + mute_epoch: &AtomicU64, + muted: bool, + on_change: &dyn Fn(bool), +) { + if input_muted.swap(muted, Ordering::AcqRel) != muted { + if muted { + mute_epoch.fetch_add(1, Ordering::AcqRel); + } + on_change(muted); + } +} + #[cfg(target_os = "macos")] mod macos { use super::*; @@ -65,7 +83,11 @@ mod macos { fn berd_airpods_capture_stop(); } - pub fn install(input_muted: Arc, on_change: F) -> Result<(), String> + pub fn install( + input_muted: Arc, + mute_epoch: Arc, + on_change: F, + ) -> Result<(), String> where F: Fn(bool) + Send + Sync + 'static, { @@ -84,10 +106,10 @@ mod macos { let handler = RcBlock::new(move |muted: Bool| { let muted = muted.as_bool(); - if input_muted.swap(muted, Ordering::AcqRel) != muted { + apply_change(&input_muted, &mute_epoch, muted, &|muted| { log::info!("AirPods input mute changed muted={muted}"); on_change(muted); - } + }); Bool::YES }); // SAFETY: The block has the generated AVFAudio signature. The API @@ -133,4 +155,16 @@ mod tests { clear(&input_muted); assert!(!input_muted.load(Ordering::Acquire)); } + + #[test] + fn mute_edge_advances_epoch_even_after_immediate_unmute() { + let input_muted = AtomicBool::new(false); + let mute_epoch = AtomicU64::new(0); + + apply_change(&input_muted, &mute_epoch, true, &|_| {}); + apply_change(&input_muted, &mute_epoch, false, &|_| {}); + + assert!(!input_muted.load(Ordering::Acquire)); + assert_eq!(mute_epoch.load(Ordering::Acquire), 1); + } } diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index fd94fc0df..a220e8a24 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -4,7 +4,7 @@ use std::{ collections::VecDeque, path::PathBuf, sync::{ - atomic::{AtomicBool, AtomicUsize, Ordering}, + atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, mpsc::{self, Receiver, SyncSender, TrySendError}, Arc, Mutex, }, @@ -82,6 +82,7 @@ enum NativeVoiceEvent { owner_window_label: String, line: String, revision: u64, + native_microphone_mute_control: bool, }, User { session_id: String, @@ -134,6 +135,7 @@ pub struct NativeVoiceState { pending: Arc>>, capture_suppressions: Arc, input_muted: Arc, + input_mute_epoch: Arc, } #[must_use = "capture suppression ends when the guard is dropped"] @@ -179,18 +181,25 @@ enum SttMessage { } struct SttPipeline { - audio_tx: SyncSender>, + audio_tx: SyncSender, audio_seen: AtomicBool, shutdown: Arc, discard_on_shutdown: Arc, input_muted: Arc, + input_mute_epoch: Arc, thread: Option>, } +struct AudioBatch { + bytes: Vec, + mute_epoch: u64, +} + impl SttPipeline { fn new( model_dir: PathBuf, input_muted: Arc, + input_mute_epoch: Arc, ) -> Result<(Self, tokio_mpsc::Receiver), String> { let (audio_tx, audio_rx) = mpsc::sync_channel(AUDIO_QUEUE_DEPTH); let (event_tx, event_rx) = tokio_mpsc::channel(64); @@ -199,6 +208,7 @@ impl SttPipeline { let worker_shutdown = Arc::clone(&shutdown); let worker_discard_on_shutdown = Arc::clone(&discard_on_shutdown); let worker_input_muted = Arc::clone(&input_muted); + let worker_input_mute_epoch = Arc::clone(&input_mute_epoch); let thread = thread::Builder::new() .name("berd-native-stt".into()) .spawn(move || { @@ -209,6 +219,7 @@ impl SttPipeline { worker_shutdown, worker_discard_on_shutdown, worker_input_muted, + worker_input_mute_epoch, ) }) .map_err(|error| format!("start native transcription: {error}"))?; @@ -219,6 +230,7 @@ impl SttPipeline { shutdown, discard_on_shutdown, input_muted, + input_mute_epoch, thread: Some(thread), }, event_rx, @@ -238,13 +250,19 @@ impl SttPipeline { if self.input_muted.load(Ordering::Acquire) { return Ok(()); } + let mute_epoch = self.input_mute_epoch.load(Ordering::Acquire); + if self.input_muted.load(Ordering::Acquire) + || mute_epoch != self.input_mute_epoch.load(Ordering::Acquire) + { + return Ok(()); + } if !self.audio_seen.swap(true, Ordering::AcqRel) { log::info!( "Native Parakeet received its first audio batch ({} bytes)", bytes.len() ); } - match self.audio_tx.try_send(bytes) { + match self.audio_tx.try_send(AudioBatch { bytes, mute_epoch }) { Ok(()) => Ok(()), Err(TrySendError::Full(_)) => Err( "Native voice audio overrun: transcription could not keep up with microphone input." @@ -433,7 +451,11 @@ pub async fn start_native_voice_conversation( return Err(error); } }; - let (pipeline, mut events) = match SttPipeline::new(model_dir, Arc::clone(&state.input_muted)) { + let (pipeline, mut events) = match SttPipeline::new( + model_dir, + Arc::clone(&state.input_muted), + Arc::clone(&state.input_mute_epoch), + ) { Ok(result) => result, Err(error) => { if microphone_claimed { @@ -442,7 +464,7 @@ pub async fn start_native_voice_conversation( return Err(error); } }; - let (revision, lifecycle_id) = { + let (revision, lifecycle_id, runtime_mute_control) = { let mut runtime = state .runtime .lock() @@ -464,7 +486,7 @@ pub async fn start_native_voice_conversation( let mute_window = webview_window.clone(); let mute_session_id = session_id.clone(); runtime.native_microphone_mute_control = - native_input_mute::start(&state.input_muted, move |muted| { + native_input_mute::start(&state.input_muted, &state.input_mute_epoch, move |muted| { let _ = mute_window.emit( EVENT_NAME, NativeVoiceEvent::InputMute { @@ -477,6 +499,7 @@ pub async fn start_native_voice_conversation( ( runtime.revision, runtime.lifecycle_id.clone().unwrap_or_default(), + runtime.native_microphone_mute_control, ) }; let _ = webview_window.emit( @@ -486,6 +509,7 @@ pub async fn start_native_voice_conversation( owner_window_label: window_label.clone(), line: "Native Parakeet voice conversation is on".to_string(), revision, + native_microphone_mute_control: runtime_mute_control, }, ); @@ -806,7 +830,7 @@ pub fn set_native_voice_input_muted( if !owns_native_mute_control(&runtime, webview_window.label(), &session_id, revision) { return Err("Native microphone mute is unavailable for this conversation.".to_string()); } - native_input_mute::set_muted(&state.input_muted, muted) + native_input_mute::set_muted(&state.input_muted, &state.input_mute_epoch, muted) } fn owns_native_mute_control( @@ -874,11 +898,12 @@ fn enqueue_pending_transcript( fn stt_worker( model_dir: PathBuf, - audio_rx: Receiver>, + audio_rx: Receiver, event_tx: tokio_mpsc::Sender, shutdown: Arc, discard_on_shutdown: Arc, input_muted: Arc, + input_mute_epoch: Arc, ) { use rubato::{Fft, FixedSync, Resampler}; use sherpa_onnx::{OfflineRecognizer, OfflineRecognizerConfig}; @@ -917,17 +942,20 @@ fn stt_worker( let mut speech = Vec::new(); let mut silence_frames = 0_usize; let mut in_speech = false; + let mut observed_mute_epoch = input_mute_epoch.load(Ordering::Acquire); while !shutdown.load(Ordering::Acquire) { - let bytes = match audio_rx.recv_timeout(Duration::from_millis(50)) { - Ok(bytes) => Some(bytes), + let batch = match audio_rx.recv_timeout(Duration::from_millis(50)) { + Ok(batch) => Some(batch), Err(mpsc::RecvTimeoutError::Timeout) => None, Err(mpsc::RecvTimeoutError::Disconnected) => break, }; let shutting_down = shutdown.load(Ordering::Acquire); - if shutting_down && (discard_on_shutdown.load(Ordering::Acquire) || bytes.is_none()) { + if shutting_down && (discard_on_shutdown.load(Ordering::Acquire) || batch.is_none()) { break; } - if !shutting_down && input_muted.load(Ordering::Acquire) { + let current_mute_epoch = input_mute_epoch.load(Ordering::Acquire); + if current_mute_epoch != observed_mute_epoch { + observed_mute_epoch = current_mute_epoch; if clear_buffered_audio( &mut input_48k, &mut leftover_16k, @@ -937,13 +965,19 @@ fn stt_worker( ) { let _ = event_tx.blocking_send(SttMessage::Speaking(false)); } + } + if !shutting_down && input_muted.load(Ordering::Acquire) { continue; } - let Some(bytes) = bytes else { + let Some(batch) = batch else { continue; }; + if batch.mute_epoch != observed_mute_epoch { + continue; + } input_48k.extend( - bytes + batch + .bytes .chunks_exact(4) .map(|sample| f32::from_le_bytes([sample[0], sample[1], sample[2], sample[3]])), ); @@ -965,7 +999,14 @@ fn stt_worker( silence_frames = 0; speech.extend_from_slice(&frame); if speech.len() >= MAX_SPEECH_SAMPLES { - flush_speech(&speech, &recognizer, &event_tx, None); + flush_speech( + &speech, + &recognizer, + &event_tx, + None, + &input_mute_epoch, + observed_mute_epoch, + ); speech.clear(); in_speech = false; let _ = event_tx.blocking_send(SttMessage::Speaking(false)); @@ -974,7 +1015,14 @@ fn stt_worker( speech.extend_from_slice(&frame); silence_frames += 1; if silence_frames >= SILENCE_FLUSH_FRAMES { - flush_speech(&speech, &recognizer, &event_tx, None); + flush_speech( + &speech, + &recognizer, + &event_tx, + None, + &input_mute_epoch, + observed_mute_epoch, + ); speech.clear(); silence_frames = 0; in_speech = false; @@ -986,7 +1034,14 @@ fn stt_worker( } if !speech.is_empty() && !discard_on_shutdown.load(Ordering::Acquire) { let (delivered_tx, delivered_rx) = mpsc::sync_channel(0); - flush_speech(&speech, &recognizer, &event_tx, Some(delivered_tx)); + flush_speech( + &speech, + &recognizer, + &event_tx, + Some(delivered_tx), + &input_mute_epoch, + observed_mute_epoch, + ); let _ = delivered_rx.recv_timeout(Duration::from_secs(5)); } } @@ -1022,6 +1077,8 @@ fn flush_speech( recognizer: &sherpa_onnx::OfflineRecognizer, event_tx: &tokio_mpsc::Sender, delivered: Option>, + input_mute_epoch: &AtomicU64, + expected_mute_epoch: u64, ) { if speech.is_empty() { return; @@ -1033,6 +1090,12 @@ fn flush_speech( .get_result() .map(|result| result.text.trim().to_string()) .unwrap_or_default(); + if input_mute_epoch.load(Ordering::Acquire) != expected_mute_epoch { + if let Some(delivered) = delivered { + let _ = delivered.send(()); + } + return; + } log::info!( "Native Parakeet finalized {} samples into {} text characters", speech.len(), @@ -1132,6 +1195,7 @@ mod tests { shutdown: Arc::new(AtomicBool::new(false)), discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::new(AtomicBool::new(false)), + input_mute_epoch: Arc::new(AtomicU64::new(0)), audio_seen: AtomicBool::new(false), thread: None, }; @@ -1147,6 +1211,7 @@ mod tests { shutdown: Arc::new(AtomicBool::new(false)), discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::new(AtomicBool::new(false)), + input_mute_epoch: Arc::new(AtomicU64::new(0)), audio_seen: AtomicBool::new(false), thread: None, }; @@ -1162,11 +1227,13 @@ mod tests { fn input_mute_discards_audio_and_unmute_resumes_queueing() { let (sender, receiver) = mpsc::sync_channel(1); let input_muted = Arc::new(AtomicBool::new(true)); + let input_mute_epoch = Arc::new(AtomicU64::new(1)); let pipeline = SttPipeline { audio_tx: sender, shutdown: Arc::new(AtomicBool::new(false)), discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::clone(&input_muted), + input_mute_epoch: Arc::clone(&input_mute_epoch), audio_seen: AtomicBool::new(false), thread: None, }; @@ -1178,7 +1245,31 @@ mod tests { input_muted.store(false, Ordering::Release); pipeline.push(vec![0; 4]).expect("unmuted audio is queued"); - assert_eq!(receiver.try_recv().expect("unmuted audio"), vec![0; 4]); + assert_eq!( + receiver.try_recv().expect("unmuted audio").bytes, + vec![0; 4] + ); + } + + #[test] + fn queued_audio_retains_epoch_across_fast_mute_unmute() { + let (sender, receiver) = mpsc::sync_channel(1); + let input_mute_epoch = Arc::new(AtomicU64::new(0)); + let pipeline = SttPipeline { + audio_tx: sender, + shutdown: Arc::new(AtomicBool::new(false)), + discard_on_shutdown: Arc::new(AtomicBool::new(false)), + input_muted: Arc::new(AtomicBool::new(false)), + input_mute_epoch: Arc::clone(&input_mute_epoch), + audio_seen: AtomicBool::new(false), + thread: None, + }; + + pipeline.push(vec![0; 4]).expect("audio queues before mute"); + input_mute_epoch.fetch_add(1, Ordering::AcqRel); + + let batch = receiver.try_recv().expect("queued audio"); + assert_ne!(batch.mute_epoch, input_mute_epoch.load(Ordering::Acquire)); } #[test] @@ -1213,6 +1304,7 @@ mod tests { shutdown: Arc::new(AtomicBool::new(false)), discard_on_shutdown: Arc::clone(&discard_on_shutdown), input_muted: Arc::clone(&input_muted), + input_mute_epoch: Arc::new(AtomicU64::new(1)), audio_seen: AtomicBool::new(false), thread: None, }; @@ -1233,6 +1325,7 @@ mod tests { shutdown: Arc::new(AtomicBool::new(false)), discard_on_shutdown: Arc::clone(&discard_on_shutdown), input_muted: Arc::clone(&input_muted), + input_mute_epoch: Arc::new(AtomicU64::new(0)), audio_seen: AtomicBool::new(false), thread: None, }; @@ -1257,6 +1350,7 @@ mod tests { shutdown: Arc::new(AtomicBool::new(false)), discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::new(AtomicBool::new(false)), + input_mute_epoch: Arc::new(AtomicU64::new(0)), audio_seen: AtomicBool::new(false), thread: None, }); @@ -1265,7 +1359,10 @@ mod tests { assert!(push_audio_for_window(&state, "other-window", vec![0; 4]).is_err()); assert!(receiver.try_recv().is_err()); push_audio_for_window(&state, "owner-window", vec![0; 4]).expect("owner can send audio"); - assert_eq!(receiver.try_recv().expect("owner audio queued"), vec![0; 4]); + assert_eq!( + receiver.try_recv().expect("owner audio queued").bytes, + vec![0; 4] + ); } #[tokio::test] @@ -1285,6 +1382,7 @@ mod tests { shutdown: Arc::clone(&shutdown), discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::new(AtomicBool::new(false)), + input_mute_epoch: Arc::new(AtomicU64::new(0)), audio_seen: AtomicBool::new(false), thread: Some(worker), }); diff --git a/src/features/voice-conversation/api/voiceConversation.ts b/src/features/voice-conversation/api/voiceConversation.ts index 3f4735a59..92cfccf07 100644 --- a/src/features/voice-conversation/api/voiceConversation.ts +++ b/src/features/voice-conversation/api/voiceConversation.ts @@ -132,6 +132,7 @@ export type VoiceConversationEvent = ownerWindowLabel: string; line: string; revision: number; + nativeMicrophoneMuteControl: boolean; } | { type: "user"; diff --git a/src/features/voice-conversation/stores/voiceConversationStore.test.ts b/src/features/voice-conversation/stores/voiceConversationStore.test.ts index 84798a75e..3519cf4ee 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.test.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.test.ts @@ -256,7 +256,7 @@ describe("voice conversation store lifecycle ordering", () => { await expect(first).resolves.toEqual(status("stopped", 2)); }); - it("does not let a stale start response regress a startup event", async () => { + it("preserves native mute control when startup wins the response race", async () => { const store = await loadStore(); const response = deferred(); mocks.start.mockReturnValue(response.promise); @@ -268,8 +268,12 @@ describe("voice conversation store lifecycle ordering", () => { ownerWindowLabel: "main", line: "type\tid\ttext", revision: 2, + nativeMicrophoneMuteControl: true, + }); + response.resolve({ + ...status("running", 2, "session-1"), + nativeMicrophoneMuteControl: true, }); - response.resolve(status("starting", 1, "session-1")); await starting; expect(store.getState()).toMatchObject({ @@ -277,6 +281,7 @@ describe("voice conversation store lifecycle ordering", () => { uiState: "listening", error: null, }); + expect(store.getState().status.nativeMicrophoneMuteControl).toBe(true); }); it("reconciles status after a failed stop", async () => { diff --git a/src/features/voice-conversation/stores/voiceConversationStore.ts b/src/features/voice-conversation/stores/voiceConversationStore.ts index 5663556f0..68b5f5bfb 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.ts @@ -245,6 +245,8 @@ export const useVoiceConversationStore = create( sessionId: event.sessionId, ownerWindowLabel: event.ownerWindowLabel, revision: event.revision, + nativeMicrophoneMuteControl: + event.nativeMicrophoneMuteControl, }, uiState: "listening", microphoneMuted: false, @@ -313,6 +315,7 @@ export const useVoiceConversationStore = create( sessionId: null, ownerWindowLabel: null, revision: event.revision, + nativeMicrophoneMuteControl: false, }, uiState: "off", userSpeaking: false, @@ -331,6 +334,7 @@ export const useVoiceConversationStore = create( sessionId: null, ownerWindowLabel: null, revision: event.revision, + nativeMicrophoneMuteControl: false, } : { ...state.status, From 84c1a8a8e3568b349c3ca69010532bc4b8c0230d Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 17:47:34 -0400 Subject: [PATCH 19/31] fix(voice): restore AirPods mute opt-in --- src-tauri/src/commands/native_input_mute.rs | 28 +++++++++---------- .../BerdAirPodsBridge/BerdAirPodsBridge.swift | 22 +++++++++++---- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/src-tauri/src/commands/native_input_mute.rs b/src-tauri/src/commands/native_input_mute.rs index 68c69a244..f6ee2f323 100644 --- a/src-tauri/src/commands/native_input_mute.rs +++ b/src-tauri/src/commands/native_input_mute.rs @@ -57,6 +57,7 @@ fn clear(input_muted: &AtomicBool) { input_muted.store(false, Ordering::Release); } +#[cfg(any(target_os = "macos", test))] fn apply_change( input_muted: &AtomicBool, mute_epoch: &AtomicU64, @@ -91,19 +92,9 @@ mod macos { where F: Fn(bool) + Send + Sync + 'static, { - // SAFETY: The Swift bridge owns one process-global AVAudioEngine and - // exposes a C-compatible lifecycle API. - if !unsafe { berd_airpods_capture_start() } { - return Err("macOS microphone capture could not start".to_string()); - } // SAFETY: Berd's minimum macOS version is 14.0, where // AVAudioApplication and these selectors are public API. let application = unsafe { AVAudioApplication::sharedInstance() }; - if let Err(error) = unsafe { application.setInputMuted_error(false) } { - unsafe { berd_airpods_capture_stop() }; - return Err(error.localizedDescription().to_string()); - } - let handler = RcBlock::new(move |muted: Bool| { let muted = muted.as_bool(); apply_change(&input_muted, &mute_epoch, muted, &|muted| { @@ -117,9 +108,18 @@ mod macos { if let Err(error) = unsafe { application.setInputMuteStateChangeHandler_error(Some(&handler)) } { - unsafe { berd_airpods_capture_stop() }; return Err(error.localizedDescription().to_string()); } + if let Err(error) = unsafe { application.setInputMuted_error(false) } { + let _ = unsafe { application.setInputMuteStateChangeHandler_error(None) }; + return Err(error.localizedDescription().to_string()); + } + // SAFETY: The Swift bridge owns one process-global AVAudioEngine and + // exposes a C-compatible lifecycle API. + if !unsafe { berd_airpods_capture_start() } { + let _ = unsafe { application.setInputMuteStateChangeHandler_error(None) }; + return Err("macOS microphone capture could not start".to_string()); + } log::info!("AirPods input mute listener started"); Ok(()) } @@ -128,14 +128,14 @@ mod macos { // SAFETY: Berd targets macOS 14+, and nil is the documented way to // cancel the process-wide handler at the end of a call lifecycle. let application = unsafe { AVAudioApplication::sharedInstance() }; - let handler_result = unsafe { application.setInputMuteStateChangeHandler_error(None) } - .map_err(|error| error.localizedDescription().to_string()); // Do not leave another Berd microphone feature inheriting the voice // conversation's last input-mute state after its handler is gone. let reset_result = unsafe { application.setInputMuted_error(false) } .map_err(|error| error.localizedDescription().to_string()); + let handler_result = unsafe { application.setInputMuteStateChangeHandler_error(None) } + .map_err(|error| error.localizedDescription().to_string()); unsafe { berd_airpods_capture_stop() }; - handler_result.and(reset_result) + reset_result.and(handler_result) } pub fn set_muted(muted: bool) -> Result<(), String> { diff --git a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift index d4475e47f..daee36820 100644 --- a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift +++ b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift @@ -8,7 +8,8 @@ private let bridgeQueue = DispatchQueue(label: "com.berd.airpods-capture") private final class AirPodsCapture: @unchecked Sendable { private var engine: AVAudioEngine? private var inputNode: AVAudioInputNode? - private var observer: NSObjectProtocol? + private var configurationObserver: NSObjectProtocol? + private var inputMuteObserver: NSObjectProtocol? private var restart: DispatchWorkItem? private var generation: UInt64 = 0 private var stopped = false @@ -54,21 +55,32 @@ private final class AirPodsCapture: @unchecked Sendable { removeTap(from: inputNode) throw error } - observer = NotificationCenter.default.addObserver( + configurationObserver = NotificationCenter.default.addObserver( forName: .AVAudioEngineConfigurationChange, object: engine, queue: nil ) { [weak self] _ in self?.scheduleRestart() } + inputMuteObserver = NotificationCenter.default.addObserver( + forName: AVAudioApplication.inputMuteStateChangeNotification, + object: nil, + queue: nil + ) { _ in + _ = AVAudioApplication.shared.isInputMuted + } self.engine = engine self.inputNode = inputNode } private func tearDownEngine() { - if let observer { - NotificationCenter.default.removeObserver(observer) - self.observer = nil + if let configurationObserver { + NotificationCenter.default.removeObserver(configurationObserver) + self.configurationObserver = nil + } + if let inputMuteObserver { + NotificationCenter.default.removeObserver(inputMuteObserver) + self.inputMuteObserver = nil } if let inputNode { removeTap(from: inputNode) } engine?.stop() From 777b19920a6d06a27642789468be85b65ed41325 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 17:49:13 -0400 Subject: [PATCH 20/31] docs(voice): preserve AirPods opt-in invariant --- .../Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift index daee36820..e20d632f2 100644 --- a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift +++ b/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdAirPodsBridge.swift @@ -62,6 +62,8 @@ private final class AirPodsCapture: @unchecked Sendable { ) { [weak self] _ in self?.scheduleRestart() } + // Observing this notification and reading the state opts this capture + // session into AirPods mute delivery. Rust's handler remains the event path. inputMuteObserver = NotificationCenter.default.addObserver( forName: AVAudioApplication.inputMuteStateChangeNotification, object: nil, From e3469165973bd527b2bca10a34c4d0227efff73d Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 18:21:43 -0400 Subject: [PATCH 21/31] fix(build): scope Swift tooling to macOS --- src-tauri/Cargo.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 39addf449..4c57c6e2d 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -18,10 +18,12 @@ members = ["crates/berd-voice", "crates/berdctl", "plugins/berdctl"] exclude = ["plugins/app-test-driver"] [build-dependencies] -swift-rs = { version = "1.0.7", features = ["build"] } tauri-build = { version = "2", features = [] } cc = "1" +[target.'cfg(target_os = "macos")'.build-dependencies] +swift-rs = { version = "1.0.7", features = ["build"] } + [dependencies] anyhow = "1" base64 = "0.22" From 00f68afffaf0154f8ff72bd0c15acae0f7fc79db Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 18:23:46 -0400 Subject: [PATCH 22/31] fix(build): compile-gate Swift linker setup --- src-tauri/build.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src-tauri/build.rs b/src-tauri/build.rs index ab8a5f53f..121ff7745 100644 --- a/src-tauri/build.rs +++ b/src-tauri/build.rs @@ -18,7 +18,6 @@ fn main() { for framework in ["Foundation", "AVFoundation", "AudioToolbox", "CoreAudio"] { println!("cargo:rustc-link-lib=framework={framework}"); } - if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("macos") { swift_rs::SwiftLinker::new("14.0") .with_package("BerdAirPodsBridge", "swift/BerdAirPodsBridge") From 01655811ecbf09423c15f5e67964f45732d4728c Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 19:03:38 -0400 Subject: [PATCH 23/31] fix(voice): serialize microphone mute intent --- .../api/voiceConversation.test.ts | 172 ++++++++++++++++-- .../api/voiceConversation.ts | 54 ++++-- .../hooks/useVoiceConversationController.ts | 6 +- .../stores/voiceConversationStore.test.ts | 119 +++++++++++- .../stores/voiceConversationStore.ts | 30 +++ 5 files changed, 342 insertions(+), 39 deletions(-) diff --git a/src/features/voice-conversation/api/voiceConversation.test.ts b/src/features/voice-conversation/api/voiceConversation.test.ts index 7f2596357..98bbf4919 100644 --- a/src/features/voice-conversation/api/voiceConversation.test.ts +++ b/src/features/voice-conversation/api/voiceConversation.test.ts @@ -23,6 +23,8 @@ vi.mock("../lib/nativeMicrophone", () => ({ import { acknowledgeVoiceConversationTranscript, + applyVoiceConversationMicrophoneMuteEvent, + applyVoiceConversationTerminalEvent, drainVoiceConversationTranscripts, getVoiceConversationStatus, listenToVoiceConversation, @@ -33,6 +35,16 @@ import { stopVoiceConversation, } from "./voiceConversation"; +function deferred() { + let resolve!: (value: T) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((promiseResolve, promiseReject) => { + resolve = promiseResolve; + reject = promiseReject; + }); + return { promise, reject, resolve }; +} + describe("voice conversation API", () => { beforeEach(() => { stopActiveMicrophoneForTest(); @@ -179,12 +191,8 @@ describe("voice conversation API", () => { expect(mocks.startMicrophone).toHaveBeenCalledOnce(); expect(mocks.stopMicrophone).not.toHaveBeenCalled(); - expect(mocks.setMicrophoneMuted.mock.calls).toEqual([ - [false], - [true], - [true], - [false], - ]); + expect(mocks.setMicrophoneMuted).toHaveBeenCalledWith(true); + expect(mocks.setMicrophoneMuted).toHaveBeenLastCalledWith(false); expect(mocks.invoke).not.toHaveBeenCalled(); }); @@ -211,6 +219,143 @@ describe("voice conversation API", () => { expect(mocks.setMicrophoneMuted).toHaveBeenLastCalledWith(true); }); + it("serializes opposite native mute intents so the latest command wins", async () => { + const status = { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + revision: 3, + nativeMicrophoneMuteControl: true, + } as const; + const first = deferred(); + const second = deferred(); + mocks.invoke + .mockReturnValueOnce(first.promise) + .mockReturnValueOnce(second.promise); + + await reconcileVoiceConversationMicrophone(status); + const mute = setVoiceConversationMicrophoneMuted(true, status); + await vi.waitFor(() => expect(mocks.invoke).toHaveBeenCalledOnce()); + expect(mocks.invoke).toHaveBeenLastCalledWith( + "set_native_voice_input_muted", + expect.objectContaining({ muted: true }), + ); + const unmute = setVoiceConversationMicrophoneMuted(false, status); + expect(mocks.setMicrophoneMuted).toHaveBeenLastCalledWith(false); + first.resolve(); + await vi.waitFor(() => expect(mocks.invoke).toHaveBeenCalledTimes(2)); + expect(mocks.setMicrophoneMuted).toHaveBeenLastCalledWith(false); + expect(mocks.invoke).toHaveBeenLastCalledWith( + "set_native_voice_input_muted", + expect.objectContaining({ muted: false }), + ); + second.resolve(); + await Promise.all([mute, unmute]); + + expect(mocks.setMicrophoneMuted).toHaveBeenLastCalledWith(false); + }); + + it("rolls browser capture back to the last applied mute after a failure", async () => { + const status = { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + revision: 3, + nativeMicrophoneMuteControl: true, + } as const; + mocks.invoke + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(new Error("mute unavailable")); + + await reconcileVoiceConversationMicrophone(status); + await setVoiceConversationMicrophoneMuted(true, status); + await expect( + setVoiceConversationMicrophoneMuted(false, status), + ).rejects.toThrow("mute unavailable"); + + expect(mocks.setMicrophoneMuted).toHaveBeenLastCalledWith(true); + }); + + it("does not let a stale UI completion replace a newer stem state", async () => { + const status = { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + revision: 3, + nativeMicrophoneMuteControl: true, + } as const; + const pending = deferred(); + mocks.invoke + .mockReturnValueOnce(pending.promise) + .mockRejectedValueOnce(new Error("mute unavailable")); + + await reconcileVoiceConversationMicrophone(status); + const staleMute = setVoiceConversationMicrophoneMuted(true, status); + await vi.waitFor(() => expect(mocks.invoke).toHaveBeenCalledOnce()); + applyVoiceConversationMicrophoneMuteEvent(false); + pending.resolve(); + await staleMute; + + await expect( + setVoiceConversationMicrophoneMuted(true, status), + ).rejects.toThrow("mute unavailable"); + expect(mocks.setMicrophoneMuted).toHaveBeenLastCalledWith(false); + }); + + it("does not run queued mute work after the conversation stops", async () => { + const running = { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + revision: 3, + nativeMicrophoneMuteControl: true, + } as const; + const stopped = { + ...running, + lifecycle: "stopped" as const, + sessionId: null, + ownerWindowLabel: null, + revision: 4, + }; + const pending = deferred(); + mocks.invoke.mockImplementation((command: string) => { + if (command === "set_native_voice_input_muted") return pending.promise; + if (command === "stop_native_voice_conversation") { + return Promise.resolve(stopped); + } + return Promise.resolve(undefined); + }); + + await reconcileVoiceConversationMicrophone(running); + const muting = setVoiceConversationMicrophoneMuted(true, running); + await vi.waitFor(() => + expect(mocks.invoke).toHaveBeenCalledWith( + "set_native_voice_input_muted", + expect.any(Object), + ), + ); + const queuedUnmute = setVoiceConversationMicrophoneMuted(false, running); + await stopVoiceConversation(); + pending.resolve(); + await Promise.all([muting, queuedUnmute]); + + expect( + mocks.invoke.mock.calls.filter( + ([command]) => command === "set_native_voice_input_muted", + ), + ).toHaveLength(1); + expect(mocks.startMicrophone).toHaveBeenCalledOnce(); + expect(mocks.stopMicrophone).toHaveBeenCalledOnce(); + }); + it("restores the previous mute state when initial capture fails", async () => { const status = { available: true, @@ -289,7 +434,7 @@ describe("voice conversation API", () => { }); }); - it("stops browser capture when native voice shuts down elsewhere", async () => { + it("stops browser capture for a validated terminal event", async () => { mocks.invoke.mockResolvedValue({ available: true, unavailableReason: null, @@ -299,18 +444,7 @@ describe("voice conversation API", () => { revision: 3, }); await startVoiceConversation("session-1"); - mocks.listen.mockImplementation(async (_name, handler) => { - handler({ - payload: { - type: "cleanShutdown", - sessionId: "session-1", - revision: 4, - }, - }); - return vi.fn(); - }); - - await listenToVoiceConversation(vi.fn()); + applyVoiceConversationTerminalEvent(); expect(mocks.stopMicrophone).toHaveBeenCalledOnce(); }); diff --git a/src/features/voice-conversation/api/voiceConversation.ts b/src/features/voice-conversation/api/voiceConversation.ts index 92cfccf07..e4071c2a6 100644 --- a/src/features/voice-conversation/api/voiceConversation.ts +++ b/src/features/voice-conversation/api/voiceConversation.ts @@ -12,6 +12,15 @@ let microphoneGeneration = 0; let microphoneStart: { generation: number; promise: Promise } | null = null; let microphoneMuted = false; +let appliedMicrophoneMuted = false; +let microphoneMuteIntent = 0; +let microphoneMuteQueue: Promise = Promise.resolve(); + +function resetMicrophoneMuteState(): void { + microphoneMuteIntent += 1; + microphoneMuted = false; + appliedMicrophoneMuted = false; +} function stopActiveMicrophone(): void { microphoneGeneration += 1; @@ -63,10 +72,14 @@ export async function setVoiceConversationMicrophoneMuted( muted: boolean, status: VoiceConversationStatus, ): Promise { - const previous = microphoneMuted; + const intent = ++microphoneMuteIntent; microphoneMuted = muted; - try { + activeMicrophone?.setMuted(muted); + const operation = microphoneMuteQueue.then(async () => { + if (intent !== microphoneMuteIntent) return; await reconcileVoiceConversationMicrophone(status); + if (intent !== microphoneMuteIntent) return; + activeMicrophone?.setMuted(microphoneMuted); if (status.nativeMicrophoneMuteControl) { await invoke("set_native_voice_input_muted", { sessionId: status.sessionId, @@ -74,23 +87,43 @@ export async function setVoiceConversationMicrophoneMuted( muted, }); } + if (intent === microphoneMuteIntent) { + appliedMicrophoneMuted = muted; + } + }); + microphoneMuteQueue = operation.catch(() => undefined); + try { + await operation; } catch (error) { - microphoneMuted = previous; - activeMicrophone?.setMuted(previous); + if (intent === microphoneMuteIntent) { + microphoneMuted = appliedMicrophoneMuted; + activeMicrophone?.setMuted(appliedMicrophoneMuted); + } throw error; } } export function applyVoiceConversationMicrophoneMuteEvent(muted: boolean) { + microphoneMuteIntent += 1; microphoneMuted = muted; + appliedMicrophoneMuted = muted; activeMicrophone?.setMuted(muted); } +export function getVoiceConversationMicrophoneMuted(): boolean { + return microphoneMuted; +} + +export function applyVoiceConversationTerminalEvent(): void { + resetMicrophoneMuteState(); + stopActiveMicrophone(); +} + export function stopActiveMicrophoneForTest(): void { if (!import.meta.env.DEV) { throw new Error("Native microphone test controls are development-only."); } - microphoneMuted = false; + resetMicrophoneMuteState(); stopActiveMicrophone(); } @@ -229,7 +262,7 @@ export function rejectVoiceConversationTranscript( export async function startVoiceConversation( sessionId: string, ): Promise { - microphoneMuted = false; + resetMicrophoneMuteState(); const { rendererId, rendererEpoch } = await getRendererInstance(); const status = await invoke( "start_native_voice_conversation", @@ -252,7 +285,7 @@ export async function startVoiceConversation( } export async function stopVoiceConversation(): Promise { - microphoneMuted = false; + resetMicrophoneMuteState(); stopActiveMicrophone(); const { rendererId, rendererEpoch } = await getRendererInstance(); return invoke("stop_native_voice_conversation", { @@ -265,13 +298,6 @@ export function listenToVoiceConversation( onEvent: (event: VoiceConversationEvent) => void, ): Promise { return listen(VOICE_CONVERSATION_EVENT, (event) => { - if ( - event.payload.type === "cleanShutdown" || - (event.payload.type === "error" && event.payload.terminal) - ) { - microphoneMuted = false; - stopActiveMicrophone(); - } onEvent(event.payload); }); } diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index 4d94878a6..2d33de276 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -751,11 +751,13 @@ export function useVoiceConversationController({ const toggleMicrophoneMute = useCallback(async () => { if (status.lifecycle !== "running") return; try { - await setMicrophoneMuted(!microphoneMuted); + await setMicrophoneMuted( + !useVoiceConversationStore.getState().microphoneMuted, + ); } catch (muteError) { addErrorNotification(status.sessionId, errorText(muteError)); } - }, [microphoneMuted, setMicrophoneMuted, status.lifecycle, status.sessionId]); + }, [setMicrophoneMuted, status.lifecycle, status.sessionId]); return useMemo( () => ({ diff --git a/src/features/voice-conversation/stores/voiceConversationStore.test.ts b/src/features/voice-conversation/stores/voiceConversationStore.test.ts index 3519cf4ee..53330d1d2 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.test.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.test.ts @@ -7,8 +7,10 @@ import type { const mocks = vi.hoisted(() => ({ applyMicrophoneMuteEvent: vi.fn(), + applyTerminalEvent: vi.fn(), acknowledge: vi.fn(), drain: vi.fn(), + getMicrophoneMuted: vi.fn(), getStatus: vi.fn(), listen: vi.fn(), reconcileMicrophone: vi.fn(), @@ -20,8 +22,10 @@ const mocks = vi.hoisted(() => ({ vi.mock("../api/voiceConversation", () => ({ applyVoiceConversationMicrophoneMuteEvent: mocks.applyMicrophoneMuteEvent, + applyVoiceConversationTerminalEvent: mocks.applyTerminalEvent, acknowledgeVoiceConversationTranscript: mocks.acknowledge, drainVoiceConversationTranscripts: mocks.drain, + getVoiceConversationMicrophoneMuted: mocks.getMicrophoneMuted, getVoiceConversationStatus: mocks.getStatus, listenToVoiceConversation: mocks.listen, reconcileVoiceConversationMicrophone: mocks.reconcileMicrophone, @@ -48,10 +52,12 @@ function status( function deferred() { let resolve!: (value: T) => void; - const promise = new Promise((resolver) => { - resolve = resolver; + let reject!: (reason?: unknown) => void; + const promise = new Promise((promiseResolve, promiseReject) => { + resolve = promiseResolve; + reject = promiseReject; }); - return { promise, resolve }; + return { promise, reject, resolve }; } describe("voice conversation store lifecycle ordering", () => { @@ -61,7 +67,9 @@ describe("voice conversation store lifecycle ordering", () => { vi.resetModules(); mocks.acknowledge.mockReset().mockResolvedValue(undefined); mocks.applyMicrophoneMuteEvent.mockReset(); + mocks.applyTerminalEvent.mockReset(); mocks.drain.mockReset().mockResolvedValue([]); + mocks.getMicrophoneMuted.mockReset().mockReturnValue(false); mocks.getStatus.mockReset().mockResolvedValue(status("stopped", 0)); mocks.start.mockReset(); mocks.stop.mockReset(); @@ -220,6 +228,23 @@ describe("voice conversation store lifecycle ordering", () => { uiState: "off", error: null, }); + expect(mocks.applyTerminalEvent).toHaveBeenCalledOnce(); + }); + + it("does not tear down capture for a stale terminal event", async () => { + const store = await loadStore(); + store.setState({ + status: status("running", 5, "session-2"), + uiState: "listening", + }); + + emit({ type: "cleanShutdown", sessionId: "session-1", revision: 4 }); + + expect(mocks.applyTerminalEvent).not.toHaveBeenCalled(); + expect(store.getState()).toMatchObject({ + status: status("running", 5, "session-2"), + uiState: "listening", + }); }); it("returns to off after a no-op stop with an unchanged revision", async () => { @@ -375,6 +400,86 @@ describe("voice conversation store lifecycle ordering", () => { }); }); + it("keeps the latest UI mute intent when requests settle out of order", async () => { + const store = await loadStore(); + store.setState({ + status: status("running", 2, "session-1"), + uiState: "listening", + }); + const mute = deferred(); + const unmute = deferred(); + mocks.setMicrophoneMuted + .mockReturnValueOnce(mute.promise) + .mockReturnValueOnce(unmute.promise); + + const muting = store.getState().setMicrophoneMuted(true); + const unmuting = store.getState().setMicrophoneMuted(false); + expect(store.getState().microphoneMuted).toBe(false); + + unmute.resolve(); + await unmuting; + mute.resolve(); + await muting; + + expect(store.getState().microphoneMuted).toBe(false); + }); + + it("rolls the latest failed intent back to the last successful mute", async () => { + const store = await loadStore(); + store.setState({ + status: status("running", 2, "session-1"), + uiState: "listening", + }); + const mute = deferred(); + const unmute = deferred(); + mocks.setMicrophoneMuted + .mockReturnValueOnce(mute.promise) + .mockReturnValueOnce(unmute.promise); + + const muting = store.getState().setMicrophoneMuted(true); + const unmuting = store.getState().setMicrophoneMuted(false); + mute.resolve(); + await muting; + mocks.getMicrophoneMuted.mockReturnValue(true); + unmute.reject(new Error("mute unavailable")); + await expect(unmuting).rejects.toThrow("mute unavailable"); + + expect(store.getState()).toMatchObject({ + microphoneMuted: true, + uiState: "error", + error: "mute unavailable", + }); + }); + + it("keeps a newer stem state after a stale UI request settles", async () => { + const store = await loadStore(); + store.setState({ + status: status("running", 2, "session-1"), + uiState: "listening", + }); + const pending = deferred(); + mocks.setMicrophoneMuted.mockReturnValueOnce(pending.promise); + + const staleMute = store.getState().setMicrophoneMuted(true); + emit({ + type: "inputMute", + sessionId: "session-1", + muted: false, + revision: 3, + }); + pending.resolve(); + await staleMute; + mocks.getMicrophoneMuted.mockReturnValue(false); + mocks.setMicrophoneMuted.mockRejectedValueOnce( + new Error("mute unavailable"), + ); + + await expect(store.getState().setMicrophoneMuted(true)).rejects.toThrow( + "mute unavailable", + ); + expect(store.getState().microphoneMuted).toBe(false); + }); + it("applies a current stem mute event to capture and UI", async () => { const store = await loadStore(); store.setState({ @@ -425,8 +530,14 @@ describe("voice conversation store lifecycle ordering", () => { store.setState({ status: status("running", 2, "session-1"), uiState: "listening", - microphoneMuted: true, }); + emit({ + type: "inputMute", + sessionId: "session-1", + muted: true, + revision: 3, + }); + mocks.getMicrophoneMuted.mockReturnValue(true); mocks.setMicrophoneMuted.mockRejectedValueOnce( new Error("microphone unavailable"), ); diff --git a/src/features/voice-conversation/stores/voiceConversationStore.ts b/src/features/voice-conversation/stores/voiceConversationStore.ts index 68b5f5bfb..b80cbfb3d 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.ts @@ -1,9 +1,11 @@ import { create } from "zustand"; import { + applyVoiceConversationTerminalEvent, applyVoiceConversationMicrophoneMuteEvent, acknowledgeVoiceConversationTranscript, drainVoiceConversationTranscripts, + getVoiceConversationMicrophoneMuted, getVoiceConversationStatus, listenToVoiceConversation, reconcileVoiceConversationMicrophone, @@ -57,6 +59,7 @@ interface VoiceConversationStore { let initialized = false; let stopInFlight: Promise | null = null; +let microphoneMuteIntent = 0; const eventSubscribers = new Set< (event: VoiceConversationEvent) => void | Promise >(); @@ -231,7 +234,16 @@ export const useVoiceConversationStore = create( if (!shouldApplyEventRevision(get().status, event.revision)) return; if (event.type === "inputMute") { + microphoneMuteIntent += 1; applyVoiceConversationMicrophoneMuteEvent(event.muted); + } else if (event.type === "startup") { + microphoneMuteIntent += 1; + } else if ( + event.type === "cleanShutdown" || + (event.type === "error" && event.terminal) + ) { + microphoneMuteIntent += 1; + applyVoiceConversationTerminalEvent(); } set((state) => { @@ -405,6 +417,7 @@ export const useVoiceConversationStore = create( }, start: async (sessionId) => { + microphoneMuteIntent += 1; set({ uiState: "starting", microphoneMuted: false, error: null }); try { const status = await startVoiceConversation(sessionId); @@ -439,6 +452,7 @@ export const useVoiceConversationStore = create( stop: () => { if (stopInFlight) return stopInFlight; + microphoneMuteIntent += 1; set({ uiState: "stopping", microphoneMuted: false, @@ -514,18 +528,34 @@ export const useVoiceConversationStore = create( setMicrophoneMuted: async (microphoneMuted) => { const current = get(); if (current.status.lifecycle !== "running") return; + const intent = ++microphoneMuteIntent; + set((state) => { + const nextState = { + ...state, + microphoneMuted, + userSpeaking: microphoneMuted ? false : state.userSpeaking, + }; + return { + microphoneMuted, + userSpeaking: nextState.userSpeaking, + uiState: activityUiState(nextState), + }; + }); try { await setVoiceConversationMicrophoneMuted( microphoneMuted, current.status, ); } catch (error) { + if (intent !== microphoneMuteIntent) return; set({ + microphoneMuted: getVoiceConversationMicrophoneMuted(), uiState: "error", error: error instanceof Error ? error.message : String(error), }); throw error; } + if (intent !== microphoneMuteIntent) return; set((state) => { if ( state.status.lifecycle !== "running" || From e0743d8f0c2b30e28df783796de936e41bbcadad Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 19:04:21 -0400 Subject: [PATCH 24/31] refactor(voice): remove diagnostic mute toast --- .../hooks/useVoiceConversationController.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index 2d33de276..cec0a633d 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -1,5 +1,4 @@ import { useCallback, useEffect, useMemo, useRef } from "react"; -import { toast } from "sonner"; import type { ChatInputSendHandler, @@ -316,12 +315,7 @@ function ensureVoiceEventDeliveryInitialized() { return; } if (event.type === "activity") return; - if (event.type === "inputMute") { - toast.message(`Microphone ${event.muted ? "muted" : "unmuted"}`, { - id: "voice-input-mute", - }); - return; - } + if (event.type === "inputMute") return; if (event.type !== "user" || !event.text.trim()) return; if ( hasDeliveredVoiceTranscript( From 557ba7d5de833bb262d976d04475f97e7a15237d Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 19:06:14 -0400 Subject: [PATCH 25/31] test(voice): restore stale startup coverage --- .../stores/voiceConversationStore.test.ts | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/features/voice-conversation/stores/voiceConversationStore.test.ts b/src/features/voice-conversation/stores/voiceConversationStore.test.ts index 53330d1d2..c0c393381 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.test.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.test.ts @@ -309,6 +309,31 @@ describe("voice conversation store lifecycle ordering", () => { expect(store.getState().status.nativeMicrophoneMuteControl).toBe(true); }); + it("ignores an older start response after a newer startup event", async () => { + const store = await loadStore(); + const response = deferred(); + mocks.start.mockReturnValue(response.promise); + + const starting = store.getState().start("session-1"); + emit({ + type: "startup", + sessionId: "session-1", + ownerWindowLabel: "main", + line: "type\tid\ttext", + revision: 2, + nativeMicrophoneMuteControl: true, + }); + response.resolve(status("starting", 1, "session-1")); + await starting; + + expect(store.getState()).toMatchObject({ + status: status("running", 2, "session-1"), + uiState: "listening", + error: null, + }); + expect(store.getState().status.nativeMicrophoneMuteControl).toBe(true); + }); + it("reconciles status after a failed stop", async () => { const store = await loadStore(); store.setState({ From 7f59ede1793fd435758e415736865c82a05c26de Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 19:25:40 -0400 Subject: [PATCH 26/31] fix(voice): preserve early stem mute state --- .../stores/voiceConversationStore.test.ts | 26 +++++++++++++++++++ .../stores/voiceConversationStore.ts | 1 - 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/features/voice-conversation/stores/voiceConversationStore.test.ts b/src/features/voice-conversation/stores/voiceConversationStore.test.ts index c0c393381..8038cff28 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.test.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.test.ts @@ -309,6 +309,32 @@ describe("voice conversation store lifecycle ordering", () => { expect(store.getState().status.nativeMicrophoneMuteControl).toBe(true); }); + it("preserves a stem mute observed immediately before startup", async () => { + const store = await loadStore(); + + emit({ + type: "inputMute", + sessionId: "session-1", + muted: true, + revision: 2, + }); + emit({ + type: "startup", + sessionId: "session-1", + ownerWindowLabel: "main", + line: "type\tid\ttext", + revision: 2, + nativeMicrophoneMuteControl: true, + }); + + expect(mocks.applyMicrophoneMuteEvent).toHaveBeenCalledWith(true); + expect(store.getState()).toMatchObject({ + status: status("running", 2, "session-1"), + uiState: "listening", + microphoneMuted: true, + }); + }); + it("ignores an older start response after a newer startup event", async () => { const store = await loadStore(); const response = deferred(); diff --git a/src/features/voice-conversation/stores/voiceConversationStore.ts b/src/features/voice-conversation/stores/voiceConversationStore.ts index b80cbfb3d..ba5f0d3d9 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.ts @@ -261,7 +261,6 @@ export const useVoiceConversationStore = create( event.nativeMicrophoneMuteControl, }, uiState: "listening", - microphoneMuted: false, error: null, }; case "user": From fbc6485c6232d24fdcc637cd8d1351d8e5f3e9d6 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 19:38:08 -0400 Subject: [PATCH 27/31] fix(voice): hydrate native mute state --- src-tauri/src/commands/native_voice.rs | 3 ++ .../api/voiceConversation.test.ts | 21 ++++++++ .../api/voiceConversation.ts | 13 +++++ .../stores/voiceConversationStore.test.ts | 49 +++++++++++++++++-- .../stores/voiceConversationStore.ts | 39 +++++++++++++-- 5 files changed, 118 insertions(+), 7 deletions(-) diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index a220e8a24..ffaa4a376 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -50,6 +50,7 @@ pub struct NativeVoiceStatus { owner_window_label: Option, revision: u64, native_microphone_mute_control: bool, + native_microphone_muted: bool, } #[derive(Clone, Debug, Serialize)] @@ -334,6 +335,8 @@ fn status(app: &AppHandle, state: &NativeVoiceState) -> NativeVoiceStatus { .map(|owner| owner.window_label.clone()), revision: runtime.revision, native_microphone_mute_control: runtime.native_microphone_mute_control, + native_microphone_muted: runtime.session_id.is_some() + && state.input_muted.load(Ordering::Acquire), } } diff --git a/src/features/voice-conversation/api/voiceConversation.test.ts b/src/features/voice-conversation/api/voiceConversation.test.ts index 98bbf4919..15c4fac47 100644 --- a/src/features/voice-conversation/api/voiceConversation.test.ts +++ b/src/features/voice-conversation/api/voiceConversation.test.ts @@ -26,7 +26,9 @@ import { applyVoiceConversationMicrophoneMuteEvent, applyVoiceConversationTerminalEvent, drainVoiceConversationTranscripts, + getVoiceConversationMicrophoneMuted, getVoiceConversationStatus, + hydrateVoiceConversationMicrophone, listenToVoiceConversation, reconcileVoiceConversationMicrophone, setVoiceConversationMicrophoneMuted, @@ -159,6 +161,25 @@ describe("voice conversation API", () => { expect(mocks.stopMicrophone).toHaveBeenCalledOnce(); }); + it("hydrates browser capture from the authoritative native mute state", async () => { + const status = { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + revision: 3, + nativeMicrophoneMuteControl: true, + nativeMicrophoneMuted: true, + } as const; + + await hydrateVoiceConversationMicrophone(status); + + expect(getVoiceConversationMicrophoneMuted()).toBe(true); + expect(mocks.startMicrophone).toHaveBeenCalledOnce(); + expect(mocks.setMicrophoneMuted).toHaveBeenLastCalledWith(true); + }); + it("does not attach browser capture in a non-owning window", async () => { const status = { available: true, diff --git a/src/features/voice-conversation/api/voiceConversation.ts b/src/features/voice-conversation/api/voiceConversation.ts index e4071c2a6..f8ca73e9e 100644 --- a/src/features/voice-conversation/api/voiceConversation.ts +++ b/src/features/voice-conversation/api/voiceConversation.ts @@ -68,6 +68,17 @@ export async function reconcileVoiceConversationMicrophone( } } +export async function hydrateVoiceConversationMicrophone( + status: VoiceConversationStatus, +): Promise { + if (status.lifecycle === "running") { + applyVoiceConversationMicrophoneMuteEvent( + status.nativeMicrophoneMuted ?? false, + ); + } + await reconcileVoiceConversationMicrophone(status); +} + export async function setVoiceConversationMicrophoneMuted( muted: boolean, status: VoiceConversationStatus, @@ -156,6 +167,8 @@ export interface VoiceConversationStatus { revision: number; /** macOS owns an input session capable of receiving headset mute controls. */ nativeMicrophoneMuteControl?: boolean; + /** Authoritative native input-mute state for renderer recovery. */ + nativeMicrophoneMuted?: boolean; } export type VoiceConversationEvent = diff --git a/src/features/voice-conversation/stores/voiceConversationStore.test.ts b/src/features/voice-conversation/stores/voiceConversationStore.test.ts index 8038cff28..186e62452 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.test.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.test.ts @@ -12,8 +12,8 @@ const mocks = vi.hoisted(() => ({ drain: vi.fn(), getMicrophoneMuted: vi.fn(), getStatus: vi.fn(), + hydrateMicrophone: vi.fn(), listen: vi.fn(), - reconcileMicrophone: vi.fn(), reject: vi.fn(), setMicrophoneMuted: vi.fn(), start: vi.fn(), @@ -27,8 +27,8 @@ vi.mock("../api/voiceConversation", () => ({ drainVoiceConversationTranscripts: mocks.drain, getVoiceConversationMicrophoneMuted: mocks.getMicrophoneMuted, getVoiceConversationStatus: mocks.getStatus, + hydrateVoiceConversationMicrophone: mocks.hydrateMicrophone, listenToVoiceConversation: mocks.listen, - reconcileVoiceConversationMicrophone: mocks.reconcileMicrophone, rejectVoiceConversationTranscript: mocks.reject, setVoiceConversationMicrophoneMuted: mocks.setMicrophoneMuted, startVoiceConversation: mocks.start, @@ -77,7 +77,7 @@ describe("voice conversation store lifecycle ordering", () => { emit = callback; return vi.fn(); }); - mocks.reconcileMicrophone.mockReset().mockResolvedValue(undefined); + mocks.hydrateMicrophone.mockReset().mockResolvedValue(undefined); mocks.reject .mockReset() .mockResolvedValue({ attempts: 1, terminal: false }); @@ -133,7 +133,48 @@ describe("voice conversation store lifecycle ordering", () => { await loadStore(); - expect(mocks.reconcileMicrophone).toHaveBeenCalledWith(running); + expect(mocks.hydrateMicrophone).toHaveBeenCalledWith(running); + }); + + it("hydrates an already-muted native lifecycle", async () => { + const running = { + ...status("running", 2, "session-1"), + nativeMicrophoneMuteControl: true, + nativeMicrophoneMuted: true, + }; + mocks.getStatus.mockResolvedValue(running); + + const store = await loadStore(); + + expect(mocks.hydrateMicrophone).toHaveBeenCalledWith(running); + expect(store.getState().microphoneMuted).toBe(true); + }); + + it("does not let stale status overwrite a newer mute event", async () => { + const response = deferred(); + mocks.getStatus.mockReturnValue(response.promise); + const { useVoiceConversationStore } = await import( + "./voiceConversationStore" + ); + + const initializing = useVoiceConversationStore.getState().init(); + await vi.waitFor(() => expect(mocks.getStatus).toHaveBeenCalledOnce()); + emit({ + type: "inputMute", + sessionId: "session-1", + muted: false, + revision: 3, + }); + response.resolve({ + ...status("running", 2, "session-1"), + nativeMicrophoneMuteControl: true, + nativeMicrophoneMuted: true, + }); + await initializing; + + expect(mocks.hydrateMicrophone).not.toHaveBeenCalled(); + expect(useVoiceConversationStore.getState().microphoneMuted).toBe(false); + expect(useVoiceConversationStore.getState().status.revision).toBe(3); }); it("refreshes availability when installation changes without a lifecycle revision", async () => { diff --git a/src/features/voice-conversation/stores/voiceConversationStore.ts b/src/features/voice-conversation/stores/voiceConversationStore.ts index ba5f0d3d9..9ec2abef9 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.ts @@ -7,8 +7,8 @@ import { drainVoiceConversationTranscripts, getVoiceConversationMicrophoneMuted, getVoiceConversationStatus, + hydrateVoiceConversationMicrophone, listenToVoiceConversation, - reconcileVoiceConversationMicrophone, rejectVoiceConversationTranscript, setVoiceConversationMicrophoneMuted, startVoiceConversation, @@ -185,8 +185,14 @@ export const useVoiceConversationStore = create( init: async () => { if (initialized) { try { + const muteIntent = microphoneMuteIntent; const status = await getVoiceConversationStatus(); - await reconcileVoiceConversationMicrophone(status); + const shouldHydrate = + shouldApplyResponseRevision(get().status, status.revision) && + muteIntent === microphoneMuteIntent; + if (shouldHydrate) { + await hydrateVoiceConversationMicrophone(status); + } set((state) => { if ( shouldApplyResponseRevision(state.status, status.revision) || @@ -200,6 +206,12 @@ export const useVoiceConversationStore = create( state.uiState === "error" ? state.uiState : uiStateForStatus(status), + microphoneMuted: + muteIntent === microphoneMuteIntent + ? status.lifecycle === "running" + ? (status.nativeMicrophoneMuted ?? false) + : false + : state.microphoneMuted, hydrated: true, }; } @@ -308,10 +320,16 @@ export const useVoiceConversationStore = create( case "inputMute": { const nextState = { ...state, + status: { + ...state.status, + sessionId: event.sessionId, + revision: event.revision, + }, microphoneMuted: event.muted, userSpeaking: event.muted ? false : state.userSpeaking, }; return { + status: nextState.status, microphoneMuted: event.muted, userSpeaking: nextState.userSpeaking, uiState: activityUiState(nextState), @@ -389,8 +407,17 @@ export const useVoiceConversationStore = create( } try { + const muteIntent = microphoneMuteIntent; const status = await getVoiceConversationStatus(); - await reconcileVoiceConversationMicrophone(status); + const shouldHydrate = + (shouldApplyResponseRevision(get().status, status.revision) || + (!get().hydrated && + get().status.revision === 0 && + get().uiState === "off")) && + muteIntent === microphoneMuteIntent; + if (shouldHydrate) { + await hydrateVoiceConversationMicrophone(status); + } set((state) => shouldApplyResponseRevision(state.status, status.revision) || (!state.hydrated && @@ -402,6 +429,12 @@ export const useVoiceConversationStore = create( state.uiState === "error" ? state.uiState : uiStateForStatus(status), + microphoneMuted: + muteIntent === microphoneMuteIntent + ? status.lifecycle === "running" + ? (status.nativeMicrophoneMuted ?? false) + : false + : state.microphoneMuted, hydrated: true, } : { hydrated: true }, From 618918904137eb45202068bd101b17776924bf19 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 19:45:30 -0400 Subject: [PATCH 28/31] fix(voice): reconcile equal-revision recovery --- .../stores/voiceConversationStore.test.ts | 53 +++++++ .../stores/voiceConversationStore.ts | 135 +++++++++++++----- 2 files changed, 150 insertions(+), 38 deletions(-) diff --git a/src/features/voice-conversation/stores/voiceConversationStore.test.ts b/src/features/voice-conversation/stores/voiceConversationStore.test.ts index 186e62452..eaeb02ebf 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.test.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.test.ts @@ -14,6 +14,7 @@ const mocks = vi.hoisted(() => ({ getStatus: vi.fn(), hydrateMicrophone: vi.fn(), listen: vi.fn(), + reconcileMicrophone: vi.fn(), reject: vi.fn(), setMicrophoneMuted: vi.fn(), start: vi.fn(), @@ -29,6 +30,7 @@ vi.mock("../api/voiceConversation", () => ({ getVoiceConversationStatus: mocks.getStatus, hydrateVoiceConversationMicrophone: mocks.hydrateMicrophone, listenToVoiceConversation: mocks.listen, + reconcileVoiceConversationMicrophone: mocks.reconcileMicrophone, rejectVoiceConversationTranscript: mocks.reject, setVoiceConversationMicrophoneMuted: mocks.setMicrophoneMuted, startVoiceConversation: mocks.start, @@ -78,6 +80,7 @@ describe("voice conversation store lifecycle ordering", () => { return vi.fn(); }); mocks.hydrateMicrophone.mockReset().mockResolvedValue(undefined); + mocks.reconcileMicrophone.mockReset().mockResolvedValue(undefined); mocks.reject .mockReset() .mockResolvedValue({ attempts: 1, terminal: false }); @@ -150,6 +153,50 @@ describe("voice conversation store lifecycle ordering", () => { expect(store.getState().microphoneMuted).toBe(true); }); + it("refreshes mute state for the same running lifecycle", async () => { + const unmuted = { + ...status("running", 2, "session-1"), + nativeMicrophoneMuteControl: true, + nativeMicrophoneMuted: false, + }; + const muted = { ...unmuted, nativeMicrophoneMuted: true }; + mocks.getStatus.mockResolvedValueOnce(unmuted).mockResolvedValueOnce(muted); + const store = await loadStore(); + + await store.getState().init(); + + expect(mocks.hydrateMicrophone).toHaveBeenLastCalledWith(muted); + expect(store.getState().microphoneMuted).toBe(true); + }); + + it("hydrates mute state when startup arrives before status", async () => { + const response = deferred(); + mocks.getStatus.mockReturnValue(response.promise); + const { useVoiceConversationStore } = await import( + "./voiceConversationStore" + ); + const initializing = useVoiceConversationStore.getState().init(); + await vi.waitFor(() => expect(mocks.getStatus).toHaveBeenCalledOnce()); + emit({ + type: "startup", + sessionId: "session-1", + ownerWindowLabel: "main", + line: "type\tid\ttext", + revision: 2, + nativeMicrophoneMuteControl: true, + }); + const muted = { + ...status("running", 2, "session-1"), + nativeMicrophoneMuteControl: true, + nativeMicrophoneMuted: true, + }; + response.resolve(muted); + await initializing; + + expect(mocks.hydrateMicrophone).toHaveBeenCalledWith(muted); + expect(useVoiceConversationStore.getState().microphoneMuted).toBe(true); + }); + it("does not let stale status overwrite a newer mute event", async () => { const response = deferred(); mocks.getStatus.mockReturnValue(response.promise); @@ -173,6 +220,12 @@ describe("voice conversation store lifecycle ordering", () => { await initializing; expect(mocks.hydrateMicrophone).not.toHaveBeenCalled(); + expect(mocks.reconcileMicrophone).toHaveBeenCalledWith( + expect.objectContaining({ + lifecycle: "running", + sessionId: "session-1", + }), + ); expect(useVoiceConversationStore.getState().microphoneMuted).toBe(false); expect(useVoiceConversationStore.getState().status.revision).toBe(3); }); diff --git a/src/features/voice-conversation/stores/voiceConversationStore.ts b/src/features/voice-conversation/stores/voiceConversationStore.ts index 9ec2abef9..ef0e5a019 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.ts @@ -9,6 +9,7 @@ import { getVoiceConversationStatus, hydrateVoiceConversationMicrophone, listenToVoiceConversation, + reconcileVoiceConversationMicrophone, rejectVoiceConversationTranscript, setVoiceConversationMicrophoneMuted, startVoiceConversation, @@ -60,6 +61,7 @@ interface VoiceConversationStore { let initialized = false; let stopInFlight: Promise | null = null; let microphoneMuteIntent = 0; +let microphoneMuteStateVersion = 0; const eventSubscribers = new Set< (event: VoiceConversationEvent) => void | Promise >(); @@ -162,6 +164,17 @@ function shouldApplyResponseRevision( return revision > current.revision; } +function isMatchingRunningSession( + current: VoiceConversationStatus, + next: VoiceConversationStatus, +) { + return ( + current.lifecycle === "running" && + next.lifecycle === "running" && + current.sessionId === next.sessionId + ); +} + export const useVoiceConversationStore = create( (set, get) => ({ status: VOICE_CONVERSATION_OFF_STATUS, @@ -185,13 +198,27 @@ export const useVoiceConversationStore = create( init: async () => { if (initialized) { try { - const muteIntent = microphoneMuteIntent; + const muteStateVersion = microphoneMuteStateVersion; const status = await getVoiceConversationStatus(); + const currentStatus = get().status; + const shouldAdopt = shouldApplyResponseRevision( + currentStatus, + status.revision, + ); + const matchingRunningSession = isMatchingRunningSession( + currentStatus, + status, + ); + const shouldReconcile = shouldAdopt || matchingRunningSession; const shouldHydrate = - shouldApplyResponseRevision(get().status, status.revision) && - muteIntent === microphoneMuteIntent; + (shouldAdopt || + (matchingRunningSession && + currentStatus.revision === status.revision)) && + muteStateVersion === microphoneMuteStateVersion; if (shouldHydrate) { await hydrateVoiceConversationMicrophone(status); + } else if (shouldReconcile) { + await reconcileVoiceConversationMicrophone(status); } set((state) => { if ( @@ -206,12 +233,11 @@ export const useVoiceConversationStore = create( state.uiState === "error" ? state.uiState : uiStateForStatus(status), - microphoneMuted: - muteIntent === microphoneMuteIntent - ? status.lifecycle === "running" - ? (status.nativeMicrophoneMuted ?? false) - : false - : state.microphoneMuted, + microphoneMuted: shouldHydrate + ? status.lifecycle === "running" + ? (status.nativeMicrophoneMuted ?? false) + : false + : state.microphoneMuted, hydrated: true, }; } @@ -222,6 +248,11 @@ export const useVoiceConversationStore = create( available: status.available, unavailableReason: status.unavailableReason, }, + microphoneMuted: shouldHydrate + ? status.lifecycle === "running" + ? (status.nativeMicrophoneMuted ?? false) + : false + : state.microphoneMuted, hydrated: true, }; } @@ -247,6 +278,7 @@ export const useVoiceConversationStore = create( if (event.type === "inputMute") { microphoneMuteIntent += 1; + microphoneMuteStateVersion += 1; applyVoiceConversationMicrophoneMuteEvent(event.muted); } else if (event.type === "startup") { microphoneMuteIntent += 1; @@ -255,6 +287,7 @@ export const useVoiceConversationStore = create( (event.type === "error" && event.terminal) ) { microphoneMuteIntent += 1; + microphoneMuteStateVersion += 1; applyVoiceConversationTerminalEvent(); } @@ -265,7 +298,7 @@ export const useVoiceConversationStore = create( ...state, status: { ...state.status, - lifecycle: "running", + lifecycle: "running" as const, sessionId: event.sessionId, ownerWindowLabel: event.ownerWindowLabel, revision: event.revision, @@ -280,7 +313,7 @@ export const useVoiceConversationStore = create( ...state, status: { ...state.status, - lifecycle: "running", + lifecycle: "running" as const, sessionId: event.sessionId, revision: event.revision, }, @@ -322,6 +355,7 @@ export const useVoiceConversationStore = create( ...state, status: { ...state.status, + lifecycle: "running" as const, sessionId: event.sessionId, revision: event.revision, }, @@ -407,38 +441,60 @@ export const useVoiceConversationStore = create( } try { - const muteIntent = microphoneMuteIntent; + const muteStateVersion = microphoneMuteStateVersion; const status = await getVoiceConversationStatus(); + const currentStatus = get().status; + const shouldAdopt = + shouldApplyResponseRevision(currentStatus, status.revision) || + (!get().hydrated && + get().status.revision === 0 && + get().uiState === "off"); + const matchingRunningSession = isMatchingRunningSession( + currentStatus, + status, + ); + const shouldReconcile = shouldAdopt || matchingRunningSession; const shouldHydrate = - (shouldApplyResponseRevision(get().status, status.revision) || - (!get().hydrated && - get().status.revision === 0 && - get().uiState === "off")) && - muteIntent === microphoneMuteIntent; + (shouldAdopt || + (matchingRunningSession && + currentStatus.revision === status.revision)) && + muteStateVersion === microphoneMuteStateVersion; if (shouldHydrate) { await hydrateVoiceConversationMicrophone(status); + } else if (shouldReconcile) { + await reconcileVoiceConversationMicrophone(status); } - set((state) => - shouldApplyResponseRevision(state.status, status.revision) || - (!state.hydrated && - state.status.revision === 0 && - state.uiState === "off") - ? { - status, - uiState: - state.uiState === "error" - ? state.uiState - : uiStateForStatus(status), - microphoneMuted: - muteIntent === microphoneMuteIntent - ? status.lifecycle === "running" - ? (status.nativeMicrophoneMuted ?? false) - : false - : state.microphoneMuted, - hydrated: true, - } - : { hydrated: true }, - ); + set((state) => { + if ( + shouldApplyResponseRevision(state.status, status.revision) || + (!state.hydrated && + state.status.revision === 0 && + state.uiState === "off") + ) { + return { + status, + uiState: + state.uiState === "error" + ? state.uiState + : uiStateForStatus(status), + microphoneMuted: shouldHydrate + ? status.lifecycle === "running" + ? (status.nativeMicrophoneMuted ?? false) + : false + : state.microphoneMuted, + hydrated: true, + }; + } + if (isMatchingRunningSession(state.status, status)) { + return { + microphoneMuted: shouldHydrate + ? (status.nativeMicrophoneMuted ?? false) + : state.microphoneMuted, + hydrated: true, + }; + } + return { hydrated: true }; + }); } catch (error) { set({ error: error instanceof Error ? error.message : String(error), @@ -450,6 +506,7 @@ export const useVoiceConversationStore = create( start: async (sessionId) => { microphoneMuteIntent += 1; + microphoneMuteStateVersion += 1; set({ uiState: "starting", microphoneMuted: false, error: null }); try { const status = await startVoiceConversation(sessionId); @@ -485,6 +542,7 @@ export const useVoiceConversationStore = create( stop: () => { if (stopInFlight) return stopInFlight; microphoneMuteIntent += 1; + microphoneMuteStateVersion += 1; set({ uiState: "stopping", microphoneMuted: false, @@ -561,6 +619,7 @@ export const useVoiceConversationStore = create( const current = get(); if (current.status.lifecycle !== "running") return; const intent = ++microphoneMuteIntent; + microphoneMuteStateVersion += 1; set((state) => { const nextState = { ...state, From c59653e0567a46156446f3d555339060b7a68afb Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 19:50:37 -0400 Subject: [PATCH 29/31] fix(voice): preserve current mute during recovery --- .../stores/voiceConversationStore.test.ts | 46 +++++++++++++--- .../stores/voiceConversationStore.ts | 52 ++++++++++++------- 2 files changed, 72 insertions(+), 26 deletions(-) diff --git a/src/features/voice-conversation/stores/voiceConversationStore.test.ts b/src/features/voice-conversation/stores/voiceConversationStore.test.ts index eaeb02ebf..6222699c3 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.test.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.test.ts @@ -197,9 +197,45 @@ describe("voice conversation store lifecycle ordering", () => { expect(useVoiceConversationStore.getState().microphoneMuted).toBe(true); }); + it("preserves a mute event that arrives while hydration is pending", async () => { + const hydration = deferred(); + const muted = { + ...status("running", 2, "session-1"), + nativeMicrophoneMuteControl: true, + nativeMicrophoneMuted: true, + }; + mocks.getStatus.mockResolvedValue(muted); + mocks.hydrateMicrophone.mockReturnValue(hydration.promise); + const { useVoiceConversationStore } = await import( + "./voiceConversationStore" + ); + + const initializing = useVoiceConversationStore.getState().init(); + await vi.waitFor(() => + expect(mocks.hydrateMicrophone).toHaveBeenCalledWith(muted), + ); + emit({ + type: "inputMute", + sessionId: "session-1", + muted: false, + revision: 2, + }); + hydration.resolve(); + await initializing; + + expect(useVoiceConversationStore.getState().microphoneMuted).toBe(false); + }); + it("does not let stale status overwrite a newer mute event", async () => { const response = deferred(); - mocks.getStatus.mockReturnValue(response.promise); + const current = { + ...status("running", 3, "session-1"), + nativeMicrophoneMuteControl: true, + nativeMicrophoneMuted: false, + }; + mocks.getStatus + .mockReturnValueOnce(response.promise) + .mockResolvedValueOnce(current); const { useVoiceConversationStore } = await import( "./voiceConversationStore" ); @@ -220,12 +256,8 @@ describe("voice conversation store lifecycle ordering", () => { await initializing; expect(mocks.hydrateMicrophone).not.toHaveBeenCalled(); - expect(mocks.reconcileMicrophone).toHaveBeenCalledWith( - expect.objectContaining({ - lifecycle: "running", - sessionId: "session-1", - }), - ); + expect(mocks.reconcileMicrophone).toHaveBeenCalledWith(current); + expect(mocks.getStatus).toHaveBeenCalledTimes(2); expect(useVoiceConversationStore.getState().microphoneMuted).toBe(false); expect(useVoiceConversationStore.getState().status.revision).toBe(3); }); diff --git a/src/features/voice-conversation/stores/voiceConversationStore.ts b/src/features/voice-conversation/stores/voiceConversationStore.ts index ef0e5a019..280a4abe3 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.ts @@ -164,17 +164,31 @@ function shouldApplyResponseRevision( return revision > current.revision; } -function isMatchingRunningSession( +function isSameRunningLifecycle( current: VoiceConversationStatus, next: VoiceConversationStatus, ) { return ( current.lifecycle === "running" && next.lifecycle === "running" && - current.sessionId === next.sessionId + current.sessionId === next.sessionId && + current.revision === next.revision && + (current.ownerWindowLabel === null || + current.ownerWindowLabel === next.ownerWindowLabel) ); } +async function getRecoveryStatus( + currentStatus: () => VoiceConversationStatus, +): Promise { + let status = await getVoiceConversationStatus(); + const current = currentStatus(); + if (current.lifecycle === "running" && status.revision < current.revision) { + status = await getVoiceConversationStatus(); + } + return status; +} + export const useVoiceConversationStore = create( (set, get) => ({ status: VOICE_CONVERSATION_OFF_STATUS, @@ -199,27 +213,27 @@ export const useVoiceConversationStore = create( if (initialized) { try { const muteStateVersion = microphoneMuteStateVersion; - const status = await getVoiceConversationStatus(); + const status = await getRecoveryStatus(() => get().status); const currentStatus = get().status; const shouldAdopt = shouldApplyResponseRevision( currentStatus, status.revision, ); - const matchingRunningSession = isMatchingRunningSession( + const sameRunningLifecycle = isSameRunningLifecycle( currentStatus, status, ); - const shouldReconcile = shouldAdopt || matchingRunningSession; + const shouldReconcile = shouldAdopt || sameRunningLifecycle; const shouldHydrate = - (shouldAdopt || - (matchingRunningSession && - currentStatus.revision === status.revision)) && + (shouldAdopt || sameRunningLifecycle) && muteStateVersion === microphoneMuteStateVersion; if (shouldHydrate) { await hydrateVoiceConversationMicrophone(status); } else if (shouldReconcile) { await reconcileVoiceConversationMicrophone(status); } + const applyHydratedMute = + shouldHydrate && muteStateVersion === microphoneMuteStateVersion; set((state) => { if ( shouldApplyResponseRevision(state.status, status.revision) || @@ -233,7 +247,7 @@ export const useVoiceConversationStore = create( state.uiState === "error" ? state.uiState : uiStateForStatus(status), - microphoneMuted: shouldHydrate + microphoneMuted: applyHydratedMute ? status.lifecycle === "running" ? (status.nativeMicrophoneMuted ?? false) : false @@ -248,7 +262,7 @@ export const useVoiceConversationStore = create( available: status.available, unavailableReason: status.unavailableReason, }, - microphoneMuted: shouldHydrate + microphoneMuted: applyHydratedMute ? status.lifecycle === "running" ? (status.nativeMicrophoneMuted ?? false) : false @@ -442,28 +456,28 @@ export const useVoiceConversationStore = create( try { const muteStateVersion = microphoneMuteStateVersion; - const status = await getVoiceConversationStatus(); + const status = await getRecoveryStatus(() => get().status); const currentStatus = get().status; const shouldAdopt = shouldApplyResponseRevision(currentStatus, status.revision) || (!get().hydrated && get().status.revision === 0 && get().uiState === "off"); - const matchingRunningSession = isMatchingRunningSession( + const sameRunningLifecycle = isSameRunningLifecycle( currentStatus, status, ); - const shouldReconcile = shouldAdopt || matchingRunningSession; + const shouldReconcile = shouldAdopt || sameRunningLifecycle; const shouldHydrate = - (shouldAdopt || - (matchingRunningSession && - currentStatus.revision === status.revision)) && + (shouldAdopt || sameRunningLifecycle) && muteStateVersion === microphoneMuteStateVersion; if (shouldHydrate) { await hydrateVoiceConversationMicrophone(status); } else if (shouldReconcile) { await reconcileVoiceConversationMicrophone(status); } + const applyHydratedMute = + shouldHydrate && muteStateVersion === microphoneMuteStateVersion; set((state) => { if ( shouldApplyResponseRevision(state.status, status.revision) || @@ -477,7 +491,7 @@ export const useVoiceConversationStore = create( state.uiState === "error" ? state.uiState : uiStateForStatus(status), - microphoneMuted: shouldHydrate + microphoneMuted: applyHydratedMute ? status.lifecycle === "running" ? (status.nativeMicrophoneMuted ?? false) : false @@ -485,9 +499,9 @@ export const useVoiceConversationStore = create( hydrated: true, }; } - if (isMatchingRunningSession(state.status, status)) { + if (isSameRunningLifecycle(state.status, status)) { return { - microphoneMuted: shouldHydrate + microphoneMuted: applyHydratedMute ? (status.nativeMicrophoneMuted ?? false) : state.microphoneMuted, hydrated: true, From f5d1030d4faa2bafb63dfd1101aae73124cc19b4 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 19:55:00 -0400 Subject: [PATCH 30/31] fix(voice): protect pending mute during recovery --- .../stores/voiceConversationStore.test.ts | 33 +++++++++++++++++++ .../stores/voiceConversationStore.ts | 14 ++++++-- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/features/voice-conversation/stores/voiceConversationStore.test.ts b/src/features/voice-conversation/stores/voiceConversationStore.test.ts index 6222699c3..7e721a1e6 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.test.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.test.ts @@ -226,6 +226,39 @@ describe("voice conversation store lifecycle ordering", () => { expect(useVoiceConversationStore.getState().microphoneMuted).toBe(false); }); + it("does not hydrate over a pending microphone mute request", async () => { + const muteRequest = deferred(); + const running = { + ...status("running", 2, "session-1"), + nativeMicrophoneMuteControl: true, + nativeMicrophoneMuted: false, + }; + mocks.getStatus.mockResolvedValue(running); + mocks.setMicrophoneMuted.mockReturnValue(muteRequest.promise); + const { useVoiceConversationStore } = await import( + "./voiceConversationStore" + ); + await useVoiceConversationStore.getState().init(); + mocks.hydrateMicrophone.mockClear(); + mocks.reconcileMicrophone.mockClear(); + + const muting = useVoiceConversationStore + .getState() + .setMicrophoneMuted(true); + await vi.waitFor(() => + expect(mocks.setMicrophoneMuted).toHaveBeenCalledWith(true, running), + ); + await useVoiceConversationStore.getState().init(); + + expect(mocks.hydrateMicrophone).not.toHaveBeenCalled(); + expect(mocks.reconcileMicrophone).toHaveBeenCalledWith(running); + expect(useVoiceConversationStore.getState().microphoneMuted).toBe(true); + + muteRequest.resolve(); + await muting; + expect(useVoiceConversationStore.getState().microphoneMuted).toBe(true); + }); + it("does not let stale status overwrite a newer mute event", async () => { const response = deferred(); const current = { diff --git a/src/features/voice-conversation/stores/voiceConversationStore.ts b/src/features/voice-conversation/stores/voiceConversationStore.ts index 280a4abe3..fa7536dcc 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.ts @@ -62,6 +62,7 @@ let initialized = false; let stopInFlight: Promise | null = null; let microphoneMuteIntent = 0; let microphoneMuteStateVersion = 0; +let pendingMicrophoneMuteRequests = 0; const eventSubscribers = new Set< (event: VoiceConversationEvent) => void | Promise >(); @@ -226,6 +227,7 @@ export const useVoiceConversationStore = create( const shouldReconcile = shouldAdopt || sameRunningLifecycle; const shouldHydrate = (shouldAdopt || sameRunningLifecycle) && + pendingMicrophoneMuteRequests === 0 && muteStateVersion === microphoneMuteStateVersion; if (shouldHydrate) { await hydrateVoiceConversationMicrophone(status); @@ -233,7 +235,9 @@ export const useVoiceConversationStore = create( await reconcileVoiceConversationMicrophone(status); } const applyHydratedMute = - shouldHydrate && muteStateVersion === microphoneMuteStateVersion; + shouldHydrate && + pendingMicrophoneMuteRequests === 0 && + muteStateVersion === microphoneMuteStateVersion; set((state) => { if ( shouldApplyResponseRevision(state.status, status.revision) || @@ -470,6 +474,7 @@ export const useVoiceConversationStore = create( const shouldReconcile = shouldAdopt || sameRunningLifecycle; const shouldHydrate = (shouldAdopt || sameRunningLifecycle) && + pendingMicrophoneMuteRequests === 0 && muteStateVersion === microphoneMuteStateVersion; if (shouldHydrate) { await hydrateVoiceConversationMicrophone(status); @@ -477,7 +482,9 @@ export const useVoiceConversationStore = create( await reconcileVoiceConversationMicrophone(status); } const applyHydratedMute = - shouldHydrate && muteStateVersion === microphoneMuteStateVersion; + shouldHydrate && + pendingMicrophoneMuteRequests === 0 && + muteStateVersion === microphoneMuteStateVersion; set((state) => { if ( shouldApplyResponseRevision(state.status, status.revision) || @@ -634,6 +641,7 @@ export const useVoiceConversationStore = create( if (current.status.lifecycle !== "running") return; const intent = ++microphoneMuteIntent; microphoneMuteStateVersion += 1; + pendingMicrophoneMuteRequests += 1; set((state) => { const nextState = { ...state, @@ -659,6 +667,8 @@ export const useVoiceConversationStore = create( error: error instanceof Error ? error.message : String(error), }); throw error; + } finally { + pendingMicrophoneMuteRequests -= 1; } if (intent !== microphoneMuteIntent) return; set((state) => { From b6ceb10f9f97b11d0d53f1c60e4d32b3fc4b34b0 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 19:56:46 -0400 Subject: [PATCH 31/31] test(voice): cover settled mute recovery race --- .../stores/voiceConversationStore.test.ts | 14 +++++++++----- .../stores/voiceConversationStore.ts | 4 ++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/features/voice-conversation/stores/voiceConversationStore.test.ts b/src/features/voice-conversation/stores/voiceConversationStore.test.ts index 7e721a1e6..ce6691b04 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.test.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.test.ts @@ -228,6 +228,7 @@ describe("voice conversation store lifecycle ordering", () => { it("does not hydrate over a pending microphone mute request", async () => { const muteRequest = deferred(); + const recoveryStatus = deferred(); const running = { ...status("running", 2, "session-1"), nativeMicrophoneMuteControl: true, @@ -241,6 +242,7 @@ describe("voice conversation store lifecycle ordering", () => { await useVoiceConversationStore.getState().init(); mocks.hydrateMicrophone.mockClear(); mocks.reconcileMicrophone.mockClear(); + mocks.getStatus.mockReturnValueOnce(recoveryStatus.promise); const muting = useVoiceConversationStore .getState() @@ -248,14 +250,16 @@ describe("voice conversation store lifecycle ordering", () => { await vi.waitFor(() => expect(mocks.setMicrophoneMuted).toHaveBeenCalledWith(true, running), ); - await useVoiceConversationStore.getState().init(); - - expect(mocks.hydrateMicrophone).not.toHaveBeenCalled(); - expect(mocks.reconcileMicrophone).toHaveBeenCalledWith(running); - expect(useVoiceConversationStore.getState().microphoneMuted).toBe(true); + const recovering = useVoiceConversationStore.getState().init(); + await vi.waitFor(() => expect(mocks.getStatus).toHaveBeenCalledTimes(2)); muteRequest.resolve(); await muting; + recoveryStatus.resolve(running); + await recovering; + + expect(mocks.hydrateMicrophone).not.toHaveBeenCalled(); + expect(mocks.reconcileMicrophone).toHaveBeenCalledWith(running); expect(useVoiceConversationStore.getState().microphoneMuted).toBe(true); }); diff --git a/src/features/voice-conversation/stores/voiceConversationStore.ts b/src/features/voice-conversation/stores/voiceConversationStore.ts index fa7536dcc..b4b48459b 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.ts @@ -214,6 +214,7 @@ export const useVoiceConversationStore = create( if (initialized) { try { const muteStateVersion = microphoneMuteStateVersion; + const muteRequestWasPending = pendingMicrophoneMuteRequests > 0; const status = await getRecoveryStatus(() => get().status); const currentStatus = get().status; const shouldAdopt = shouldApplyResponseRevision( @@ -227,6 +228,7 @@ export const useVoiceConversationStore = create( const shouldReconcile = shouldAdopt || sameRunningLifecycle; const shouldHydrate = (shouldAdopt || sameRunningLifecycle) && + !muteRequestWasPending && pendingMicrophoneMuteRequests === 0 && muteStateVersion === microphoneMuteStateVersion; if (shouldHydrate) { @@ -460,6 +462,7 @@ export const useVoiceConversationStore = create( try { const muteStateVersion = microphoneMuteStateVersion; + const muteRequestWasPending = pendingMicrophoneMuteRequests > 0; const status = await getRecoveryStatus(() => get().status); const currentStatus = get().status; const shouldAdopt = @@ -474,6 +477,7 @@ export const useVoiceConversationStore = create( const shouldReconcile = shouldAdopt || sameRunningLifecycle; const shouldHydrate = (shouldAdopt || sameRunningLifecycle) && + !muteRequestWasPending && pendingMicrophoneMuteRequests === 0 && muteStateVersion === microphoneMuteStateVersion; if (shouldHydrate) {