diff --git a/src-tauri/capabilities/voice-buddy.json b/src-tauri/capabilities/voice-buddy.json new file mode 100644 index 000000000..3ca7cbfa5 --- /dev/null +++ b/src-tauri/capabilities/voice-buddy.json @@ -0,0 +1,7 @@ +{ + "$schema": "../gen/schemas/desktop-schema.json", + "identifier": "voice-buddy", + "description": "Capability for the always-on-top voice conversation buddy", + "windows": ["voice-buddy"], + "permissions": ["core:default", "core:window:allow-start-dragging"] +} diff --git a/src-tauri/gen/schemas/capabilities.json b/src-tauri/gen/schemas/capabilities.json index 854f600a3..343a796e1 100644 --- a/src-tauri/gen/schemas/capabilities.json +++ b/src-tauri/gen/schemas/capabilities.json @@ -1 +1 @@ -{"default":{"identifier":"default","description":"Capability for the main window","local":true,"windows":["main"],"permissions":["core:default","core:window:allow-start-dragging","core:window:allow-toggle-maximize","core:window:allow-show","core:window:allow-close","core:window:allow-set-size","core:window:allow-set-min-size","deep-link:default","opener:default",{"identifier":"opener:allow-open-url","allow":[{"url":"linear://*"},{"url":"https://linear.app/*"}]},{"identifier":"opener:allow-open-path","allow":[{"path":"$HOME/**"},{"path":"$HOME/.goose/**"},{"path":"$TEMP/**"},{"path":"/Volumes/**"},{"path":"/mnt/**"},{"path":"/workspace/**"},{"path":"/workspaces/**"},{"path":"/opt/**"},{"path":"/srv/**"},{"path":"*:/**"}]},"window-state:allow-restore-state","window-state:allow-save-window-state","dialog:allow-open","dialog:allow-save","clipboard-manager:allow-write-text","app-test-driver:default","berdctl:default","updater:default","process:allow-restart","notification:default"]},"session-window":{"identifier":"session-window","description":"Capability for popped-out chat session windows","local":true,"windows":["session:*"],"permissions":["core:default","core:window:allow-start-dragging","core:window:allow-show","core:window:allow-close","core:window:allow-set-focus","core:window:allow-set-size","core:window:allow-set-min-size","opener:default",{"identifier":"opener:allow-open-url","allow":[{"url":"linear://*"},{"url":"https://linear.app/*"}]},{"identifier":"opener:allow-open-path","allow":[{"path":"$DOWNLOAD"}]},"dialog:allow-open","dialog:allow-save","berdctl:allow-status"],"platforms":["macOS"]}} \ No newline at end of file +{"default":{"identifier":"default","description":"Capability for the main window","local":true,"windows":["main"],"permissions":["core:default","core:window:allow-start-dragging","core:window:allow-toggle-maximize","core:window:allow-show","core:window:allow-close","core:window:allow-set-size","core:window:allow-set-min-size","deep-link:default","opener:default",{"identifier":"opener:allow-open-url","allow":[{"url":"linear://*"},{"url":"https://linear.app/*"}]},{"identifier":"opener:allow-open-path","allow":[{"path":"$HOME/**"},{"path":"$HOME/.goose/**"},{"path":"$TEMP/**"},{"path":"/Volumes/**"},{"path":"/mnt/**"},{"path":"/workspace/**"},{"path":"/workspaces/**"},{"path":"/opt/**"},{"path":"/srv/**"},{"path":"*:/**"}]},"window-state:allow-restore-state","window-state:allow-save-window-state","dialog:allow-open","dialog:allow-save","clipboard-manager:allow-write-text","app-test-driver:default","berdctl:default","updater:default","process:allow-restart","notification:default"]},"session-window":{"identifier":"session-window","description":"Capability for popped-out chat session windows","local":true,"windows":["session:*"],"permissions":["core:default","core:window:allow-start-dragging","core:window:allow-show","core:window:allow-close","core:window:allow-set-focus","core:window:allow-set-size","core:window:allow-set-min-size","opener:default",{"identifier":"opener:allow-open-url","allow":[{"url":"linear://*"},{"url":"https://linear.app/*"}]},{"identifier":"opener:allow-open-path","allow":[{"path":"$DOWNLOAD"}]},"dialog:allow-open","dialog:allow-save","berdctl:allow-status"],"platforms":["macOS"]},"voice-buddy":{"identifier":"voice-buddy","description":"Capability for the always-on-top voice conversation buddy","local":true,"windows":["voice-buddy"],"permissions":["core:default","core:window:allow-start-dragging"]}} \ No newline at end of file diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index 485664fe5..7b1c1c700 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -51,6 +51,7 @@ pub mod system; pub mod telemetry; pub mod terminal; pub mod updates; +pub mod voice_buddy; pub mod voice_capture; pub mod whoami; pub mod window_session; diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index ffaa4a376..d49660cb5 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -1,7 +1,7 @@ //! Native Parakeet speech recognition for Desktop voice conversations. use std::{ - collections::VecDeque, + collections::{HashMap, VecDeque}, path::PathBuf, sync::{ atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, @@ -12,7 +12,7 @@ use std::{ time::Duration, }; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use tauri::{AppHandle, Emitter, Manager, State, WebviewWindow}; use tokio::sync::mpsc as tokio_mpsc; @@ -20,7 +20,7 @@ use super::{ native_input_mute, pocket_voice::parakeet_model_dir, voice_capture::VoiceCaptureState, }; -const EVENT_NAME: &str = "voice-conversation:event"; +pub(crate) const EVENT_NAME: &str = "voice-conversation:event"; const MAX_AUDIO_BATCH_BYTES: usize = 100 * 1024; const AUDIO_QUEUE_DEPTH: usize = 50; const MAX_PENDING_TRANSCRIPTS: usize = 64; @@ -32,6 +32,26 @@ const VAD_THRESHOLD: f32 = 0.5; // At 16 kHz with 256-sample frames this is 1.2 seconds. const SILENCE_FLUSH_FRAMES: usize = 75; +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MicrophoneMuteRequest { + session_id: String, + expected_revision: u64, + muted: bool, + renderer_id: String, + renderer_epoch: u64, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AssistantSpeakingRequest { + session_id: String, + expected_revision: u64, + speaking: bool, + renderer_id: String, + renderer_epoch: u64, +} + #[derive(Clone, Copy, Debug, Default, Serialize)] #[serde(rename_all = "kebab-case")] enum Lifecycle { @@ -48,9 +68,8 @@ pub struct NativeVoiceStatus { lifecycle: Lifecycle, session_id: Option, owner_window_label: Option, + microphone_muted: bool, revision: u64, - native_microphone_mute_control: bool, - native_microphone_muted: bool, } #[derive(Clone, Debug, Serialize)] @@ -77,13 +96,12 @@ pub struct TranscriptRejection { rename_all = "camelCase", rename_all_fields = "camelCase" )] -enum NativeVoiceEvent { +pub(crate) enum NativeVoiceEvent { Startup { session_id: String, owner_window_label: String, line: String, revision: u64, - native_microphone_mute_control: bool, }, User { session_id: String, @@ -98,7 +116,7 @@ enum NativeVoiceEvent { activity: &'static str, revision: u64, }, - InputMute { + MicrophoneMute { session_id: String, muted: bool, revision: u64, @@ -107,6 +125,9 @@ enum NativeVoiceEvent { session_id: String, revision: u64, }, + ControlsDismissed { + revision: u64, + }, Error { session_id: Option, message: String, @@ -122,19 +143,61 @@ struct Runtime { revision: u64, owner: Option, pipeline: Option, + controls_ready: bool, + controls_suppressed: bool, + controls_visibility_generation: u64, native_microphone_mute_control: bool, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct ControlsVisibilityTarget { + pub(crate) suppressed: bool, + pub(crate) generation: u64, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ControlsVisibilityAcknowledgement { + Inactive, + Ready, + Superseded(ControlsVisibilityTarget), +} + #[derive(Clone)] struct RuntimeOwner { window_label: String, } +#[derive(Clone)] +struct VoiceStartBlock { + token: String, + window_label: String, + renderer_id: String, + renderer_epoch: u64, +} + +type StopSnapshot = ( + Option, + u64, + Option, + Option<(RuntimeOwner, String)>, +); + +struct StopCompletion { + session_id: String, + controls_revision: u64, + next_revision: u64, + owner: RuntimeOwner, + owner_id: String, +} + #[derive(Clone, Default)] pub struct NativeVoiceState { runtime: Arc>, + stop_serial: Arc>, + start_blocks: Arc>>>, pending: Arc>>, capture_suppressions: Arc, + microphone_muted: Arc, input_muted: Arc, input_mute_epoch: Arc, } @@ -156,6 +219,78 @@ impl Drop for CaptureSuppressionGuard { } impl NativeVoiceState { + fn block_starts( + &self, + session_id: String, + window_label: String, + renderer_id: String, + renderer_epoch: u64, + ) -> Result { + let token = uuid::Uuid::new_v4().to_string(); + self.start_blocks + .lock() + .map_err(|_| "native voice start block lock was poisoned".to_string())? + .entry(session_id) + .or_default() + .push(VoiceStartBlock { + token: token.clone(), + window_label, + renderer_id, + renderer_epoch, + }); + Ok(token) + } + + fn release_start_block(&self, session_id: &str, token: &str) -> Result<(), String> { + let mut blocks = self + .start_blocks + .lock() + .map_err(|_| "native voice start block lock was poisoned".to_string())?; + let Some(session_blocks) = blocks.get_mut(session_id) else { + return Ok(()); + }; + session_blocks.retain(|block| block.token != token); + if session_blocks.is_empty() { + blocks.remove(session_id); + } + Ok(()) + } + + fn release_start_blocks_for_window(&self, window_label: &str) { + let Ok(mut blocks) = self.start_blocks.lock() else { + return; + }; + blocks.retain(|_, session_blocks| { + session_blocks.retain(|block| block.window_label != window_label); + !session_blocks.is_empty() + }); + } + + pub(crate) fn release_start_blocks_for_replaced_renderer( + &self, + window_label: &str, + renderer_id: &str, + renderer_epoch: u64, + ) { + let Ok(mut blocks) = self.start_blocks.lock() else { + return; + }; + blocks.retain(|_, session_blocks| { + session_blocks.retain(|block| { + block.window_label != window_label + || (block.renderer_id == renderer_id && block.renderer_epoch == renderer_epoch) + }); + !session_blocks.is_empty() + }); + } + + #[cfg(test)] + fn starts_blocked(&self, session_id: &str) -> bool { + self.start_blocks + .lock() + .is_ok_and(|blocks| blocks.contains_key(session_id)) + } + pub fn suppress_capture(&self) -> CaptureSuppressionGuard { let previous = self.capture_suppressions.fetch_add(1, Ordering::SeqCst); log::info!( @@ -170,6 +305,328 @@ impl NativeVoiceState { fn capture_is_suppressed(&self) -> bool { self.capture_suppressions.load(Ordering::SeqCst) > 0 } + + pub fn microphone_is_muted(&self) -> bool { + self.microphone_muted.load(Ordering::SeqCst) || self.input_muted.load(Ordering::Acquire) + } + + pub fn active_session_target(&self) -> Option<(String, String)> { + let runtime = self.runtime.lock().ok()?; + Some(( + runtime.session_id.clone()?, + runtime.owner.as_ref()?.window_label.clone(), + )) + } + + pub fn active_session_lifecycle_target(&self) -> Option<(String, String, u64)> { + let runtime = self.runtime.lock().ok()?; + Some(( + runtime.session_id.clone()?, + runtime.owner.as_ref()?.window_label.clone(), + runtime.revision, + )) + } + + pub(crate) fn controls_visibility_target( + &self, + session_id: &str, + expected_revision: u64, + ) -> Result, String> { + let runtime = self + .runtime + .lock() + .map_err(|_| "native voice state lock was poisoned".to_string())?; + if runtime.session_id.as_deref() != Some(session_id) + || runtime.revision != expected_revision + { + return Ok(None); + } + Ok(Some(ControlsVisibilityTarget { + suppressed: runtime.controls_suppressed, + generation: runtime.controls_visibility_generation, + })) + } + + pub(crate) fn acknowledge_controls_visibility( + &self, + session_id: &str, + expected_revision: u64, + applied_generation: u64, + ) -> Result { + let mut runtime = self + .runtime + .lock() + .map_err(|_| "native voice state lock was poisoned".to_string())?; + if runtime.session_id.as_deref() != Some(session_id) + || runtime.revision != expected_revision + { + return Ok(ControlsVisibilityAcknowledgement::Inactive); + } + runtime.controls_ready = true; + if runtime.controls_visibility_generation == applied_generation { + Ok(ControlsVisibilityAcknowledgement::Ready) + } else { + Ok(ControlsVisibilityAcknowledgement::Superseded( + ControlsVisibilityTarget { + suppressed: runtime.controls_suppressed, + generation: runtime.controls_visibility_generation, + }, + )) + } + } + + pub fn controls_ready_for(&self, session_id: &str, revision: u64) -> bool { + self.runtime.lock().ok().is_some_and(|runtime| { + runtime.session_id.as_deref() == Some(session_id) + && runtime.revision == revision + && runtime.controls_ready + }) + } + + pub fn set_controls_suppressed( + &self, + caller_window_label: &str, + session_id: &str, + expected_revision: u64, + suppressed: bool, + ) -> Result, String> { + let mut runtime = self + .runtime + .lock() + .map_err(|_| "native voice state lock was poisoned".to_string())?; + if runtime.session_id.as_deref() != Some(session_id) + || runtime.revision != expected_revision + { + return Ok(None); + } + if runtime + .owner + .as_ref() + .map(|owner| owner.window_label.as_str()) + != Some(caller_window_label) + { + return Err("Only the voice conversation owner can change control visibility.".into()); + } + let previous_suppression = runtime.controls_suppressed; + if previous_suppression != suppressed { + runtime.controls_suppressed = suppressed; + runtime.controls_visibility_generation = + runtime.controls_visibility_generation.wrapping_add(1); + } + Ok(Some(( + runtime.controls_ready && !suppressed, + previous_suppression, + ))) + } + + pub fn rollback_controls_suppression( + &self, + session_id: &str, + expected_revision: u64, + failed_suppression: bool, + previous_suppression: bool, + ) { + if let Ok(mut runtime) = self.runtime.lock() { + if runtime.session_id.as_deref() == Some(session_id) + && runtime.revision == expected_revision + && runtime.controls_suppressed == failed_suppression + { + runtime.controls_suppressed = previous_suppression; + runtime.controls_visibility_generation = + runtime.controls_visibility_generation.wrapping_add(1); + } + } + } + + pub fn is_active_for_session(&self, session_id: &str) -> bool { + self.runtime + .lock() + .ok() + .and_then(|runtime| runtime.session_id.clone()) + .is_some_and(|active_session_id| active_session_id == session_id) + } + + fn set_microphone_muted_target( + &self, + caller_window_label: &str, + session_id: &str, + expected_revision: u64, + muted: bool, + ) -> Result, String> { + let owner_window_label = { + let runtime = self + .runtime + .lock() + .map_err(|_| "native voice state lock was poisoned".to_string())?; + if runtime.session_id.as_deref() != Some(session_id) + || runtime.revision != expected_revision + { + return Ok(None); + } + let owner_window_label = runtime + .owner + .as_ref() + .map(|owner| owner.window_label.clone()) + .ok_or_else(|| "The native voice conversation has no owning window.".to_string())?; + if caller_window_label != owner_window_label + && caller_window_label != super::voice_buddy::WINDOW_LABEL + { + return Err( + "Only the voice conversation owner or floating controls can mute the microphone." + .to_string(), + ); + } + let native_microphone_mute_control = runtime.native_microphone_mute_control; + if native_microphone_mute_control { + native_input_mute::set_muted(&self.input_muted, &self.input_mute_epoch, muted)?; + } + // Native input mute is authoritative when installed so a hardware + // unmute cannot be masked by a stale renderer fallback latch. + self.microphone_muted.store( + software_microphone_mute(native_microphone_mute_control, muted), + Ordering::SeqCst, + ); + owner_window_label + }; + Ok(Some(owner_window_label)) + } + + pub fn set_microphone_muted( + &self, + app: &AppHandle, + caller_window_label: &str, + session_id: &str, + expected_revision: u64, + muted: bool, + ) -> Result<(), String> { + let Some(owner_window_label) = self.set_microphone_muted_target( + caller_window_label, + session_id, + expected_revision, + muted, + )? + else { + return Ok(()); + }; + let event = NativeVoiceEvent::MicrophoneMute { + session_id: session_id.to_string(), + muted, + revision: expected_revision, + }; + if let Some(window) = app.get_webview_window(&owner_window_label) { + let _ = window.emit(EVENT_NAME, event.clone()); + } + super::voice_buddy::emit(app, event); + Ok(()) + } + + fn assistant_activity_target( + &self, + caller_window_label: &str, + session_id: &str, + expected_revision: u64, + ) -> Result, String> { + let runtime = self + .runtime + .lock() + .map_err(|_| "native voice state lock was poisoned".to_string())?; + if runtime.session_id.as_deref() != Some(session_id) + || runtime.revision != expected_revision + { + return Ok(None); + } + let owner_window_label = runtime + .owner + .as_ref() + .map(|owner| owner.window_label.clone()) + .ok_or_else(|| "The native voice conversation has no owning window.".to_string())?; + if owner_window_label != caller_window_label { + return Err("Only the voice conversation owner can report assistant activity.".into()); + } + Ok(Some((owner_window_label, runtime.revision))) + } + + fn set_assistant_speaking( + &self, + app: &AppHandle, + caller_window_label: &str, + session_id: &str, + expected_revision: u64, + speaking: bool, + ) -> Result<(), String> { + let Some((owner_window_label, revision)) = + self.assistant_activity_target(caller_window_label, session_id, expected_revision)? + else { + return Ok(()); + }; + let event = NativeVoiceEvent::Activity { + session_id: session_id.to_string(), + activity: if speaking { + "assistant-speaking" + } else { + "assistant-idle" + }, + revision, + }; + if let Some(window) = app.get_webview_window(&owner_window_label) { + let _ = window.emit(EVENT_NAME, event.clone()); + } + super::voice_buddy::emit(app, event); + Ok(()) + } + + fn take_stop_snapshot( + &self, + expected_lifecycle: Option<(&str, u64)>, + ) -> Result, String> { + let mut runtime = self + .runtime + .lock() + .map_err(|_| "native voice state lock was poisoned".to_string())?; + if expected_lifecycle.is_some_and(|(session_id, revision)| { + runtime.session_id.as_deref() != Some(session_id) || runtime.revision != revision + }) { + return Ok(None); + } + if runtime.session_id.is_none() { + return Ok(None); + } + let owner = runtime.owner.clone(); + let session_id = runtime.session_id.clone(); + let owner_id = session_id.as_deref().map(native_owner_id); + Ok(Some(( + session_id, + runtime.revision, + runtime.pipeline.take(), + owner.zip(owner_id), + ))) + } + + fn owner_matches_lifecycle( + &self, + caller_window_label: &str, + session_id: &str, + expected_revision: u64, + ) -> Result { + let runtime = self + .runtime + .lock() + .map_err(|_| "native voice state lock was poisoned".to_string())?; + if runtime.session_id.as_deref() != Some(session_id) + || runtime.revision != expected_revision + { + return Ok(false); + } + if runtime + .owner + .as_ref() + .map(|owner| owner.window_label.as_str()) + != Some(caller_window_label) + { + return Err("Only the voice conversation owner can stop it.".to_string()); + } + Ok(true) + } } enum SttMessage { @@ -188,6 +645,7 @@ struct SttPipeline { discard_on_shutdown: Arc, input_muted: Arc, input_mute_epoch: Arc, + shutdown_mute_epoch: Arc, thread: Option>, } @@ -206,10 +664,12 @@ impl SttPipeline { let (event_tx, event_rx) = tokio_mpsc::channel(64); let shutdown = Arc::new(AtomicBool::new(false)); let discard_on_shutdown = Arc::new(AtomicBool::new(false)); + let shutdown_mute_epoch = Arc::new(AtomicU64::new(0)); let worker_shutdown = Arc::clone(&shutdown); let worker_discard_on_shutdown = Arc::clone(&discard_on_shutdown); let worker_input_muted = Arc::clone(&input_muted); let worker_input_mute_epoch = Arc::clone(&input_mute_epoch); + let worker_shutdown_mute_epoch = Arc::clone(&shutdown_mute_epoch); let thread = thread::Builder::new() .name("berd-native-stt".into()) .spawn(move || { @@ -221,6 +681,7 @@ impl SttPipeline { worker_discard_on_shutdown, worker_input_muted, worker_input_mute_epoch, + worker_shutdown_mute_epoch, ) }) .map_err(|error| format!("start native transcription: {error}"))?; @@ -232,6 +693,7 @@ impl SttPipeline { discard_on_shutdown, input_muted, input_mute_epoch, + shutdown_mute_epoch, thread: Some(thread), }, event_rx, @@ -281,6 +743,13 @@ impl SttPipeline { } fn signal_shutdown(&self) { + if self.shutdown.load(Ordering::Acquire) { + return; + } + self.shutdown_mute_epoch.store( + self.input_mute_epoch.load(Ordering::Acquire), + Ordering::Release, + ); self.latch_muted_shutdown(); self.shutdown.store(true, Ordering::Release); } @@ -304,11 +773,29 @@ impl Drop for SttPipeline { } } +#[cfg(not(test))] +const STT_WORKER_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); +#[cfg(test)] +const STT_WORKER_SHUTDOWN_TIMEOUT: Duration = Duration::from_millis(100); + async fn shutdown_pipeline(mut pipeline: SttPipeline) { let worker = pipeline.begin_shutdown(); drop(pipeline); if let Some(worker) = worker { - let _ = tauri::async_runtime::spawn_blocking(move || worker.join()).await; + let deadline = tokio::time::Instant::now() + STT_WORKER_SHUTDOWN_TIMEOUT; + while !worker.is_finished() { + if tokio::time::Instant::now() >= deadline { + log::error!( + "Native voice recognizer did not stop within {:?}; detaching it", + STT_WORKER_SHUTDOWN_TIMEOUT + ); + return; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + if worker.join().is_err() { + log::error!("Native voice recognizer worker panicked during shutdown"); + } } } @@ -333,10 +820,8 @@ fn status(app: &AppHandle, state: &NativeVoiceState) -> NativeVoiceStatus { .owner .as_ref() .map(|owner| owner.window_label.clone()), + microphone_muted: state.microphone_is_muted(), revision: runtime.revision, - native_microphone_mute_control: runtime.native_microphone_mute_control, - native_microphone_muted: runtime.session_id.is_some() - && state.input_muted.load(Ordering::Acquire), } } @@ -348,6 +833,39 @@ pub fn get_native_voice_conversation_status( status(&app, &state) } +#[tauri::command] +pub fn block_native_voice_conversation_starts( + state: State<'_, NativeVoiceState>, + capture: State<'_, VoiceCaptureState>, + webview_window: WebviewWindow, + session_id: String, + renderer_id: String, + renderer_epoch: u64, +) -> Result { + let session_id = session_id.trim().to_string(); + if session_id.is_empty() || session_id.len() > 256 { + return Err("session id must be between 1 and 256 bytes".to_string()); + } + let window_label = webview_window.label().to_string(); + capture.with_active_renderer(&window_label, &renderer_id, renderer_epoch, || { + state.block_starts( + session_id, + window_label.clone(), + renderer_id.clone(), + renderer_epoch, + ) + }) +} + +#[tauri::command] +pub fn release_native_voice_conversation_start_block( + state: State<'_, NativeVoiceState>, + session_id: String, + token: String, +) -> Result<(), String> { + state.release_start_block(&session_id, &token) +} + #[tauri::command] pub fn drain_native_voice_conversation_transcripts( state: State<'_, NativeVoiceState>, @@ -439,7 +957,7 @@ pub async fn start_native_voice_conversation( } let window_label = webview_window.label().to_string(); let owner_id = native_owner_id(&session_id); - let microphone_claimed = capture.claim_microphone( + let mut microphone_claimed = capture.claim_microphone( window_label.clone(), renderer_id.clone(), renderer_epoch, @@ -467,15 +985,37 @@ pub async fn start_native_voice_conversation( return Err(error); } }; - let (revision, lifecycle_id, runtime_mute_control) = { + let lifecycle_guard = state.stop_serial.lock().await; + match refresh_microphone_claim( + capture.inner(), + &window_label, + &renderer_id, + renderer_epoch, + &owner_id, + &mut microphone_claimed, + ) { + Ok(()) => {} + Err(error) => { + drop(lifecycle_guard); + if microphone_claimed { + capture.release_microphone(&window_label, &renderer_id, renderer_epoch, &owner_id); + } + return Err(error); + } + } + let install_result = (|| -> Result<(u64, String), String> { + let start_blocks = state + .start_blocks + .lock() + .map_err(|_| "native voice start block lock was poisoned".to_string())?; + if start_blocks.contains_key(&session_id) { + return Err("Voice cannot start while this chat is being archived.".to_string()); + } let mut runtime = state .runtime .lock() .map_err(|_| "native voice state lock was poisoned".to_string())?; if runtime.session_id.is_some() { - if microphone_claimed { - capture.release_microphone(&window_label, &renderer_id, renderer_epoch, &owner_id); - } return Err("A native voice conversation is already active.".to_string()); } runtime.revision = runtime.revision.wrapping_add(1); @@ -485,26 +1025,70 @@ pub async fn start_native_voice_conversation( window_label: window_label.clone(), }); runtime.pipeline = Some(pipeline); + runtime.controls_ready = false; + // Voice always starts from its owning session, where the in-session + // controls are already available. The owner renderer reveals the + // floating controls when that session stops being foreground. + runtime.controls_suppressed = true; + runtime.controls_visibility_generation = 0; + state.microphone_muted.store(false, Ordering::SeqCst); let runtime_revision = runtime.revision; + let mute_app = app.clone(); let mute_window = webview_window.clone(); let mute_session_id = session_id.clone(); runtime.native_microphone_mute_control = native_input_mute::start(&state.input_muted, &state.input_mute_epoch, move |muted| { - let _ = mute_window.emit( - EVENT_NAME, - NativeVoiceEvent::InputMute { - session_id: mute_session_id.clone(), - muted, - revision: runtime_revision, - }, - ); + let event = NativeVoiceEvent::MicrophoneMute { + session_id: mute_session_id.clone(), + muted, + revision: runtime_revision, + }; + let _ = mute_window.emit(EVENT_NAME, event.clone()); + super::voice_buddy::emit(&mute_app, event); }); - ( + Ok(( runtime.revision, runtime.lifecycle_id.clone().unwrap_or_default(), - runtime.native_microphone_mute_control, - ) + )) + })(); + let (revision, lifecycle_id) = match install_result { + Ok(lifecycle) => lifecycle, + Err(error) => { + drop(lifecycle_guard); + if microphone_claimed { + capture.release_microphone(&window_label, &renderer_id, renderer_epoch, &owner_id); + } + return Err(error); + } }; + if app.get_webview_window(&window_label).is_none() + || state.active_session_lifecycle_target() + != Some((session_id.clone(), window_label.clone(), revision)) + { + drop(lifecycle_guard); + state + .stop_active_for_lifecycle(&app, capture.inner(), &session_id, revision) + .await?; + return Err("The voice conversation owner closed during startup.".to_string()); + } + if let Err(error) = super::voice_buddy::install(&app) { + drop(lifecycle_guard); + state.stop_active(&app, &capture).await?; + return Err(format!( + "Could not show the floating voice controls: {error}" + )); + } + if app.get_webview_window(&window_label).is_none() + || state.active_session_lifecycle_target() + != Some((session_id.clone(), window_label.clone(), revision)) + { + drop(lifecycle_guard); + state + .stop_active_for_lifecycle(&app, capture.inner(), &session_id, revision) + .await?; + return Err("The voice conversation owner closed during startup.".to_string()); + } + drop(lifecycle_guard); let _ = webview_window.emit( EVENT_NAME, NativeVoiceEvent::Startup { @@ -512,14 +1096,23 @@ pub async fn start_native_voice_conversation( owner_window_label: window_label.clone(), line: "Native Parakeet voice conversation is on".to_string(), revision, - native_microphone_mute_control: runtime_mute_control, }, ); - - let event_app = app.clone(); + super::voice_buddy::emit( + &app, + NativeVoiceEvent::Startup { + session_id: session_id.clone(), + owner_window_label: window_label.clone(), + line: "Native Parakeet voice conversation is on".to_string(), + revision, + }, + ); + + let event_app = app.clone(); let event_window = webview_window.clone(); let runtime = Arc::clone(&state.runtime); let pending = Arc::clone(&state.pending); + let event_state = state.inner().clone(); let input_muted = Arc::clone(&state.input_muted); tauri::async_runtime::spawn(async move { while let Some(event) = events.recv().await { @@ -532,18 +1125,17 @@ pub async fn start_native_voice_conversation( } match event { SttMessage::Speaking(speaking) => { - let _ = event_window.emit( - EVENT_NAME, - NativeVoiceEvent::Activity { - session_id: session_id.clone(), - activity: if speaking { - "user-speaking" - } else { - "user-idle" - }, - revision, + let event = NativeVoiceEvent::Activity { + session_id: session_id.clone(), + activity: if speaking { + "user-speaking" + } else { + "user-idle" }, - ); + revision, + }; + let _ = event_window.emit(EVENT_NAME, event.clone()); + super::voice_buddy::emit(&event_app, event); } SttMessage::Final { text, delivered } => { let transcript = PendingTranscript { @@ -554,9 +1146,21 @@ pub async fn start_native_voice_conversation( revision, delivery_attempts: 0, }; - let evicted = pending.lock().ok().and_then(|mut queue| { - enqueue_pending_transcript(&mut queue, transcript.clone()) - }); + let Ok((accepted, evicted)) = enqueue_transcript_if_active( + &runtime, + &pending, + &session_id, + revision, + transcript.clone(), + ) else { + break; + }; + if !accepted { + if let Some(delivered) = delivered { + let _ = delivered.send(()); + } + break; + } if evicted.is_some() { let _ = event_window.emit( EVENT_NAME, @@ -584,6 +1188,7 @@ pub async fn start_native_voice_conversation( } } SttMessage::Failed(message) => { + let _stop_guard = event_state.stop_serial.lock().await; let pipeline = { let Ok(mut current) = runtime.lock() else { break; @@ -604,18 +1209,29 @@ pub async fn start_native_voice_conversation( if let Some(pipeline) = pipeline { shutdown_pipeline(pipeline).await; } + event_state.microphone_muted.store(false, Ordering::SeqCst); event_app .state::() .release_owner(&window_label, &owner_id); - let _ = event_window.emit( - EVENT_NAME, - NativeVoiceEvent::Error { - session_id: Some(session_id.clone()), - message, - revision: revision.wrapping_add(1), - terminal: true, - }, + let terminal_event = NativeVoiceEvent::Error { + session_id: Some(session_id.clone()), + message, + revision: revision.wrapping_add(1), + terminal: true, + }; + let _ = event_window.emit(EVENT_NAME, terminal_event.clone()); + super::voice_buddy::emit(&event_app, terminal_event); + let shutdown_event = NativeVoiceEvent::CleanShutdown { + session_id: session_id.clone(), + revision: revision.wrapping_add(1), + }; + let _ = event_window.emit(EVENT_NAME, shutdown_event.clone()); + super::voice_buddy::dismiss_after_terminal_event( + &event_app, + revision, + shutdown_event, ); + super::voice_buddy::restore_hidden_owner(&event_app, &window_label); break; } } @@ -625,6 +1241,61 @@ pub async fn start_native_voice_conversation( } #[tauri::command] +pub fn set_native_voice_microphone_muted( + app: AppHandle, + state: State<'_, NativeVoiceState>, + capture: State<'_, VoiceCaptureState>, + webview_window: WebviewWindow, + request: MicrophoneMuteRequest, +) -> Result { + let apply = || { + state.set_microphone_muted( + &app, + webview_window.label(), + &request.session_id, + request.expected_revision, + request.muted, + ) + }; + if webview_window.label() == super::voice_buddy::WINDOW_LABEL { + apply()?; + } else { + capture.with_active_renderer( + webview_window.label(), + &request.renderer_id, + request.renderer_epoch, + apply, + )?; + } + Ok(status(&app, &state)) +} + +#[tauri::command] +pub fn set_native_voice_assistant_speaking( + app: AppHandle, + state: State<'_, NativeVoiceState>, + capture: State<'_, VoiceCaptureState>, + webview_window: WebviewWindow, + request: AssistantSpeakingRequest, +) -> Result<(), String> { + capture.with_active_renderer( + webview_window.label(), + &request.renderer_id, + request.renderer_epoch, + || { + state.set_assistant_speaking( + &app, + webview_window.label(), + &request.session_id, + request.expected_revision, + request.speaking, + ) + }, + ) +} + +#[tauri::command] +#[allow(clippy::too_many_arguments)] // Tauri injects four guards beside the exact lifecycle payload. pub async fn stop_native_voice_conversation( app: AppHandle, state: State<'_, NativeVoiceState>, @@ -632,59 +1303,14 @@ pub async fn stop_native_voice_conversation( webview_window: WebviewWindow, renderer_id: String, renderer_epoch: u64, + session_id: String, + expected_revision: u64, ) -> Result { capture.activate_renderer(webview_window.label(), &renderer_id, renderer_epoch)?; - let (session_id, revision, pipeline, owner) = { - let mut runtime = state - .runtime - .lock() - .map_err(|_| "native voice state lock was poisoned".to_string())?; - let owner = runtime.owner.clone(); - let session_id = runtime.session_id.clone(); - let owner_id = session_id.as_deref().map(native_owner_id); - let revision = runtime.revision; - ( - session_id, - revision, - runtime.pipeline.take(), - owner.zip(owner_id), - ) - }; - // Keep the lifecycle current while the worker flushes its final buffered - // utterance into the durable pending queue. - if let Some(pipeline) = pipeline { - shutdown_pipeline(pipeline).await; - } - let revision = { - let mut runtime = state - .runtime - .lock() - .map_err(|_| "native voice state lock was poisoned".to_string())?; - if runtime.revision == revision && runtime.session_id == session_id { - native_input_mute::stop(&state.input_muted); - runtime.native_microphone_mute_control = false; - runtime.session_id = None; - runtime.lifecycle_id = None; - runtime.owner = None; - runtime.revision = runtime.revision.wrapping_add(1); - } - runtime.revision - }; - if let Some((owner, owner_id)) = owner.as_ref() { - capture.release_owner(&owner.window_label, owner_id); - } - if let Some(session_id) = session_id { - let target = owner - .as_ref() - .and_then(|(owner, _)| app.get_webview_window(&owner.window_label)) - .unwrap_or(webview_window); - let _ = target.emit( - EVENT_NAME, - NativeVoiceEvent::CleanShutdown { - session_id, - revision, - }, - ); + if state.owner_matches_lifecycle(webview_window.label(), &session_id, expected_revision)? { + state + .stop_active_for_lifecycle(&app, &capture, &session_id, expected_revision) + .await?; } Ok(status(&app, &state)) } @@ -693,12 +1319,179 @@ fn native_owner_id(session_id: &str) -> String { format!("native-voice:{session_id}") } +fn refresh_microphone_claim( + capture: &VoiceCaptureState, + window_label: &str, + renderer_id: &str, + renderer_epoch: u64, + owner_id: &str, + microphone_claimed: &mut bool, +) -> Result<(), String> { + let claimed_after_wait = capture.claim_microphone( + window_label.to_string(), + renderer_id.to_string(), + renderer_epoch, + owner_id.to_string(), + )?; + *microphone_claimed |= claimed_after_wait; + Ok(()) +} + impl NativeVoiceState { + pub async fn stop_active( + &self, + app: &AppHandle, + capture: &VoiceCaptureState, + ) -> Result<(), String> { + self.stop_active_inner(app, capture, None).await.map(|_| ()) + } + + pub async fn stop_active_for_lifecycle( + &self, + app: &AppHandle, + capture: &VoiceCaptureState, + expected_session_id: &str, + expected_revision: u64, + ) -> Result { + self.stop_active_inner( + app, + capture, + Some((expected_session_id, expected_revision, None)), + ) + .await + } + + pub async fn stop_active_if_lifecycle( + &self, + app: &AppHandle, + capture: &VoiceCaptureState, + expected_session_id: &str, + expected_revision: u64, + failure_message: &str, + ) -> Result { + self.stop_active_inner( + app, + capture, + Some(( + expected_session_id, + expected_revision, + Some(failure_message), + )), + ) + .await + } + + async fn stop_active_inner( + &self, + app: &AppHandle, + capture: &VoiceCaptureState, + expected_lifecycle: Option<(&str, u64, Option<&str>)>, + ) -> Result { + let _stop_guard = self.stop_serial.lock().await; + let failure_message = expected_lifecycle.and_then(|(_, _, message)| message); + let completion = self + .stop_lifecycle_locked( + expected_lifecycle.map(|(session_id, revision, _)| (session_id, revision)), + ) + .await?; + let Some(StopCompletion { + session_id, + controls_revision, + next_revision, + owner, + owner_id, + }) = completion + else { + return Ok(false); + }; + if let Some(failure_message) = failure_message { + let failure_event = NativeVoiceEvent::Error { + session_id: Some(session_id.clone()), + message: failure_message.to_string(), + revision: next_revision, + terminal: true, + }; + if let Some(target) = app.get_webview_window(&owner.window_label) { + let _ = target.emit(EVENT_NAME, failure_event.clone()); + } + super::voice_buddy::emit(app, failure_event); + } + self.microphone_muted.store(false, Ordering::SeqCst); + capture.release_owner(&owner.window_label, &owner_id); + let shutdown_event = NativeVoiceEvent::CleanShutdown { + session_id, + revision: next_revision, + }; + if let Some(target) = app.get_webview_window(&owner.window_label) { + let _ = target.emit(EVENT_NAME, shutdown_event.clone()); + } + super::voice_buddy::dismiss_after_terminal_event(app, controls_revision, shutdown_event); + super::voice_buddy::restore_hidden_owner(app, &owner.window_label); + Ok(true) + } + + #[cfg(test)] + async fn stop_lifecycle( + &self, + expected_lifecycle: Option<(&str, u64)>, + ) -> Result, String> { + let _stop_guard = self.stop_serial.lock().await; + self.stop_lifecycle_locked(expected_lifecycle).await + } + + async fn stop_lifecycle_locked( + &self, + expected_lifecycle: Option<(&str, u64)>, + ) -> Result, String> { + let Some((session_id, revision, pipeline, owner)) = + self.take_stop_snapshot(expected_lifecycle)? + else { + return Ok(None); + }; + // Keep the lifecycle current through the bounded shutdown window so a + // cooperative worker can flush its final utterance durably. A worker + // that misses the deadline is detached; its revision-bound late events + // are discarded rather than leaking into a replacement lifecycle. + if let Some(pipeline) = pipeline { + shutdown_pipeline(pipeline).await; + } + let (stopped, next_revision) = { + let mut runtime = self + .runtime + .lock() + .map_err(|_| "native voice state lock was poisoned".to_string())?; + let stopped = runtime.revision == revision && runtime.session_id == session_id; + if stopped { + native_input_mute::stop(&self.input_muted); + runtime.native_microphone_mute_control = false; + runtime.session_id = None; + runtime.lifecycle_id = None; + runtime.owner = None; + runtime.revision = runtime.revision.wrapping_add(1); + } + (stopped, runtime.revision) + }; + if !stopped { + return Ok(None); + } + let (Some(session_id), Some((owner, owner_id))) = (session_id, owner) else { + return Ok(None); + }; + Ok(Some(StopCompletion { + session_id, + controls_revision: revision, + next_revision, + owner, + owner_id, + })) + } + pub async fn stop_for_model_removal( &self, app: &AppHandle, capture: &VoiceCaptureState, ) -> Result<(), String> { + let _stop_guard = self.stop_serial.lock().await; let (session_id, revision, pipeline, owner) = { let mut runtime = self .runtime @@ -729,65 +1522,137 @@ impl NativeVoiceState { } runtime.revision }; + self.microphone_muted.store(false, Ordering::SeqCst); if let (Some(owner), Some(session_id)) = (owner, session_id) { capture.release_owner(&owner.window_label, &native_owner_id(&session_id)); + let shutdown_event = NativeVoiceEvent::CleanShutdown { + session_id, + revision: next_revision, + }; if let Some(window) = app.get_webview_window(&owner.window_label) { - let _ = window.emit( - EVENT_NAME, - NativeVoiceEvent::CleanShutdown { - session_id, - revision: next_revision, - }, - ); + let _ = window.emit(EVENT_NAME, shutdown_event.clone()); } + super::voice_buddy::dismiss_after_terminal_event(app, revision, shutdown_event); + super::voice_buddy::restore_hidden_owner(app, &owner.window_label); + } else { + super::voice_buddy::dismiss_stale_after_terminal(app, next_revision); } Ok(()) } - pub fn stop_for_window_destroyed(&self, window_label: &str) -> bool { - let (session_id, revision, pipeline) = { - let Ok(mut runtime) = self.runtime.lock() else { - return false; - }; - if runtime + pub fn capture_destroyed_owner_lifecycle(&self, window_label: &str) -> Option<(String, u64)> { + self.release_start_blocks_for_window(window_label); + let runtime = self.runtime.lock().ok()?; + if runtime + .owner + .as_ref() + .is_none_or(|owner| owner.window_label != window_label) + { + return None; + } + runtime + .session_id + .clone() + .map(|session_id| (session_id, runtime.revision)) + } + + #[cfg(test)] + async fn stop_destroyed_owner_lifecycle( + &self, + window_label: &str, + expected_session_id: &str, + expected_revision: u64, + ) -> Result, String> { + self.stop_destroyed_owner_lifecycle_with_cleanup( + window_label, + expected_session_id, + expected_revision, + |_| {}, + ) + .await + } + + async fn stop_destroyed_owner_lifecycle_with_cleanup( + &self, + window_label: &str, + expected_session_id: &str, + expected_revision: u64, + cleanup: impl FnOnce(&StopCompletion), + ) -> Result, String> { + let _stop_guard = self.stop_serial.lock().await; + let completion = self + .stop_destroyed_owner_lifecycle_locked( + window_label, + expected_session_id, + expected_revision, + ) + .await?; + if let Some(completion) = completion.as_ref() { + cleanup(completion); + } + Ok(completion) + } + + async fn stop_destroyed_owner_lifecycle_locked( + &self, + window_label: &str, + expected_session_id: &str, + expected_revision: u64, + ) -> Result, String> { + let owner_matches = { + let runtime = self + .runtime + .lock() + .map_err(|_| "native voice state lock was poisoned".to_string())?; + runtime .owner .as_ref() - .is_none_or(|owner| owner.window_label != window_label) - { - return false; - } - let pipeline = runtime.pipeline.take(); - if let Some(pipeline) = pipeline.as_ref() { - pipeline.signal_shutdown(); - } - native_input_mute::stop(&self.input_muted); - runtime.native_microphone_mute_control = false; - (runtime.session_id.clone(), runtime.revision, pipeline) + .is_some_and(|owner| owner.window_label == window_label) }; - if pipeline.is_none() { - if let Ok(mut runtime) = self.runtime.lock() { - if runtime.revision == revision && runtime.session_id == session_id { - runtime.session_id = None; - runtime.lifecycle_id = None; - runtime.owner = None; - runtime.revision = runtime.revision.wrapping_add(1); - } - } - return true; + if !owner_matches { + return Ok(None); } - let runtime = Arc::clone(&self.runtime); - tauri::async_runtime::spawn(async move { - shutdown_pipeline(pipeline.expect("pipeline checked above")).await; - if let Ok(mut runtime) = runtime.lock() { - if runtime.revision == revision && runtime.session_id == session_id { - runtime.session_id = None; - runtime.lifecycle_id = None; - runtime.owner = None; - runtime.revision = runtime.revision.wrapping_add(1); - } - } - }); - true + let completion = self + .stop_lifecycle_locked(Some((expected_session_id, expected_revision))) + .await?; + if completion.is_some() { + self.microphone_muted.store(false, Ordering::SeqCst); + } + Ok(completion) + } + + pub async fn stop_for_window_destroyed( + &self, + app: &AppHandle, + capture: &VoiceCaptureState, + pocket_voice: &super::pocket_voice::PocketVoiceState, + window_label: &str, + expected_session_id: &str, + expected_revision: u64, + ) -> Result { + let Some(completion) = self + .stop_destroyed_owner_lifecycle_with_cleanup( + window_label, + expected_session_id, + expected_revision, + |completion| { + capture.release_owner(&completion.owner.window_label, &completion.owner_id); + pocket_voice.stop_for_window_destroyed(); + }, + ) + .await? + else { + return Ok(false); + }; + super::voice_buddy::dismiss_after_terminal_event( + app, + completion.controls_revision, + NativeVoiceEvent::CleanShutdown { + session_id: completion.session_id, + revision: completion.next_revision, + }, + ); + Ok(true) } pub fn stop_for_app_exit(&self) { @@ -807,6 +1672,7 @@ impl NativeVoiceState { ) }; drop(pipeline); + self.microphone_muted.store(false, Ordering::SeqCst); if let Ok(mut runtime) = self.runtime.lock() { if runtime.revision == revision && runtime.session_id == session_id { runtime.session_id = None; @@ -818,37 +1684,45 @@ impl NativeVoiceState { } } -#[tauri::command] -pub fn set_native_voice_input_muted( - state: State<'_, NativeVoiceState>, - webview_window: WebviewWindow, - session_id: String, - revision: u64, - muted: bool, -) -> Result<(), String> { - let runtime = state - .runtime - .lock() - .map_err(|_| "native voice state lock was poisoned".to_string())?; - if !owns_native_mute_control(&runtime, webview_window.label(), &session_id, revision) { - return Err("Native microphone mute is unavailable for this conversation.".to_string()); +pub fn handle_voice_owner_window_destroyed(app: &AppHandle, window_label: &str) { + app.state::() + .release_window(window_label); + let destroyed_lifecycle = app + .state::() + .capture_destroyed_owner_lifecycle(window_label); + let app_for_native_close = app.clone(); + let label_for_native_close = window_label.to_string(); + if let Some((session_id, revision)) = destroyed_lifecycle { + tauri::async_runtime::spawn(async move { + let native_voice = app_for_native_close.state::(); + let capture = app_for_native_close.state::(); + let pocket_voice = + app_for_native_close.state::(); + match native_voice + .stop_for_window_destroyed( + &app_for_native_close, + capture.inner(), + pocket_voice.inner(), + &label_for_native_close, + &session_id, + revision, + ) + .await + { + Ok(true) => {} + Ok(false) => {} + Err(error) => { + log::error!("Failed to stop voice for destroyed owner window: {error}"); + } + } + }); } - native_input_mute::set_muted(&state.input_muted, &state.input_mute_epoch, muted) + app.state::() + .stop_for_window_destroyed(window_label); } -fn owns_native_mute_control( - runtime: &Runtime, - window_label: &str, - session_id: &str, - revision: u64, -) -> bool { - runtime.native_microphone_mute_control - && runtime.session_id.as_deref() == Some(session_id) - && runtime.revision == revision - && runtime - .owner - .as_ref() - .is_some_and(|owner| owner.window_label == window_label) +fn software_microphone_mute(native_microphone_mute_control: bool, muted: bool) -> bool { + !native_microphone_mute_control && muted } #[tauri::command] @@ -879,7 +1753,7 @@ fn push_audio_for_window( { return Err("Only the owning window may send native voice audio.".to_string()); } - if state.capture_is_suppressed() { + if state.capture_is_suppressed() || state.microphone_is_muted() { return Ok(()); } if let Some(pipeline) = runtime.pipeline.as_ref() { @@ -899,6 +1773,29 @@ fn enqueue_pending_transcript( evicted } +fn enqueue_transcript_if_active( + runtime: &Mutex, + pending: &Mutex>, + expected_session_id: &str, + expected_revision: u64, + transcript: PendingTranscript, +) -> Result<(bool, Option), String> { + let runtime = runtime + .lock() + .map_err(|_| "native voice state lock was poisoned".to_string())?; + if runtime.session_id.as_deref() != Some(expected_session_id) + || runtime.revision != expected_revision + { + return Ok((false, None)); + } + let mut pending = pending + .lock() + .map_err(|_| "pending transcript lock was poisoned".to_string())?; + let evicted = enqueue_pending_transcript(&mut pending, transcript); + Ok((true, evicted)) +} + +#[allow(clippy::too_many_arguments)] // Worker boundary keeps channel and mute lifecycle inputs explicit. fn stt_worker( model_dir: PathBuf, audio_rx: Receiver, @@ -907,6 +1804,7 @@ fn stt_worker( discard_on_shutdown: Arc, input_muted: Arc, input_mute_epoch: Arc, + shutdown_mute_epoch: Arc, ) { use rubato::{Fft, FixedSync, Resampler}; use sherpa_onnx::{OfflineRecognizer, OfflineRecognizerConfig}; @@ -952,11 +1850,8 @@ fn stt_worker( Err(mpsc::RecvTimeoutError::Timeout) => None, Err(mpsc::RecvTimeoutError::Disconnected) => break, }; - let shutting_down = shutdown.load(Ordering::Acquire); - if shutting_down && (discard_on_shutdown.load(Ordering::Acquire) || batch.is_none()) { - break; - } - let current_mute_epoch = input_mute_epoch.load(Ordering::Acquire); + let (shutting_down, current_mute_epoch) = + sample_effective_mute_epoch(&input_mute_epoch, &shutdown, &shutdown_mute_epoch); if current_mute_epoch != observed_mute_epoch { observed_mute_epoch = current_mute_epoch; if clear_buffered_audio( @@ -969,6 +1864,9 @@ fn stt_worker( let _ = event_tx.blocking_send(SttMessage::Speaking(false)); } } + if shutting_down && (discard_on_shutdown.load(Ordering::Acquire) || batch.is_none()) { + break; + } if !shutting_down && input_muted.load(Ordering::Acquire) { continue; } @@ -1008,6 +1906,8 @@ fn stt_worker( &event_tx, None, &input_mute_epoch, + &shutdown, + &shutdown_mute_epoch, observed_mute_epoch, ); speech.clear(); @@ -1024,6 +1924,8 @@ fn stt_worker( &event_tx, None, &input_mute_epoch, + &shutdown, + &shutdown_mute_epoch, observed_mute_epoch, ); speech.clear(); @@ -1043,12 +1945,28 @@ fn stt_worker( &event_tx, Some(delivered_tx), &input_mute_epoch, + &shutdown, + &shutdown_mute_epoch, observed_mute_epoch, ); let _ = delivered_rx.recv_timeout(Duration::from_secs(5)); } } +fn sample_effective_mute_epoch( + input_mute_epoch: &AtomicU64, + shutdown: &AtomicBool, + shutdown_mute_epoch: &AtomicU64, +) -> (bool, u64) { + let live_mute_epoch = input_mute_epoch.load(Ordering::Acquire); + let shutting_down = shutdown.load(Ordering::Acquire); + if shutting_down { + (true, shutdown_mute_epoch.load(Ordering::Acquire)) + } else { + (false, live_mute_epoch) + } +} + fn clear_buffered_audio( input_48k: &mut Vec, leftover_16k: &mut Vec, @@ -1075,12 +1993,15 @@ fn resample(resampler: &mut rubato::Fft, samples: &[f32]) -> Vec { .unwrap_or_default() } +#[allow(clippy::too_many_arguments)] // Recognition needs both live and shutdown mute clocks. fn flush_speech( speech: &[f32], recognizer: &sherpa_onnx::OfflineRecognizer, event_tx: &tokio_mpsc::Sender, delivered: Option>, input_mute_epoch: &AtomicU64, + shutdown: &AtomicBool, + shutdown_mute_epoch: &AtomicU64, expected_mute_epoch: u64, ) { if speech.is_empty() { @@ -1093,7 +2014,9 @@ fn flush_speech( .get_result() .map(|result| result.text.trim().to_string()) .unwrap_or_default(); - if input_mute_epoch.load(Ordering::Acquire) != expected_mute_epoch { + if sample_effective_mute_epoch(input_mute_epoch, shutdown, shutdown_mute_epoch).1 + != expected_mute_epoch + { if let Some(delivered) = delivered { let _ = delivered.send(()); } @@ -1126,21 +2049,10 @@ mod tests { use super::*; #[test] - fn native_mute_control_is_bound_to_window_session_and_revision() { - let runtime = Runtime { - session_id: Some("session-1".to_string()), - revision: 4, - owner: Some(RuntimeOwner { - window_label: "main".to_string(), - }), - native_microphone_mute_control: true, - ..Runtime::default() - }; - - assert!(owns_native_mute_control(&runtime, "main", "session-1", 4)); - assert!(!owns_native_mute_control(&runtime, "other", "session-1", 4)); - assert!(!owns_native_mute_control(&runtime, "main", "session-2", 4)); - assert!(!owns_native_mute_control(&runtime, "main", "session-1", 5)); + fn native_mute_control_does_not_latch_the_software_fallback() { + assert!(!software_microphone_mute(true, true)); + assert!(software_microphone_mute(false, true)); + assert!(!software_microphone_mute(false, false)); } #[test] @@ -1162,32 +2074,415 @@ mod tests { } #[test] - fn window_destroy_stops_only_its_owned_voice_lifecycle() { + fn assistant_activity_is_bound_to_the_exact_voice_lifecycle() { let state = NativeVoiceState::default(); { let mut runtime = state.runtime.lock().expect("lock native runtime"); runtime.session_id = Some("session-1".to_string()); - runtime.lifecycle_id = Some("lifecycle-1".to_string()); + runtime.revision = 7; runtime.owner = Some(RuntimeOwner { - window_label: "session-window".to_string(), + window_label: "main".to_string(), }); } - assert!(!state.stop_for_window_destroyed("other-window")); assert_eq!( state - .runtime - .lock() - .expect("lock native runtime") - .session_id - .as_deref(), - Some("session-1") + .assistant_activity_target("main", "session-1", 7) + .expect("current activity target"), + Some(("main".to_string(), 7)), + ); + assert_eq!( + state + .assistant_activity_target("main", "session-1", 6) + .expect("stale activity is ignored"), + None, ); + assert!(state + .assistant_activity_target("session:other", "session-1", 7) + .is_err()); - assert!(state.stop_for_window_destroyed("session-window")); - let runtime = state.runtime.lock().expect("lock native runtime"); - assert!(runtime.session_id.is_none()); + state.runtime.lock().expect("lock native runtime").revision = 8; + assert_eq!( + state + .assistant_activity_target("main", "session-1", 7) + .expect("prior lifecycle activity is ignored after restart"), + None, + ); + } + + #[test] + fn stale_controls_watchdog_cannot_take_a_restarted_voice_lifecycle() { + let state = NativeVoiceState::default(); + { + let mut runtime = state.runtime.lock().expect("lock native runtime"); + runtime.session_id = Some("session-b".to_string()); + runtime.revision = 8; + runtime.owner = Some(RuntimeOwner { + window_label: "main".to_string(), + }); + } + + assert!(state + .take_stop_snapshot(Some(("session-a", 7))) + .expect("stale watchdog check") + .is_none()); + assert_eq!( + state.active_session_lifecycle_target(), + Some(("session-b".to_string(), "main".to_string(), 8)), + ); + } + + #[tokio::test] + async fn concurrent_stops_flush_one_final_transcript_once() { + let state = NativeVoiceState::default(); + let (audio_tx, _audio_rx) = mpsc::sync_channel(1); + let shutdown = Arc::new(AtomicBool::new(false)); + let worker_shutdown = Arc::clone(&shutdown); + let pending = Arc::clone(&state.pending); + let worker = thread::spawn(move || { + while !worker_shutdown.load(Ordering::Acquire) { + thread::yield_now(); + } + pending + .lock() + .expect("lock pending transcripts") + .push_back(PendingTranscript { + session_id: "session-1".to_string(), + lifecycle_id: "lifecycle-1".to_string(), + id: "final-1".to_string(), + text: "final words".to_string(), + revision: 4, + delivery_attempts: 0, + }); + }); + { + let mut runtime = state.runtime.lock().expect("lock native runtime"); + runtime.session_id = Some("session-1".to_string()); + runtime.lifecycle_id = Some("lifecycle-1".to_string()); + runtime.revision = 4; + runtime.owner = Some(RuntimeOwner { + window_label: "main".to_string(), + }); + runtime.pipeline = Some(SttPipeline { + audio_tx, + audio_seen: AtomicBool::new(false), + shutdown, + discard_on_shutdown: Arc::new(AtomicBool::new(false)), + input_muted: Arc::new(AtomicBool::new(false)), + input_mute_epoch: Arc::new(AtomicU64::new(0)), + shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), + thread: Some(worker), + }); + } + + let first_state = state.clone(); + let second_state = state.clone(); + let first = tokio::spawn(async move { + first_state + .stop_lifecycle(Some(("session-1", 4))) + .await + .expect("first stop") + }); + let second = tokio::spawn(async move { + second_state + .stop_lifecycle(Some(("session-1", 4))) + .await + .expect("second stop") + }); + let (first, second) = tokio::join!(first, second); + let completions = [first.expect("join first"), second.expect("join second")]; + + assert_eq!( + completions.iter().filter(|result| result.is_some()).count(), + 1 + ); + let pending = state.pending.lock().expect("lock pending transcripts"); + assert_eq!(pending.len(), 1); + assert_eq!( + pending.front().map(|item| item.id.as_str()), + Some("final-1") + ); + } + + #[tokio::test] + async fn non_cooperative_worker_cannot_block_stop_or_replacement_lifecycle() { + let state = NativeVoiceState::default(); + let (audio_tx, _audio_rx) = mpsc::sync_channel(1); + let worker_release = Arc::new(AtomicBool::new(false)); + let release = Arc::clone(&worker_release); + let worker = thread::spawn(move || { + while !release.load(Ordering::Acquire) { + thread::sleep(Duration::from_millis(5)); + } + }); + { + let mut runtime = state.runtime.lock().expect("lock native runtime"); + runtime.session_id = Some("session-1".to_string()); + runtime.lifecycle_id = Some("lifecycle-1".to_string()); + runtime.revision = 4; + runtime.owner = Some(RuntimeOwner { + window_label: "main".to_string(), + }); + runtime.pipeline = Some(SttPipeline { + audio_tx, + audio_seen: AtomicBool::new(false), + shutdown: Arc::new(AtomicBool::new(false)), + discard_on_shutdown: Arc::new(AtomicBool::new(false)), + input_muted: Arc::new(AtomicBool::new(false)), + input_mute_epoch: Arc::new(AtomicU64::new(0)), + shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), + thread: Some(worker), + }); + } + + let completion = tokio::time::timeout( + Duration::from_millis(500), + state.stop_lifecycle(Some(("session-1", 4))), + ) + .await + .expect("stop is bounded") + .expect("stop succeeds"); + assert!(completion.is_some()); + + { + let mut runtime = state.runtime.lock().expect("lock native runtime"); + assert!(runtime.session_id.is_none()); + runtime.session_id = Some("session-2".to_string()); + runtime.lifecycle_id = Some("lifecycle-2".to_string()); + runtime.revision = 6; + runtime.owner = Some(RuntimeOwner { + window_label: "main".to_string(), + }); + } + assert_eq!( + state.active_session_lifecycle_target(), + Some(("session-2".to_string(), "main".to_string(), 6)) + ); + assert!(state + .take_stop_snapshot(Some(("session-1", 4))) + .expect("late stale lifecycle is ignored") + .is_none()); + let (accepted, evicted) = enqueue_transcript_if_active( + &state.runtime, + &state.pending, + "session-1", + 4, + PendingTranscript { + session_id: "session-1".to_string(), + lifecycle_id: "lifecycle-1".to_string(), + id: "late-final".to_string(), + text: "late words".to_string(), + revision: 4, + delivery_attempts: 0, + }, + ) + .expect("late transcript lifecycle check"); + assert!(!accepted); + assert!(evicted.is_none()); + assert!(state.pending.lock().expect("lock pending queue").is_empty()); + + worker_release.store(true, Ordering::Release); + } + + #[test] + fn microphone_mute_is_authorized_and_lifecycle_bound() { + let state = NativeVoiceState::default(); + { + let mut runtime = state.runtime.lock().expect("lock native runtime"); + runtime.session_id = Some("session-b".to_string()); + runtime.revision = 8; + runtime.owner = Some(RuntimeOwner { + window_label: "main".to_string(), + }); + } + + assert_eq!( + state + .set_microphone_muted_target("main", "session-a", 7, true) + .expect("stale mute is ignored"), + None, + ); + assert!(!state.microphone_is_muted()); + assert!(state + .set_microphone_muted_target("other", "session-b", 8, true) + .is_err()); + assert!(!state.microphone_is_muted()); + assert_eq!( + state + .set_microphone_muted_target( + super::super::voice_buddy::WINDOW_LABEL, + "session-b", + 8, + true, + ) + .expect("floating controls can mute"), + Some("main".to_string()), + ); + assert!(state.microphone_is_muted()); + } + + #[test] + fn owner_stop_authorization_is_lifecycle_bound() { + let state = NativeVoiceState::default(); + { + let mut runtime = state.runtime.lock().expect("lock native runtime"); + runtime.session_id = Some("session-b".to_string()); + runtime.revision = 8; + runtime.owner = Some(RuntimeOwner { + window_label: "main".to_string(), + }); + } + + assert!(!state + .owner_matches_lifecycle("main", "session-a", 7) + .expect("stale owner stop is ignored")); + assert!(state + .owner_matches_lifecycle("other", "session-b", 8) + .is_err()); + assert!(state + .owner_matches_lifecycle("main", "session-b", 8) + .expect("owner can stop current lifecycle")); + } + + #[tokio::test] + async fn window_destroy_stops_only_its_owned_voice_lifecycle() { + let state = NativeVoiceState::default(); + state.microphone_muted.store(true, Ordering::SeqCst); + { + let mut runtime = state.runtime.lock().expect("lock native runtime"); + runtime.session_id = Some("session-1".to_string()); + runtime.lifecycle_id = Some("lifecycle-1".to_string()); + runtime.owner = Some(RuntimeOwner { + window_label: "session-window".to_string(), + }); + } + + assert!(state + .capture_destroyed_owner_lifecycle("other-window") + .is_none()); + assert_eq!( + state + .runtime + .lock() + .expect("lock native runtime") + .session_id + .as_deref(), + Some("session-1") + ); + + let completion = state + .stop_destroyed_owner_lifecycle("session-window", "session-1", 0) + .await + .expect("stop destroyed owner") + .expect("owned lifecycle stops"); + assert_eq!(completion.controls_revision, 0); + assert_eq!(completion.next_revision, 1); + let runtime = state.runtime.lock().expect("lock native runtime"); + assert!(runtime.session_id.is_none()); assert!(runtime.owner.is_none()); + assert!(!state.microphone_muted.load(Ordering::SeqCst)); + } + + #[test] + fn floating_controls_follow_only_the_exact_owner_lifecycle() { + let state = NativeVoiceState::default(); + { + let mut runtime = state.runtime.lock().expect("lock native runtime"); + runtime.session_id = Some("session-1".to_string()); + runtime.revision = 4; + runtime.owner = Some(RuntimeOwner { + window_label: "main".to_string(), + }); + runtime.controls_suppressed = true; + } + + assert_eq!( + state + .controls_visibility_target("session-1", 4) + .expect("read control visibility"), + Some(ControlsVisibilityTarget { + suppressed: true, + generation: 0, + }), + ); + assert_eq!( + state + .acknowledge_controls_visibility("session-1", 4, 0) + .expect("acknowledge controls visibility"), + ControlsVisibilityAcknowledgement::Ready, + ); + assert!(state.controls_ready_for("session-1", 4)); + assert_eq!( + state + .acknowledge_controls_visibility("session-1", 3, 0) + .expect("stale readiness is ignored"), + ControlsVisibilityAcknowledgement::Inactive, + ); + assert_eq!( + state + .set_controls_suppressed("main", "session-1", 4, false) + .expect("owner reveals controls"), + Some((true, true)), + ); + state.rollback_controls_suppression("session-1", 4, false, true); + assert_eq!( + state + .controls_visibility_target("session-1", 4) + .expect("failed visibility is rolled back"), + Some(ControlsVisibilityTarget { + suppressed: true, + generation: 2, + }), + ); + assert_eq!( + state + .set_controls_suppressed("main", "session-1", 3, true) + .expect("stale lifecycle is ignored"), + None, + ); + assert!(state + .set_controls_suppressed("other-window", "session-1", 4, true) + .is_err()); + } + + #[test] + fn floating_controls_remain_ready_while_visibility_converges() { + let state = NativeVoiceState::default(); + { + let mut runtime = state.runtime.lock().expect("lock native runtime"); + runtime.session_id = Some("session-1".to_string()); + runtime.revision = 4; + runtime.owner = Some(RuntimeOwner { + window_label: "main".to_string(), + }); + runtime.controls_suppressed = true; + } + + let mut target = state + .controls_visibility_target("session-1", 4) + .expect("read initial visibility") + .expect("active lifecycle"); + for _ in 0..4 { + state + .set_controls_suppressed("main", "session-1", 4, !target.suppressed) + .expect("change visibility"); + target = match state + .acknowledge_controls_visibility("session-1", 4, target.generation) + .expect("acknowledge superseded visibility") + { + ControlsVisibilityAcknowledgement::Superseded(next_target) => next_target, + acknowledgement => panic!("expected superseded target, got {acknowledgement:?}"), + }; + assert!(state.controls_ready_for("session-1", 4)); + } + + assert_eq!( + state + .acknowledge_controls_visibility("session-1", 4, target.generation) + .expect("acknowledge newest visibility"), + ControlsVisibilityAcknowledgement::Ready, + ); + assert!(state.controls_ready_for("session-1", 4)); } #[test] @@ -1199,6 +2494,7 @@ mod tests { discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::new(AtomicBool::new(false)), input_mute_epoch: Arc::new(AtomicU64::new(0)), + shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), audio_seen: AtomicBool::new(false), thread: None, }; @@ -1215,6 +2511,7 @@ mod tests { discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::new(AtomicBool::new(false)), input_mute_epoch: Arc::new(AtomicU64::new(0)), + shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), audio_seen: AtomicBool::new(false), thread: None, }; @@ -1237,6 +2534,7 @@ mod tests { discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::clone(&input_muted), input_mute_epoch: Arc::clone(&input_mute_epoch), + shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), audio_seen: AtomicBool::new(false), thread: None, }; @@ -1264,6 +2562,7 @@ mod tests { discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::new(AtomicBool::new(false)), input_mute_epoch: Arc::clone(&input_mute_epoch), + shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), audio_seen: AtomicBool::new(false), thread: None, }; @@ -1308,6 +2607,7 @@ mod tests { discard_on_shutdown: Arc::clone(&discard_on_shutdown), input_muted: Arc::clone(&input_muted), input_mute_epoch: Arc::new(AtomicU64::new(1)), + shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), audio_seen: AtomicBool::new(false), thread: None, }; @@ -1322,21 +2622,42 @@ mod tests { fn unmuted_shutdown_keeps_final_utterance_after_later_mute_event() { let (sender, _receiver) = mpsc::sync_channel(1); let input_muted = Arc::new(AtomicBool::new(false)); + let input_mute_epoch = Arc::new(AtomicU64::new(0)); let discard_on_shutdown = Arc::new(AtomicBool::new(false)); let mut pipeline = SttPipeline { audio_tx: sender, shutdown: Arc::new(AtomicBool::new(false)), discard_on_shutdown: Arc::clone(&discard_on_shutdown), input_muted: Arc::clone(&input_muted), - input_mute_epoch: Arc::new(AtomicU64::new(0)), + input_mute_epoch: Arc::clone(&input_mute_epoch), + shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), audio_seen: AtomicBool::new(false), thread: None, }; pipeline.begin_shutdown(); input_muted.store(true, Ordering::Release); + input_mute_epoch.fetch_add(1, Ordering::AcqRel); + pipeline.signal_shutdown(); assert!(!discard_on_shutdown.load(Ordering::Acquire)); + assert_eq!(pipeline.shutdown_mute_epoch.load(Ordering::Acquire), 0); + assert_eq!( + sample_effective_mute_epoch( + &input_mute_epoch, + &AtomicBool::new(false), + &pipeline.shutdown_mute_epoch, + ), + (false, 1), + ); + assert_eq!( + sample_effective_mute_epoch( + &input_mute_epoch, + &pipeline.shutdown, + &pipeline.shutdown_mute_epoch, + ), + (true, 0), + ); } #[test] @@ -1354,6 +2675,7 @@ mod tests { discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::new(AtomicBool::new(false)), input_mute_epoch: Arc::new(AtomicU64::new(0)), + shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), audio_seen: AtomicBool::new(false), thread: None, }); @@ -1361,6 +2683,16 @@ mod tests { assert!(push_audio_for_window(&state, "other-window", vec![0; 4]).is_err()); assert!(receiver.try_recv().is_err()); + state.microphone_muted.store(true, Ordering::SeqCst); + push_audio_for_window(&state, "owner-window", vec![0; 4]) + .expect("muted owner audio is ignored"); + assert!(receiver.try_recv().is_err()); + state.microphone_muted.store(false, Ordering::SeqCst); + state.input_muted.store(true, Ordering::SeqCst); + push_audio_for_window(&state, "owner-window", vec![0; 4]) + .expect("native-muted owner audio is ignored"); + assert!(receiver.try_recv().is_err()); + state.input_muted.store(false, Ordering::SeqCst); push_audio_for_window(&state, "owner-window", vec![0; 4]).expect("owner can send audio"); assert_eq!( receiver.try_recv().expect("owner audio queued").bytes, @@ -1369,8 +2701,9 @@ mod tests { } #[tokio::test] - async fn window_destroy_schedules_blocked_worker_join_off_callback() { + async fn window_destroy_awaits_bounded_worker_shutdown_off_callback() { let state = NativeVoiceState::default(); + state.microphone_muted.store(true, Ordering::SeqCst); let (sender, _receiver) = mpsc::sync_channel(1); let shutdown = Arc::new(AtomicBool::new(false)); let worker = thread::spawn(|| thread::sleep(Duration::from_millis(250))); @@ -1386,16 +2719,21 @@ mod tests { discard_on_shutdown: Arc::new(AtomicBool::new(false)), input_muted: Arc::new(AtomicBool::new(false)), input_mute_epoch: Arc::new(AtomicU64::new(0)), + shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), audio_seen: AtomicBool::new(false), thread: Some(worker), }); } - let started = std::time::Instant::now(); - assert!(state.stop_for_window_destroyed("owner-window")); - assert!(started.elapsed() < Duration::from_millis(50)); + let completion = state + .stop_destroyed_owner_lifecycle("owner-window", "session-1", 0) + .await + .expect("stop destroyed owner") + .expect("owned lifecycle stops"); + assert_eq!(completion.controls_revision, 0); + assert_eq!(completion.next_revision, 1); assert!(shutdown.load(Ordering::Acquire)); - tokio::time::sleep(Duration::from_millis(300)).await; + assert!(!state.microphone_muted.load(Ordering::SeqCst)); assert!(state .runtime .lock() @@ -1404,6 +2742,179 @@ mod tests { .is_none()); } + #[tokio::test] + async fn owner_destroy_waits_for_start_serialization_before_stopping_exact_lifecycle() { + let state = NativeVoiceState::default(); + let startup_guard = state.stop_serial.lock().await; + { + let mut runtime = state.runtime.lock().expect("lock native runtime"); + runtime.session_id = Some("session-a".to_string()); + runtime.lifecycle_id = Some("lifecycle-a".to_string()); + runtime.revision = 4; + runtime.owner = Some(RuntimeOwner { + window_label: "owner-window".to_string(), + }); + } + + let close_state = state.clone(); + let close = tokio::spawn(async move { + close_state + .stop_destroyed_owner_lifecycle("owner-window", "session-a", 4) + .await + .expect("stop destroyed owner") + }); + tokio::task::yield_now().await; + assert_eq!( + state.active_session_lifecycle_target(), + Some(("session-a".to_string(), "owner-window".to_string(), 4,)) + ); + + { + let mut runtime = state.runtime.lock().expect("lock native runtime"); + runtime.session_id = Some("session-b".to_string()); + runtime.lifecycle_id = Some("lifecycle-b".to_string()); + runtime.revision = 6; + runtime.owner = Some(RuntimeOwner { + window_label: "owner-window".to_string(), + }); + } + drop(startup_guard); + assert!(close.await.expect("join owner close").is_none()); + assert!(state + .take_stop_snapshot(Some(("session-a", 4))) + .expect("stale A cleanup is rejected") + .is_none()); + assert_eq!( + state.active_session_lifecycle_target(), + Some(("session-b".to_string(), "owner-window".to_string(), 6,)) + ); + } + + #[tokio::test] + async fn owner_destroy_keeps_cleanup_inside_start_stop_serialization() { + let state = NativeVoiceState::default(); + { + let mut runtime = state.runtime.lock().expect("lock native runtime"); + runtime.session_id = Some("session-a".to_string()); + runtime.lifecycle_id = Some("lifecycle-a".to_string()); + runtime.revision = 4; + runtime.owner = Some(RuntimeOwner { + window_label: "owner-window".to_string(), + }); + } + + let cleanup_ran = AtomicBool::new(false); + let completion = state + .stop_destroyed_owner_lifecycle_with_cleanup("owner-window", "session-a", 4, |_| { + assert!(state.stop_serial.try_lock().is_err()); + cleanup_ran.store(true, Ordering::SeqCst); + }) + .await + .expect("stop destroyed owner") + .expect("owned lifecycle stops"); + + assert_eq!(completion.next_revision, 5); + assert!(cleanup_ran.load(Ordering::SeqCst)); + } + + #[test] + fn queued_start_reclaims_microphone_after_destroyed_owner_cleanup() { + let capture = VoiceCaptureState::default(); + let first_epoch = capture.register_renderer_for_test("owner-window", "renderer-a"); + let owner_id = native_owner_id("session-a"); + assert!(capture + .claim_microphone( + "owner-window".to_string(), + "renderer-a".to_string(), + first_epoch, + owner_id.clone(), + ) + .expect("initial lifecycle claims microphone")); + + let second_epoch = capture.register_renderer_for_test("owner-window", "renderer-b"); + let mut replacement_claimed = capture + .claim_microphone( + "owner-window".to_string(), + "renderer-b".to_string(), + second_epoch, + owner_id.clone(), + ) + .expect("replacement renderer inherits the native claim"); + assert!(!replacement_claimed); + assert!(capture.release_owner("owner-window", &owner_id)); + + refresh_microphone_claim( + &capture, + "owner-window", + "renderer-b", + second_epoch, + &owner_id, + &mut replacement_claimed, + ) + .expect("queued replacement reclaims after serialized cleanup"); + + assert!(replacement_claimed); + assert!(!capture + .claim_microphone( + "owner-window".to_string(), + "renderer-b".to_string(), + second_epoch, + owner_id, + ) + .expect("replacement keeps the microphone claim")); + } + + #[test] + fn archive_start_blocks_are_process_wide_and_window_scoped() { + let state = NativeVoiceState::default(); + let shared_state = state.clone(); + let first_token = state + .block_starts( + "session-1".to_string(), + "main".to_string(), + "renderer-1".to_string(), + 1, + ) + .expect("block starts from main"); + let second_token = shared_state + .block_starts( + "session-1".to_string(), + "session-window".to_string(), + "renderer-2".to_string(), + 1, + ) + .expect("block starts from session window"); + + assert!(shared_state.starts_blocked("session-1")); + state + .release_start_block("session-1", &first_token) + .expect("release main block"); + assert!(shared_state.starts_blocked("session-1")); + + shared_state.release_start_blocks_for_window("session-window"); + assert!(!state.starts_blocked("session-1")); + state + .release_start_block("session-1", &second_token) + .expect("stale release is harmless"); + } + + #[test] + fn renderer_replacement_clears_abandoned_archive_start_blocks() { + let state = NativeVoiceState::default(); + state + .block_starts( + "session-1".to_string(), + "main".to_string(), + "renderer-1".to_string(), + 1, + ) + .expect("block starts"); + + state.release_start_blocks_for_replaced_renderer("main", "renderer-2", 2); + + assert!(!state.starts_blocked("session-1")); + } + #[test] fn retained_transcripts_are_capped_and_fail_terminally() { let mut pending = VecDeque::new(); @@ -1467,5 +2978,14 @@ mod tests { "deliveryAttempts": 0, }), ); + + assert_eq!( + serde_json::to_value(NativeVoiceEvent::ControlsDismissed { revision: 3 }) + .expect("serialize controls-dismissed event"), + serde_json::json!({ + "type": "controlsDismissed", + "revision": 3, + }), + ); } } diff --git a/src-tauri/src/commands/notifications.rs b/src-tauri/src/commands/notifications.rs index e657294fb..4defc296d 100644 --- a/src-tauri/src/commands/notifications.rs +++ b/src-tauri/src/commands/notifications.rs @@ -2,9 +2,9 @@ use std::path::{Component, Path, PathBuf}; #[cfg(target_os = "macos")] use std::process::Command; -use tauri::AppHandle; #[cfg(target_os = "macos")] use tauri::Manager; +use tauri::{AppHandle, State}; struct CompletionNotificationRequest { session_id: String, @@ -27,10 +27,14 @@ struct CompletionNotificationState { #[tauri::command] pub fn show_completion_notification( app: AppHandle, + voice_state: State<'_, crate::commands::native_voice::NativeVoiceState>, session_id: String, body: String, sound: Option, ) -> Result<(), String> { + if voice_state.is_active_for_session(&session_id) { + return Ok(()); + } show_platform_completion_notification( app, CompletionNotificationRequest { @@ -41,6 +45,14 @@ pub fn show_completion_notification( ) } +#[tauri::command] +pub fn should_suppress_completion_notification( + voice_state: State<'_, crate::commands::native_voice::NativeVoiceState>, + session_id: String, +) -> bool { + voice_state.is_active_for_session(&session_id) +} + #[cfg(target_os = "macos")] pub fn init_completion_notifications(app: &tauri::AppHandle) -> Result<(), String> { macos_completion::init_completion_notifications(app) diff --git a/src-tauri/src/commands/voice_buddy.rs b/src-tauri/src/commands/voice_buddy.rs new file mode 100644 index 000000000..411b4541f --- /dev/null +++ b/src-tauri/src/commands/voice_buddy.rs @@ -0,0 +1,614 @@ +//! Cross-platform always-on-top controls for the process-wide voice conversation. + +use serde::{Deserialize, Serialize}; +use tauri::{ + AppHandle, Emitter, Manager, PhysicalPosition, WebviewUrl, WebviewWindow, WebviewWindowBuilder, + WindowEvent, +}; + +use super::{ + native_voice::{ControlsVisibilityAcknowledgement, NativeVoiceEvent, NativeVoiceState}, + voice_capture::VoiceCaptureState, +}; + +pub const WINDOW_LABEL: &str = "voice-buddy"; +pub const OPEN_SESSION_EVENT: &str = "voice-conversation:open-session"; +const WINDOW_WIDTH: f64 = 176.0; +const WINDOW_HEIGHT: f64 = 56.0; +const SCREEN_INSET: i32 = 24; + +fn controls_url(revision: u64) -> String { + format!("index.html?voiceBuddy=1&voiceRevision={revision}") +} + +fn controls_revision_from_url(url: &str) -> Option { + url.split_once('?')? + .1 + .split('#') + .next()? + .split('&') + .find_map(|pair| pair.strip_prefix("voiceRevision=")?.parse().ok()) +} + +fn controls_window_matches_revision(url: &str, controls_revision: u64) -> bool { + controls_revision_from_url(url) == Some(controls_revision) +} + +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct OpenSessionPayload { + session_id: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ControlsVisibilityRequest { + session_id: String, + expected_revision: u64, + suppressed: bool, + renderer_id: String, + renderer_epoch: u64, +} + +fn focus_window(window: &WebviewWindow) { + let _ = window.show(); + let _ = window.unminimize(); + let _ = window.set_focus(); +} + +fn require_controls_window(window_label: &str) -> Result<(), String> { + if window_label != WINDOW_LABEL { + return Err("Only the floating voice controls can use this command.".to_string()); + } + Ok(()) +} + +fn should_restore_owner(owner_visible: bool) -> bool { + cfg!(not(target_os = "macos")) && !owner_visible +} + +pub fn restore_hidden_owner(app: &AppHandle, owner_window_label: &str) { + if let Some(owner) = app + .get_webview_window(owner_window_label) + .filter(|owner| should_restore_owner(owner.is_visible().unwrap_or(false))) + { + focus_window(&owner); + } +} + +pub fn open_active_session(app: &AppHandle) -> Result<(), String> { + let state = app.state::(); + let Some((session_id, owner_window_label)) = state.active_session_target() else { + return Ok(()); + }; + let window = app + .get_webview_window(&owner_window_label) + .ok_or_else(|| "The voice session window is no longer available.".to_string())?; + focus_window(&window); + if owner_window_label == "main" { + window + .emit(OPEN_SESSION_EVENT, OpenSessionPayload { session_id }) + .map_err(|error| error.to_string())?; + } + Ok(()) +} + +fn position_near_bottom_right(app: &AppHandle, window: &WebviewWindow) { + let owner_monitor = app + .state::() + .active_session_target() + .and_then(|(_, label)| app.get_webview_window(&label)) + .and_then(|owner| owner.current_monitor().ok().flatten()); + let Some(monitor) = owner_monitor.or_else(|| window.primary_monitor().ok().flatten()) else { + return; + }; + let work_area = monitor.work_area(); + let Ok(window_size) = window.outer_size() else { + return; + }; + let x = work_area.position.x + + i32::try_from(work_area.size.width.saturating_sub(window_size.width)).unwrap_or_default() + - SCREEN_INSET; + let y = work_area.position.y + + i32::try_from(work_area.size.height.saturating_sub(window_size.height)) + .unwrap_or_default() + - SCREEN_INSET; + let _ = window.set_position(PhysicalPosition::new(x, y)); +} + +fn make_macos_transparent(window: &WebviewWindow) -> Result<(), String> { + #[cfg(target_os = "macos")] + { + use objc2::msg_send; + use objc2::runtime::{AnyClass, AnyObject}; + use objc2_foundation::NSString; + + window + .with_webview(|platform_webview| unsafe { + let webview = platform_webview.inner() as *mut AnyObject; + if webview.is_null() { + return; + } + + let ns_window: *mut AnyObject = msg_send![&*webview, window]; + if !ns_window.is_null() { + let _: () = msg_send![&*ns_window, setOpaque: false]; + if let Some(ns_color) = AnyClass::get(c"NSColor") { + let clear_color: *mut AnyObject = msg_send![ns_color, clearColor]; + let _: () = msg_send![&*ns_window, setBackgroundColor: clear_color]; + } + } + + if let Some(ns_number) = AnyClass::get(c"NSNumber") { + let key = NSString::from_str("drawsBackground"); + let no_value: *mut AnyObject = msg_send![ns_number, numberWithBool: false]; + let _: () = msg_send![&*webview, setValue: no_value, forKey: &*key]; + } + }) + .map_err(|error| error.to_string())?; + } + + #[cfg(not(target_os = "macos"))] + let _ = window; + Ok(()) +} + +pub fn install(app: &AppHandle) -> Result<(), String> { + if let Some(window) = app.get_webview_window(WINDOW_LABEL) { + window + .destroy() + .map_err(|error| format!("Could not replace stale floating voice controls: {error}"))?; + if app.get_webview_window(WINDOW_LABEL).is_some() { + return Err("Stale floating voice controls could not be replaced.".to_string()); + } + } + let (session_id, owner_window_label, revision) = app + .state::() + .active_session_lifecycle_target() + .ok_or_else(|| "No native voice conversation is active.".to_string())?; + + let builder = WebviewWindowBuilder::new( + app, + WINDOW_LABEL, + WebviewUrl::App(controls_url(revision).into()), + ) + .title("Berd voice conversation") + .inner_size(WINDOW_WIDTH, WINDOW_HEIGHT) + .resizable(false) + .maximizable(false) + .minimizable(false) + .decorations(false) + .shadow(false) + .always_on_top(true) + .skip_taskbar(true) + .focused(false) + .visible(false); + #[cfg(not(target_os = "macos"))] + let builder = builder.transparent(true); + let window = builder.build().map_err(|error| error.to_string())?; + make_macos_transparent(&window)?; + window.on_window_event(|event| { + if let WindowEvent::CloseRequested { api, .. } = event { + api.prevent_close(); + } + }); + position_near_bottom_right(app, &window); + let fallback_app = app.clone(); + tauri::async_runtime::spawn(async move { + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + let state = fallback_app.state::(); + if !state.controls_ready_for(&session_id, revision) { + log::error!( + "Floating voice controls did not become ready; stopping the voice conversation" + ); + if state.active_session_lifecycle_target() + != Some((session_id.clone(), owner_window_label.clone(), revision)) + { + return; + } + let capture = fallback_app.state::(); + if let Err(error) = state + .stop_active_if_lifecycle( + &fallback_app, + capture.inner(), + &session_id, + revision, + "Voice controls could not open, so the voice conversation was stopped.", + ) + .await + { + log::error!("Failed to stop voice after controls readiness timeout: {error}"); + } + } + }); + Ok(()) +} + +fn active_controls_match(active_revision: Option, controls_url: Option<&str>) -> bool { + active_revision.is_some_and(|revision| { + controls_url.is_some_and(|url| controls_window_matches_revision(url, revision)) + }) +} + +fn should_destroy_stale_candidate( + candidate_url: Option<&str>, + active_revision: Option, +) -> bool { + candidate_url.is_some_and(|url| !active_controls_match(active_revision, Some(url))) +} + +fn verify_stale_candidate_removed( + candidate_url: &str, + current_url: Result, String>, + destroy_result: Result<(), String>, +) -> Result<(), String> { + let current_url = current_url?; + if current_url != Some(candidate_url) { + return Ok(()); + } + destroy_result?; + Err("Stale floating voice controls remained after removal.".to_string()) +} + +pub fn matches_active_lifecycle(app: &AppHandle) -> bool { + let controls_url = app + .get_webview_window(WINDOW_LABEL) + .and_then(|window| window.url().ok()); + let active_revision = app + .state::() + .active_session_lifecycle_target() + .map(|(_, _, revision)| revision); + active_controls_match( + active_revision, + controls_url.as_ref().map(|url| url.as_str()), + ) +} + +pub fn should_preserve_main_for_voice( + active_owner_window_label: Option<&str>, + controls_match_active_voice: bool, +) -> bool { + active_owner_window_label == Some("main") || controls_match_active_voice +} + +pub fn destroy_stale_for_main_close(app: &AppHandle) -> Result<(), String> { + let Some(window) = app.get_webview_window(WINDOW_LABEL) else { + return Ok(()); + }; + let candidate_url = window.url().map_err(|error| error.to_string())?; + let active_revision = app + .state::() + .active_session_lifecycle_target() + .map(|(_, _, revision)| revision); + if !should_destroy_stale_candidate(Some(candidate_url.as_str()), active_revision) { + return Ok(()); + } + let destroy_result = window + .destroy() + .map_err(|error| format!("Could not remove stale floating voice controls: {error}")); + let current_window = app.get_webview_window(WINDOW_LABEL); + let current_url = current_window + .as_ref() + .map(|current| current.url().map_err(|error| error.to_string())) + .transpose(); + let current_url = current_url + .as_ref() + .map(|url| url.as_ref().map(|url| url.as_str())) + .map_err(Clone::clone); + verify_stale_candidate_removed(candidate_url.as_str(), current_url, destroy_result) +} + +fn reconcile_terminal_controls( + emit_terminal: impl FnOnce(), + destroy: impl FnOnce() -> Result<(), String>, + hide: impl FnOnce() -> Result<(), String>, +) { + emit_terminal(); + if let Err(error) = destroy() { + log::error!("Failed to remove stopped floating voice controls: {error}"); + if let Err(hide_error) = hide() { + log::error!("Failed to hide stopped floating voice controls: {hide_error}"); + } + } +} + +pub fn dismiss_after_terminal_event( + app: &AppHandle, + controls_revision: u64, + payload: T, +) { + let Some(window) = app.get_webview_window(WINDOW_LABEL) else { + return; + }; + let window_matches_lifecycle = window + .url() + .ok() + .is_some_and(|url| controls_window_matches_revision(url.as_str(), controls_revision)); + reconcile_terminal_controls( + || { + let _ = window.emit(super::native_voice::EVENT_NAME, payload); + }, + || { + if window_matches_lifecycle { + window.destroy().map_err(|error| error.to_string()) + } else { + Ok(()) + } + }, + || { + if window_matches_lifecycle { + window.hide().map_err(|error| error.to_string()) + } else { + Ok(()) + } + }, + ); +} + +pub fn dismiss_stale_after_terminal(app: &AppHandle, terminal_revision: u64) { + let Some(window) = app.get_webview_window(WINDOW_LABEL) else { + return; + }; + reconcile_terminal_controls( + || { + let _ = window.emit( + super::native_voice::EVENT_NAME, + NativeVoiceEvent::ControlsDismissed { + revision: terminal_revision, + }, + ); + }, + || window.destroy().map_err(|error| error.to_string()), + || window.hide().map_err(|error| error.to_string()), + ); +} + +pub fn emit(app: &AppHandle, payload: T) { + if let Some(window) = app.get_webview_window(WINDOW_LABEL) { + let _ = window.emit(super::native_voice::EVENT_NAME, payload); + } +} + +#[tauri::command] +pub fn open_voice_conversation_session(app: AppHandle) -> Result<(), String> { + open_active_session(&app) +} + +#[tauri::command] +pub async fn show_voice_conversation_controls( + window: WebviewWindow, + state: tauri::State<'_, NativeVoiceState>, + capture: tauri::State<'_, VoiceCaptureState>, + session_id: String, + expected_revision: u64, +) -> Result<(), String> { + if window.label() != WINDOW_LABEL { + return Err("Only the floating voice controls can show this window.".to_string()); + } + let Some((active_session_id, owner_window_label, active_revision)) = + state.active_session_lifecycle_target() + else { + return Ok(()); + }; + if active_session_id != session_id || active_revision != expected_revision { + return Ok(()); + } + let Some(mut target) = state.controls_visibility_target(&session_id, expected_revision)? else { + return Ok(()); + }; + loop { + let apply_result = if target.suppressed { + window.hide() + } else { + window.show() + }; + if let Err(error) = apply_result { + if state.active_session_lifecycle_target() + == Some(( + session_id.clone(), + owner_window_label.clone(), + expected_revision, + )) + { + state + .stop_active_if_lifecycle( + window.app_handle(), + capture.inner(), + &session_id, + expected_revision, + "Voice controls could not open, so the voice conversation was stopped.", + ) + .await + .map_err(|stop_error| { + format!( + "The floating voice controls could not be prepared ({error}), and the voice conversation could not be stopped: {stop_error}" + ) + })?; + } + return Err(error.to_string()); + } + match state.acknowledge_controls_visibility( + &session_id, + expected_revision, + target.generation, + )? { + ControlsVisibilityAcknowledgement::Inactive + | ControlsVisibilityAcknowledgement::Ready => return Ok(()), + ControlsVisibilityAcknowledgement::Superseded(next_target) => { + target = next_target; + } + } + } +} + +#[tauri::command] +pub fn set_voice_conversation_controls_suppressed( + window: WebviewWindow, + state: tauri::State<'_, NativeVoiceState>, + capture: tauri::State<'_, VoiceCaptureState>, + request: ControlsVisibilityRequest, +) -> Result<(), String> { + capture.with_active_renderer( + window.label(), + &request.renderer_id, + request.renderer_epoch, + || { + let Some((should_show, previous_suppression)) = state.set_controls_suppressed( + window.label(), + &request.session_id, + request.expected_revision, + request.suppressed, + )? + else { + return Ok(()); + }; + let Some(controls) = window.app_handle().get_webview_window(WINDOW_LABEL) else { + state.rollback_controls_suppression( + &request.session_id, + request.expected_revision, + request.suppressed, + previous_suppression, + ); + if should_show { + open_active_session(window.app_handle()).map_err(|recovery_error| { + format!( + "The floating voice controls are no longer available, and the voice session could not be restored: {recovery_error}" + ) + })?; + } + return Err("The floating voice controls are no longer available.".to_string()); + }; + let result = if should_show { + controls.show() + } else { + controls.hide() + }; + if let Err(error) = result { + state.rollback_controls_suppression( + &request.session_id, + request.expected_revision, + request.suppressed, + previous_suppression, + ); + if should_show { + open_active_session(window.app_handle()).map_err(|recovery_error| { + format!( + "The floating voice controls could not be shown ({error}), and the voice session could not be restored: {recovery_error}" + ) + })?; + } + return Err(error.to_string()); + } + Ok(()) + }, + ) +} + +#[tauri::command] +pub async fn stop_voice_conversation_from_buddy( + app: AppHandle, + state: tauri::State<'_, NativeVoiceState>, + capture: tauri::State<'_, VoiceCaptureState>, + window: WebviewWindow, + session_id: String, + expected_revision: u64, +) -> Result<(), String> { + require_controls_window(window.label())?; + state + .stop_active_for_lifecycle(&app, capture.inner(), &session_id, expected_revision) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hang_up_accepts_only_the_floating_controls_window() { + assert!(require_controls_window(WINDOW_LABEL).is_ok()); + assert!(require_controls_window("main").is_err()); + assert!(require_controls_window("session:other").is_err()); + } + + #[test] + fn stale_controls_do_not_match_an_inactive_or_replacement_lifecycle() { + let controls = controls_url(4); + assert!(active_controls_match(Some(4), Some(&controls))); + assert!(!active_controls_match(None, Some(&controls))); + assert!(!active_controls_match(Some(6), Some(&controls))); + assert!(!active_controls_match(Some(4), None)); + } + + #[test] + fn stale_cleanup_never_targets_controls_created_after_candidate_capture() { + assert!(!should_destroy_stale_candidate(None, Some(4))); + assert!(!should_destroy_stale_candidate(None, None)); + assert!(should_destroy_stale_candidate( + Some(&controls_url(3)), + Some(4), + )); + assert!(verify_stale_candidate_removed( + &controls_url(3), + Ok(Some(&controls_url(4))), + Err("old handle is gone".to_string()), + ) + .is_ok()); + assert!(verify_stale_candidate_removed( + &controls_url(3), + Ok(Some(&controls_url(3))), + Err("old handle is stuck".to_string()), + ) + .is_err()); + assert!(verify_stale_candidate_removed( + &controls_url(3), + Err("could not inspect current controls".to_string()), + Err("old handle is stuck".to_string()), + ) + .is_err()); + } + + #[test] + fn main_is_preserved_during_owner_startup_without_controls() { + assert!(should_preserve_main_for_voice(Some("main"), false)); + assert!(should_preserve_main_for_voice(Some("session:1"), true)); + assert!(!should_preserve_main_for_voice(Some("session:1"), false)); + assert!(!should_preserve_main_for_voice(None, false)); + } + + #[test] + fn hidden_owner_restoration_policy_is_platform_specific() { + assert!(!should_restore_owner(true)); + assert_eq!(should_restore_owner(false), cfg!(not(target_os = "macos")),); + } + + #[test] + fn terminal_controls_emit_before_destroy_and_hide_on_failure() { + let emitted = std::cell::Cell::new(false); + let hidden = std::cell::Cell::new(false); + + reconcile_terminal_controls( + || emitted.set(true), + || Err("destroy failed".to_string()), + || { + hidden.set(true); + Ok(()) + }, + ); + + assert!(emitted.get()); + assert!(hidden.get()); + } + + #[test] + fn floating_controls_match_active_revision_not_terminal_revision() { + let url = format!("tauri://localhost/{}", controls_url(42)); + assert_eq!(controls_revision_from_url(&url), Some(42)); + assert!(controls_window_matches_revision(&url, 42)); + assert!(!controls_window_matches_revision(&url, 43)); + assert_eq!( + controls_revision_from_url("tauri://localhost/index.html?voiceBuddy=1"), + None + ); + } +} diff --git a/src-tauri/src/commands/voice_capture.rs b/src-tauri/src/commands/voice_capture.rs index 2453e2ff3..56d15c66c 100644 --- a/src-tauri/src/commands/voice_capture.rs +++ b/src-tauri/src/commands/voice_capture.rs @@ -93,6 +93,31 @@ impl CaptureState { } impl VoiceCaptureState { + #[cfg(test)] + pub(crate) fn register_renderer_for_test(&self, window_label: &str, renderer_id: &str) -> u64 { + self.state + .lock() + .expect("capture lock") + .register_renderer(window_label.to_string(), renderer_id.to_string()) + .expect("register renderer") + } + + pub(crate) fn with_active_renderer( + &self, + window_label: &str, + renderer_id: &str, + renderer_epoch: u64, + operation: impl FnOnce() -> Result, + ) -> Result { + validate_id("renderer", renderer_id)?; + let mut state = self + .state + .lock() + .map_err(|_| "Voice capture state lock was poisoned".to_string())?; + state.activate_renderer(window_label, renderer_id, renderer_epoch)?; + operation() + } + pub fn activate_renderer( &self, window_label: &str, @@ -190,15 +215,20 @@ impl VoiceCaptureState { #[tauri::command] pub fn register_voice_renderer_instance( state: State<'_, VoiceCaptureState>, + native_voice: State<'_, super::native_voice::NativeVoiceState>, webview_window: WebviewWindow, renderer_id: String, ) -> Result { validate_id("renderer", &renderer_id)?; - state + let window_label = webview_window.label().to_string(); + let mut capture_state = state .state .lock() - .map_err(|_| "Voice capture state lock was poisoned".to_string())? - .register_renderer(webview_window.label().to_string(), renderer_id) + .map_err(|_| "Voice capture state lock was poisoned".to_string())?; + let epoch = capture_state.register_renderer(window_label.clone(), renderer_id.clone())?; + native_voice.release_start_blocks_for_replaced_renderer(&window_label, &renderer_id, epoch); + drop(capture_state); + Ok(epoch) } fn validate_id(label: &str, value: &str) -> Result<(), String> { @@ -335,4 +365,36 @@ mod tests { ) .expect("replacement renderer reclaims microphone")); } + + #[test] + fn replaced_renderer_cannot_run_a_late_voice_operation() { + let capture = VoiceCaptureState::default(); + let first_epoch = capture + .state + .lock() + .expect("capture lock") + .register_renderer("main".into(), "renderer-1".into()) + .expect("register first renderer"); + capture + .activate_renderer("main", "renderer-1", first_epoch) + .expect("activate first renderer"); + let second_epoch = capture + .state + .lock() + .expect("capture lock") + .register_renderer("main".into(), "renderer-2".into()) + .expect("register replacement renderer"); + capture + .activate_renderer("main", "renderer-2", second_epoch) + .expect("activate replacement renderer"); + let operation_ran = std::cell::Cell::new(false); + + assert!(capture + .with_active_renderer("main", "renderer-1", first_epoch, || { + operation_ran.set(true); + Ok(()) + }) + .is_err()); + assert!(!operation_ran.get()); + } } diff --git a/src-tauri/src/commands/window_session.rs b/src-tauri/src/commands/window_session.rs index 6af73a640..298be24a2 100644 --- a/src-tauri/src/commands/window_session.rs +++ b/src-tauri/src/commands/window_session.rs @@ -677,20 +677,10 @@ pub fn open_session_window( let label_for_close = label.clone(); window.on_window_event(move |event| { if matches!(event, WindowEvent::Destroyed) { - app_for_close - .state::() - .release_window(&label_for_close); - let stopped_native_voice = app_for_close - .state::() - .stop_for_window_destroyed(&label_for_close); - if stopped_native_voice { - app_for_close - .state::() - .stop_for_window_destroyed(); - } - app_for_close - .state::() - .stop_for_window_destroyed(&label_for_close); + crate::commands::native_voice::handle_voice_owner_window_destroyed( + &app_for_close, + &label_for_close, + ); reg_for_close.release_label(&label_for_close); let _ = emit_snapshot(&app_for_close, ®_for_close); } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 8672de987..b01ce27d8 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -23,9 +23,9 @@ use services::{bundled_agents, bundled_skills, distro_bundle::DistroBundleState} use std::path::PathBuf; #[cfg(target_os = "macos")] use tauri::menu::{AboutMetadataBuilder, MenuBuilder, SubmenuBuilder}; -use tauri::{Manager, RunEvent}; #[cfg(target_os = "macos")] -use tauri::{WebviewWindow, WindowEvent}; +use tauri::WebviewWindow; +use tauri::{Manager, RunEvent, WindowEvent}; use tauri_plugin_window_state::StateFlags; #[cfg(target_os = "macos")] @@ -376,6 +376,8 @@ pub fn run() { // Surface WKWebView renderer memory and detect silent OOM reaps. services::renderer_monitor::start(app.handle().clone()); + attach_main_window_lifecycle(app); + // Build a custom macOS application menu so that the app submenu, // "About" item, and "Quit" item use the product name "Berd" // instead of the Cargo binary name. @@ -383,7 +385,6 @@ pub fn run() { { set_dev_dock_icon(); refresh_traffic_light_position_on_window_changes(app); - attach_main_window_lifecycle(app); let app_menu = SubmenuBuilder::new(app, "Berd") .about_with_text( @@ -640,13 +641,21 @@ pub fn run() { commands::siri_voice::finish_siri_voice_stream, commands::siri_voice::stop_siri_voice, commands::native_voice::get_native_voice_conversation_status, + commands::native_voice::block_native_voice_conversation_starts, + commands::native_voice::release_native_voice_conversation_start_block, + commands::native_voice::set_native_voice_microphone_muted, + commands::native_voice::set_native_voice_assistant_speaking, commands::native_voice::drain_native_voice_conversation_transcripts, commands::native_voice::acknowledge_native_voice_conversation_transcript, commands::native_voice::reject_native_voice_conversation_transcript, commands::native_voice::start_native_voice_conversation, commands::native_voice::stop_native_voice_conversation, commands::native_voice::push_native_voice_audio, - commands::native_voice::set_native_voice_input_muted, + commands::voice_buddy::open_voice_conversation_session, + commands::voice_buddy::show_voice_conversation_controls, + commands::voice_buddy::set_voice_conversation_controls_suppressed, + commands::voice_buddy::stop_voice_conversation_from_buddy, + commands::notifications::should_suppress_completion_notification, commands::voice_capture::register_voice_renderer_instance, commands::window_session::get_session_window_support, commands::window_session::open_session_window, @@ -711,7 +720,6 @@ fn refresh_traffic_light_position_on_window_changes(app: &tauri::App) { } } -#[cfg(target_os = "macos")] fn attach_main_window_lifecycle(app: &tauri::App) { let Some(main) = app.get_webview_window("main") else { return; @@ -719,13 +727,45 @@ fn attach_main_window_lifecycle(app: &tauri::App) { let app_handle = app.handle().clone(); main.on_window_event(move |event| { + if matches!(event, WindowEvent::Destroyed) { + commands::native_voice::handle_voice_owner_window_destroyed(&app_handle, "main"); + return; + } if let WindowEvent::CloseRequested { api, .. } = event { let has_secondary_window = app_handle .webview_windows() .keys() - .any(|label| label != "main"); + .any(|label| label != "main" && label != commands::voice_buddy::WINDOW_LABEL); + let active_voice_owner_window_label = app_handle + .state::() + .active_session_lifecycle_target() + .map(|(_, owner_window_label, _)| owner_window_label); + let controls_match_active_voice = + commands::voice_buddy::matches_active_lifecycle(&app_handle); + let preserve_for_voice = commands::voice_buddy::should_preserve_main_for_voice( + active_voice_owner_window_label.as_deref(), + controls_match_active_voice, + ); + let stale_controls_cleanup_failed = if !preserve_for_voice { + commands::voice_buddy::destroy_stale_for_main_close(&app_handle) + .inspect_err(|error| { + log::error!("Failed to remove stale voice controls on main close: {error}"); + }) + .is_err() + } else { + false + }; + + if stale_controls_cleanup_failed && !cfg!(target_os = "macos") { + app_handle.exit(0); + return; + } + + let should_preserve = preserve_for_voice + || (cfg!(target_os = "macos") + && (has_secondary_window || stale_controls_cleanup_failed)); - if has_secondary_window { + if should_preserve { api.prevent_close(); if let Some(main) = app_handle.get_webview_window("main") { let _ = main.hide(); diff --git a/src/app/AppShell.berdctl.test.tsx b/src/app/AppShell.berdctl.test.tsx index f85f08521..09ef3ccd6 100644 --- a/src/app/AppShell.berdctl.test.tsx +++ b/src/app/AppShell.berdctl.test.tsx @@ -35,6 +35,21 @@ const mockDeletePersonaSource = vi.hoisted(() => vi.fn()); const mockLoadSessionMessages = vi.hoisted(() => vi.fn()); const mockToastError = vi.hoisted(() => vi.fn()); +vi.mock( + "@/features/voice-conversation/api/voiceConversation", + async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/features/voice-conversation/api/voiceConversation") + >()), + blockNativeVoiceConversationStarts: vi + .fn() + .mockResolvedValue("archive-token"), + releaseNativeVoiceConversationStartBlock: vi + .fn() + .mockResolvedValue(undefined), + }), +); + vi.mock("sonner", () => ({ toast: { error: (...args: unknown[]) => mockToastError(...args), diff --git a/src/app/AppShell.navigation.test.tsx b/src/app/AppShell.navigation.test.tsx index 095c4cb48..48c98896f 100644 --- a/src/app/AppShell.navigation.test.tsx +++ b/src/app/AppShell.navigation.test.tsx @@ -29,6 +29,14 @@ import { OPEN_SETTINGS_EVENT } from "@/features/settings/lib/settingsEvents"; import { SHORTCUT_PREFERENCES_STORAGE_KEY } from "@/features/shortcuts/lib/shortcutRegistry"; import { useShortcutsDialogStore } from "@/features/shortcuts/stores/shortcutsDialogStore"; import { useProjectStore } from "@/features/projects/stores/projectStore"; +import { + useVoiceConversationStore, + VOICE_CONVERSATION_OFF_STATUS, +} from "@/features/voice-conversation/stores/voiceConversationStore"; +import { + blockNativeVoiceConversationStarts, + releaseNativeVoiceConversationStartBlock, +} from "@/features/voice-conversation/api/voiceConversation"; import { dispatchOnboarding } from "@/features/onboarding/model"; import { resetHomeWidgetStoreForTests, @@ -53,6 +61,21 @@ import { useRuntimeConfigStore } from "@/shared/runtime-config/runtimeConfigStor import { gooseServeSelectionFromExecutionTarget } from "@/features/chat/lib/gooseServeExecutionTarget"; import { ASSISTIVE_UX_STORAGE_KEY } from "@/shared/assistive-ux/registry"; +vi.mock( + "@/features/voice-conversation/api/voiceConversation", + async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/features/voice-conversation/api/voiceConversation") + >()), + blockNativeVoiceConversationStarts: vi + .fn() + .mockResolvedValue("archive-token"), + releaseNativeVoiceConversationStartBlock: vi + .fn() + .mockResolvedValue(undefined), + }), +); + import { DEFAULT_RUNTIME_CONFIG, type RuntimeConfig, @@ -60,7 +83,6 @@ import { import { AppShell, shouldStopVoiceConversationOnExperimentChange, - shouldStopVoiceConversationOnSessionChange, } from "./AppShell"; import type { NavigationPanesViewProps } from "@/app/views/NavigationPanesView"; import type { AppShellContent as AppShellContentType } from "./ui/AppShellContent"; @@ -88,6 +110,7 @@ const gitMocks = vi.hoisted(() => ({ removeWorktree: vi.fn(), })); const mockIsExternalAgentReady = vi.hoisted(() => vi.fn()); +const originalStopVoiceConversation = useVoiceConversationStore.getState().stop; const mockAgentStatus = vi.hoisted(() => ({ readyAgentIds: new Set(["goose"]), })); @@ -872,46 +895,6 @@ describe("AppShell global navigation", () => { ).toBe(false); }); - it("stops voice only when navigation leaves its bound chat", () => { - const base = { - previousSessionId: "session-1", - boundSessionId: "session-1", - lifecycle: "running", - }; - - expect( - shouldStopVoiceConversationOnSessionChange({ - ...base, - nextSessionId: "session-2", - }), - ).toBe(true); - expect( - shouldStopVoiceConversationOnSessionChange({ - ...base, - nextSessionId: null, - }), - ).toBe(true); - expect( - shouldStopVoiceConversationOnSessionChange({ - ...base, - nextSessionId: "session-1", - }), - ).toBe(false); - expect( - shouldStopVoiceConversationOnSessionChange({ - ...base, - nextSessionId: "session-2", - boundSessionId: "session-elsewhere", - }), - ).toBe(false); - expect( - shouldStopVoiceConversationOnSessionChange({ - ...base, - nextSessionId: "session-2", - lifecycle: "stopped", - }), - ).toBe(false); - }); afterEach(cleanup); beforeEach(() => { @@ -934,6 +917,17 @@ describe("AppShell global navigation", () => { mockSessionWindowSupport.supported = false; mockFocusSessionWindow.mockReset(); useSessionWindowStore.getState().setSnapshot([]); + useVoiceConversationStore.setState({ + status: VOICE_CONVERSATION_OFF_STATUS, + microphoneMuted: false, + stop: originalStopVoiceConversation, + }); + vi.mocked(blockNativeVoiceConversationStarts) + .mockReset() + .mockResolvedValue("archive-token"); + vi.mocked(releaseNativeVoiceConversationStartBlock) + .mockReset() + .mockResolvedValue(undefined); mockListExtensions.mockReset(); mockListExtensions.mockResolvedValue([]); mockAcpCreateSession.mockReset(); @@ -2544,6 +2538,320 @@ describe("AppShell global navigation", () => { expect(gitMocks.removeWorktree).not.toHaveBeenCalled(); }); + it("releases a late voice-start lease without archiving after the deadline", async () => { + vi.useFakeTimers(); + const lease = deferred(); + vi.mocked(blockNativeVoiceConversationStarts).mockReturnValueOnce( + lease.promise, + ); + useChatSessionStore.setState({ + sessions: [makeManagedWorktreeSession("stalled-voice-lease")], + }); + renderAppShell(); + + const outcome = getAppNavigationController().archiveSession( + "session-1", + "reject", + Date.now() + 4_000, + ); + await vi.advanceTimersByTimeAsync(1_000); + + await expect(outcome).resolves.toEqual({ + ok: false, + reason: "timed_out", + }); + expect(mockAcpArchiveSession).not.toHaveBeenCalled(); + + lease.resolve("late-archive-token"); + await vi.waitFor(() => + expect(releaseNativeVoiceConversationStartBlock).toHaveBeenCalledWith( + "session-1", + "late-archive-token", + ), + ); + vi.useRealTimers(); + }); + + it.each([ + "reject", + "discard", + ] as const)("does not use the %s archive policy on a background voice session", async (cleanupPolicy) => { + const stopVoiceConversation = vi.fn().mockResolvedValue(undefined); + useChatSessionStore.setState({ + sessions: [makeManagedWorktreeSession("background-voice")], + }); + useVoiceConversationStore.setState({ + status: { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + microphoneMuted: false, + revision: 1, + }, + stop: stopVoiceConversation, + }); + renderAppShell(); + + const outcome = await getAppNavigationController().archiveSession( + "session-1", + cleanupPolicy, + ); + + expect(outcome).toEqual({ + ok: false, + reason: "target_session_running", + }); + expect(mockAcpArchiveSession).not.toHaveBeenCalled(); + expect(stopVoiceConversation).not.toHaveBeenCalled(); + }); + + it("stops active voice before confirmed archival", async () => { + const stoppedStatus = { + available: true, + unavailableReason: null, + lifecycle: "stopped" as const, + sessionId: null, + ownerWindowLabel: null, + microphoneMuted: false, + revision: 2, + }; + const stopRequest = deferred(); + const stopVoiceConversation = vi.fn(async () => { + const status = await stopRequest.promise; + useVoiceConversationStore.setState({ status }); + return status; + }); + useChatSessionStore.setState({ + sessions: [ + { + id: "session-1", + title: "Voice archive", + executionTarget: { harnessId: "goose" }, + workingDir: "~/voice-archive", + createdAt: "2026-08-21T00:00:00.000Z", + updatedAt: "2026-08-21T00:00:00.000Z", + messageCount: 1, + }, + ], + }); + useVoiceConversationStore.setState({ + status: { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + microphoneMuted: false, + revision: 1, + }, + stop: stopVoiceConversation as never, + }); + renderAppShell(); + + const outcome = getAppNavigationController().archiveSession( + "session-1", + "confirm", + ); + await waitFor(() => expect(stopVoiceConversation).toHaveBeenCalledOnce()); + expect(mockAcpArchiveSession).not.toHaveBeenCalled(); + + stopRequest.resolve(stoppedStatus); + await expect(outcome).resolves.toEqual({ ok: true }); + expect(mockAcpArchiveSession).toHaveBeenCalledWith("session-1"); + expect(stopVoiceConversation.mock.invocationCallOrder[0]).toBeLessThan( + mockAcpArchiveSession.mock.invocationCallOrder[0], + ); + }); + + it("keeps the session unarchived when voice restarts during shutdown", async () => { + const replacementStatus = { + available: true, + unavailableReason: null, + lifecycle: "running" as const, + sessionId: "session-1", + ownerWindowLabel: "main", + microphoneMuted: false, + revision: 2, + }; + const stopVoiceConversation = vi.fn(async () => { + useVoiceConversationStore.setState({ status: replacementStatus }); + return replacementStatus; + }); + useChatSessionStore.setState({ + sessions: [ + { + id: "session-1", + title: "Voice archive", + executionTarget: { harnessId: "goose" }, + workingDir: "~/voice-archive", + createdAt: "2026-08-21T00:00:00.000Z", + updatedAt: "2026-08-21T00:00:00.000Z", + messageCount: 1, + }, + ], + }); + useVoiceConversationStore.setState({ + status: { ...replacementStatus, revision: 1 }, + stop: stopVoiceConversation, + }); + renderAppShell(); + + await expect( + getAppNavigationController().archiveSession("session-1", "confirm"), + ).resolves.toEqual({ ok: false, reason: "voice_stop_failed" }); + + expect(mockAcpArchiveSession).not.toHaveBeenCalled(); + expect(mockToastError).toHaveBeenCalledWith( + "Couldn't stop voice, so the chat wasn't archived", + { description: "Voice is still active for this chat." }, + ); + }); + + it("rechecks the archive deadline after a delayed voice stop", async () => { + vi.useFakeTimers(); + const stoppedStatus = { + available: true, + unavailableReason: null, + lifecycle: "stopped" as const, + sessionId: null, + ownerWindowLabel: null, + microphoneMuted: false, + revision: 2, + }; + const stopRequest = deferred(); + const stopVoiceConversation = vi.fn(async () => { + const status = await stopRequest.promise; + useVoiceConversationStore.setState({ status }); + return status; + }); + useChatSessionStore.setState({ + sessions: [ + { + id: "session-1", + title: "Voice archive deadline", + executionTarget: { harnessId: "goose" }, + workingDir: "~/voice-archive-deadline", + createdAt: "2026-08-21T00:00:00.000Z", + updatedAt: "2026-08-21T00:00:00.000Z", + messageCount: 1, + }, + ], + }); + useVoiceConversationStore.setState({ + status: { + ...stoppedStatus, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + revision: 1, + }, + stop: stopVoiceConversation as never, + }); + renderAppShell(); + + const outcome = getAppNavigationController().archiveSession( + "session-1", + "confirm", + Date.now() + 5_000, + ); + await vi.waitFor( + () => expect(stopVoiceConversation).toHaveBeenCalledOnce(), + { + timeout: 500, + }, + ); + await vi.advanceTimersByTimeAsync(2_000); + stopRequest.resolve(stoppedStatus); + + await expect(outcome).resolves.toEqual({ + ok: false, + reason: "timed_out", + }); + expect(mockAcpArchiveSession).not.toHaveBeenCalled(); + vi.useRealTimers(); + }); + + it("keeps the session unarchived when voice cannot stop", async () => { + const stopError = new Error("microphone shutdown failed"); + const stopVoiceConversation = vi.fn().mockRejectedValue(stopError); + useChatSessionStore.setState({ + sessions: [ + { + id: "session-1", + title: "Voice archive", + executionTarget: { harnessId: "goose" }, + workingDir: "~/voice-archive", + createdAt: "2026-08-21T00:00:00.000Z", + updatedAt: "2026-08-21T00:00:00.000Z", + messageCount: 1, + }, + ], + }); + useVoiceConversationStore.setState({ + status: { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + microphoneMuted: false, + revision: 1, + }, + stop: stopVoiceConversation, + }); + renderAppShell(); + + await expect( + getAppNavigationController().archiveSession("session-1", "confirm"), + ).resolves.toEqual({ ok: false, reason: "voice_stop_failed" }); + expect(mockAcpArchiveSession).not.toHaveBeenCalled(); + expect(mockToastError).toHaveBeenCalledWith( + "Couldn't stop voice, so the chat wasn't archived", + { description: "microphone shutdown failed" }, + ); + }); + + it("rechecks background voice immediately before auto-archive", async () => { + const inspection = deferred(); + mockPathExists.mockResolvedValue(true); + gitMocks.getGitState.mockReturnValue(inspection.promise); + useChatSessionStore.setState({ + sessions: [makeManagedWorktreeSession("voice-starts-during-inspection")], + }); + renderAppShell(); + + const outcome = getAppNavigationController().archiveSession( + "session-1", + "reject", + ); + await waitFor(() => { + expect(gitMocks.getGitState).toHaveBeenCalled(); + }); + + useVoiceConversationStore.setState({ + status: { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + microphoneMuted: false, + revision: 1, + }, + }); + inspection.resolve( + managedWorktreeGitState("voice-starts-during-inspection"), + ); + + await expect(outcome).resolves.toEqual({ + ok: false, + reason: "target_session_running", + }); + expect(mockAcpArchiveSession).not.toHaveBeenCalled(); + }); + it("rechecks running state before noninteractive archival", async () => { const inspection = deferred(); mockPathExists.mockResolvedValue(true); @@ -2576,6 +2884,19 @@ describe("AppShell global navigation", () => { it("archives the active session with Cmd+E", async () => { const user = userEvent.setup(); + const stopVoiceConversation = vi.fn(async () => { + const status = { + available: true, + unavailableReason: null, + lifecycle: "stopped" as const, + sessionId: null, + ownerWindowLabel: null, + microphoneMuted: false, + revision: 2, + }; + useVoiceConversationStore.setState({ status }); + return status; + }); const session: ChatSession = { id: "session-1", title: "Active chat", @@ -2589,6 +2910,18 @@ describe("AppShell global navigation", () => { sessions: [session], activeSessionId: null, }); + useVoiceConversationStore.setState({ + status: { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + microphoneMuted: false, + revision: 1, + }, + stop: stopVoiceConversation, + }); renderAppShell(); @@ -2603,6 +2936,7 @@ describe("AppShell global navigation", () => { expect(screen.getByTestId("active-view")).toHaveTextContent("home"); }); expect(mockAcpArchiveSession).toHaveBeenCalledWith("session-1"); + expect(stopVoiceConversation).toHaveBeenCalledOnce(); expect(useChatSessionStore.getState().activeSessionId).toBeNull(); expect( useChatSessionStore.getState().getSession("session-1")?.archivedAt, @@ -5546,6 +5880,7 @@ describe("AppShell global navigation", () => { it("cycles sessions with Ctrl+Tab and Ctrl+Shift+Tab", async () => { const user = userEvent.setup(); + const stopVoiceConversation = vi.fn(); const sessionBase = { executionTarget: { harnessId: "goose" }, workingDir: "~/goose artifacts", @@ -5569,6 +5904,18 @@ describe("AppShell global navigation", () => { ] as ChatSession[], activeSessionId: null, }); + useVoiceConversationStore.setState({ + status: { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + microphoneMuted: false, + revision: 1, + }, + stop: stopVoiceConversation, + }); renderAppShell(); @@ -5601,5 +5948,6 @@ describe("AppShell global navigation", () => { expect(screen.getByTestId("rendered-session-id")).toHaveTextContent( "session-1", ); + expect(stopVoiceConversation).not.toHaveBeenCalled(); }); }); diff --git a/src/app/AppShell.tsx b/src/app/AppShell.tsx index c08504812..ef3188482 100644 --- a/src/app/AppShell.tsx +++ b/src/app/AppShell.tsx @@ -223,7 +223,11 @@ import { isDesignSystemExplorerEnabled } from "@/features/design-system/lib/desi import { useExperiment } from "@/features/experiments/experimentPreferences"; import { OnboardingFlow } from "@/features/onboarding/ui/OnboardingFlow"; import { useOnboardingState } from "@/features/onboarding/model"; -import { useVoiceConversationStore } from "@/features/voice-conversation/stores/voiceConversationStore"; +import { + blockVoiceConversationStarts, + useVoiceConversationStore, +} from "@/features/voice-conversation/stores/voiceConversationStore"; +import { listenToVoiceConversationOpenSession } from "@/features/voice-conversation/api/voiceConversation"; import { usePocketVoiceSetup } from "@/features/voice-conversation/hooks/usePocketVoiceSetup"; import { useSiriVoiceSetup } from "@/features/voice-conversation/hooks/useSiriVoiceSetup"; import { PocketVoiceSetupDialog } from "@/features/voice-conversation/ui/PocketVoiceSetupDialog"; @@ -360,27 +364,35 @@ function getSessionArchiveInterruptionReason( : null; } -type GlobalComposerPlacement = "docked" | "centered" | "handoff"; +async function acquireVoiceStartBlockBeforeDeadline( + sessionId: string, + deadlineMs?: number, +): Promise<(() => Promise) | null> { + const acquisition = blockVoiceConversationStarts(sessionId); + if (deadlineMs == null) return acquisition; -export function shouldStopVoiceConversationOnSessionChange({ - previousSessionId, - nextSessionId, - boundSessionId, - lifecycle, -}: { - previousSessionId: string | null; - nextSessionId: string | null; - boundSessionId: string | null; - lifecycle: string; -}): boolean { - return ( - previousSessionId !== null && - previousSessionId !== nextSessionId && - boundSessionId === previousSessionId && - lifecycle !== "stopped" && - lifecycle !== "unavailable" - ); + const remainingMs = deadlineMs - MUTATION_DEADLINE_MARGIN_MS - Date.now(); + if (remainingMs <= 0) { + void acquisition.then((release) => release()).catch(() => undefined); + return null; + } + + let timeoutId: number | undefined; + const timeout = new Promise((resolve) => { + timeoutId = window.setTimeout(() => resolve(null), remainingMs); + }); + const release = await Promise.race([acquisition, timeout]); + if (timeoutId !== undefined) window.clearTimeout(timeoutId); + if (!release) { + void acquisition + .then((lateRelease) => lateRelease()) + .catch(() => undefined); + } + return release; } + +type GlobalComposerPlacement = "docked" | "centered" | "handoff"; + const current = (id: string, label: string): TopBarBreadcrumb => ({ id, label, @@ -764,18 +776,7 @@ export function AppShell({ ) { voice.clearRequestedStart(previousSessionId); } - if ( - !shouldStopVoiceConversationOnSessionChange({ - previousSessionId, - nextSessionId: activeSessionId, - boundSessionId: voice.status.sessionId, - lifecycle: voice.status.lifecycle, - }) - ) { - return; - } - void stopVoiceConversation().catch(() => undefined); - }, [activeSessionId, stopVoiceConversation]); + }, [activeSessionId]); const sidebarIsResizing = isResizing; const sidebarDockedPanelOuterWidth = sidebarPanelOuterWidth; const sidebarDockedOuterWidth = sidebarCollapsed ? 0 : sidebarPanelOuterWidth; @@ -3618,6 +3619,19 @@ export function AppShell({ if (!session || session.id !== sessionId) { return { ok: false as const, reason: "session_not_found" as const }; } + await useVoiceConversationStore.getState().init(); + const voiceBeforeArchive = useVoiceConversationStore.getState().status; + if ( + cleanupPolicy !== "confirm" && + voiceBeforeArchive.sessionId === sessionId && + voiceBeforeArchive.lifecycle !== "stopped" && + voiceBeforeArchive.lifecycle !== "unavailable" + ) { + return { + ok: false as const, + reason: "target_session_running" as const, + }; + } let plans: InspectedSessionWorkspaceCleanupPlan[] = []; if (hasSessionWorkspaceCleanupTargets(session)) { @@ -3692,87 +3706,150 @@ export function AppShell({ reason: "blocked_unsaved_changes" as const, }; } - + const releaseVoiceStartBlock = + await acquireVoiceStartBlockBeforeDeadline(sessionId, deadlineMs); + if (!releaseVoiceStartBlock) { + return { ok: false as const, reason: "timed_out" as const }; + } try { - await useChatSessionStore - .getState() - .archiveSession(sessionId, fallbackSession); - const homeWidgetState = useHomeWidgetStore.getState(); - const pinnedWidget = homeWidgetState.instances.find( - (instance) => - instance.type === "chatPin" && - instance.state?.sessionId === sessionId, + const postLeaseInterruption = getSessionArchiveInterruptionReason( + sessionId, + cleanupPolicy, + deadlineMs, ); - if (pinnedWidget) { - homeWidgetState.removeWidget(pinnedWidget.id); + if (postLeaseInterruption) { + return { ok: false as const, reason: postLeaseInterruption }; } - } catch (error) { - if (cleanupPolicy === "confirm") { - toast.error( - formatAcpErrorMessage( - error, - t("chat:notifications.archiveError"), - ), - ); + await useVoiceConversationStore.getState().init(); + const voiceBeforeMutation = + useVoiceConversationStore.getState().status; + const targetOwnsVoice = + voiceBeforeMutation.sessionId === sessionId && + voiceBeforeMutation.lifecycle !== "stopped" && + voiceBeforeMutation.lifecycle !== "unavailable"; + if (cleanupPolicy !== "confirm" && targetOwnsVoice) { + return { + ok: false as const, + reason: "target_session_running" as const, + }; + } + if (targetOwnsVoice) { + try { + const stoppedStatus = await useVoiceConversationStore + .getState() + .stop(); + const currentStatus = useVoiceConversationStore.getState().status; + const targetStillOwnsVoice = [stoppedStatus, currentStatus].some( + (status) => + status.sessionId === sessionId && + status.lifecycle !== "stopped" && + status.lifecycle !== "unavailable", + ); + if (targetStillOwnsVoice) { + throw new Error("Voice is still active for this chat."); + } + } catch (error) { + console.error("Failed to stop voice before archiving:", error); + toast.error(t("chat:notifications.voiceStopBeforeArchiveError"), { + description: formatAcpErrorMessage(error), + }); + return { + ok: false as const, + reason: "voice_stop_failed" as const, + }; + } } - return { - ok: false as const, - reason: - error instanceof SessionNotFoundError - ? ("session_not_found" as const) - : ("backend_archive_failed" as const), - }; - } - let cleanupFailureReason: - | "target_session_running" - | "workspace_cleanup_failed" - | "timed_out" - | null = null; - try { - await cleanupSessionWorkspaces(plans, { - getInterruptionReason: () => - getSessionArchiveInterruptionReason( - sessionId, - cleanupPolicy, - deadlineMs, - ), - }); - } catch (error) { - cleanupFailureReason = - error instanceof SessionWorkspaceCleanupInterruptedError - ? error.reason - : "workspace_cleanup_failed"; - console.error( - "Failed to clean up archived session Git resources:", - error, + const preMutationInterruption = getSessionArchiveInterruptionReason( + sessionId, + cleanupPolicy, + deadlineMs, ); - if (cleanupPolicy === "confirm") { - toast.error( - formatAcpErrorMessage( - error, - t("chat:notifications.gitCleanupError"), - ), + if (preMutationInterruption) { + return { ok: false as const, reason: preMutationInterruption }; + } + try { + await useChatSessionStore + .getState() + .archiveSession(sessionId, fallbackSession); + const homeWidgetState = useHomeWidgetStore.getState(); + const pinnedWidget = homeWidgetState.instances.find( + (instance) => + instance.type === "chatPin" && + instance.state?.sessionId === sessionId, ); + if (pinnedWidget) { + homeWidgetState.removeWidget(pinnedWidget.id); + } + } catch (error) { + if (cleanupPolicy === "confirm") { + toast.error( + formatAcpErrorMessage( + error, + t("chat:notifications.archiveError"), + ), + ); + } + return { + ok: false as const, + reason: + error instanceof SessionNotFoundError + ? ("session_not_found" as const) + : ("backend_archive_failed" as const), + }; + } + let cleanupFailureReason: + | "target_session_running" + | "workspace_cleanup_failed" + | "timed_out" + | null = null; + try { + await cleanupSessionWorkspaces(plans, { + getInterruptionReason: () => + getSessionArchiveInterruptionReason( + sessionId, + cleanupPolicy, + deadlineMs, + ), + }); + } catch (error) { + cleanupFailureReason = + error instanceof SessionWorkspaceCleanupInterruptedError + ? error.reason + : "workspace_cleanup_failed"; + console.error( + "Failed to clean up archived session Git resources:", + error, + ); + if (cleanupPolicy === "confirm") { + toast.error( + formatAcpErrorMessage( + error, + t("chat:notifications.gitCleanupError"), + ), + ); + } } - } - const wasActiveSession = - useChatSessionStore.getState().activeSessionId === sessionId; - cleanupChatSession(sessionId); - if (useSessionWindowStore.getState().isOpenInWindow(sessionId)) { - releaseSession(sessionId).catch((error: unknown) => - console.error("Failed to release session window:", error), - ); - } - if (wasActiveSession) { - setActiveSession(null); - setActiveView("home"); - } + const wasActiveSession = + useChatSessionStore.getState().activeSessionId === sessionId; + cleanupChatSession(sessionId); + if (useSessionWindowStore.getState().isOpenInWindow(sessionId)) { + releaseSession(sessionId).catch((error: unknown) => + console.error("Failed to release session window:", error), + ); + } + if (wasActiveSession) { + setActiveSession(null); + setActiveView("home"); + } - return cleanupFailureReason - ? { ok: true as const, cleanupIncomplete: cleanupFailureReason } - : { ok: true as const }; + return cleanupFailureReason + ? { ok: true as const, cleanupIncomplete: cleanupFailureReason } + : { ok: true as const }; + } finally { + await releaseVoiceStartBlock(); + } } finally { releaseArchiveQueue(); } @@ -3895,6 +3972,31 @@ export function AppShell({ [activeView, guardAppNavigation, isMultiWindowEnabled, selectSessionDirect], ); + useEffect(() => { + let cancelled = false; + let unlisten: (() => void) | null = null; + void listenToVoiceConversationOpenSession((sessionId) => { + const voice = useVoiceConversationStore.getState().status; + if (voice.lifecycle === "running" && voice.sessionId === sessionId) { + handleSelectSession(sessionId); + } + }) + .then((cleanup) => { + if (cancelled) cleanup(); + else unlisten = cleanup; + }) + .catch((error) => { + console.error( + "Failed to listen for voice session open requests:", + error, + ); + }); + return () => { + cancelled = true; + unlisten?.(); + }; + }, [handleSelectSession]); + const handleSelectSearchResult = useCallback( (sessionId: string, messageId?: string, query?: string) => { guardAppNavigation(() => { diff --git a/src/app/lib/rendererDiagnostics.ts b/src/app/lib/rendererDiagnostics.ts index 22343711a..5dbc59250 100644 --- a/src/app/lib/rendererDiagnostics.ts +++ b/src/app/lib/rendererDiagnostics.ts @@ -5,7 +5,7 @@ const SECRET_VALUE_PATTERN = /\b(authorization|refresh_token|access_token|secret_key|api_key|apikey|password|secret|token)\b\s*[:=]\s*(['"]?)[^,\s;&'"]+/gi; interface RendererDiagnosticsContext { - windowKind: "main" | "session"; + windowKind: "main" | "session" | "voice-buddy"; } let installed = false; diff --git a/src/features/berdctl/bridge/appNavigationController.ts b/src/features/berdctl/bridge/appNavigationController.ts index 71cf368d6..63821983e 100644 --- a/src/features/berdctl/bridge/appNavigationController.ts +++ b/src/features/berdctl/bridge/appNavigationController.ts @@ -9,6 +9,7 @@ export type CommandFailureReason = | "focus_failed" | "workspace_cleanup_failed" | "timed_out" + | "voice_stop_failed" | "backend_archive_failed"; export type CommandOutcome = | { diff --git a/src/features/berdctl/commands/impl/archiveSession.ts b/src/features/berdctl/commands/impl/archiveSession.ts index aea6afb46..7db954a66 100644 --- a/src/features/berdctl/commands/impl/archiveSession.ts +++ b/src/features/berdctl/commands/impl/archiveSession.ts @@ -99,6 +99,8 @@ function archiveFailureMessage( return sessionNotFoundMessage(sessionId); case "backend_archive_failed": return backendArchiveFailedMessage("session", sessionId); + case "voice_stop_failed": + return `Could not stop voice for session "${sessionId}"; the session was not archived.`; case "target_session_running": return `Refusing to archive session "${sessionId}" because it started running or opened in another window; wait for the turn to finish or close that window.`; case "cleanup_requires_discard": diff --git a/src/features/berdctl/commands/impl/openSession.ts b/src/features/berdctl/commands/impl/openSession.ts index 5645586d3..8c1cdcc92 100644 --- a/src/features/berdctl/commands/impl/openSession.ts +++ b/src/features/berdctl/commands/impl/openSession.ts @@ -59,6 +59,7 @@ function openFailureMessage( case "focus_failed": return "The session is open in a separate window that could not be focused; tell the user."; case "backend_archive_failed": + case "voice_stop_failed": case "target_session_running": case "cleanup_requires_discard": case "git_inspection_failed": diff --git a/src/features/berdctl/commands/types.ts b/src/features/berdctl/commands/types.ts index 978e442a2..31027947b 100644 --- a/src/features/berdctl/commands/types.ts +++ b/src/features/berdctl/commands/types.ts @@ -80,6 +80,7 @@ export const COMMAND_ERROR_CODES = [ "blocked_unsaved_changes", "backend_read_failed", "backend_archive_failed", + "voice_stop_failed", "cleanup_requires_discard", "git_inspection_failed", "workspace_cleanup_failed", diff --git a/src/features/chat/types.ts b/src/features/chat/types.ts index ff145c1c0..7d23d4f53 100644 --- a/src/features/chat/types.ts +++ b/src/features/chat/types.ts @@ -95,6 +95,7 @@ export interface ChatInputVoiceConversation { | "error"; boundSessionId: string | null; active: boolean; + ownsActiveConversation?: boolean; microphoneMuted: boolean; error?: string | null; disabled?: boolean; diff --git a/src/features/chat/ui/ChatInputToolbar.tsx b/src/features/chat/ui/ChatInputToolbar.tsx index 8d03d40a0..2fdc786dc 100644 --- a/src/features/chat/ui/ChatInputToolbar.tsx +++ b/src/features/chat/ui/ChatInputToolbar.tsx @@ -1,13 +1,14 @@ import { useMemo, useState } from "react"; import { Mic, - Headphones, + MicOff, + Phone, + PhoneOff, ArrowUp, File, FolderOpen, Settings2, Plus, - Volume2, } from "lucide-react"; import { useTranslation } from "react-i18next"; import { useLocaleFormatting } from "@/shared/i18n"; @@ -63,35 +64,6 @@ interface ChatInputToolbarComposerActions { voiceConversation?: ChatInputVoiceConversation; } -function UserVoiceActivityIndicator() { - return ( -