diff --git a/.env.example b/.env.example index 696d3a061..f4ba0cc8a 100644 --- a/.env.example +++ b/.env.example @@ -51,6 +51,14 @@ RELAY_URL=ws://localhost:3000 # (use `just web` for Vite HMR instead). # BUZZ_WEB_DIR=./web/dist +# ----------------------------------------------------------------------------- +# Transcription (dictation) +# ----------------------------------------------------------------------------- +# OpenAI API key for real-time voice transcription in the composer. +# When absent, the dictation mic button is hidden. +# BUZZ_OPENAI_API_KEY=sk-... +# BUZZ_TRANSCRIPTION_MODEL=whisper-1 + # ----------------------------------------------------------------------------- # Git (NIP-34 bare repositories) # ----------------------------------------------------------------------------- diff --git a/crates/buzz-relay/Cargo.toml b/crates/buzz-relay/Cargo.toml index 1134b2c70..54f432f82 100644 --- a/crates/buzz-relay/Cargo.toml +++ b/crates/buzz-relay/Cargo.toml @@ -72,6 +72,7 @@ url = { workspace = true } moka = { workspace = true } metrics = { workspace = true } metrics-exporter-prometheus = { workspace = true } +reqwest = { workspace = true } [features] dev = ["buzz-auth/dev"] diff --git a/crates/buzz-relay/src/api/bridge.rs b/crates/buzz-relay/src/api/bridge.rs index 59093a060..625b052e2 100644 --- a/crates/buzz-relay/src/api/bridge.rs +++ b/crates/buzz-relay/src/api/bridge.rs @@ -25,7 +25,7 @@ use super::{api_error, internal_error, not_found}; /// /// Returns the authenticated public key and an event ID for replay detection. /// For X-Pubkey dev mode, the event ID is a zero hash (no replay concern). -fn verify_bridge_auth( +pub(crate) fn verify_bridge_auth( headers: &HeaderMap, method: &str, url: &str, @@ -76,7 +76,7 @@ fn verify_bridge_auth( /// `AppState`, not process-local memory. Any Redis/guard error fails closed: /// without the shared `SET NX EX` proof, a stateless worker cannot admit the /// NIP-98 request safely. -async fn check_nip98_replay( +pub(crate) async fn check_nip98_replay( state: &AppState, tenant: &TenantContext, event_id_bytes: [u8; 32], @@ -135,7 +135,11 @@ async fn check_nip98_replay_with_guard( /// pass and the relay would proceed against the wrong tenant's auth context), /// and (b) reject every legitimate request whose community host isn't the /// single configured one. Substituting `tenant.host()` closes both directions. -fn nip98_expected_url(config_relay_url: &str, tenant: &TenantContext, path: &str) -> String { +pub(crate) fn nip98_expected_url( + config_relay_url: &str, + tenant: &TenantContext, + path: &str, +) -> String { let scheme = if config_relay_url.trim_start().starts_with("wss://") { "https" } else { diff --git a/crates/buzz-relay/src/api/mod.rs b/crates/buzz-relay/src/api/mod.rs index 10a88002b..139efd78a 100644 --- a/crates/buzz-relay/src/api/mod.rs +++ b/crates/buzz-relay/src/api/mod.rs @@ -1,10 +1,11 @@ -//! HTTP API — media, git, NIP-05, and the Nostr HTTP bridge. +//! HTTP API — media, git, NIP-05, transcription, and the Nostr HTTP bridge. pub mod bridge; pub mod events; pub mod git; pub mod media; pub mod nip05; +pub mod transcribe; // Re-export imeta helpers used by ingest pipeline. pub use crate::handlers::imeta::{validate_imeta_tags, verify_imeta_blobs}; diff --git a/crates/buzz-relay/src/api/transcribe.rs b/crates/buzz-relay/src/api/transcribe.rs new file mode 100644 index 000000000..0d188f012 --- /dev/null +++ b/crates/buzz-relay/src/api/transcribe.rs @@ -0,0 +1,465 @@ +//! Transcription session endpoint — proxies OpenAI Realtime API client-secret minting. +//! +//! When `BUZZ_OPENAI_API_KEY` is configured, the relay can mint ephemeral client +//! secrets for the OpenAI Realtime API. The desktop app uses these to establish a +//! WebRTC connection for real-time speech-to-text dictation. +//! +//! Both endpoints require NIP-98 auth (same as `/events`, `/query`, `/count`). + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use axum::{ + extract::State, + http::{HeaderMap, StatusCode}, + response::Json, +}; +use serde::Serialize; +use serde_json::Value; + +use crate::state::AppState; + +use super::api_error; + +const OPENAI_REALTIME_CLIENT_SECRETS_URL: &str = + "https://api.openai.com/v1/realtime/client_secrets"; + +/// Rate-limit window for transcription session minting. +const TRANSCRIBE_RATE_WINDOW: Duration = Duration::from_secs(60); + +/// Response for `GET /transcribe/status`. +#[derive(Serialize)] +pub struct TranscribeStatus { + configured: bool, + model: String, +} + +/// Response for `POST /transcribe/session`. +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TranscribeSession { + client_secret: String, + model: String, +} + +/// `GET /transcribe/status` — check if transcription is configured. +/// +/// Requires NIP-98 auth. Returns whether the relay has an OpenAI API key +/// configured for real-time transcription **and** the caller is allowed to +/// mint sessions (i.e. is a relay member). This prevents non-members from +/// being prompted for microphone access only to hit a 403 on session creation. +pub async fn transcribe_status( + State(state): State>, + headers: HeaderMap, +) -> Result, (StatusCode, Json)> { + let pubkey = authenticate(&state, &headers, "/transcribe/status", "GET").await?; + + // Check both: API key configured AND caller can actually mint sessions. + let configured = if state.config.openai_api_key.is_some() { + // On membership-required relays, `authenticate` already verified + // membership. On open relays, we need to check explicitly since + // `/transcribe/session` enforces hard membership regardless. + if state.config.require_relay_membership { + true + } else { + can_mint_session(&state, &headers, &pubkey).await + } + } else { + false + }; + + Ok(Json(TranscribeStatus { + configured, + model: state.config.transcription_model.clone(), + })) +} + +/// `POST /transcribe/session` — create an ephemeral OpenAI Realtime session. +/// +/// Requires NIP-98 auth **and** relay membership (even on open relays). +/// Each session mints a metered OpenAI Realtime connection on the relay +/// operator's bill, so we require the caller to be an actual relay member +/// regardless of `BUZZ_REQUIRE_RELAY_MEMBERSHIP`. +pub async fn create_transcribe_session( + State(state): State>, + headers: HeaderMap, +) -> Result, (StatusCode, Json)> { + let pubkey = authenticate(&state, &headers, "/transcribe/session", "POST").await?; + + // Hard membership gate — billable endpoint requires actual relay membership + // even on open relays where `enforce_relay_membership` would short-circuit. + require_relay_member(&state, &headers, &pubkey).await?; + + // Per-pubkey rate limit — each session mints a metered OpenAI Realtime + // connection on the operator's bill. + if transcribe_rate_limited(&state, &pubkey) { + metrics::counter!("buzz_transcribe_session_rejections_total", "reason" => "rate_limit") + .increment(1); + return Err(api_error( + StatusCode::TOO_MANY_REQUESTS, + "transcription session rate limit exceeded — try again shortly", + )); + } + + let api_key = state.config.openai_api_key.as_deref().ok_or_else(|| { + api_error( + StatusCode::SERVICE_UNAVAILABLE, + "transcription is not configured on this relay", + ) + })?; + + let model = state.config.transcription_model.clone(); + + let client = openai_client().map_err(|(status, msg)| api_error(status, msg))?; + + // Build the session payload — VAD configuration depends on the model. + // `gpt-realtime-whisper` requires manual audio commit (no turn detection), + // while other models (e.g. `whisper-1`) use server-side VAD. + let session_payload = build_session_payload(&model); + + let response = client + .post(OPENAI_REALTIME_CLIENT_SECRETS_URL) + .header("Authorization", format!("Bearer {api_key}")) + .header("Content-Type", "application/json") + .json(&session_payload) + .send() + .await + .map_err(|e| { + tracing::error!("OpenAI realtime session request failed: {e}"); + api_error( + StatusCode::BAD_GATEWAY, + "failed to create transcription session", + ) + })?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + tracing::error!("OpenAI realtime session error ({status}): {body}"); + return Err(api_error( + StatusCode::BAD_GATEWAY, + "OpenAI rejected the transcription session request", + )); + } + + let body: Value = response.json().await.map_err(|e| { + tracing::error!("OpenAI realtime session response parse error: {e}"); + api_error( + StatusCode::BAD_GATEWAY, + "invalid response from transcription service", + ) + })?; + + let client_secret = extract_client_secret(&body).ok_or_else(|| { + tracing::error!("OpenAI realtime session response missing client_secret: {body}"); + api_error( + StatusCode::BAD_GATEWAY, + "transcription service returned unexpected response", + ) + })?; + + Ok(Json(TranscribeSession { + client_secret, + model, + })) +} + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +/// Check whether the caller can mint a transcription session (i.e. is a relay +/// member or delegated by one). Used by `/transcribe/status` to avoid showing +/// the mic button to users who would get a 403 on session creation. +async fn can_mint_session( + state: &AppState, + headers: &HeaderMap, + pubkey: &nostr::PublicKey, +) -> bool { + // Try the same membership check that `require_relay_member` uses. + require_relay_member(state, headers, pubkey).await.is_ok() +} + +/// Build the OpenAI Realtime session payload with model-appropriate VAD config. +/// +/// - `gpt-realtime-whisper` (live transcription model): omits `turn_detection` +/// entirely — audio is committed manually by the client per OpenAI guidance. +/// - Other models (e.g. `whisper-1`): use `server_vad` for automatic turn +/// detection. +fn build_session_payload(model: &str) -> Value { + let uses_manual_commit = model.contains("realtime-whisper"); + + let audio_input = if uses_manual_commit { + // Manual commit mode — omit turn_detection entirely. + serde_json::json!({ + "transcription": { "model": model } + }) + } else { + // Server VAD mode — include turn_detection. + serde_json::json!({ + "transcription": { "model": model }, + "turn_detection": { "type": "server_vad" } + }) + }; + + serde_json::json!({ + "session": { + "type": "transcription", + "audio": { + "input": audio_input + } + }, + "expires_after": { + "anchor": "created_at", + "seconds": 60 + } + }) +} + +/// Authenticate the request using the same NIP-98 / X-Pubkey pattern as the +/// bridge endpoints, plus replay detection and relay membership enforcement. +/// Returns the authenticated pubkey on success. +async fn authenticate( + state: &AppState, + headers: &HeaderMap, + path: &str, + method: &str, +) -> Result)> { + let raw_host = headers + .get("host") + .and_then(|v| v.to_str().ok()) + .unwrap_or(""); + let tenant = crate::tenant::bind_community(&state.db, raw_host) + .await + .map_err(|_| { + api_error( + StatusCode::NOT_FOUND, + "relay: no community is configured for this host", + ) + })?; + + let url = super::bridge::nip98_expected_url(&state.config.relay_url, &tenant, path); + // Always require NIP-98 signed auth for transcribe endpoints — these mint + // billable OpenAI sessions, so we cannot trust the unauthenticated X-Pubkey + // dev fallback (which is spoofable) regardless of BUZZ_REQUIRE_AUTH_TOKEN. + let (pubkey, event_id_bytes) = super::bridge::verify_bridge_auth( + headers, method, &url, None, true, // force NIP-98 — billable endpoint + )?; + super::bridge::check_nip98_replay(state, &tenant, event_id_bytes).await?; + + // Enforce relay membership (with NIP-OA fallback via x-auth-tag header). + let pubkey_bytes = pubkey.to_bytes().to_vec(); + let auth_tag = headers.get("x-auth-tag").and_then(|v| v.to_str().ok()); + super::relay_members::enforce_relay_membership( + state, + tenant.community(), + &pubkey_bytes, + auth_tag, + ) + .await?; + + Ok(pubkey) +} + +/// Hard relay-membership check for billable endpoints. +/// +/// Unlike `enforce_relay_membership` (which short-circuits on open relays), +/// this always verifies that the pubkey is an actual relay member or is +/// delegated via NIP-OA by a member. This prevents arbitrary NIP-98 signers +/// from minting metered sessions on the operator's bill. +async fn require_relay_member( + state: &AppState, + headers: &HeaderMap, + pubkey: &nostr::PublicKey, +) -> Result<(), (StatusCode, Json)> { + // If the relay already requires membership globally, the `authenticate` + // call above handled it — no need to double-check. + if state.config.require_relay_membership { + return Ok(()); + } + + // On open relays we still require membership for this billable endpoint. + let raw_host = headers + .get("host") + .and_then(|v| v.to_str().ok()) + .unwrap_or(""); + let tenant = crate::tenant::bind_community(&state.db, raw_host) + .await + .map_err(|_| { + api_error( + StatusCode::NOT_FOUND, + "relay: no community is configured for this host", + ) + })?; + + let pubkey_hex = pubkey.to_hex(); + let is_member = state + .db + .is_relay_member(tenant.community(), &pubkey_hex) + .await + .map_err(|e| { + tracing::error!("transcribe membership check failed: {e}"); + api_error(StatusCode::INTERNAL_SERVER_ERROR, "membership check failed") + })?; + + if is_member { + return Ok(()); + } + + // NIP-OA fallback: check if the agent's owner is a member. + if state.config.allow_nip_oa_auth { + let auth_tag = headers.get("x-auth-tag").and_then(|v| v.to_str().ok()); + if let Some(owner) = + super::relay_members::extract_nip_oa_owner(&pubkey.to_bytes(), auth_tag) + { + let owner_hex = owner.to_hex(); + let owner_is_member = state + .db + .is_relay_member(tenant.community(), &owner_hex) + .await + .map_err(|e| { + tracing::error!("transcribe owner membership check failed: {e}"); + api_error(StatusCode::INTERNAL_SERVER_ERROR, "membership check failed") + })?; + if owner_is_member { + return Ok(()); + } + } + } + + Err(api_error( + StatusCode::FORBIDDEN, + "relay membership required for transcription sessions", + )) +} + +/// Per-pubkey sliding-window rate limiter for transcription session minting. +fn transcribe_rate_limited(state: &AppState, pubkey: &nostr::PublicKey) -> bool { + let key: [u8; 32] = pubkey.to_bytes(); + let now = Instant::now(); + let limit = state.config.transcribe_sessions_per_minute; + let mut entry = state.transcribe_rate_limiter.entry(key).or_insert((0, now)); + let (count, window_start) = entry.value_mut(); + if now.duration_since(*window_start) >= TRANSCRIBE_RATE_WINDOW { + *count = 1; + *window_start = now; + return false; + } + if *count >= limit { + return true; + } + *count += 1; + false +} + +/// Shared HTTP client for OpenAI requests (connection pooling). +fn openai_client() -> Result<&'static reqwest::Client, (StatusCode, &'static str)> { + static CLIENT: std::sync::OnceLock = std::sync::OnceLock::new(); + // `get_or_init` only runs the closure once; if TLS backend init fails we + // propagate an API error instead of panicking in production. + if let Some(c) = CLIENT.get() { + return Ok(c); + } + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(10)) + .build() + .map_err(|_| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + "failed to initialize HTTP client", + ) + })?; + // Another thread may have raced us — `get_or_init` is fine here since we + // already proved construction succeeds. + Ok(CLIENT.get_or_init(|| client)) +} + +fn extract_client_secret(value: &Value) -> Option { + // Shape 1: { "client_secret": { "value": "..." } } + if let Some(cs) = value.get("client_secret") { + if let Some(v) = cs.get("value").and_then(|v| v.as_str()) { + return Some(v.to_string()); + } + // Shape 2: { "client_secret": "..." } + if let Some(v) = cs.as_str() { + return Some(v.to_string()); + } + } + // Shape 3: { "value": "..." } + value + .get("value") + .and_then(|v| v.as_str()) + .map(String::from) +} + +#[cfg(test)] +mod tests { + use super::extract_client_secret; + use serde_json::json; + + #[test] + fn parses_nested_client_secret() { + let body = json!({ "client_secret": { "value": "sec_abc123", "expires_at": 9999 } }); + assert_eq!(extract_client_secret(&body), Some("sec_abc123".to_string())); + } + + #[test] + fn parses_direct_string_client_secret() { + let body = json!({ "client_secret": "sec_direct" }); + assert_eq!(extract_client_secret(&body), Some("sec_direct".to_string())); + } + + #[test] + fn parses_top_level_value() { + let body = json!({ "value": "sec_toplevel" }); + assert_eq!( + extract_client_secret(&body), + Some("sec_toplevel".to_string()) + ); + } + + #[test] + fn returns_none_for_missing_secret() { + let body = json!({ "id": "sess_123", "model": "gpt-4o" }); + assert_eq!(extract_client_secret(&body), None); + } + + #[test] + fn build_session_payload_whisper1_includes_server_vad() { + use super::build_session_payload; + let payload = build_session_payload("whisper-1"); + let td = &payload["session"]["audio"]["input"]["turn_detection"]; + assert_eq!(td["type"], "server_vad"); + } + + #[test] + fn build_session_payload_sets_short_expires_after() { + use super::build_session_payload; + let payload = build_session_payload("whisper-1"); + assert_eq!(payload["expires_after"]["anchor"], "created_at"); + assert_eq!(payload["expires_after"]["seconds"], 60); + let payload2 = build_session_payload("gpt-realtime-whisper"); + assert_eq!(payload2["expires_after"]["anchor"], "created_at"); + assert_eq!(payload2["expires_after"]["seconds"], 60); + } + + #[test] + fn build_session_payload_realtime_whisper_omits_turn_detection() { + use super::build_session_payload; + let payload = build_session_payload("gpt-realtime-whisper"); + let td = &payload["session"]["audio"]["input"]["turn_detection"]; + assert!( + td.is_null(), + "turn_detection should be absent for realtime-whisper model" + ); + } + + #[test] + fn build_session_payload_realtime_whisper_variant_omits_turn_detection() { + use super::build_session_payload; + let payload = build_session_payload("gpt-4o-realtime-whisper-20250512"); + let td = &payload["session"]["audio"]["input"]["turn_detection"]; + assert!( + td.is_null(), + "turn_detection should be absent for any realtime-whisper variant" + ); + } +} diff --git a/crates/buzz-relay/src/config.rs b/crates/buzz-relay/src/config.rs index eaa67a052..31f754fbc 100644 --- a/crates/buzz-relay/src/config.rs +++ b/crates/buzz-relay/src/config.rs @@ -120,6 +120,12 @@ pub struct Config { pub media_max_concurrent_uploads_per_pubkey: u32, /// Maximum media upload starts accepted from one pubkey per minute. pub media_uploads_per_minute: u32, + /// Maximum transcription sessions one pubkey can mint per minute. + /// Each session opens a metered OpenAI Realtime connection on the operator's + /// bill, so this bounds cost exposure from a single member. + pub transcribe_sessions_per_minute: u32, + /// Transcription model to use (default: whisper-1). + pub transcription_model: String, /// Optional override for ephemeral channel TTL (in seconds). /// When set, any channel created with a TTL tag will use this value instead @@ -152,6 +158,11 @@ pub struct Config { /// Used to authenticate internal policy endpoint requests. pub git_hook_hmac_secret: String, + /// Optional OpenAI API key for real-time transcription (dictation). + /// When absent, the `/transcribe/session` endpoint returns 503 and the + /// desktop mic button stays hidden. + pub openai_api_key: Option, + /// Optional path to the web UI `dist/` directory. /// When set, the relay serves the SPA from this directory for browser requests. /// When unset, no static file serving happens (relay behaves as before). @@ -184,7 +195,7 @@ impl Config { let bind_addr = parse_bind_addr(&bind_addr_raw)?; let database_url = std::env::var("DATABASE_URL") - .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".to_string()); + .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".to_string()); // sadscan:disable np.postgres.1 let redis_url = std::env::var("REDIS_URL").unwrap_or_else(|_| "redis://localhost:6379".to_string()); @@ -342,6 +353,17 @@ impl Config { .filter(|&v| v > 0) .unwrap_or(30); + let transcribe_sessions_per_minute: u32 = + std::env::var("BUZZ_TRANSCRIBE_SESSIONS_PER_MINUTE") + .ok() + .and_then(|v| v.parse().ok()) + .filter(|&v| v > 0) + .unwrap_or(5); + let transcription_model: String = std::env::var("BUZZ_TRANSCRIPTION_MODEL") + .ok() + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| "whisper-1".to_string()); + let ephemeral_ttl_override = std::env::var("BUZZ_EPHEMERAL_TTL_OVERRIDE") .ok() .and_then(|v| v.parse::().ok()) @@ -380,6 +402,11 @@ impl Config { let secret: [u8; 32] = rand::random(); hex::encode(secret) }); + let openai_api_key = std::env::var("BUZZ_OPENAI_API_KEY") + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()); + // Web UI static file serving let web_dir = std::env::var("BUZZ_WEB_DIR") .ok() @@ -433,6 +460,8 @@ impl Config { media_max_concurrent_uploads, media_max_concurrent_uploads_per_pubkey, media_uploads_per_minute, + transcribe_sessions_per_minute, + transcription_model, ephemeral_ttl_override, git_repo_path, git_max_pack_bytes, @@ -440,6 +469,7 @@ impl Config { git_max_repos_per_pubkey, git_max_concurrent_ops, git_hook_hmac_secret, + openai_api_key, web_dir, }) } diff --git a/crates/buzz-relay/src/router.rs b/crates/buzz-relay/src/router.rs index fc9e1ec38..b3d188f37 100644 --- a/crates/buzz-relay/src/router.rs +++ b/crates/buzz-relay/src/router.rs @@ -62,6 +62,15 @@ pub fn build_router(state: Arc) -> Router { .route("/count", post(api::bridge::count_events)) // Webhook trigger (secret-authenticated, no NIP-98) .route("/hooks/{id}", post(api::bridge::workflow_webhook)) + // Transcription (dictation) — proxies OpenAI Realtime client-secret minting + .route( + "/transcribe/status", + get(api::transcribe::transcribe_status), + ) + .route( + "/transcribe/session", + post(api::transcribe::create_transcribe_session), + ) // Huddle audio WebSocket route .route( "/huddle/{channel_id}/audio", @@ -93,6 +102,7 @@ pub fn build_router(state: Arc) -> Router { || path.starts_with("/internal/") || path.starts_with("/.well-known/") || path.starts_with("/huddle/") + || path.starts_with("/transcribe/") || path == "/health" || path == "/_liveness" || path == "/_readiness" diff --git a/crates/buzz-relay/src/state.rs b/crates/buzz-relay/src/state.rs index a674e6d7c..232de96d7 100644 --- a/crates/buzz-relay/src/state.rs +++ b/crates/buzz-relay/src/state.rs @@ -303,6 +303,10 @@ pub struct AppState { /// Per-uploader sliding-window rate limiter for media upload starts. /// Key: uploader pubkey bytes (32). Value: (count, window_start). pub media_upload_rate_limiter: Arc>, + /// Per-pubkey sliding-window rate limiter for transcription session minting. + /// Key: requester pubkey bytes (32). Value: (count, window_start). + /// Bounds the cost of OpenAI Realtime sessions minted on the operator's bill. + pub transcribe_rate_limiter: Arc>, /// Current in-flight media uploads per uploader pubkey. pub media_uploads_in_flight: Arc>, /// Cache for observer agent-owner authorization (kind 24200). @@ -441,6 +445,7 @@ impl AppState { observer_rate_limiter: Arc::new(DashMap::new()), mesh_connect_rate_limiter: Arc::new(DashMap::new()), media_upload_rate_limiter: Arc::new(DashMap::new()), + transcribe_rate_limiter: Arc::new(DashMap::new()), media_uploads_in_flight: Arc::new(DashMap::new()), observer_owner_cache: Arc::new( moka::sync::Cache::builder() diff --git a/desktop/src/features/dictation/api/transcribeSession.ts b/desktop/src/features/dictation/api/transcribeSession.ts new file mode 100644 index 000000000..276ca495a --- /dev/null +++ b/desktop/src/features/dictation/api/transcribeSession.ts @@ -0,0 +1,69 @@ +import { getRelayHttpUrl, signRelayEvent } from "@/shared/api/tauri"; + +export interface TranscribeStatus { + configured: boolean; + model: string; +} + +export interface TranscribeSession { + clientSecret: string; + model: string; +} + +/** NIP-98 event kind for HTTP request authorization. */ +const NIP98_KIND = 27235; + +/** + * Build a NIP-98 `Authorization: Nostr ` header for an HTTP request. + * + * The relay verifies the signed event's `u` tag against its own + * host-derived expected URL, so `url` must be the exact absolute URL being + * fetched (scheme + host + path). The `method` tag must match the request. + */ +async function nip98AuthHeader(url: string, method: string): Promise { + const nonce = crypto.randomUUID(); + const event = await signRelayEvent({ + kind: NIP98_KIND, + content: "", + tags: [ + ["u", url], + ["method", method], + ["nonce", nonce], + ], + }); + const json = JSON.stringify(event); + // btoa needs a binary string; encode UTF-8 first so non-ASCII survives. + const base64 = btoa(String.fromCharCode(...new TextEncoder().encode(json))); + return `Nostr ${base64}`; +} + +export async function getTranscribeStatus(): Promise { + const baseUrl = await getRelayHttpUrl(); + const url = `${baseUrl}/transcribe/status`; + const response = await fetch(url, { + headers: { Authorization: await nip98AuthHeader(url, "GET") }, + }); + if (!response.ok) { + throw new Error(`Transcribe status check failed: ${response.status}`); + } + return response.json(); +} + +export async function createTranscribeSession(): Promise { + const baseUrl = await getRelayHttpUrl(); + const url = `${baseUrl}/transcribe/session`; + const response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: await nip98AuthHeader(url, "POST"), + }, + }); + if (!response.ok) { + const body = await response.text().catch(() => ""); + throw new Error( + `Failed to create transcribe session (${response.status}): ${body}`, + ); + } + return response.json(); +} diff --git a/desktop/src/features/dictation/hooks/useComposerDictation.ts b/desktop/src/features/dictation/hooks/useComposerDictation.ts new file mode 100644 index 000000000..b9d8fd3c2 --- /dev/null +++ b/desktop/src/features/dictation/hooks/useComposerDictation.ts @@ -0,0 +1,75 @@ +import type * as React from "react"; +import { useEffect, useRef } from "react"; +import { useDictation } from "./useDictation"; + +interface UseComposerDictationOptions { + /** Ref to a function that syncs contentRef from the Tiptap editor and returns it. */ + syncContentRef: React.MutableRefObject<() => string>; + /** Whether the composer is currently disabled (read-only, etc.). */ + disabled?: boolean; + disabledRef: React.MutableRefObject; + isSendingRef: React.MutableRefObject; + isUploadingRef: React.MutableRefObject; + /** Updates contentRef + isContentEmpty state. */ + setComposerContent: (text: string) => void; + /** Ref to a function that updates the Tiptap editor document. */ + setEditorContentRef: React.MutableRefObject<(text: string) => void>; + submitMessageRef: React.MutableRefObject<() => void>; + /** When this key changes (channel/thread switch), active dictation is stopped. */ + draftKey?: string | null; +} + +/** + * Thin wrapper around `useDictation` pre-wired for the MessageComposer's + * state management (syncContentRef, setComposerContent, editor, submitMessageRef). + */ +export function useComposerDictation({ + syncContentRef, + disabled = false, + disabledRef, + isSendingRef, + isUploadingRef, + setComposerContent, + setEditorContentRef, + submitMessageRef, + draftKey, +}: UseComposerDictationOptions) { + const isSendBlockedRef = useRef(false); + isSendBlockedRef.current = + disabledRef.current || isSendingRef.current || isUploadingRef.current; + + const dictation = useDictation({ + getText: () => syncContentRef.current(), + setText: (text) => { + setComposerContent(text); + setEditorContentRef.current(text); + }, + onSend: (text) => { + setComposerContent(text); + setEditorContentRef.current(text); + // Submit synchronously — the content ref is already set above, so + // syncComposerContentFromEditor() will serialize the editor which now + // holds the dictated text. + submitMessageRef.current(); + }, + isSendBlockedRef, + }); + + // Cancel dictation when the channel/thread changes so that transcript events + // from a stale WebRTC session don't leak into the wrong draft. + // biome-ignore lint/correctness/useExhaustiveDependencies: draftKey is the sole trigger + useEffect(() => { + dictation.cancelRecording(); + }, [draftKey]); + + // Auto-cancel dictation when the composer becomes disabled mid-recording + // (e.g. channel becomes read-only, parent send state disables thread composer). + // Without this, the WebRTC session keeps running with no way to stop it. + useEffect(() => { + if (disabled && dictation.isRecording) { + dictation.cancelRecording(); + } + }, [disabled, dictation.isRecording, dictation.cancelRecording]); + + return dictation; +} diff --git a/desktop/src/features/dictation/hooks/useDictation.ts b/desktop/src/features/dictation/hooks/useDictation.ts new file mode 100644 index 000000000..5e21fed16 --- /dev/null +++ b/desktop/src/features/dictation/hooks/useDictation.ts @@ -0,0 +1,87 @@ +import type * as React from "react"; +import { useCallback, useMemo, useRef } from "react"; +import { + DEFAULT_AUTO_SUBMIT_PHRASE, + getAutoSubmitMatch, + parseAutoSubmitPhrases, + replaceTrailingTranscribedText, +} from "../lib/voiceInput"; +import { useRealtimeDictation } from "./useRealtimeDictation"; + +interface UseDictationOptions { + /** Returns the current composer text (must be fresh — synced from editor). */ + getText: () => string; + /** Set composer text */ + setText: (value: string) => void; + /** Send the message */ + onSend: (text: string) => void; + /** Ref that is `true` when sending is blocked (uploading, preparing mention, etc.) */ + isSendBlockedRef?: React.MutableRefObject; +} + +export function useDictation({ + getText, + setText, + onSend, + isSendBlockedRef, +}: UseDictationOptions) { + const autoSubmitPhrases = useMemo( + () => parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE), + [], + ); + const stopRecordingRef = useRef<() => void>(() => {}); + const lastTranscriptRef = useRef(""); + + const handleTranscript = useCallback( + (transcript: string) => { + const previous = lastTranscriptRef.current; + const latest = getText(); + const merged = replaceTrailingTranscribedText( + latest, + previous, + transcript, + ); + const match = getAutoSubmitMatch(transcript, autoSubmitPhrases); + + if (!match) { + setText(merged); + lastTranscriptRef.current = transcript; + return; + } + + const textWithoutPhrase = replaceTrailingTranscribedText( + latest, + previous, + match.textWithoutPhrase, + ); + if (!textWithoutPhrase.trim()) return; + + stopRecordingRef.current(); + + if (isSendBlockedRef?.current) { + setText(textWithoutPhrase); + return; + } + + // Set the text first so the composer shows the final dictated content, + // then trigger send. We intentionally do NOT clear the composer here — + // the send flow in MessageComposer handles clearing on successful send. + // If a mention dialog opens (non-member mention), the text stays in the + // composer so the user doesn't lose their dictated message. + setText(textWithoutPhrase.trim()); + onSend(textWithoutPhrase.trim()); + lastTranscriptRef.current = ""; + }, + [autoSubmitPhrases, getText, onSend, isSendBlockedRef, setText], + ); + + const dictation = useRealtimeDictation({ + onRecordingStart: () => { + lastTranscriptRef.current = ""; + }, + onTranscriptText: handleTranscript, + }); + stopRecordingRef.current = dictation.stopRecording; + + return dictation; +} diff --git a/desktop/src/features/dictation/hooks/useRealtimeDictation.ts b/desktop/src/features/dictation/hooks/useRealtimeDictation.ts new file mode 100644 index 000000000..959798b9e --- /dev/null +++ b/desktop/src/features/dictation/hooks/useRealtimeDictation.ts @@ -0,0 +1,373 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import { toast } from "sonner"; +import { + createTranscribeSession, + getTranscribeStatus, +} from "../api/transcribeSession"; +import { + type AudioBufferCapture, + type TranscriptEvent, + type TranscriptSegmentState, + BUFFER_COMMITTED_EVENT, + TRANSCRIPT_COMPLETED_EVENT, + TRANSCRIPT_DELTA_EVENT, + commitAudioBuffer, + connectPeerConnection, + createAudioBufferCapture, + createPeerConnection, + createTranscriptSegmentState, + flushAudioBuffer, + getTranscriptText, + mergeTranscriptEvent, + requiresManualCommit, +} from "../lib/realtimeAudio"; + +interface UseRealtimeDictationOptions { + disabled?: boolean; + onRecordingStart?: () => void; + onTranscriptText: (text: string) => void; +} + +function closeResources(resources: { + audioCapture?: AudioBufferCapture | null; + dataChannel?: RTCDataChannel | null; + peerConnection?: RTCPeerConnection | null; + stream?: MediaStream | null; +}) { + resources.audioCapture?.close(); + resources.dataChannel?.close(); + resources.peerConnection?.close(); + for (const track of resources.stream?.getTracks() ?? []) { + track.stop(); + } +} + +export function useRealtimeDictation({ + disabled = false, + onRecordingStart, + onTranscriptText, +}: UseRealtimeDictationOptions) { + const [isRecording, setIsRecording] = useState(false); + const [isStarting, setIsStarting] = useState(false); + const [isTranscribing, setIsTranscribing] = useState(false); + const [isConfigured, setIsConfigured] = useState(false); + + const peerConnectionRef = useRef(null); + const dataChannelRef = useRef(null); + const streamRef = useRef(null); + const audioCaptureRef = useRef(null); + const segmentStateRef = useRef( + createTranscriptSegmentState(), + ); + const activeRunIdRef = useRef(0); + const manualCommitRef = useRef(false); + const commitIntervalRef = useRef | null>(null); + const onRecordingStartRef = useRef(onRecordingStart); + const onTranscriptTextRef = useRef(onTranscriptText); + + onRecordingStartRef.current = onRecordingStart; + onTranscriptTextRef.current = onTranscriptText; + + const isEnabled = !disabled && isConfigured; + + // Check if transcription is configured on mount + useEffect(() => { + let cancelled = false; + getTranscribeStatus() + .then((status) => { + if (!cancelled) setIsConfigured(status.configured); + }) + .catch(() => { + if (!cancelled) setIsConfigured(false); + }); + return () => { + cancelled = true; + }; + }, []); + + /** + * Tear down recording resources. + * + * @param invalidateRun - When true, immediately marks the run stale so + * late transcript events are rejected (used by send/edit-save/navigation). + * When false (user-initiated stop), the run stays valid during the grace + * window so the final commit's transcript is delivered to the composer. + */ + const cleanupResources = useCallback((invalidateRun = true) => { + // Clear periodic commit interval if active. + if (commitIntervalRef.current) { + clearInterval(commitIntervalRef.current); + commitIntervalRef.current = null; + } + + // For manual-commit models (no server VAD), commit any buffered audio + // before tearing down the connection so OpenAI processes the final chunk. + // We keep the data channel open briefly to receive the transcript response. + const dc = dataChannelRef.current; + const needsCommit = + manualCommitRef.current && dc && dc.readyState === "open"; + manualCommitRef.current = false; + + // Send final commit for manual-commit models. + if (needsCommit && dc) { + commitAudioBuffer(dc); + } + + // Determine whether we need a grace window before full teardown. + // User-initiated stop (!invalidateRun) needs a grace period for BOTH: + // - Manual-commit models: final commit's transcript response + // - Server-VAD models: in-flight VAD completion events + const needsGrace = !invalidateRun && dc && dc.readyState === "open"; + + if (needsGrace && dc) { + // Stop the mic immediately so no new audio is sent. + for (const track of streamRef.current?.getTracks() ?? []) { + track.stop(); + } + streamRef.current = null; + audioCaptureRef.current?.close(); + audioCaptureRef.current = null; + // Delay WebRTC teardown briefly so final transcript events arrive. + const pc = peerConnectionRef.current; + peerConnectionRef.current = null; + dataChannelRef.current = null; + const runId = activeRunIdRef.current; + + setTimeout(() => { + // After the grace window, invalidate the run and tear down. + if (activeRunIdRef.current === runId) { + activeRunIdRef.current += 1; + } + dc.close(); + pc?.close(); + }, 3000); + return; + } + + // Immediate teardown: send/navigation cancellation, or no open channel. + activeRunIdRef.current += 1; + closeResources({ + audioCapture: audioCaptureRef.current, + dataChannel: dataChannelRef.current, + peerConnection: peerConnectionRef.current, + stream: streamRef.current, + }); + audioCaptureRef.current = null; + dataChannelRef.current = null; + peerConnectionRef.current = null; + streamRef.current = null; + }, []); + + /** Full cleanup — invalidates the run (used by send/navigation). */ + const cleanup = useCallback(() => { + cleanupResources(true); + setIsRecording(false); + setIsStarting(false); + setIsTranscribing(false); + }, [cleanupResources]); + + /** User-initiated stop — preserves final transcript for manual-commit models. */ + const userStop = useCallback(() => { + cleanupResources(false); + setIsRecording(false); + setIsStarting(false); + // Note: isTranscribing stays true briefly while the final transcript arrives. + }, [cleanupResources]); + + useEffect(() => cleanupResources, [cleanupResources]); + + const handleRealtimeEvent = useCallback( + (runId: number, event: TranscriptEvent) => { + // Ignore events from a stale run (e.g. user sent/stopped while + // transcripts were still in-flight from the data channel). + if (activeRunIdRef.current !== runId) return; + + if (event.type === "error") { + console.error("OpenAI realtime server error", event); + toast.error(event.error?.message ?? "Voice input error"); + return; + } + + if ( + event.type !== TRANSCRIPT_DELTA_EVENT && + event.type !== TRANSCRIPT_COMPLETED_EVENT && + event.type !== BUFFER_COMMITTED_EVENT + ) { + return; + } + + const prevText = getTranscriptText(segmentStateRef.current); + const merged = mergeTranscriptEvent(segmentStateRef.current, event); + + if (merged === prevText) return; + + onTranscriptTextRef.current(merged); + setIsTranscribing(event.type !== TRANSCRIPT_COMPLETED_EVENT); + }, + [], + ); + + const startRecording = useCallback(async () => { + if (!isEnabled || isStarting || isRecording) return; + + const runId = activeRunIdRef.current + 1; + activeRunIdRef.current = runId; + const isStaleRun = () => activeRunIdRef.current !== runId; + + let stream: MediaStream | null = null; + let audioCapture: AudioBufferCapture | null = null; + let peerConnection: RTCPeerConnection | null = null; + let dataChannel: RTCDataChannel | null = null; + + setIsStarting(true); + segmentStateRef.current = createTranscriptSegmentState(); + onRecordingStartRef.current?.(); + + try { + // 1. Capture mic immediately for instant feedback + stream = await navigator.mediaDevices.getUserMedia({ + audio: { + autoGainControl: true, + echoCancellation: true, + noiseSuppression: true, + }, + }); + if (isStaleRun()) { + closeResources({ stream }); + return; + } + streamRef.current = stream; + setIsRecording(true); + + // 2. Buffer PCM via AudioWorklet while network calls proceed + audioCapture = await createAudioBufferCapture(stream); + if (isStaleRun()) { + closeResources({ audioCapture, stream }); + return; + } + audioCaptureRef.current = audioCapture; + + // 3. Create session via relay + const session = await createTranscribeSession(); + if (isStaleRun()) { + closeResources({ audioCapture, stream }); + return; + } + manualCommitRef.current = requiresManualCommit(session.model); + + // 4. Set up WebRTC + peerConnection = createPeerConnection(); + peerConnectionRef.current = peerConnection; + const activeStream = stream; + stream.getAudioTracks().forEach((track) => { + peerConnection?.addTrack(track, activeStream); + }); + + dataChannel = peerConnection.createDataChannel("oai-events"); + dataChannelRef.current = dataChannel; + dataChannel.addEventListener("message", (message) => { + try { + handleRealtimeEvent(runId, JSON.parse(String(message.data))); + } catch { + // Ignore non-JSON events + } + }); + + // Flush buffered audio once data channel opens + const channelToFlush = dataChannel; + const captureToFlush = audioCapture; + const useManualCommit = manualCommitRef.current; + dataChannel.addEventListener("open", () => { + // If the user stopped (or restarted) recording between the SDP + // exchange and the channel opening, drop this run's buffered audio. + if (isStaleRun()) { + captureToFlush.close(); + return; + } + flushAudioBuffer(channelToFlush, captureToFlush.chunks); + // For manual-commit models, commit the initial buffered audio and + // start a periodic commit interval so streaming transcripts flow + // during recording (server VAD models commit automatically). + if (useManualCommit) { + commitAudioBuffer(channelToFlush); + // Commit every 2s to produce streaming transcript segments while + // the user is still speaking. Each commit triggers a transcription + // of the audio accumulated since the last commit. + commitIntervalRef.current = setInterval(() => { + if (channelToFlush.readyState === "open") { + commitAudioBuffer(channelToFlush); + } + }, 2000); + } + captureToFlush.close(); + audioCaptureRef.current = null; + }); + + // 5. SDP exchange + await connectPeerConnection({ + peerConnection, + clientSecret: session.clientSecret, + }); + if (isStaleRun()) { + closeResources({ audioCapture, dataChannel, peerConnection, stream }); + return; + } + } catch (error) { + closeResources({ audioCapture, dataChannel, peerConnection, stream }); + if (!isStaleRun()) { + audioCaptureRef.current = null; + dataChannelRef.current = null; + peerConnectionRef.current = null; + streamRef.current = null; + setIsRecording(false); + setIsTranscribing(false); + + const message = + error instanceof Error ? error.message : "Voice input failed"; + if (/not allowed|denied|permission/i.test(message)) { + toast.error("Microphone access denied", { + description: + "Allow microphone access in System Settings to use dictation.", + }); + } else if (/not found|no audio/i.test(message)) { + toast.error("No microphone found", { + description: "Connect a microphone and try again.", + }); + } else { + toast.error("Voice input failed", { description: message }); + } + } + } finally { + if (!isStaleRun()) setIsStarting(false); + } + }, [handleRealtimeEvent, isEnabled, isRecording, isStarting]); + + /** User-initiated stop (mic button) — preserves final transcript. */ + const stopRecording = useCallback(() => userStop(), [userStop]); + + /** + * Cancel recording and reject all pending transcripts. Used by send, + * edit-save, and navigation to prevent late events from refilling the + * composer after the content has been dispatched. + */ + const cancelRecording = useCallback(() => cleanup(), [cleanup]); + + const toggleRecording = useCallback(() => { + if (isRecording || isStarting) { + stopRecording(); + return; + } + void startRecording(); + }, [isRecording, isStarting, startRecording, stopRecording]); + + return { + isEnabled, + isRecording, + isStarting, + isTranscribing, + startRecording, + stopRecording, + cancelRecording, + toggleRecording, + }; +} diff --git a/desktop/src/features/dictation/index.ts b/desktop/src/features/dictation/index.ts new file mode 100644 index 000000000..1578aaf65 --- /dev/null +++ b/desktop/src/features/dictation/index.ts @@ -0,0 +1,3 @@ +export { useComposerDictation } from "./hooks/useComposerDictation"; +export { useDictation } from "./hooks/useDictation"; +export { DictationButton } from "./ui/DictationButton"; diff --git a/desktop/src/features/dictation/lib/realtimeAudio.test.mjs b/desktop/src/features/dictation/lib/realtimeAudio.test.mjs new file mode 100644 index 000000000..0121ef436 --- /dev/null +++ b/desktop/src/features/dictation/lib/realtimeAudio.test.mjs @@ -0,0 +1,334 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; + +// Inline the logic to keep the test self-contained and avoid bundler issues. +const TRANSCRIPT_DELTA_EVENT = + "conversation.item.input_audio_transcription.delta"; +const TRANSCRIPT_COMPLETED_EVENT = + "conversation.item.input_audio_transcription.completed"; +const BUFFER_COMMITTED_EVENT = "input_audio_buffer.committed"; + +function createTranscriptSegmentState() { + return { itemOrder: [], items: new Map() }; +} + +function getOrCreateItem(state, itemId, previousItemId) { + let seg = state.items.get(itemId); + if (!seg) { + seg = { pending: "", finalized: null }; + state.items.set(itemId, seg); + if (previousItemId) { + const prevIndex = state.itemOrder.indexOf(previousItemId); + if (prevIndex !== -1) { + state.itemOrder.splice(prevIndex + 1, 0, itemId); + } else { + state.itemOrder.push(itemId); + } + } else { + state.itemOrder.push(itemId); + } + } + return seg; +} + +function getTranscriptText(state) { + let result = ""; + for (const id of state.itemOrder) { + const seg = state.items.get(id); + if (!seg) continue; + const text = seg.finalized ?? seg.pending; + if (!text) continue; + if (result && !result.endsWith(" ") && !text.startsWith(" ")) { + result += " "; + } + result += text; + } + return result; +} + +function mergeTranscriptEvent(state, event) { + const itemId = event.item_id ?? "__default__"; + + if (event.type === BUFFER_COMMITTED_EVENT) { + getOrCreateItem(state, itemId, event.previous_item_id ?? undefined); + } else if (event.type === TRANSCRIPT_DELTA_EVENT) { + const seg = getOrCreateItem(state, itemId); + const delta = event.delta ?? ""; + if (delta) { + seg.pending += delta; + } + } else if (event.type === TRANSCRIPT_COMPLETED_EVENT) { + const seg = getOrCreateItem(state, itemId); + seg.finalized = event.transcript ?? ""; + } + + return getTranscriptText(state); +} + +describe("mergeTranscriptEvent", () => { + it("accumulates delta events for a single item", () => { + const state = createTranscriptSegmentState(); + const r1 = mergeTranscriptEvent(state, { + type: TRANSCRIPT_DELTA_EVENT, + item_id: "item_1", + delta: "hello ", + }); + assert.equal(r1, "hello "); + + const r2 = mergeTranscriptEvent(state, { + type: TRANSCRIPT_DELTA_EVENT, + item_id: "item_1", + delta: "world", + }); + assert.equal(r2, "hello world"); + }); + + it("replaces deltas with finalized text on completed event", () => { + const state = createTranscriptSegmentState(); + mergeTranscriptEvent(state, { + type: TRANSCRIPT_DELTA_EVENT, + item_id: "item_1", + delta: "hello world", + }); + + const result = mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_1", + transcript: "Hello, world.", + }); + assert.equal(result, "Hello, world."); + }); + + it("handles multiple items in order", () => { + const state = createTranscriptSegmentState(); + + // First item + mergeTranscriptEvent(state, { + type: TRANSCRIPT_DELTA_EVENT, + item_id: "item_1", + delta: "first ", + }); + mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_1", + transcript: "First.", + }); + + // Second item + mergeTranscriptEvent(state, { + type: TRANSCRIPT_DELTA_EVENT, + item_id: "item_2", + delta: "second", + }); + assert.equal(state.items.get("item_2").pending, "second"); + + const result = mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_2", + transcript: "Second.", + }); + assert.equal(result, "First. Second."); + }); + + it("handles out-of-order completed events by item id", () => { + const state = createTranscriptSegmentState(); + + // Both items start with deltas + mergeTranscriptEvent(state, { + type: TRANSCRIPT_DELTA_EVENT, + item_id: "item_1", + delta: "first", + }); + mergeTranscriptEvent(state, { + type: TRANSCRIPT_DELTA_EVENT, + item_id: "item_2", + delta: "second", + }); + + // item_2 completes before item_1 + mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_2", + transcript: "Second.", + }); + + // item_1 still shows pending + let result = mergeTranscriptEvent(state, { + type: TRANSCRIPT_DELTA_EVENT, + item_id: "item_1", + delta: " more", + }); + assert.equal(result, "first more Second."); + + // item_1 finally completes + result = mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_1", + transcript: "First more.", + }); + assert.equal(result, "First more. Second."); + }); + + it("does not duplicate text on completed event", () => { + const state = createTranscriptSegmentState(); + + mergeTranscriptEvent(state, { + type: TRANSCRIPT_DELTA_EVENT, + item_id: "item_1", + delta: "hello world", + }); + + const result = mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_1", + transcript: "Hello, world.", + }); + assert.equal(result, "Hello, world."); + }); + + it("falls back to __default__ when item_id is missing", () => { + const state = createTranscriptSegmentState(); + mergeTranscriptEvent(state, { + type: TRANSCRIPT_DELTA_EVENT, + delta: "no id", + }); + const result = mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + transcript: "No id.", + }); + assert.equal(result, "No id."); + }); + + it("preserves committed item order when completions arrive out of order", () => { + const state = createTranscriptSegmentState(); + + // Server commits items in order: item_1 then item_2 + mergeTranscriptEvent(state, { + type: BUFFER_COMMITTED_EVENT, + item_id: "item_1", + previous_item_id: null, + }); + mergeTranscriptEvent(state, { + type: BUFFER_COMMITTED_EVENT, + item_id: "item_2", + previous_item_id: "item_1", + }); + + // But item_2's completion arrives first + mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_2", + transcript: "Second.", + }); + + // Then item_1's completion arrives + const result = mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_1", + transcript: "First.", + }); + + // Order must follow committed order, not arrival order + assert.equal(result, "First. Second."); + }); + + it("inserts item after previous_item_id even if later items already exist", () => { + const state = createTranscriptSegmentState(); + + // Commit item_1 + mergeTranscriptEvent(state, { + type: BUFFER_COMMITTED_EVENT, + item_id: "item_1", + }); + + // Commit item_3 (item_2 not yet committed) + mergeTranscriptEvent(state, { + type: BUFFER_COMMITTED_EVENT, + item_id: "item_3", + previous_item_id: "item_1", + }); + + // Now commit item_2 between item_1 and item_3 + // (previous_item_id = item_1, so it goes after item_1) + mergeTranscriptEvent(state, { + type: BUFFER_COMMITTED_EVENT, + item_id: "item_2", + previous_item_id: "item_1", + }); + + // Add transcripts + mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_1", + transcript: "One.", + }); + mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_2", + transcript: "Two.", + }); + const result = mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_3", + transcript: "Three.", + }); + + // item_2 was inserted after item_1 (before item_3) + assert.equal(result, "One. Two. Three."); + }); + + it("handles completion-only flow with committed order", () => { + const state = createTranscriptSegmentState(); + + // Server commits both items + mergeTranscriptEvent(state, { + type: BUFFER_COMMITTED_EVENT, + item_id: "item_A", + }); + mergeTranscriptEvent(state, { + type: BUFFER_COMMITTED_EVENT, + item_id: "item_B", + previous_item_id: "item_A", + }); + + // Only completed events arrive (no deltas) — in reverse order + mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_B", + transcript: "Bravo.", + }); + const result = mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_A", + transcript: "Alpha.", + }); + + assert.equal(result, "Alpha. Bravo."); + }); +}); + +// Inline the logic to keep the test self-contained. +function requiresManualCommit(model) { + return model.includes("realtime-whisper"); +} + +describe("requiresManualCommit", () => { + it("returns true for gpt-realtime-whisper", () => { + assert.equal(requiresManualCommit("gpt-realtime-whisper"), true); + }); + + it("returns true for versioned realtime-whisper model", () => { + assert.equal( + requiresManualCommit("gpt-4o-realtime-whisper-20250512"), + true, + ); + }); + + it("returns false for whisper-1", () => { + assert.equal(requiresManualCommit("whisper-1"), false); + }); + + it("returns false for gpt-4o-transcribe", () => { + assert.equal(requiresManualCommit("gpt-4o-transcribe"), false); + }); +}); diff --git a/desktop/src/features/dictation/lib/realtimeAudio.ts b/desktop/src/features/dictation/lib/realtimeAudio.ts new file mode 100644 index 000000000..4c46fec7b --- /dev/null +++ b/desktop/src/features/dictation/lib/realtimeAudio.ts @@ -0,0 +1,256 @@ +import { + REALTIME_BUFFER_PROCESSOR_NAME, + createWorkletBlobUrl, +} from "./realtimeBufferWorklet"; + +export const OPENAI_REALTIME_WEBRTC_URL = + "https://api.openai.com/v1/realtime/calls"; +export const TRANSCRIPT_DELTA_EVENT = + "conversation.item.input_audio_transcription.delta"; +export const TRANSCRIPT_COMPLETED_EVENT = + "conversation.item.input_audio_transcription.completed"; +export const BUFFER_COMMITTED_EVENT = "input_audio_buffer.committed"; + +const MAX_BUFFER_CHUNKS = 500; // ~10s at 20ms per chunk + +export type TranscriptEvent = { + type?: string; + item_id?: string; + previous_item_id?: string; + content_index?: number; + delta?: string; + transcript?: string; + message?: string; + error?: { message?: string }; +}; + +export function createPeerConnection(): RTCPeerConnection { + return new RTCPeerConnection(); +} + +export async function connectPeerConnection(options: { + peerConnection: RTCPeerConnection; + clientSecret: string; +}): Promise { + const offer = await options.peerConnection.createOffer(); + await options.peerConnection.setLocalDescription(offer); + + const response = await fetch(OPENAI_REALTIME_WEBRTC_URL, { + method: "POST", + headers: { + Authorization: `Bearer ${options.clientSecret}`, + "Content-Type": "application/sdp", + }, + body: offer.sdp ?? "", + }); + + const body = await response.text(); + if (!response.ok) { + throw new Error( + `OpenAI realtime connection failed (${response.status}): ${body}`, + ); + } + + await options.peerConnection.setRemoteDescription({ + type: "answer", + sdp: body, + }); +} + +/** + * Per-item tracking for a single transcription turn. OpenAI's Realtime API + * sends delta and completed events tagged with an `item_id`; completed events + * for different turns can arrive out of order, so we reconcile by item id. + */ +interface ItemSegment { + /** Accumulated delta text (replaced by finalized on completion). */ + pending: string; + /** Finalized text (set once the completed event arrives). */ + finalized: string | null; +} + +/** + * State for tracking transcription segments keyed by item id. + * Maintains insertion order so the full transcript is reconstructed + * in the order items were first seen. + */ +export interface TranscriptSegmentState { + /** Ordered item ids (insertion order = utterance order). */ + itemOrder: string[]; + /** Per-item segment data. */ + items: Map; +} + +export function createTranscriptSegmentState(): TranscriptSegmentState { + return { itemOrder: [], items: new Map() }; +} + +/** Get the current full transcript text from segment state. */ +export function getTranscriptText(state: TranscriptSegmentState): string { + let result = ""; + for (const id of state.itemOrder) { + const seg = state.items.get(id); + if (!seg) continue; + const text = seg.finalized ?? seg.pending; + if (!text) continue; + if (result && !result.endsWith(" ") && !text.startsWith(" ")) { + result += " "; + } + result += text; + } + return result; +} + +/** + * Internal: get or create the segment for an item. + * When `previousItemId` is provided (from committed events), the new item is + * inserted after that item in `itemOrder` to preserve the server's utterance + * order — even if transcript events for a later item arrive first. + */ +function getOrCreateItem( + state: TranscriptSegmentState, + itemId: string, + previousItemId?: string, +): ItemSegment { + let seg = state.items.get(itemId); + if (!seg) { + seg = { pending: "", finalized: null }; + state.items.set(itemId, seg); + if (previousItemId) { + const prevIndex = state.itemOrder.indexOf(previousItemId); + if (prevIndex !== -1) { + state.itemOrder.splice(prevIndex + 1, 0, itemId); + } else { + // Previous item not yet seen — append (best effort). + state.itemOrder.push(itemId); + } + } else { + state.itemOrder.push(itemId); + } + } + return seg; +} + +/** + * Merge a transcript event into the segment state, keyed by `item_id`. + * + * - Committed events: register the item in the correct order using + * `previous_item_id` from the server, before any transcript arrives. + * - Delta events: append to the item's `pending` text. + * - Completed events: store `finalized` text, replacing accumulated deltas. + * + * Returns the full merged text across all items in order. + */ +export function mergeTranscriptEvent( + state: TranscriptSegmentState, + event: TranscriptEvent, +): string { + // Use item_id from the event; fall back to a synthetic key for events + // that lack one (shouldn't happen in practice, but be defensive). + const itemId = event.item_id ?? "__default__"; + + if (event.type === BUFFER_COMMITTED_EVENT) { + // Register the item in the correct position before transcripts arrive. + getOrCreateItem(state, itemId, event.previous_item_id ?? undefined); + } else if (event.type === TRANSCRIPT_DELTA_EVENT) { + const seg = getOrCreateItem(state, itemId); + const delta = event.delta ?? ""; + if (delta) { + seg.pending += delta; + } + } else if (event.type === TRANSCRIPT_COMPLETED_EVENT) { + const seg = getOrCreateItem(state, itemId); + seg.finalized = event.transcript ?? ""; + } + + // Reconstruct full text from all items in order. + return getTranscriptText(state); +} + +// ── Audio buffer capture ────────────────────────────────────────────────── + +export interface AudioBufferCapture { + chunks: Int16Array[]; + close(): void; +} + +export async function createAudioBufferCapture( + stream: MediaStream, +): Promise { + const audioContext = new AudioContext(); + const blobUrl = createWorkletBlobUrl(); + try { + await audioContext.audioWorklet.addModule(blobUrl); + } finally { + URL.revokeObjectURL(blobUrl); + } + + const source = audioContext.createMediaStreamSource(stream); + const worklet = new AudioWorkletNode( + audioContext, + REALTIME_BUFFER_PROCESSOR_NAME, + ); + source.connect(worklet); + worklet.connect(audioContext.destination); + + const chunks: Int16Array[] = []; + worklet.port.onmessage = (event: MessageEvent) => { + if (chunks.length < MAX_BUFFER_CHUNKS) { + chunks.push(new Int16Array(event.data)); + } + }; + + return { + chunks, + close() { + worklet.disconnect(); + source.disconnect(); + void audioContext.close(); + }, + }; +} + +// ── Flush buffered PCM into the data channel ────────────────────────────── + +function int16ToBase64(pcm: Int16Array): string { + const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength); + let binary = ""; + for (let i = 0; i < bytes.length; i++) { + binary += String.fromCharCode(bytes[i]); + } + return btoa(binary); +} + +export function flushAudioBuffer( + dataChannel: RTCDataChannel, + chunks: Int16Array[], +): void { + for (const chunk of chunks) { + dataChannel.send( + JSON.stringify({ + type: "input_audio_buffer.append", + audio: int16ToBase64(chunk), + }), + ); + } + chunks.length = 0; +} + +/** + * Send `input_audio_buffer.commit` to finalize buffered audio for transcription. + * + * Required when server VAD is disabled (e.g. `realtime-whisper` models) — + * without a commit, appended audio is never processed. For models using + * server VAD, the server commits automatically on speech boundaries. + */ +export function commitAudioBuffer(dataChannel: RTCDataChannel): void { + dataChannel.send(JSON.stringify({ type: "input_audio_buffer.commit" })); +} + +/** + * Whether a transcription model requires manual audio commit (no server VAD). + * Models containing "realtime-whisper" use manual commit per OpenAI guidance. + */ +export function requiresManualCommit(model: string): boolean { + return model.includes("realtime-whisper"); +} diff --git a/desktop/src/features/dictation/lib/realtimeBufferWorklet.ts b/desktop/src/features/dictation/lib/realtimeBufferWorklet.ts new file mode 100644 index 000000000..ac8ffaabd --- /dev/null +++ b/desktop/src/features/dictation/lib/realtimeBufferWorklet.ts @@ -0,0 +1,53 @@ +const TARGET_SAMPLE_RATE = 24000; +const FRAME_SAMPLES = 480; // 20ms at 24kHz + +export const REALTIME_BUFFER_PROCESSOR_NAME = "realtime-buffer-processor"; + +export const REALTIME_BUFFER_WORKLET_SOURCE = /* js */ ` +class RealtimeBufferProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this._ratio = sampleRate / ${TARGET_SAMPLE_RATE}; + this._offset = 0; + this._buf = new Float32Array(${FRAME_SAMPLES}); + this._idx = 0; + } + + process(inputs) { + const input = inputs[0]?.[0]; + if (!input) return true; + + while (this._offset < input.length) { + const i = Math.floor(this._offset); + const frac = this._offset - i; + const s0 = input[i]; + const s1 = i + 1 < input.length ? input[i + 1] : s0; + this._buf[this._idx++] = s0 + frac * (s1 - s0); + + if (this._idx >= ${FRAME_SAMPLES}) { + const pcm = new Int16Array(${FRAME_SAMPLES}); + for (let j = 0; j < ${FRAME_SAMPLES}; j++) { + const s = Math.max(-1, Math.min(1, this._buf[j])); + pcm[j] = s < 0 ? s * 0x8000 : s * 0x7fff; + } + this.port.postMessage(pcm.buffer, [pcm.buffer]); + this._idx = 0; + } + this._offset += this._ratio; + } + this._offset -= input.length; + return true; + } +} + +registerProcessor('${REALTIME_BUFFER_PROCESSOR_NAME}', RealtimeBufferProcessor); +`; + +/** Create a blob URL that can be passed to `audioWorklet.addModule()`. */ +export function createWorkletBlobUrl(): string { + return URL.createObjectURL( + new Blob([REALTIME_BUFFER_WORKLET_SOURCE], { + type: "application/javascript", + }), + ); +} diff --git a/desktop/src/features/dictation/lib/voiceInput.test.mjs b/desktop/src/features/dictation/lib/voiceInput.test.mjs new file mode 100644 index 000000000..151104cbd --- /dev/null +++ b/desktop/src/features/dictation/lib/voiceInput.test.mjs @@ -0,0 +1,150 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + DEFAULT_AUTO_SUBMIT_PHRASE, + getAutoSubmitMatch, + parseAutoSubmitPhrases, + replaceTrailingTranscribedText, +} from "./voiceInput.ts"; + +// ── parseAutoSubmitPhrases ────────────────────────────────────────────────── + +test("parseAutoSubmitPhrases_returnsEmptyForNullish", () => { + assert.deepEqual(parseAutoSubmitPhrases(null), []); + assert.deepEqual(parseAutoSubmitPhrases(undefined), []); + assert.deepEqual(parseAutoSubmitPhrases(""), []); +}); + +test("parseAutoSubmitPhrases_splitsNormalizesAndDedupes", () => { + assert.deepEqual(parseAutoSubmitPhrases("Submit, send it, submit, "), [ + "submit", + "send it", + ]); +}); + +test("parseAutoSubmitPhrases_stripsTrailingPunctuation", () => { + assert.deepEqual(parseAutoSubmitPhrases("submit!"), ["submit"]); +}); + +// ── replaceTrailingTranscribedText ────────────────────────────────────────── + +test("replaceTrailingTranscribedText_appendsWhenNoPrevious", () => { + assert.equal( + replaceTrailingTranscribedText("Hello", "", "world"), + "Hello world", + ); +}); + +test("replaceTrailingTranscribedText_appendsToEmptyBase", () => { + assert.equal(replaceTrailingTranscribedText("", "", "hello"), "hello"); +}); + +test("replaceTrailingTranscribedText_replacesTrailingInterim", () => { + // Interim "hello wor" is refined to "hello world". + assert.equal( + replaceTrailingTranscribedText("hello wor", "hello wor", "hello world"), + "hello world", + ); +}); + +test("replaceTrailingTranscribedText_preservesTextTypedBeforeDictation", () => { + // User typed "Note: " then dictated; the manual prefix must survive. + assert.equal( + replaceTrailingTranscribedText("Note: hi", "hi", "hi there"), + "Note: hi there", + ); +}); + +test("replaceTrailingTranscribedText_appendsWhenPreviousNoLongerMatches", () => { + // If the previous transcript isn't the trailing text anymore, append. + assert.equal( + replaceTrailingTranscribedText("edited text", "old", "new"), + "edited text new", + ); +}); + +test("replaceTrailingTranscribedText_noDoubleSpaceBeforePunctuation", () => { + assert.equal( + replaceTrailingTranscribedText("Hello", "", ", world"), + "Hello, world", + ); +}); + +// ── getAutoSubmitMatch ────────────────────────────────────────────────────── + +test("getAutoSubmitMatch_returnsNullWhenPhraseAbsent", () => { + assert.equal( + getAutoSubmitMatch("hello there", parseAutoSubmitPhrases("submit")), + null, + ); +}); + +test("getAutoSubmitMatch_returnsNullWhenPhrasesEmpty", () => { + // DEFAULT_AUTO_SUBMIT_PHRASE is empty (auto-submit disabled by default). + assert.equal( + getAutoSubmitMatch( + "send this message submit", + parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE), + ), + null, + ); +}); + +test("getAutoSubmitMatch_matchesTrailingPhraseAndStripsIt", () => { + const match = getAutoSubmitMatch( + "send this message submit", + parseAutoSubmitPhrases("submit"), + ); + assert.ok(match); + assert.equal(match.matchedPhrase, "submit"); + assert.equal(match.textWithoutPhrase, "send this message"); +}); + +test("getAutoSubmitMatch_ignoresPhraseMidSentence", () => { + // "submit" is not at the end, so it must not auto-send. + assert.equal( + getAutoSubmitMatch( + "submit the form later", + parseAutoSubmitPhrases("submit"), + ), + null, + ); +}); + +test("getAutoSubmitMatch_requiresWordBoundaryBeforePhrase", () => { + // "resubmit" ends with "submit" but is not a standalone word → no match. + assert.equal( + getAutoSubmitMatch("resubmit", parseAutoSubmitPhrases("submit")), + null, + ); +}); + +test("getAutoSubmitMatch_toleratesTrailingPunctuation", () => { + const match = getAutoSubmitMatch( + "ship it submit.", + parseAutoSubmitPhrases("submit"), + ); + assert.ok(match); + assert.equal(match.textWithoutPhrase, "ship it"); +}); + +test("getAutoSubmitMatch_matchesMultiWordPhrase", () => { + const match = getAutoSubmitMatch( + "please do this send it", + parseAutoSubmitPhrases("send it"), + ); + assert.ok(match); + assert.equal(match.matchedPhrase, "send it"); + assert.equal(match.textWithoutPhrase, "please do this"); +}); + +test("getAutoSubmitMatch_prefersLongestPhrase", () => { + const match = getAutoSubmitMatch( + "text please submit now", + parseAutoSubmitPhrases("submit now, now"), + ); + assert.ok(match); + assert.equal(match.matchedPhrase, "submit now"); + assert.equal(match.textWithoutPhrase, "text please"); +}); diff --git a/desktop/src/features/dictation/lib/voiceInput.ts b/desktop/src/features/dictation/lib/voiceInput.ts new file mode 100644 index 000000000..1c287cef7 --- /dev/null +++ b/desktop/src/features/dictation/lib/voiceInput.ts @@ -0,0 +1,114 @@ +/** + * Default auto-submit phrase. Empty string disables auto-submit — the user + * must manually press Enter/Send after dictation. The infrastructure for + * configurable phrases is in place (parseAutoSubmitPhrases, getAutoSubmitMatch) + * and can be wired to a user setting when we're ready to ship auto-submit. + */ +export const DEFAULT_AUTO_SUBMIT_PHRASE = ""; + +const TRAILING_PUNCTUATION_REGEX = /[\s"'`.,!?;:)\]}]+$/u; + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function normalizePhrase(value: string): string { + return value + .toLowerCase() + .replace(/\s+/g, " ") + .trim() + .replace(TRAILING_PUNCTUATION_REGEX, "") + .trim(); +} + +export function parseAutoSubmitPhrases( + rawValue: string | null | undefined, +): string[] { + if (!rawValue) return []; + return Array.from( + new Set( + rawValue + .split(",") + .map((value) => normalizePhrase(value)) + .filter(Boolean), + ), + ); +} + +function appendTranscribedText(baseText: string, fragment: string): string { + const normalizedFragment = fragment.replace(/\s+/g, " ").trim(); + if (!normalizedFragment) return baseText; + if (!baseText.trim()) return normalizedFragment; + if (/[\s([{/-]$/.test(baseText) || /^[,.;!?)]/.test(normalizedFragment)) { + return `${baseText}${normalizedFragment}`; + } + return `${baseText} ${normalizedFragment}`; +} + +export function replaceTrailingTranscribedText( + fullText: string, + previousTranscribedText: string, + nextTranscribedText: string, +): string { + if (!previousTranscribedText) { + return appendTranscribedText(fullText, nextTranscribedText); + } + + if (fullText.endsWith(previousTranscribedText)) { + return appendTranscribedText( + fullText.slice(0, -previousTranscribedText.length), + nextTranscribedText, + ); + } + + const trimmedPreviousText = previousTranscribedText.trim(); + if (trimmedPreviousText && fullText.endsWith(trimmedPreviousText)) { + return appendTranscribedText( + fullText.slice(0, -trimmedPreviousText.length), + nextTranscribedText, + ); + } + + return appendTranscribedText(fullText, nextTranscribedText); +} + +export function getAutoSubmitMatch( + transcribedText: string, + autoSubmitPhrases: string[], +): { matchedPhrase: string; textWithoutPhrase: string } | null { + const normalizedTranscribedText = normalizePhrase(transcribedText); + if (!normalizedTranscribedText) return null; + + const sortedPhrases = [...autoSubmitPhrases].sort( + (left, right) => right.length - left.length, + ); + + for (const phrase of sortedPhrases) { + if (!normalizedTranscribedText.endsWith(phrase)) continue; + + const phraseStartIndex = normalizedTranscribedText.length - phrase.length; + if ( + phraseStartIndex > 0 && + normalizedTranscribedText[phraseStartIndex - 1] !== " " + ) { + continue; + } + + const trimmedText = transcribedText.replace(TRAILING_PUNCTUATION_REGEX, ""); + const phraseWords = phrase.split(" ").filter(Boolean).map(escapeRegExp); + const phrasePattern = new RegExp( + `(^|\\s)(${phraseWords.join("\\s+")})\\s*$`, + "iu", + ); + const rawMatch = trimmedText.match(phrasePattern); + const phraseStartOffset = + rawMatch && rawMatch.index !== undefined + ? rawMatch.index + (rawMatch[1]?.length ?? 0) + : trimmedText.length - phrase.length; + const textWithoutPhrase = trimmedText.slice(0, phraseStartOffset).trimEnd(); + + return { matchedPhrase: phrase, textWithoutPhrase }; + } + + return null; +} diff --git a/desktop/src/features/dictation/ui/DictationButton.tsx b/desktop/src/features/dictation/ui/DictationButton.tsx new file mode 100644 index 000000000..387a0ee32 --- /dev/null +++ b/desktop/src/features/dictation/ui/DictationButton.tsx @@ -0,0 +1,63 @@ +import { Mic } from "lucide-react"; +import { Button } from "@/shared/ui/button"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/shared/ui/tooltip"; +import { cn } from "@/shared/lib/cn"; + +interface DictationState { + isEnabled: boolean; + isRecording: boolean; + isStarting: boolean; + isTranscribing: boolean; + toggleRecording: () => void; +} + +interface DictationButtonProps { + dictation: DictationState; + disabled?: boolean; +} + +export function DictationButton({ + dictation, + disabled = false, +}: DictationButtonProps) { + if (!dictation.isEnabled) return null; + + const tooltipText = dictation.isRecording + ? "Stop recording" + : dictation.isTranscribing + ? "Transcribing…" + : "Dictate message"; + + // Allow the stop action whenever the mic is live (isRecording), even if + // the session setup is still in progress (isStarting) or the composer is + // disabled. Only block the button when idle + disabled, or when startup + // hasn't captured the mic yet (isStarting && !isRecording). + const isDisabled = dictation.isRecording + ? false + : disabled || dictation.isStarting; + + return ( + + + + + {tooltipText} + + ); +} diff --git a/desktop/src/features/messages/ui/MessageComposer.tsx b/desktop/src/features/messages/ui/MessageComposer.tsx index f30d039ff..aac744783 100644 --- a/desktop/src/features/messages/ui/MessageComposer.tsx +++ b/desktop/src/features/messages/ui/MessageComposer.tsx @@ -51,6 +51,7 @@ import { MessageComposerToolbar } from "./MessageComposerToolbar"; import { NonMemberMentionDialog } from "./NonMemberMentionDialog"; import { useMentionSendFlow } from "./useMentionSendFlow"; import { useComposerContentState } from "./useComposerContentState"; +import { DictationButton, useComposerDictation } from "@/features/dictation"; type MessageComposerProps = { channelId?: string | null; @@ -205,7 +206,6 @@ function MessageComposerImpl({ onEditSaveRef.current = onEditSave; onEditLastOwnMessageRef.current = onEditLastOwnMessage; editTargetRef.current = editTarget; - const isAutocompleteOpenRef = React.useRef(false); isAutocompleteOpenRef.current = mentions.isMentionOpen || @@ -213,8 +213,21 @@ function MessageComposerImpl({ emojiAutocomplete.isEmojiAutocompleteOpen; const submitMessageRef = React.useRef<() => void>(() => {}); + const setEditorContentRef = React.useRef<(text: string) => void>(() => {}); + const stopDictationRef = React.useRef<() => void>(() => {}); + const dictation = useComposerDictation({ + syncContentRef: syncContentRefFromEditorRef, + disabled, + disabledRef, + isSendingRef, + isUploadingRef, + setComposerContent, + setEditorContentRef, + submitMessageRef, + draftKey: effectiveDraftKey, + }); + stopDictationRef.current = dictation.cancelRecording; const composerScrollRef = React.useRef(null); - // Set after `useLinkEditor` exists below; the editor's link-click handler // delegates through this ref to break the hook ordering cycle (the editor // needs `onEditLink`, but the link editor needs the editor's `richText`). @@ -271,6 +284,7 @@ function MessageComposerImpl({ }, }); + setEditorContentRef.current = richText.setContent; const linkEditor = useLinkEditor(richText); syncContentRefFromEditorRef.current = () => { const markdown = richText.getMarkdown(); @@ -523,6 +537,7 @@ function MessageComposerImpl({ // Edit mode if (editTargetRef.current && onEditSaveRef.current) { if (isSendingRef.current || isUploadingRef.current) return; + stopDictationRef.current(); // stop dictation so late transcripts don't refill during edit save const currentPendingImeta = media.pendingImetaRef.current; const hasMedia = currentPendingImeta.length > 0; // Empty text + zero attachments is a no-op (don't let edit become an @@ -539,11 +554,8 @@ function MessageComposerImpl({ spoileredAttachmentUrls, ); - // NIP-30: attach `["emoji", shortcode, url]` tags for custom emoji in the - // edited body, exactly like the send path. Without this an edited message - // ships with no emoji tags, so the receiver can't resolve a `:shortcode:` - // and renders the literal text. `?? []` preserves edit semantics (a - // defined-but-empty media set means "wipe attachments"). + // NIP-30 emoji tags for the edited body (mirrors the send path). + // `?? []` preserves edit semantics (empty = "wipe attachments"). const outgoingTags = mergeOutgoingTags( mediaTags, @@ -586,6 +598,8 @@ function MessageComposerImpl({ return; } + stopDictationRef.current(); // stop dictation so late transcripts don't refill + const capturedThreadContext = onCaptureSendContext?.() ?? null; // If a thread-reply composer reported no reply target at submit time, // bail here rather than discovering the null later after async awaits. @@ -632,10 +646,8 @@ function MessageComposerImpl({ ); // ── Keyboard handling ─────────────────────────────────────────────── - // Tiptap handles formatting shortcuts (⌘B, ⌘I, etc.) natively. - // Plain Enter → submit is now handled inside the Tiptap `submitOnEnter` - // extension (fires before ProseMirror's splitBlock). This wrapper only - // handles autocomplete arrow/enter keys and Escape for edit mode. + // Autocomplete arrow/enter keys and Escape for edit mode only; plain + // Enter → submit is in the Tiptap `submitOnEnter` extension. const handleEditorKeyDown = React.useCallback( (event: React.KeyboardEvent) => { // Let autocomplete handle keys first @@ -943,7 +955,12 @@ function MessageComposerImpl({ + + {toolbarExtraActions} + + } formattingDisabled={disabled} isEmojiPickerOpen={isEmojiPickerOpen} isFormattingOpen={isFormattingOpen} @@ -979,5 +996,4 @@ function MessageComposerImpl({ ); } - export const MessageComposer = React.memo(MessageComposerImpl); diff --git a/desktop/tests/e2e/human-edit-agent-content.spec.ts b/desktop/tests/e2e/human-edit-agent-content.spec.ts index 93bc13621..0994fce23 100644 --- a/desktop/tests/e2e/human-edit-agent-content.spec.ts +++ b/desktop/tests/e2e/human-edit-agent-content.spec.ts @@ -99,11 +99,15 @@ test("owner can edit their owned agent's message", async ({ page }) => { // Edit banner must appear confirming edit mode is active. await expect(page.getByTestId("edit-target")).toBeVisible({ timeout: 5_000 }); - // Clear the composer and type the new content, then submit. + // Wait for the editor to be populated with the original message content + // (edit mode calls richText.setContent which is async in Tiptap's + // transaction pipeline). Then select-all and type the replacement. const input = page.getByTestId("message-input"); - await input.clear(); - await input.fill(editedContent); - await input.press("Enter"); + await expect(input).not.toBeEmpty({ timeout: 5_000 }); + await input.click(); + await page.keyboard.press("ControlOrMeta+A"); + await page.keyboard.type(editedContent); + await page.keyboard.press("Enter"); // Edit mode must exit (banner gone) and the updated content must render. await expect(page.getByTestId("edit-target")).toBeHidden();