diff --git a/Cargo.lock b/Cargo.lock index 008f8eab0..a04c90b57 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -696,6 +696,7 @@ name = "basic_room" version = "0.1.0" dependencies = [ "env_logger 0.11.10", + "hound", "livekit", "livekit-api", "log", @@ -4019,6 +4020,7 @@ dependencies = [ "semver", "serde", "serde_json", + "serial_test", "test-case", "test-log", "thiserror 1.0.69", @@ -6720,6 +6722,15 @@ dependencies = [ "regex", ] +[[package]] +name = "scc" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46e6f046b7fef48e2660c57ed794263155d713de679057f2d0c169bfc6e756cc" +dependencies = [ + "sdd", +] + [[package]] name = "schannel" version = "0.1.29" @@ -6802,6 +6813,12 @@ dependencies = [ "tiny-skia", ] +[[package]] +name = "sdd" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca" + [[package]] name = "sec1" version = "0.7.3" @@ -6939,6 +6956,32 @@ dependencies = [ "serde", ] +[[package]] +name = "serial_test" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "911bd979bf1070a3f3aa7b691a3b3e9968f339ceeec89e08c280a8a22207a32f" +dependencies = [ + "futures-executor", + "futures-util", + "log", + "once_cell", + "parking_lot", + "scc", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a7d91949b85b0d2fb687445e448b40d322b6b3e4af6b44a29b21d9a5f33e6d9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "sha1" version = "0.10.6" diff --git a/docs/ADM_PROXY_DESIGN.md b/docs/ADM_PROXY_DESIGN.md new file mode 100644 index 000000000..b729e0819 --- /dev/null +++ b/docs/ADM_PROXY_DESIGN.md @@ -0,0 +1,940 @@ +# Audio Device Module (ADM) Proxy Design Document + +## Overview + +This document describes the design and implementation of the Audio Device Module (ADM) Proxy system in the LiveKit Rust SDK. The ADM Proxy enables platform audio device access while maintaining backward compatibility with manual audio pushing via `NativeAudioSource`. + +--- + +## Goals + +### Problem Statement + +WebRTC's `AudioDeviceModule` (ADM) is traditionally configured at `PeerConnectionFactory` creation time. The SDK needs to support two audio capture methods: + +1. **Manual audio push** (default): Applications push audio frames via `NativeAudioSource` +2. **Platform audio capture**: WebRTC captures from the system microphone automatically + +These two methods must coexist without interference. + +### Design Goals + +| Goal | Description | +|------|-------------| +| **Dual Audio Support** | Support both `NativeAudioSource` (manual push) and platform microphone capture | +| **Multiple Audio Tracks** | Allow multiple audio tracks with different sources simultaneously | +| **Backward Compatible** | Existing code using `NativeAudioSource` continues to work unchanged | +| **Clean Public API** | Expose a simple `PlatformAudio` interface for device management | +| **FFI Support** | Platform audio available for FFI clients (Python, Unity, etc.) | + +### Non-Goals / Limitations + +| Limitation | Description | +|------------|-------------| +| **Index-based device IDs** | Device indices may change on hot-plug | +| **Process-global** | Audio configuration is process-global, not per-room | + +--- + +## Architecture + +### High-Level Design + +The SDK uses a **recording gate** pattern rather than mode switching: + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Audio Architecture │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌─────────────────────────────────────────────────────────────────────┐ │ +│ │ PeerConnectionFactory (created once at startup) │ │ +│ │ └─ AdmProxy (wraps Platform ADM) │ │ +│ │ ├─ Platform ADM: Always created and initialized │ │ +│ │ └─ recording_enabled_: Gate for microphone access (default OFF) │ │ +│ └─────────────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ┌─────────────────┼─────────────────────┐ │ +│ ▼ ▼ ▼ │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ +│ │ Device Track │ │ Native Track │ │ Native Track │ │ +│ │ (Microphone) │ │ (TTS) │ │ (Screen Cap) │ │ +│ └──────────────┘ └──────────────┘ └──────────────┘ │ +│ │ │ │ │ +│ │ │ │ │ +│ Uses AudioState Uses AddSink Uses AddSink │ +│ (is_external=false) (is_external=true) (is_external=true) │ +│ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +### Key Components + +1. **AdmProxy**: Wraps WebRTC's platform ADM with a recording gate +2. **PlatformAudio**: Rust API for enabling platform audio and device management +3. **NativeAudioSource**: Existing API for manual audio frame pushing +4. **external_audio_source.patch**: WebRTC patch to prevent audio mixing conflicts + +### Recording Gate Pattern + +Instead of swapping ADM implementations, the SDK uses a simple boolean gate: + +```cpp +// adm_proxy.h +class AdmProxy : public webrtc::AudioDeviceModule { + // Platform ADM is ALWAYS created at startup + webrtc::scoped_refptr platform_adm_; + + // Gate controls whether microphone recording is active + // Default: FALSE - NativeAudioSource works without interference + bool recording_enabled_ = false; +}; +``` + +When `recording_enabled_ = false`: +- `InitRecording()` returns success but does nothing +- `StartRecording()` returns success but does nothing +- Microphone is not accessed +- `NativeAudioSource` works normally + +When `recording_enabled_ = true` (via `PlatformAudio::new()`): +- `InitRecording()` initializes the microphone +- `StartRecording()` starts microphone capture +- Device audio flows to tracks using `RtcAudioSource::Device` + +--- + +## WebRTC Patching + +The SDK applies a patch to WebRTC to support multiple audio sources without conflicts. This section explains the patch and why it's necessary. + +### The Problem + +WebRTC's `AudioState` routes device-captured audio to **all** `AudioSendStream` instances by default. This causes problems when mixing device audio with manually-pushed audio: + +``` +Without Patch: + ADM (microphone) → AudioState → ALL AudioSendStreams + NativeAudioSource → Same AudioSendStreams + = DOUBLE FEEDING! (device audio + manual audio mixed incorrectly) +``` + +### The Solution: external_audio_source.patch + +Located at: `webrtc-sys/libwebrtc/patches/external_audio_source.patch` + +The patch adds an `is_external_source()` method to `AudioSourceInterface`: + +```cpp +// api/media_stream_interface.h +class AudioSourceInterface : public MediaSourceInterface { + // Returns true if this source delivers audio externally (via AddSink), + // bypassing the ADM/AudioState audio distribution path. + virtual bool is_external_source() const { return false; } +}; +``` + +### Patch Details + +**1. AudioSourceInterface addition** (`api/media_stream_interface.h`): +```cpp +// Returns true if this source delivers audio externally (via AddSink), +// bypassing the ADM/AudioState audio distribution path. +// When true, AudioSendStream should not register with AudioState. +virtual bool is_external_source() const { return false; } +``` + +**2. AudioSendStream::Config flag** (`call/audio_send_stream.h`): +```cpp +struct Config { + // When true, this stream uses an external audio source (not ADM). + // AudioState will NOT send device-captured audio to this stream. + bool external_source = false; +}; +``` + +**3. AudioSendStream lifecycle changes** (`audio/audio_send_stream.cc`): +```cpp +void AudioSendStream::Start() { + // Only register with AudioState if not using external source. + // External sources deliver audio directly via AddSink. + if (!config_.external_source) { + audio_state()->AddSendingStream(this, ...); + } +} + +void AudioSendStream::Stop() { + if (!config_.external_source) { + audio_state()->RemoveSendingStream(this); + } +} +``` + +**4. Automatic detection** (`media/engine/webrtc_voice_engine.cc`): +```cpp +void WebRtcAudioSendStream::SetSource(AudioSource* source) { + // Check if this is an external audio source + if (source->is_external_source() && !config_.external_source) { + config_.external_source = true; + stream_->Reconfigure(config_, nullptr); + } + source->SetSink(this); +} +``` + +### SDK Implementation + +**NativeAudioSource** (`webrtc-sys/include/livekit/audio_track.h`): +```cpp +class AudioTrackSource::InternalSource : public webrtc::LocalAudioSource { + // Override to indicate this is an external audio source. + // This prevents AudioState from sending device audio to streams using this source. + bool is_external_source() const override { return true; } +}; +``` + +**Device Source**: Uses WebRTC's built-in `LocalAudioSource` which returns `false` (default). + +### Audio Flow with Patch + +``` +With Patch: + ADM (microphone) → AudioState → Only streams with is_external=false (Device tracks) + NativeAudioSource → Only streams with is_external=true (Native tracks) + = CLEAN SEPARATION! +``` + +### Why Not platform_audio_source.patch? + +An alternative approach would be `platform_audio_source.patch` that creates a new source type consuming from an ADM sink. This was considered but rejected: + +| Approach | Pros | Cons | +|----------|------|------| +| **external_audio_source.patch** (chosen) | Minimal patch, uses standard WebRTC AudioState for device audio | Single device track per ADM | +| **platform_audio_source.patch** | Unified source model, multiple device tracks | More complex, extra buffering/latency, larger patch | + +The current approach is preferred because: +1. **Minimal WebRTC modification**: Only adds a boolean flag +2. **Uses standard audio path**: Device audio uses WebRTC's battle-tested AudioState +3. **Low latency**: No extra buffering for device audio +4. **Simpler**: Less code to maintain + +--- + +## Audio Modes + +### Quick Reference + +| Source Type | Use Case | Audio Flow | AEC Works? | +|-------------|----------|------------|------------| +| `RtcAudioSource::Native` | TTS, file streaming, agents | Manual push via `capture_frame()` | No | +| `RtcAudioSource::Device` | VoIP, microphone capture | Automatic via platform ADM | Yes | + +### Mode 1: Manual Audio Push (Default) + +Use `NativeAudioSource` to push audio frames manually. This is the default mode and requires no special setup. + +```rust +use livekit::webrtc::prelude::*; +use livekit::webrtc::audio_source::native::NativeAudioSource; +use livekit::webrtc::audio_source::RtcAudioSource; +use livekit::prelude::*; + +// Create audio source for manual frame pushing +let source = NativeAudioSource::new( + AudioSourceOptions::default(), + 48000, // sample rate + 2, // channels + 100, // queue size in ms +); + +// Push frames manually +source.capture_frame(&audio_frame).await; + +// Create track +let track = LocalAudioTrack::create_audio_track( + "audio", + RtcAudioSource::Native(source) +); + +// Publish the track +room.local_participant() + .publish_track(LocalTrack::Audio(track), TrackPublishOptions::default()) + .await?; +``` + +**Characteristics:** +- `recording_enabled_ = false` (default) +- ADM recording operations are no-ops +- Audio is pushed via `capture_frame()` +- `is_external_source() = true` prevents AudioState interference +- AEC does NOT work (no valid playout reference) + +**Suitable for:** +- Server-side agents +- Text-to-speech (TTS) audio +- Audio from files or network streams +- Testing without audio hardware + +--- + +### Mode 2: Platform Audio Capture + +Use `PlatformAudio` to capture from the system microphone. WebRTC handles device management automatically. + +```rust +use livekit::prelude::*; + +// Create PlatformAudio instance (enables recording gate) +let audio = PlatformAudio::new()?; + +// Enumerate devices +println!("Recording devices:"); +for i in 0..audio.recording_devices() as u16 { + println!(" [{}] {}", i, audio.recording_device_name(i)); +} + +// Select device +audio.set_recording_device(0)?; + +// Connect to room +let (room, events) = Room::connect(&url, &token, RoomOptions::default()).await?; + +// Create track using Device source (Platform ADM handles capture) +let track = LocalAudioTrack::create_audio_track("microphone", audio.rtc_source()); + +// Publish +room.local_participant() + .publish_track(LocalTrack::Audio(track), TrackPublishOptions::default()) + .await?; + +// ... use room ... + +// PlatformAudio dropped automatically when out of scope +``` + +**Characteristics:** +- `PlatformAudio::new()` sets `recording_enabled_ = true` +- ADM recording operations work normally +- Audio captured automatically from selected microphone +- `is_external_source() = false` allows AudioState routing +- AEC works correctly + +**Platform implementations:** +| Platform | Backend | +|----------|---------| +| macOS/iOS | CoreAudio / VPIO | +| Windows | WASAPI | +| Linux | PulseAudio / ALSA | +| Android | AAudio / OpenSL ES | + +--- + +### Combining Both Modes + +You can use both `NativeAudioSource` and `PlatformAudio` simultaneously for different tracks: + +```rust +use livekit::prelude::*; +use livekit::webrtc::audio_source::native::NativeAudioSource; + +// Track A: Microphone via platform audio +let mic = PlatformAudio::new()?; +let mic_track = LocalAudioTrack::create_audio_track("mic", mic.rtc_source()); + +// Track B: TTS via manual pushing +let tts_source = NativeAudioSource::new(opts, 48000, 2, 100); +let tts_track = LocalAudioTrack::create_audio_track( + "tts", + RtcAudioSource::Native(tts_source), +); + +// Publish both - they don't interfere with each other +room.local_participant().publish_track(LocalTrack::Audio(mic_track), opts).await?; +room.local_participant().publish_track(LocalTrack::Audio(tts_track), opts).await?; +``` + +This works because: +1. `mic_track` uses `is_external_source() = false` → receives ADM audio via AudioState +2. `tts_track` uses `is_external_source() = true` → receives audio via `capture_frame()` +3. The `external_audio_source.patch` ensures they don't mix + +--- + +## Remote Audio Playback + +Understanding how remote audio reaches speakers is important for choosing the right audio mode. + +### Without PlatformAudio (Manual Playback) + +When using only `NativeAudioSource` (the default mode), remote audio does **not** automatically play to speakers. You must explicitly create an `AudioStream` to receive audio frames from remote tracks: + +```rust +use livekit::prelude::*; +use libwebrtc::audio_stream::native::NativeAudioStream; +use futures_util::StreamExt; + +// When a remote track is received +let RoomEvent::TrackSubscribed { track, .. } = event else { continue }; +let RemoteTrack::Audio(remote_audio) = track.into() else { continue }; + +// Create an AudioStream to pull audio from the remote track +let mut stream = NativeAudioStream::new( + remote_audio.rtc_track(), + 48000, // desired sample rate + 2, // desired channels +); + +// Poll the stream to receive audio frames +while let Some(frame) = stream.next().await { + // frame.data: Vec - PCM audio samples + // frame.sample_rate: u32 + // frame.num_channels: u32 + // frame.samples_per_channel: u32 + + // Application must route this audio to speakers manually + // (e.g., via cpal, rodio, or platform audio APIs) +} +``` + +**How it works internally:** + +1. `NativeAudioStream::new()` creates a `NativeAudioSink` and registers it with the remote track via `audio.add_sink(&sink)` +2. WebRTC calls the sink's `on_data()` callback when decoded audio frames arrive +3. Frames are queued (bounded queue with configurable size, default 10 frames / ~100ms) +4. Application polls the stream to receive frames +5. Application is responsible for routing audio to the actual speaker device + +**Use case:** Server-side agents, headless applications, or apps that need custom audio routing. + +### With PlatformAudio (Automatic Playback) + +When `PlatformAudio` is active, remote audio automatically plays through the system speakers via WebRTC's audio mixer and the ADM's playout path: + +```rust +use livekit::prelude::*; + +// Create PlatformAudio (enables both recording AND playout via ADM) +let audio = PlatformAudio::new()?; + +// Optionally select speaker device +audio.set_playout_device(0)?; + +// Connect to room - remote audio will automatically play through speakers +let (room, mut events) = Room::connect(&url, &token, RoomOptions::default()).await?; + +// Remote tracks automatically play - no AudioStream needed for speaker output +while let Some(event) = events.recv().await { + match event { + RoomEvent::TrackSubscribed { track, .. } => { + // Audio track automatically plays to speakers + // No additional code needed for playback + } + _ => {} + } +} +``` + +**How it works internally:** + +1. WebRTC's `AudioReceiveStream` decodes incoming audio +2. Audio is mixed by WebRTC's internal audio mixer +3. ADM's `NeedMorePlayData()` is called by the audio device thread +4. Mixed audio is delivered to the platform speaker device + +**Track mute/unmute:** Remote track mute state is handled by WebRTC internally. Muted tracks don't contribute to the mix. + +### Comparison + +| Aspect | Without PlatformAudio | With PlatformAudio | +|--------|----------------------|-------------------| +| Remote audio to speakers | Manual via `NativeAudioStream` | Automatic via ADM | +| Application code needed | Create stream + route to speaker | None | +| Latency | Depends on app implementation | Optimized by WebRTC | +| Audio mixing | Application handles | WebRTC handles | +| Device selection | Application handles | `set_playout_device()` | +| AEC reference | Not available | Available | + +### Hybrid Approach + +You can combine both approaches - use `PlatformAudio` for automatic speaker playback while also creating `NativeAudioStream` for audio processing/analysis: + +```rust +let audio = PlatformAudio::new()?; // Enables automatic playback + +// Remote audio plays automatically to speakers +// Additionally, create a stream for audio analysis +let stream = NativeAudioStream::new(remote_track.rtc_track(), 48000, 1); +tokio::spawn(async move { + while let Some(frame) = stream.next().await { + // Analyze audio (e.g., VAD, transcription) + // Audio still plays to speakers via ADM + } +}); +``` + +--- + +## Public API + +### PlatformAudio + +```rust +/// Platform audio device management for microphone capture and speaker playout. +#[derive(Clone)] +pub struct PlatformAudio { ... } + +impl PlatformAudio { + /// Creates a new PlatformAudio instance. + /// Enables ADM recording for microphone capture. + /// Multiple instances share the same underlying ADM. + pub fn new() -> AudioResult; + + /// Get the RTC audio source for creating tracks. + /// Returns `RtcAudioSource::Device`. + pub fn rtc_source(&self) -> RtcAudioSource; + + // === Device Enumeration === + + /// Get the number of playout (speaker) devices. + pub fn playout_devices(&self) -> i16; + + /// Get the number of recording (microphone) devices. + pub fn recording_devices(&self) -> i16; + + /// Get the name of a playout device by index. + pub fn playout_device_name(&self, index: u16) -> String; + + /// Get the name of a recording device by index. + pub fn recording_device_name(&self, index: u16) -> String; + + // === Device Selection === + + /// Set the active playout device. + pub fn set_playout_device(&self, index: u16) -> AudioResult<()>; + + /// Set the active recording device. + pub fn set_recording_device(&self, index: u16) -> AudioResult<()>; + + /// Switch playout device during active session (hot-swap). + pub fn switch_playout_device(&self, index: u16) -> AudioResult<()>; + + /// Switch recording device during active session (hot-swap). + pub fn switch_recording_device(&self, index: u16) -> AudioResult<()>; + + // === Audio Processing === + + /// Configure audio processing (AEC, AGC, NS). + pub fn configure_audio_processing(&self, opts: AudioProcessingOptions) -> AudioResult<()>; + + /// Enable or disable echo cancellation. + pub fn set_echo_cancellation(&self, enabled: bool) -> AudioResult<()>; + + /// Enable or disable noise suppression. + pub fn set_noise_suppression(&self, enabled: bool) -> AudioResult<()>; + + /// Enable or disable automatic gain control. + pub fn set_auto_gain_control(&self, enabled: bool) -> AudioResult<()>; + + /// Explicitly release platform audio resources. + pub fn release(self); +} +``` + +### AudioError + +```rust +/// Errors that can occur during audio operations. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AudioError { + /// Platform audio could not be initialized. + PlatformInitFailed, + + /// The specified device index is invalid. + InvalidDeviceIndex, + + /// An audio operation failed. + OperationFailed(String), +} + +/// Result type for audio operations. +pub type AudioResult = Result; +``` + +### RtcAudioSource + +```rust +/// Audio source type for creating LocalAudioTrack. +pub enum RtcAudioSource { + /// Manual audio push via NativeAudioSource. + Native(NativeAudioSource), + + /// Platform device audio (microphone capture via ADM). + Device, +} +``` + +--- + +## FFI API + +The SDK provides a Protocol Buffers-based FFI interface for foreign language clients (Python, Unity, Node.js, etc.). The FFI uses a handle-based model where clients create a `PlatformAudio` handle and use it for all subsequent operations. + +### Protocol Messages + +Located at: `livekit-ffi/protocol/audio_manager.proto` + +```protobuf +// Create a new PlatformAudio instance +message NewPlatformAudioRequest {} + +message NewPlatformAudioResponse { + oneof message { + OwnedPlatformAudio platform_audio = 1; // Handle on success + string error = 2; // Error message on failure + } +} + +message OwnedPlatformAudio { + FfiOwnedHandle handle = 1; + PlatformAudioInfo info = 2; +} + +message PlatformAudioInfo { + int32 recording_device_count = 1; + int32 playout_device_count = 2; +} + +// Enumerate audio devices +message GetAudioDevicesRequest { + uint64 platform_audio_handle = 1; +} + +message GetAudioDevicesResponse { + repeated AudioDeviceInfo playout_devices = 1; + repeated AudioDeviceInfo recording_devices = 2; + optional string error = 3; +} + +message AudioDeviceInfo { + uint32 index = 1; + string name = 2; +} + +// Set recording device +message SetRecordingDeviceRequest { + uint64 platform_audio_handle = 1; + uint32 index = 2; +} + +message SetRecordingDeviceResponse { + optional string error = 1; +} + +// Set playout device +message SetPlayoutDeviceRequest { + uint64 platform_audio_handle = 1; + uint32 index = 2; +} + +message SetPlayoutDeviceResponse { + optional string error = 1; +} +``` + +### FFI Usage Pattern + +**1. Create PlatformAudio Handle:** +``` +Request: NewPlatformAudioRequest {} +Response: OwnedPlatformAudio { handle: 123, info: { recording: 2, playout: 3 } } +``` + +**2. Enumerate Devices:** +``` +Request: GetAudioDevicesRequest { platform_audio_handle: 123 } +Response: { + recording_devices: [ + { index: 0, name: "MacBook Pro Microphone" }, + { index: 1, name: "External USB Microphone" } + ], + playout_devices: [ + { index: 0, name: "MacBook Pro Speakers" }, + { index: 1, name: "External Headphones" } + ] +} +``` + +**3. Select Devices:** +``` +Request: SetRecordingDeviceRequest { platform_audio_handle: 123, index: 0 } +Response: SetRecordingDeviceResponse { error: null } + +Request: SetPlayoutDeviceRequest { platform_audio_handle: 123, index: 1 } +Response: SetPlayoutDeviceResponse { error: null } +``` + +**4. Create Audio Track:** +Use the handle to create an audio track with `RtcAudioSource::Device`. + +**5. Release Handle:** +When done, dispose the handle using `DisposeRequest`. The ADM recording is disabled when all handles are released. + +### Handle Lifecycle + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ FFI Client Lifecycle │ +├─────────────────────────────────────────────────────────────────┤ +│ │ +│ 1. NewPlatformAudioRequest() │ +│ └─→ Creates PlatformAudio, enables ADM recording │ +│ └─→ Returns handle_id (e.g., 123) │ +│ │ +│ 2. GetAudioDevicesRequest(handle=123) │ +│ └─→ Enumerates available microphones and speakers │ +│ │ +│ 3. SetRecordingDeviceRequest(handle=123, index=0) │ +│ └─→ Selects which microphone to use │ +│ │ +│ 4. Create audio track with Device source │ +│ └─→ Track captures from selected microphone │ +│ │ +│ 5. DisposeRequest(handle=123) │ +│ └─→ Releases PlatformAudio, disables ADM if last handle │ +│ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +### Reference Counting + +Multiple FFI clients can create `PlatformAudio` handles. All handles share the same underlying ADM: + +``` +Client A: NewPlatformAudioRequest() → handle_1 (ref_count: 1) +Client B: NewPlatformAudioRequest() → handle_2 (ref_count: 2) +Client A: DisposeRequest(handle_1) (ref_count: 1) +Client B: DisposeRequest(handle_2) (ref_count: 0, ADM disabled) +``` + +### Error Handling + +FFI responses include optional error messages: + +| Error | Meaning | +|-------|---------| +| `"Invalid device index"` | Device index >= device count | +| `"Platform audio initialization failed"` | ADM could not be created | +| `"Handle not found"` | Invalid or already disposed handle | + +--- + +## Implementation Details + +### AdmProxy Class + +```cpp +// webrtc-sys/include/livekit/adm_proxy.h +class AdmProxy : public webrtc::AudioDeviceModule { + public: + explicit AdmProxy(const webrtc::Environment& env, + webrtc::Thread* worker_thread); + ~AdmProxy() override; + + // Check if platform ADM was successfully initialized + bool is_initialized() const; + + // Control whether recording (microphone) is enabled. + // When disabled, InitRecording/StartRecording are no-ops. + void set_recording_enabled(bool enabled); + bool recording_enabled() const; + + // All AudioDeviceModule methods delegate to platform_adm_ + // Recording methods check recording_enabled_ first + + private: + const webrtc::Environment& env_; + webrtc::Thread* worker_thread_; + + // The underlying platform ADM (always created at startup) + webrtc::scoped_refptr platform_adm_; + bool adm_initialized_ = false; + + // Recording gate - defaults to FALSE + bool recording_enabled_ = false; +}; +``` + +### Recording Gate Implementation + +```cpp +// webrtc-sys/src/adm_proxy.cpp + +int32_t AdmProxy::InitRecording() { + if (!platform_adm_) return -1; + if (!recording_enabled_) { + // Return success but don't actually initialize + return 0; + } + return platform_adm_->InitRecording(); +} + +int32_t AdmProxy::StartRecording() { + if (!platform_adm_) return -1; + if (!recording_enabled_) { + // Return success but don't actually start + return 0; + } + return platform_adm_->StartRecording(); +} + +bool AdmProxy::Recording() const { + if (!platform_adm_) return false; + if (!recording_enabled_) return false; + return platform_adm_->Recording(); +} +``` + +### PlatformAudio Reference Counting + +```rust +// livekit/src/audio.rs + +lazy_static! { + static ref PLATFORM_ADM_HANDLE: Mutex> = Mutex::new(Weak::new()); +} + +struct PlatformAdmHandle { + runtime: Arc, +} + +impl PlatformAudio { + pub fn new() -> AudioResult { + let mut handle_ref = PLATFORM_ADM_HANDLE.lock(); + + // Reuse existing handle if available + if let Some(handle) = handle_ref.upgrade() { + return Ok(Self { handle }); + } + + // Create new handle and enable recording + let runtime = LkRuntime::instance(); + runtime.set_adm_recording_enabled(true); + + let handle = Arc::new(PlatformAdmHandle { runtime }); + *handle_ref = Arc::downgrade(&handle); + + Ok(Self { handle }) + } +} +``` + +--- + +## Backward Compatibility + +### NativeAudioSource Unchanged + +Existing code using `NativeAudioSource` works without any changes: + +```rust +// This code works exactly as before +let source = NativeAudioSource::new(opts, 48000, 2, 100); +source.capture_frame(&frame).await; +let track = LocalAudioTrack::create_audio_track("audio", RtcAudioSource::Native(source)); +``` + +Why it continues to work: +1. `recording_enabled_ = false` by default → ADM recording is disabled +2. `is_external_source() = true` → AudioState doesn't interfere +3. No code changes required in user applications + +### Migration from AudioManager + +If you previously used `AudioManager`, migrate to `PlatformAudio`: + +**Before:** +```rust +let audio = AudioManager::instance(); +audio.set_mode(AudioMode::Platform)?; +let track = LocalAudioTrack::create_audio_track("mic", RtcAudioSource::Device); +``` + +**After:** +```rust +let audio = PlatformAudio::new()?; +let track = LocalAudioTrack::create_audio_track("mic", audio.rtc_source()); +``` + +--- + +## Platform-Specific Notes + +### iOS + +- Creates a VPIO (Voice Processing IO) AudioUnit +- Only one VPIO can exist per process +- Drop all `PlatformAudio` instances to release the microphone +- Other audio frameworks (e.g., expo-audio-studio) get silence while VPIO is active + +### Android + +- Hardware AEC is unreliable on many devices +- Default is software audio processing (`prefer_hardware_processing = false`) +- Use `AudioProcessingOptions` to configure + +### Desktop (macOS, Windows, Linux) + +- Hardware audio processing not available +- WebRTC's software APM is always used +- Device hot-plug supported via `switch_recording_device()` + +--- + +## File Structure + +``` +rust-sdks/ +├── webrtc-sys/ +│ ├── include/livekit/ +│ │ ├── adm_proxy.h # AdmProxy class with recording gate +│ │ ├── audio_track.h # NativeAudioSource with is_external_source() +│ │ └── peer_connection_factory.h +│ ├── src/ +│ │ ├── adm_proxy.cpp # Recording gate implementation +│ │ ├── audio_track.cpp +│ │ ├── peer_connection_factory.cpp +│ │ └── peer_connection_factory.rs # FFI bindings +│ └── libwebrtc/ +│ └── patches/ +│ └── external_audio_source.patch # WebRTC patch for multi-source support +│ +├── libwebrtc/ +│ └── src/ +│ ├── audio_source.rs # RtcAudioSource enum +│ └── peer_connection_factory.rs +│ +├── livekit/ +│ └── src/ +│ ├── prelude.rs +│ ├── audio.rs # PlatformAudio, AudioProcessingOptions +│ └── rtc_engine/ +│ └── lk_runtime.rs # Runtime with ADM control methods +│ +└── livekit-ffi/ + ├── protocol/ + │ ├── audio_manager.proto # PlatformAudio FFI messages + │ ├── ffi.proto # Main FFI request/response definitions + │ └── handle.proto # FfiOwnedHandle message + └── src/ + └── server/ + └── requests.rs # FFI request handlers (FfiPlatformAudio) +``` + +--- + +## References + +- [WebRTC AudioDeviceModule Documentation](https://webrtc.googlesource.com/src/+/main/modules/audio_device/g3doc/audio_device_module.md) +- [LiveKit Swift SDK - AudioManager](https://docs.livekit.io/client-sdk-swift/AudioManager/) +- [LiveKit Android SDK - AudioOptions](https://docs.livekit.io/reference/client-sdk-android/livekit-android-sdk/io.livekit.android/-audio-options/) diff --git a/examples/basic_room/Cargo.toml b/examples/basic_room/Cargo.toml index 9b8651984..e4997ece5 100644 --- a/examples/basic_room/Cargo.toml +++ b/examples/basic_room/Cargo.toml @@ -10,3 +10,4 @@ env_logger = { workspace = true } livekit = { workspace = true, features = ["rustls-tls-native-roots"] } livekit-api = { workspace = true, features = ["rustls-tls-native-roots"] } log = { workspace = true } +hound = "3.5" diff --git a/examples/basic_room/src/main.rs b/examples/basic_room/src/main.rs index e82b784c2..15051bfb8 100644 --- a/examples/basic_room/src/main.rs +++ b/examples/basic_room/src/main.rs @@ -1,18 +1,159 @@ +use livekit::options::TrackPublishOptions; use livekit::prelude::*; +use livekit::webrtc::audio_source::native::NativeAudioSource; +use livekit::webrtc::audio_source::AudioSourceOptions; use livekit_api::access_token; use std::env; +use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; +use tokio::signal; -// Connect to a room using the specified env variables -// and print all incoming events +// Usage: +// cargo run -p basic_room -- --list-devices # List audio devices and exit +// cargo run -p basic_room -- --platform-audio # Publish microphone using PlatformAudio +// cargo run -p basic_room -- --platform-audio-and-file # Publish both mic + WAV file +// cargo run -p basic_room -- --file # Publish just WAV file (no mic) +// cargo run -p basic_room # Connect without audio publishing #[tokio::main] async fn main() { env_logger::init(); + let args: Vec = env::args().collect(); + let list_devices = args.iter().any(|arg| arg == "--list-devices"); + let use_platform_audio = args.iter().any(|arg| arg == "--platform-audio"); + + // Check for --platform-audio-and-file + let platform_audio_and_file_path = args + .iter() + .position(|arg| arg == "--platform-audio-and-file") + .and_then(|i| args.get(i + 1)) + .map(|s| s.clone()); + + // Check for --file (file only, no microphone) + let file_only_path = args + .iter() + .position(|arg| arg == "--file") + .and_then(|i| args.get(i + 1)) + .map(|s| s.clone()); + + let use_platform_audio_and_file = platform_audio_and_file_path.is_some(); + let file_path = platform_audio_and_file_path.or(file_only_path.clone()); + + // --list-devices: enumerate audio devices and exit + if list_devices { + let audio = match PlatformAudio::new() { + Ok(audio) => audio, + Err(e) => { + eprintln!("Failed to initialize platform audio: {}", e); + return; + } + }; + + println!("Recording devices (microphones):"); + let recording_count = audio.recording_devices(); + if recording_count == 0 { + println!(" (none)"); + } else { + for i in 0..recording_count as u16 { + println!(" [{}] {}", i, audio.recording_device_name(i)); + } + } + + println!("\nPlayout devices (speakers):"); + let playout_count = audio.playout_devices(); + if playout_count == 0 { + println!(" (none)"); + } else { + for i in 0..playout_count as u16 { + println!(" [{}] {}", i, audio.playout_device_name(i)); + } + } + + println!("\nAudio processing:"); + println!(" Hardware AEC available: {}", audio.is_hardware_aec_available()); + println!(" Hardware AGC available: {}", audio.is_hardware_agc_available()); + println!(" Hardware NS available: {}", audio.is_hardware_ns_available()); + + return; + } + let url = env::var("LIVEKIT_URL").expect("LIVEKIT_URL is not set"); let api_key = env::var("LIVEKIT_API_KEY").expect("LIVEKIT_API_KEY is not set"); let api_secret = env::var("LIVEKIT_API_SECRET").expect("LIVEKIT_API_SECRET is not set"); + // Determine what to publish + let publish_mic = use_platform_audio || use_platform_audio_and_file; + + // Create PlatformAudio if needed (must be created BEFORE connecting to room) + let platform_audio = if publish_mic { + let audio = PlatformAudio::new().expect("Failed to initialize platform audio"); + log::info!("Platform audio initialized"); + + let recording_count = audio.recording_devices(); + let playout_count = audio.playout_devices(); + + log::info!("Recording devices: {}", recording_count); + for i in 0..recording_count as u16 { + log::info!(" [{}] {}", i, audio.recording_device_name(i)); + } + + log::info!("Playout devices: {}", playout_count); + for i in 0..playout_count as u16 { + log::info!(" [{}] {}", i, audio.playout_device_name(i)); + } + + if recording_count > 0 { + audio.set_recording_device(0).expect("Failed to set recording device"); + } + if playout_count > 0 { + audio.set_playout_device(0).expect("Failed to set playout device"); + } + + audio + .configure_audio_processing(AudioProcessingOptions { + echo_cancellation: true, + noise_suppression: true, + auto_gain_control: true, + prefer_hardware_processing: false, + }) + .expect("Failed to configure audio processing"); + + Some(audio) + } else { + None + }; + + // Load WAV file if specified + // Note: ADM recording is disabled by default, so when using --file mode (NativeAudioSource only), + // the microphone will not be activated. It's only enabled when PlatformAudio::new() is called. + let wav_data = if let Some(ref path) = file_path { + Some(load_wav_file(path).expect("Failed to load WAV file")) + } else { + None + }; + + // Create NativeAudioSource for file playback if needed + // Use queue_size_ms > 0 for buffered path - internal AudioTask delivers frames every 10ms + // This should provide more consistent timing when ADM recording is also active + let file_source = if let Some(ref wav) = wav_data { + log::info!( + "Creating NativeAudioSource: sample_rate={}, channels={}", + wav.sample_rate, + wav.channels + ); + Some(NativeAudioSource::new( + AudioSourceOptions::default(), + wav.sample_rate, + wav.channels, + 0, // Fast path: direct delivery to avoid race condition with ADM + )) + } else { + None + }; + let token = access_token::AccessToken::with_api_key(&api_key, &api_secret) .with_identity("rust-bot") .with_name("Rust Bot") @@ -25,7 +166,102 @@ async fn main() { .unwrap(); let (room, mut rx) = Room::connect(&url, &token, RoomOptions::default()).await.unwrap(); - log::info!("Connected to room: {} - {}", room.name(), String::from(room.sid().await)); + log::info!("Connected to room: {}", room.name()); + + // DIAGNOSTIC: Publish file audio track FIRST (before microphone) + // This helps diagnose if the first track sets global audio configuration + let running = Arc::new(AtomicBool::new(true)); + let file_task = if let (Some(source), Some(wav)) = (file_source.as_ref(), wav_data.clone()) { + let track = LocalAudioTrack::create_audio_track( + "file_audio", + RtcAudioSource::Native(source.clone()), + ); + + // Ensure the track is unmuted before publishing + track.unmute(); + log::info!( + "File track state before publish: enabled={}, muted={}", + track.is_enabled(), + track.is_muted() + ); + + let publication = room + .local_participant() + .publish_track( + LocalTrack::Audio(track.clone()), + TrackPublishOptions { source: TrackSource::Unknown, ..Default::default() }, + ) + .await + .expect("Failed to publish file audio track"); + + // Ensure track is enabled and unmuted after publishing + track.enable(); + track.unmute(); + log::info!( + "File track state after publish: enabled={}, muted={}, publication_muted={}", + track.is_enabled(), + track.is_muted(), + publication.is_muted() + ); + + log::info!( + "Published file audio track FIRST: {} Hz, {} channels, {} samples", + wav.sample_rate, + wav.channels, + wav.samples.len() + ); + + // Wait for the file track to be fully set up before publishing microphone + log::info!("Waiting 500ms for file audio track setup before publishing mic..."); + tokio::time::sleep(Duration::from_millis(500)).await; + + let source_clone = source.clone(); + let running_clone = running.clone(); + Some(tokio::spawn(async move { + // Additional wait for playback to ensure everything is connected + log::info!("Starting WAV playback..."); + play_wav_file(source_clone, wav, running_clone).await; + })) + } else { + None + }; + + // Publish microphone track SECOND (after file track is set up) + // + // DIAGNOSTIC FINDINGS: + // - SKIP_MIC_PUBLISH=1 still crashes because ADM recording is still active + // - The race condition is between ADM's audio thread and NativeAudioSource's tokio thread + // - To avoid the crash, use --file mode (no PlatformAudio, no ADM recording) + // + let skip_mic_publish = std::env::var("SKIP_MIC_PUBLISH").is_ok(); + + if let Some(ref audio) = platform_audio { + if skip_mic_publish { + log::warn!("DIAGNOSTIC: PlatformAudio is active (ADM recording enabled) but NOT publishing mic track"); + log::warn!("If audio still plays at wrong speed, the issue is ADM configuration"); + log::warn!("If audio plays correctly, the issue is the device audio track publishing"); + } else { + let track = LocalAudioTrack::create_audio_track("microphone", audio.rtc_source()); + + log::info!("Publishing microphone track SECOND (after file track)..."); + room.local_participant() + .publish_track( + LocalTrack::Audio(track), + TrackPublishOptions { source: TrackSource::Microphone, ..Default::default() }, + ) + .await + .expect("Failed to publish microphone track"); + + log::info!("Published microphone track using PlatformAudio"); + + if file_task.is_some() { + log::info!("Both tracks published: file (48kHz) FIRST, then microphone"); + log::warn!( + "WARNING: Publishing both simultaneously may cause sample rate conflicts!" + ); + } + } + } room.local_participant() .publish_data(DataPacket { @@ -36,7 +272,212 @@ async fn main() { .await .unwrap(); - while let Some(msg) = rx.recv().await { - log::info!("Event: {:?}", msg); + log::info!("Entering event loop - press Ctrl+C to stop"); + + // Handle Ctrl+C gracefully + let ctrl_c = async { + signal::ctrl_c().await.expect("Failed to listen for Ctrl+C"); + log::info!("Received Ctrl+C signal"); + }; + + tokio::select! { + _ = ctrl_c => { + log::info!("Shutting down gracefully..."); + } + _ = async { + while let Some(msg) = rx.recv().await { + log::info!("Event: {:?}", msg); + } + } => { + log::info!("Event loop ended"); + } + } + + // Stop playback task + log::info!("Stopping playback..."); + running.store(false, Ordering::SeqCst); + if let Some(task) = file_task { + log::info!("Waiting for playback task to finish..."); + let _ = task.await; + } + + // Disconnect from the room gracefully + log::info!("Disconnecting from room..."); + room.close().await; + log::info!("Disconnected. Goodbye!"); +} + +#[derive(Clone)] +struct WavData { + sample_rate: u32, + channels: u32, + samples: Vec, +} + +fn load_wav_file>(path: P) -> Result> { + let path = path.as_ref(); + log::info!("Loading WAV file: {}", path.display()); + + let reader = hound::WavReader::open(path)?; + let spec = reader.spec(); + + log::info!( + "WAV spec: {} Hz, {} channels, {} bits, {:?}", + spec.sample_rate, + spec.channels, + spec.bits_per_sample, + spec.sample_format + ); + + let samples: Vec = match spec.sample_format { + hound::SampleFormat::Int => { + if spec.bits_per_sample == 16 { + reader.into_samples::().filter_map(|s| s.ok()).collect() + } else if spec.bits_per_sample == 32 { + reader + .into_samples::() + .filter_map(|s| s.ok()) + .map(|s| (s >> 16) as i16) + .collect() + } else if spec.bits_per_sample == 8 { + reader + .into_samples::() + .filter_map(|s| s.ok()) + .map(|s| (s as i16) << 8) + .collect() + } else { + return Err(format!("Unsupported bit depth: {}", spec.bits_per_sample).into()); + } + } + hound::SampleFormat::Float => reader + .into_samples::() + .filter_map(|s| s.ok()) + .map(|s| (s * i16::MAX as f32) as i16) + .collect(), + }; + + log::info!("Loaded {} samples from WAV file", samples.len()); + + Ok(WavData { sample_rate: spec.sample_rate, channels: spec.channels as u32, samples }) +} + +async fn play_wav_file(source: NativeAudioSource, wav: WavData, running: Arc) { + log::info!("=== WAV PLAYBACK TASK STARTED ==="); + + let samples_per_channel_per_frame = (wav.sample_rate / 100) as usize; // 10ms frames + let samples_per_frame = samples_per_channel_per_frame * wav.channels as usize; + let total_duration_secs = + wav.samples.len() as f64 / (wav.sample_rate as f64 * wav.channels as f64); + + log::info!( + "WAV playback config: sample_rate={}, channels={}, samples_per_channel_per_frame={}, samples_per_frame={}, total_samples={}, duration={:.2}s", + wav.sample_rate, + wav.channels, + samples_per_channel_per_frame, + samples_per_frame, + wav.samples.len(), + total_duration_secs + ); + log::info!( + "NativeAudioSource config: sample_rate={}, num_channels={}", + source.sample_rate(), + source.num_channels() + ); + + // Use interval for accurate timing instead of sleep + let mut interval = tokio::time::interval(Duration::from_millis(10)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + let mut position = 0; + let mut frame_count = 0u64; + let start_time = std::time::Instant::now(); + + while running.load(Ordering::SeqCst) { + interval.tick().await; + + let end = (position + samples_per_frame).min(wav.samples.len()); + let frame_samples: Vec = if end > position { + wav.samples[position..end].to_vec() + } else { + // Restart from beginning (loop) + position = 0; + let end = samples_per_frame.min(wav.samples.len()); + wav.samples[0..end].to_vec() + }; + + // Pad with silence if needed + let mut padded = frame_samples; + while padded.len() < samples_per_frame { + padded.push(0); + } + + // Check if audio data is not silent (first few frames) + if frame_count < 5 { + let max_sample = padded.iter().map(|s| s.abs()).max().unwrap_or(0); + let avg_sample: i32 = + padded.iter().map(|s| (*s as i32).abs()).sum::() / padded.len() as i32; + log::info!( + "Frame {} audio data: max={}, avg={}, first_samples={:?}", + frame_count, + max_sample, + avg_sample, + &padded[..8.min(padded.len())] + ); + } + + let frame = livekit::webrtc::audio_frame::AudioFrame { + data: padded.into(), + sample_rate: wav.sample_rate, + num_channels: wav.channels, + samples_per_channel: samples_per_channel_per_frame as u32, + }; + + match source.capture_frame(&frame).await { + Ok(()) => { + // Log first 10 frames to verify playback is working + if frame_count < 10 { + log::info!( + "Frame {} captured successfully (position={}, sample_rate={}, channels={}, samples_per_ch={})", + frame_count, position, frame.sample_rate, frame.num_channels, frame.samples_per_channel + ); + } + } + Err(e) => { + log::warn!("Failed to capture frame {}: {}", frame_count, e); + } + } + + position += samples_per_frame; + frame_count += 1; + + // Log timing every 100 frames (1 second) + if frame_count % 100 == 0 { + let elapsed = start_time.elapsed(); + let expected_ms = frame_count * 10; + let actual_ms = elapsed.as_millis() as u64; + let drift_ms = actual_ms as i64 - expected_ms as i64; + log::info!( + "Playback progress: frame={}, elapsed={}ms, expected={}ms, drift={}ms", + frame_count, + actual_ms, + expected_ms, + drift_ms + ); + } + + if position >= wav.samples.len() { + position = 0; // Loop + log::info!( + "WAV playback looping after {} frames ({:.1}s)", + frame_count, + frame_count as f64 * 0.01 + ); + } } + + log::info!( + "=== WAV PLAYBACK TASK STOPPED after {} frames ({:.1}s) ===", + frame_count, + frame_count as f64 * 0.01 + ); } diff --git a/libwebrtc/src/audio_source.rs b/libwebrtc/src/audio_source.rs index 248cc2d8c..5bb7a65da 100644 --- a/libwebrtc/src/audio_source.rs +++ b/libwebrtc/src/audio_source.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use livekit_protocol::enum_dispatch; - use crate::imp::audio_source as imp_as; #[derive(Default, Debug)] @@ -23,21 +21,152 @@ pub struct AudioSourceOptions { pub auto_gain_control: bool, } +/// Audio source type for creating audio tracks. +/// +/// Choose the appropriate source based on your use case: +/// +/// | Use Case | Source | Description | +/// |----------|--------|-------------| +/// | Manual audio (TTS, files) | `RtcAudioSource::Native(source)` | Push frames manually | +/// | Microphone capture | `RtcAudioSource::Device` | Automatic via Platform ADM | +/// | Both (mic + screen) | Use both types | Multiple tracks supported | +/// +/// # Combining Sources +/// +/// You can have multiple audio tracks with different source types: +/// - Track A: `RtcAudioSource::Device` for microphone (via `PlatformAudioSource`) +/// - Track B: `RtcAudioSource::Native` for screen capture or TTS #[non_exhaustive] #[derive(Debug, Clone)] pub enum RtcAudioSource { + /// Native audio source for manual audio frame capture. + /// + /// Use this with Synthetic ADM mode (the default). You push audio frames + /// manually via `NativeAudioSource::capture_frame()`. + /// + /// # Example + /// + /// ```rust,no_run + /// use livekit::webrtc::audio_source::native::NativeAudioSource; + /// use livekit::webrtc::audio_source::{AudioSourceOptions, RtcAudioSource}; + /// + /// let source = NativeAudioSource::new( + /// AudioSourceOptions::default(), + /// 48000, 2, 100, + /// ); + /// source.capture_frame(&frame).await?; + /// + /// let track = LocalAudioTrack::create_audio_track( + /// "audio", + /// RtcAudioSource::Native(source), + /// ); + /// ``` #[cfg(not(target_arch = "wasm32"))] Native(native::NativeAudioSource), + + /// Device audio source - uses Platform ADM for automatic microphone capture. + /// + /// Use this with Platform ADM mode. WebRTC automatically captures audio from + /// the selected recording device (microphone). You do NOT push frames manually. + /// + /// # Requirements + /// + /// 1. **Enable Platform ADM first:** + /// ```rust,no_run + /// use livekit::{AudioManager, AudioMode}; + /// let audio = AudioManager::instance(); + /// audio.set_mode(AudioMode::Platform)?; + /// ``` + /// + /// 2. **Optionally select a device:** + /// ```rust,no_run + /// audio.set_recording_device(0)?; + /// ``` + /// + /// 3. **Create track with `Device` source:** + /// ```rust,no_run + /// use livekit::webrtc::audio_source::RtcAudioSource; + /// let track = LocalAudioTrack::create_audio_track( + /// "microphone", + /// RtcAudioSource::Device, + /// ); + /// ``` + /// + /// 4. **Reset after disconnect (IMPORTANT for iOS):** + /// ```rust,no_run + /// room.disconnect().await; + /// audio.reset(); // Releases VPIO AudioUnit + /// ``` + /// + /// # Combining with NativeAudioSource + /// + /// You CAN use `NativeAudioSource` alongside Platform ADM to have multiple + /// audio tracks with different sources (e.g., microphone + screen capture). + /// See `PlatformAudioSource` for the recommended approach. + /// + /// # Important + /// + /// Do NOT forget to call `AudioManager::reset()` after disconnecting, + /// especially on iOS where VPIO must be released for other audio frameworks. + /// + /// # Platform Support + /// + /// - **iOS**: CoreAudio with VPIO (Voice Processing IO) + /// - **macOS**: CoreAudio + /// - **Windows**: WASAPI + /// - **Linux**: PulseAudio / ALSA + /// - **Android**: AAudio / OpenSL ES + #[cfg(not(target_arch = "wasm32"))] + Device, } impl RtcAudioSource { - enum_dispatch!( - [Native]; - fn set_audio_options(self: &Self, options: AudioSourceOptions) -> (); - fn audio_options(self: &Self) -> AudioSourceOptions; - fn sample_rate(self: &Self) -> u32; - fn num_channels(self: &Self) -> u32; - ); + /// Set audio processing options. + /// Note: For `Device` source, options are controlled by the Platform ADM. + pub fn set_audio_options(&self, options: AudioSourceOptions) { + match self { + #[cfg(not(target_arch = "wasm32"))] + RtcAudioSource::Native(source) => source.set_audio_options(options), + #[cfg(not(target_arch = "wasm32"))] + RtcAudioSource::Device => { + // Device source options are managed by the Platform ADM + // This is a no-op + } + } + } + + /// Get audio processing options. + /// Note: For `Device` source, returns default options (actual options are managed by ADM). + pub fn audio_options(&self) -> AudioSourceOptions { + match self { + #[cfg(not(target_arch = "wasm32"))] + RtcAudioSource::Native(source) => source.audio_options(), + #[cfg(not(target_arch = "wasm32"))] + RtcAudioSource::Device => AudioSourceOptions::default(), + } + } + + /// Get the sample rate. + /// Note: For `Device` source, returns 48000 (default WebRTC sample rate). + pub fn sample_rate(&self) -> u32 { + match self { + #[cfg(not(target_arch = "wasm32"))] + RtcAudioSource::Native(source) => source.sample_rate(), + #[cfg(not(target_arch = "wasm32"))] + RtcAudioSource::Device => 48000, // Default WebRTC sample rate + } + } + + /// Get the number of channels. + /// Note: For `Device` source, returns 1 (mono). + pub fn num_channels(&self) -> u32 { + match self { + #[cfg(not(target_arch = "wasm32"))] + RtcAudioSource::Native(source) => source.num_channels(), + #[cfg(not(target_arch = "wasm32"))] + RtcAudioSource::Device => 1, // Default to mono + } + } } #[cfg(not(target_arch = "wasm32"))] diff --git a/libwebrtc/src/native/peer_connection_factory.rs b/libwebrtc/src/native/peer_connection_factory.rs index 4edc63047..8389c12da 100644 --- a/libwebrtc/src/native/peer_connection_factory.rs +++ b/libwebrtc/src/native/peer_connection_factory.rs @@ -94,6 +94,18 @@ impl PeerConnectionFactory { } } + /// Create an audio track that uses the Platform ADM for capture. + /// + /// This requires that `enable_platform_adm()` was called first. + /// The track will capture audio from the selected recording device. + pub fn create_device_audio_track(&self, label: &str) -> RtcAudioTrack { + RtcAudioTrack { + handle: imp_at::RtcAudioTrack { + sys_handle: self.sys_handle.create_device_audio_track(label.to_string()), + }, + } + } + pub fn get_rtp_sender_capabilities(&self, media_type: MediaType) -> RtpCapabilities { self.sys_handle.rtp_sender_capabilities(media_type.into()).into() } @@ -101,6 +113,151 @@ impl PeerConnectionFactory { pub fn get_rtp_receiver_capabilities(&self, media_type: MediaType) -> RtpCapabilities { self.sys_handle.rtp_receiver_capabilities(media_type.into()).into() } + + // ===== Device Management Methods ===== + + /// Get the number of playout (output) devices + pub fn playout_devices(&self) -> i16 { + self.sys_handle.playout_devices() + } + + /// Get the number of recording (input) devices + pub fn recording_devices(&self) -> i16 { + self.sys_handle.recording_devices() + } + + /// Get the name of a playout device by index + pub fn playout_device_name(&self, index: u16) -> String { + self.sys_handle.playout_device_name(index) + } + + /// Get the name of a recording device by index + pub fn recording_device_name(&self, index: u16) -> String { + self.sys_handle.recording_device_name(index) + } + + /// Set the playout device by index + /// Returns 0 on success, negative on error. + pub fn set_playout_device(&self, index: u16) -> i32 { + self.sys_handle.set_playout_device(index) + } + + /// Set the recording device by index + /// Returns 0 on success, negative on error. + pub fn set_recording_device(&self, index: u16) -> i32 { + self.sys_handle.set_recording_device(index) + } + + /// Stop recording (clears initialized state, allowing device switch) + pub fn stop_recording(&self) -> i32 { + self.sys_handle.stop_recording() + } + + /// Initialize recording + pub fn init_recording(&self) -> i32 { + self.sys_handle.init_recording() + } + + /// Start recording + pub fn start_recording(&self) -> i32 { + self.sys_handle.start_recording() + } + + /// Check if recording is initialized + pub fn recording_is_initialized(&self) -> bool { + self.sys_handle.recording_is_initialized() + } + + /// Stop playout (clears initialized state, allowing device switch) + pub fn stop_playout(&self) -> i32 { + self.sys_handle.stop_playout() + } + + /// Initialize playout + pub fn init_playout(&self) -> i32 { + self.sys_handle.init_playout() + } + + /// Start playout + pub fn start_playout(&self) -> i32 { + self.sys_handle.start_playout() + } + + /// Check if playout is initialized + pub fn playout_is_initialized(&self) -> bool { + self.sys_handle.playout_is_initialized() + } + + // ===== Built-in Audio Processing Methods ===== + // These control hardware AEC/AGC/NS on platforms that support it (iOS, some Android) + + /// Check if built-in (hardware) AEC is available on this device. + /// + /// Returns true on iOS (VPIO) and some Android devices. + /// Returns false on desktop platforms (macOS, Windows, Linux). + pub fn builtin_aec_is_available(&self) -> bool { + self.sys_handle.builtin_aec_is_available() + } + + /// Check if built-in (hardware) AGC is available on this device. + /// + /// Returns true on iOS (VPIO) and some Android devices. + /// Returns false on desktop platforms (macOS, Windows, Linux). + pub fn builtin_agc_is_available(&self) -> bool { + self.sys_handle.builtin_agc_is_available() + } + + /// Check if built-in (hardware) NS is available on this device. + /// + /// Returns true on iOS (VPIO) and some Android devices. + /// Returns false on desktop platforms (macOS, Windows, Linux). + pub fn builtin_ns_is_available(&self) -> bool { + self.sys_handle.builtin_ns_is_available() + } + + /// Enable or disable built-in (hardware) AEC. + /// + /// When disabled on platforms that support it, WebRTC's software AEC + /// will be used instead. + /// + /// Returns 0 on success, negative on error. + pub fn enable_builtin_aec(&self, enable: bool) -> i32 { + self.sys_handle.enable_builtin_aec(enable) + } + + /// Enable or disable built-in (hardware) AGC. + /// + /// When disabled on platforms that support it, WebRTC's software AGC + /// will be used instead. + /// + /// Returns 0 on success, negative on error. + pub fn enable_builtin_agc(&self, enable: bool) -> i32 { + self.sys_handle.enable_builtin_agc(enable) + } + + /// Enable or disable built-in (hardware) NS. + /// + /// When disabled on platforms that support it, WebRTC's software NS + /// will be used instead. + /// + /// Returns 0 on success, negative on error. + pub fn enable_builtin_ns(&self, enable: bool) -> i32 { + self.sys_handle.enable_builtin_ns(enable) + } + + /// Control whether ADM recording (microphone) is enabled. + /// + /// When disabled, WebRTC's calls to InitRecording/StartRecording will be no-ops. + /// Use this when only using NativeAudioSource (no microphone capture needed). + /// This prevents the microphone from interfering with the audio pipeline. + pub fn set_adm_recording_enabled(&self, enabled: bool) { + self.sys_handle.set_adm_recording_enabled(enabled) + } + + /// Check if ADM recording (microphone) is enabled. + pub fn adm_recording_enabled(&self) -> bool { + self.sys_handle.adm_recording_enabled() + } } #[cfg(test)] diff --git a/libwebrtc/src/peer_connection_factory.rs b/libwebrtc/src/peer_connection_factory.rs index 12f6d24bc..a006cf324 100644 --- a/libwebrtc/src/peer_connection_factory.rs +++ b/libwebrtc/src/peer_connection_factory.rs @@ -94,6 +94,46 @@ pub mod native { pub trait PeerConnectionFactoryExt { fn create_video_track(&self, label: &str, source: NativeVideoSource) -> RtcVideoTrack; fn create_audio_track(&self, label: &str, source: NativeAudioSource) -> RtcAudioTrack; + + /// Create an audio track that uses the Platform ADM for capture. + /// The track will capture audio from the selected recording device. + fn create_device_audio_track(&self, label: &str) -> RtcAudioTrack; + + // Device enumeration + fn playout_devices(&self) -> i16; + fn recording_devices(&self) -> i16; + fn playout_device_name(&self, index: u16) -> String; + fn recording_device_name(&self, index: u16) -> String; + + // Device selection + fn set_playout_device(&self, index: u16) -> i32; + fn set_recording_device(&self, index: u16) -> i32; + + // Recording control (for device switching while active) + fn stop_recording(&self) -> i32; + fn init_recording(&self) -> i32; + fn start_recording(&self) -> i32; + fn recording_is_initialized(&self) -> bool; + + // Playout control (for device switching while active) + fn stop_playout(&self) -> i32; + fn init_playout(&self) -> i32; + fn start_playout(&self) -> i32; + fn playout_is_initialized(&self) -> bool; + + // Built-in audio processing (hardware AEC/AGC/NS) + // Only available on iOS and some Android devices + fn builtin_aec_is_available(&self) -> bool; + fn builtin_agc_is_available(&self) -> bool; + fn builtin_ns_is_available(&self) -> bool; + fn enable_builtin_aec(&self, enable: bool) -> i32; + fn enable_builtin_agc(&self, enable: bool) -> i32; + fn enable_builtin_ns(&self, enable: bool) -> i32; + + // ADM recording control + // Use this to disable microphone when only using NativeAudioSource + fn set_adm_recording_enabled(&self, enabled: bool); + fn adm_recording_enabled(&self) -> bool; } impl PeerConnectionFactoryExt for PeerConnectionFactory { @@ -104,5 +144,97 @@ pub mod native { fn create_audio_track(&self, label: &str, source: NativeAudioSource) -> RtcAudioTrack { self.handle.create_audio_track(label, source) } + + fn create_device_audio_track(&self, label: &str) -> RtcAudioTrack { + self.handle.create_device_audio_track(label) + } + + fn playout_devices(&self) -> i16 { + self.handle.playout_devices() + } + + fn recording_devices(&self) -> i16 { + self.handle.recording_devices() + } + + fn playout_device_name(&self, index: u16) -> String { + self.handle.playout_device_name(index) + } + + fn recording_device_name(&self, index: u16) -> String { + self.handle.recording_device_name(index) + } + + fn set_playout_device(&self, index: u16) -> i32 { + self.handle.set_playout_device(index) + } + + fn set_recording_device(&self, index: u16) -> i32 { + self.handle.set_recording_device(index) + } + + fn stop_recording(&self) -> i32 { + self.handle.stop_recording() + } + + fn init_recording(&self) -> i32 { + self.handle.init_recording() + } + + fn start_recording(&self) -> i32 { + self.handle.start_recording() + } + + fn recording_is_initialized(&self) -> bool { + self.handle.recording_is_initialized() + } + + fn stop_playout(&self) -> i32 { + self.handle.stop_playout() + } + + fn init_playout(&self) -> i32 { + self.handle.init_playout() + } + + fn start_playout(&self) -> i32 { + self.handle.start_playout() + } + + fn playout_is_initialized(&self) -> bool { + self.handle.playout_is_initialized() + } + + fn builtin_aec_is_available(&self) -> bool { + self.handle.builtin_aec_is_available() + } + + fn builtin_agc_is_available(&self) -> bool { + self.handle.builtin_agc_is_available() + } + + fn builtin_ns_is_available(&self) -> bool { + self.handle.builtin_ns_is_available() + } + + fn enable_builtin_aec(&self, enable: bool) -> i32 { + self.handle.enable_builtin_aec(enable) + } + + fn enable_builtin_agc(&self, enable: bool) -> i32 { + self.handle.enable_builtin_agc(enable) + } + + fn enable_builtin_ns(&self, enable: bool) -> i32 { + self.handle.enable_builtin_ns(enable) + } + + fn set_adm_recording_enabled(&self, enabled: bool) { + self.handle.set_adm_recording_enabled(enabled) + } + + fn adm_recording_enabled(&self) -> bool { + self.handle.adm_recording_enabled() + } } } diff --git a/livekit-ffi-node-bindings/proto/audio_frame_pb.d.ts b/livekit-ffi-node-bindings/proto/audio_frame_pb.d.ts index 4ddf1e471..0cdebc5ed 100644 --- a/livekit-ffi-node-bindings/proto/audio_frame_pb.d.ts +++ b/livekit-ffi-node-bindings/proto/audio_frame_pb.d.ts @@ -136,9 +136,19 @@ export declare enum AudioStreamType { */ export declare enum AudioSourceType { /** + * Push-based audio source - manually capture frames via CaptureAudioFrameRequest + * * @generated from enum value: AUDIO_SOURCE_NATIVE = 0; */ AUDIO_SOURCE_NATIVE = 0, + + /** + * Platform ADM-based audio source - captures from microphone automatically + * Requires PlatformAudio to be created first to enable ADM recording + * + * @generated from enum value: AUDIO_SOURCE_PLATFORM = 1; + */ + AUDIO_SOURCE_PLATFORM = 1, } /** @@ -368,6 +378,14 @@ export declare class NewAudioSourceRequest extends Message); static readonly runtime: typeof proto2; @@ -1364,6 +1382,14 @@ export declare class AudioSourceOptions extends Message { */ autoGainControl?: boolean; + /** + * Prefer hardware audio processing (e.g., iOS VPIO). Lower latency. + * Only applies to AudioSourcePlatform. Default: true. + * + * @generated from field: optional bool prefer_hardware = 4; + */ + preferHardware?: boolean; + constructor(data?: PartialMessage); static readonly runtime: typeof proto2; @@ -1618,3 +1644,375 @@ export declare class LoadAudioFilterPluginResponse extends Message | undefined, b: LoadAudioFilterPluginResponse | PlainMessage | undefined): boolean; } +/** + * Information about an audio device. + * + * @generated from message livekit.proto.AudioDeviceInfo + */ +export declare class AudioDeviceInfo extends Message { + /** + * Device index (0-based). + * + * @generated from field: required uint32 index = 1; + */ + index?: number; + + /** + * Device name as reported by the operating system. + * + * @generated from field: required string name = 2; + */ + name?: string; + + constructor(data?: PartialMessage); + + static readonly runtime: typeof proto2; + static readonly typeName = "livekit.proto.AudioDeviceInfo"; + static readonly fields: FieldList; + + static fromBinary(bytes: Uint8Array, options?: Partial): AudioDeviceInfo; + + static fromJson(jsonValue: JsonValue, options?: Partial): AudioDeviceInfo; + + static fromJsonString(jsonString: string, options?: Partial): AudioDeviceInfo; + + static equals(a: AudioDeviceInfo | PlainMessage | undefined, b: AudioDeviceInfo | PlainMessage | undefined): boolean; +} + +/** + * Information about a PlatformAudio instance. + * + * @generated from message livekit.proto.PlatformAudioInfo + */ +export declare class PlatformAudioInfo extends Message { + /** + * Number of available recording (microphone) devices. + * + * @generated from field: required int32 recording_device_count = 1; + */ + recordingDeviceCount?: number; + + /** + * Number of available playout (speaker) devices. + * + * @generated from field: required int32 playout_device_count = 2; + */ + playoutDeviceCount?: number; + + constructor(data?: PartialMessage); + + static readonly runtime: typeof proto2; + static readonly typeName = "livekit.proto.PlatformAudioInfo"; + static readonly fields: FieldList; + + static fromBinary(bytes: Uint8Array, options?: Partial): PlatformAudioInfo; + + static fromJson(jsonValue: JsonValue, options?: Partial): PlatformAudioInfo; + + static fromJsonString(jsonString: string, options?: Partial): PlatformAudioInfo; + + static equals(a: PlatformAudioInfo | PlainMessage | undefined, b: PlatformAudioInfo | PlainMessage | undefined): boolean; +} + +/** + * Owned PlatformAudio handle with info. + * + * @generated from message livekit.proto.OwnedPlatformAudio + */ +export declare class OwnedPlatformAudio extends Message { + /** + * @generated from field: required livekit.proto.FfiOwnedHandle handle = 1; + */ + handle?: FfiOwnedHandle; + + /** + * @generated from field: required livekit.proto.PlatformAudioInfo info = 2; + */ + info?: PlatformAudioInfo; + + constructor(data?: PartialMessage); + + static readonly runtime: typeof proto2; + static readonly typeName = "livekit.proto.OwnedPlatformAudio"; + static readonly fields: FieldList; + + static fromBinary(bytes: Uint8Array, options?: Partial): OwnedPlatformAudio; + + static fromJson(jsonValue: JsonValue, options?: Partial): OwnedPlatformAudio; + + static fromJsonString(jsonString: string, options?: Partial): OwnedPlatformAudio; + + static equals(a: OwnedPlatformAudio | PlainMessage | undefined, b: OwnedPlatformAudio | PlainMessage | undefined): boolean; +} + +/** + * Create a new PlatformAudio instance. + * + * This enables the platform ADM for microphone capture and speaker playout. + * If another PlatformAudio instance exists, this reuses the same underlying ADM. + * + * The returned handle must be kept alive while platform audio is needed. + * When all handles are released, the ADM is automatically disabled. + * + * @generated from message livekit.proto.NewPlatformAudioRequest + */ +export declare class NewPlatformAudioRequest extends Message { + constructor(data?: PartialMessage); + + static readonly runtime: typeof proto2; + static readonly typeName = "livekit.proto.NewPlatformAudioRequest"; + static readonly fields: FieldList; + + static fromBinary(bytes: Uint8Array, options?: Partial): NewPlatformAudioRequest; + + static fromJson(jsonValue: JsonValue, options?: Partial): NewPlatformAudioRequest; + + static fromJsonString(jsonString: string, options?: Partial): NewPlatformAudioRequest; + + static equals(a: NewPlatformAudioRequest | PlainMessage | undefined, b: NewPlatformAudioRequest | PlainMessage | undefined): boolean; +} + +/** + * @generated from message livekit.proto.NewPlatformAudioResponse + */ +export declare class NewPlatformAudioResponse extends Message { + /** + * @generated from oneof livekit.proto.NewPlatformAudioResponse.message + */ + message: { + /** + * The PlatformAudio handle on success. + * + * @generated from field: livekit.proto.OwnedPlatformAudio platform_audio = 1; + */ + value: OwnedPlatformAudio; + case: "platformAudio"; + } | { + /** + * Error message if creation failed. + * + * @generated from field: string error = 2; + */ + value: string; + case: "error"; + } | { case: undefined; value?: undefined }; + + constructor(data?: PartialMessage); + + static readonly runtime: typeof proto2; + static readonly typeName = "livekit.proto.NewPlatformAudioResponse"; + static readonly fields: FieldList; + + static fromBinary(bytes: Uint8Array, options?: Partial): NewPlatformAudioResponse; + + static fromJson(jsonValue: JsonValue, options?: Partial): NewPlatformAudioResponse; + + static fromJsonString(jsonString: string, options?: Partial): NewPlatformAudioResponse; + + static equals(a: NewPlatformAudioResponse | PlainMessage | undefined, b: NewPlatformAudioResponse | PlainMessage | undefined): boolean; +} + +/** + * Get available audio devices. + * + * Returns lists of available recording (microphone) and playout (speaker) devices. + * + * @generated from message livekit.proto.GetAudioDevicesRequest + */ +export declare class GetAudioDevicesRequest extends Message { + /** + * The PlatformAudio handle. + * + * @generated from field: required uint64 platform_audio_handle = 1; + */ + platformAudioHandle?: bigint; + + constructor(data?: PartialMessage); + + static readonly runtime: typeof proto2; + static readonly typeName = "livekit.proto.GetAudioDevicesRequest"; + static readonly fields: FieldList; + + static fromBinary(bytes: Uint8Array, options?: Partial): GetAudioDevicesRequest; + + static fromJson(jsonValue: JsonValue, options?: Partial): GetAudioDevicesRequest; + + static fromJsonString(jsonString: string, options?: Partial): GetAudioDevicesRequest; + + static equals(a: GetAudioDevicesRequest | PlainMessage | undefined, b: GetAudioDevicesRequest | PlainMessage | undefined): boolean; +} + +/** + * @generated from message livekit.proto.GetAudioDevicesResponse + */ +export declare class GetAudioDevicesResponse extends Message { + /** + * Available playout devices (speakers/headphones). + * + * @generated from field: repeated livekit.proto.AudioDeviceInfo playout_devices = 1; + */ + playoutDevices: AudioDeviceInfo[]; + + /** + * Available recording devices (microphones). + * + * @generated from field: repeated livekit.proto.AudioDeviceInfo recording_devices = 2; + */ + recordingDevices: AudioDeviceInfo[]; + + /** + * Error message if enumeration failed, empty/absent on success. + * + * @generated from field: optional string error = 3; + */ + error?: string; + + constructor(data?: PartialMessage); + + static readonly runtime: typeof proto2; + static readonly typeName = "livekit.proto.GetAudioDevicesResponse"; + static readonly fields: FieldList; + + static fromBinary(bytes: Uint8Array, options?: Partial): GetAudioDevicesResponse; + + static fromJson(jsonValue: JsonValue, options?: Partial): GetAudioDevicesResponse; + + static fromJsonString(jsonString: string, options?: Partial): GetAudioDevicesResponse; + + static equals(a: GetAudioDevicesResponse | PlainMessage | undefined, b: GetAudioDevicesResponse | PlainMessage | undefined): boolean; +} + +/** + * Set the recording device (microphone). + * + * Call this before creating audio tracks to select which microphone to use. + * Device indices are 0-based and must be less than the recording device count. + * + * @generated from message livekit.proto.SetRecordingDeviceRequest + */ +export declare class SetRecordingDeviceRequest extends Message { + /** + * The PlatformAudio handle. + * + * @generated from field: required uint64 platform_audio_handle = 1; + */ + platformAudioHandle?: bigint; + + /** + * Device index from GetAudioDevicesResponse.recording_devices. + * + * @generated from field: required uint32 index = 2; + */ + index?: number; + + constructor(data?: PartialMessage); + + static readonly runtime: typeof proto2; + static readonly typeName = "livekit.proto.SetRecordingDeviceRequest"; + static readonly fields: FieldList; + + static fromBinary(bytes: Uint8Array, options?: Partial): SetRecordingDeviceRequest; + + static fromJson(jsonValue: JsonValue, options?: Partial): SetRecordingDeviceRequest; + + static fromJsonString(jsonString: string, options?: Partial): SetRecordingDeviceRequest; + + static equals(a: SetRecordingDeviceRequest | PlainMessage | undefined, b: SetRecordingDeviceRequest | PlainMessage | undefined): boolean; +} + +/** + * @generated from message livekit.proto.SetRecordingDeviceResponse + */ +export declare class SetRecordingDeviceResponse extends Message { + /** + * Error message if the operation failed: + * - "Invalid device index" if index >= recording device count + * - Other platform-specific errors + * Empty/absent on success. + * + * @generated from field: optional string error = 1; + */ + error?: string; + + constructor(data?: PartialMessage); + + static readonly runtime: typeof proto2; + static readonly typeName = "livekit.proto.SetRecordingDeviceResponse"; + static readonly fields: FieldList; + + static fromBinary(bytes: Uint8Array, options?: Partial): SetRecordingDeviceResponse; + + static fromJson(jsonValue: JsonValue, options?: Partial): SetRecordingDeviceResponse; + + static fromJsonString(jsonString: string, options?: Partial): SetRecordingDeviceResponse; + + static equals(a: SetRecordingDeviceResponse | PlainMessage | undefined, b: SetRecordingDeviceResponse | PlainMessage | undefined): boolean; +} + +/** + * Set the playout device (speaker/headphones). + * + * Call this before connecting to select which speaker to use for audio output. + * Device indices are 0-based and must be less than the playout device count. + * + * @generated from message livekit.proto.SetPlayoutDeviceRequest + */ +export declare class SetPlayoutDeviceRequest extends Message { + /** + * The PlatformAudio handle. + * + * @generated from field: required uint64 platform_audio_handle = 1; + */ + platformAudioHandle?: bigint; + + /** + * Device index from GetAudioDevicesResponse.playout_devices. + * + * @generated from field: required uint32 index = 2; + */ + index?: number; + + constructor(data?: PartialMessage); + + static readonly runtime: typeof proto2; + static readonly typeName = "livekit.proto.SetPlayoutDeviceRequest"; + static readonly fields: FieldList; + + static fromBinary(bytes: Uint8Array, options?: Partial): SetPlayoutDeviceRequest; + + static fromJson(jsonValue: JsonValue, options?: Partial): SetPlayoutDeviceRequest; + + static fromJsonString(jsonString: string, options?: Partial): SetPlayoutDeviceRequest; + + static equals(a: SetPlayoutDeviceRequest | PlainMessage | undefined, b: SetPlayoutDeviceRequest | PlainMessage | undefined): boolean; +} + +/** + * @generated from message livekit.proto.SetPlayoutDeviceResponse + */ +export declare class SetPlayoutDeviceResponse extends Message { + /** + * Error message if the operation failed: + * - "Invalid device index" if index >= playout device count + * - Other platform-specific errors + * Empty/absent on success. + * + * @generated from field: optional string error = 1; + */ + error?: string; + + constructor(data?: PartialMessage); + + static readonly runtime: typeof proto2; + static readonly typeName = "livekit.proto.SetPlayoutDeviceResponse"; + static readonly fields: FieldList; + + static fromBinary(bytes: Uint8Array, options?: Partial): SetPlayoutDeviceResponse; + + static fromJson(jsonValue: JsonValue, options?: Partial): SetPlayoutDeviceResponse; + + static fromJsonString(jsonString: string, options?: Partial): SetPlayoutDeviceResponse; + + static equals(a: SetPlayoutDeviceResponse | PlainMessage | undefined, b: SetPlayoutDeviceResponse | PlainMessage | undefined): boolean; +} + diff --git a/livekit-ffi-node-bindings/proto/audio_frame_pb.js b/livekit-ffi-node-bindings/proto/audio_frame_pb.js index 2e33235ea..ade631c1e 100644 --- a/livekit-ffi-node-bindings/proto/audio_frame_pb.js +++ b/livekit-ffi-node-bindings/proto/audio_frame_pb.js @@ -82,6 +82,7 @@ const AudioSourceType = /*@__PURE__*/ proto2.makeEnum( "livekit.proto.AudioSourceType", [ {no: 0, name: "AUDIO_SOURCE_NATIVE"}, + {no: 1, name: "AUDIO_SOURCE_PLATFORM"}, ], ); @@ -156,6 +157,7 @@ const NewAudioSourceRequest = /*@__PURE__*/ proto2.makeMessageType( { no: 3, name: "sample_rate", kind: "scalar", T: 13 /* ScalarType.UINT32 */, req: true }, { no: 4, name: "num_channels", kind: "scalar", T: 13 /* ScalarType.UINT32 */, req: true }, { no: 5, name: "queue_size_ms", kind: "scalar", T: 13 /* ScalarType.UINT32 */, opt: true }, + { no: 6, name: "platform_audio_handle", kind: "scalar", T: 4 /* ScalarType.UINT64 */, opt: true }, ], ); @@ -517,6 +519,7 @@ const AudioSourceOptions = /*@__PURE__*/ proto2.makeMessageType( { no: 1, name: "echo_cancellation", kind: "scalar", T: 8 /* ScalarType.BOOL */, req: true }, { no: 2, name: "noise_suppression", kind: "scalar", T: 8 /* ScalarType.BOOL */, req: true }, { no: 3, name: "auto_gain_control", kind: "scalar", T: 8 /* ScalarType.BOOL */, req: true }, + { no: 4, name: "prefer_hardware", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, ], ); @@ -613,6 +616,150 @@ const LoadAudioFilterPluginResponse = /*@__PURE__*/ proto2.makeMessageType( ], ); +/** + * Information about an audio device. + * + * @generated from message livekit.proto.AudioDeviceInfo + */ +const AudioDeviceInfo = /*@__PURE__*/ proto2.makeMessageType( + "livekit.proto.AudioDeviceInfo", + () => [ + { no: 1, name: "index", kind: "scalar", T: 13 /* ScalarType.UINT32 */, req: true }, + { no: 2, name: "name", kind: "scalar", T: 9 /* ScalarType.STRING */, req: true }, + ], +); + +/** + * Information about a PlatformAudio instance. + * + * @generated from message livekit.proto.PlatformAudioInfo + */ +const PlatformAudioInfo = /*@__PURE__*/ proto2.makeMessageType( + "livekit.proto.PlatformAudioInfo", + () => [ + { no: 1, name: "recording_device_count", kind: "scalar", T: 5 /* ScalarType.INT32 */, req: true }, + { no: 2, name: "playout_device_count", kind: "scalar", T: 5 /* ScalarType.INT32 */, req: true }, + ], +); + +/** + * Owned PlatformAudio handle with info. + * + * @generated from message livekit.proto.OwnedPlatformAudio + */ +const OwnedPlatformAudio = /*@__PURE__*/ proto2.makeMessageType( + "livekit.proto.OwnedPlatformAudio", + () => [ + { no: 1, name: "handle", kind: "message", T: FfiOwnedHandle, req: true }, + { no: 2, name: "info", kind: "message", T: PlatformAudioInfo, req: true }, + ], +); + +/** + * Create a new PlatformAudio instance. + * + * This enables the platform ADM for microphone capture and speaker playout. + * If another PlatformAudio instance exists, this reuses the same underlying ADM. + * + * The returned handle must be kept alive while platform audio is needed. + * When all handles are released, the ADM is automatically disabled. + * + * @generated from message livekit.proto.NewPlatformAudioRequest + */ +const NewPlatformAudioRequest = /*@__PURE__*/ proto2.makeMessageType( + "livekit.proto.NewPlatformAudioRequest", + [], +); + +/** + * @generated from message livekit.proto.NewPlatformAudioResponse + */ +const NewPlatformAudioResponse = /*@__PURE__*/ proto2.makeMessageType( + "livekit.proto.NewPlatformAudioResponse", + () => [ + { no: 1, name: "platform_audio", kind: "message", T: OwnedPlatformAudio, oneof: "message" }, + { no: 2, name: "error", kind: "scalar", T: 9 /* ScalarType.STRING */, oneof: "message" }, + ], +); + +/** + * Get available audio devices. + * + * Returns lists of available recording (microphone) and playout (speaker) devices. + * + * @generated from message livekit.proto.GetAudioDevicesRequest + */ +const GetAudioDevicesRequest = /*@__PURE__*/ proto2.makeMessageType( + "livekit.proto.GetAudioDevicesRequest", + () => [ + { no: 1, name: "platform_audio_handle", kind: "scalar", T: 4 /* ScalarType.UINT64 */, req: true }, + ], +); + +/** + * @generated from message livekit.proto.GetAudioDevicesResponse + */ +const GetAudioDevicesResponse = /*@__PURE__*/ proto2.makeMessageType( + "livekit.proto.GetAudioDevicesResponse", + () => [ + { no: 1, name: "playout_devices", kind: "message", T: AudioDeviceInfo, repeated: true }, + { no: 2, name: "recording_devices", kind: "message", T: AudioDeviceInfo, repeated: true }, + { no: 3, name: "error", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, + ], +); + +/** + * Set the recording device (microphone). + * + * Call this before creating audio tracks to select which microphone to use. + * Device indices are 0-based and must be less than the recording device count. + * + * @generated from message livekit.proto.SetRecordingDeviceRequest + */ +const SetRecordingDeviceRequest = /*@__PURE__*/ proto2.makeMessageType( + "livekit.proto.SetRecordingDeviceRequest", + () => [ + { no: 1, name: "platform_audio_handle", kind: "scalar", T: 4 /* ScalarType.UINT64 */, req: true }, + { no: 2, name: "index", kind: "scalar", T: 13 /* ScalarType.UINT32 */, req: true }, + ], +); + +/** + * @generated from message livekit.proto.SetRecordingDeviceResponse + */ +const SetRecordingDeviceResponse = /*@__PURE__*/ proto2.makeMessageType( + "livekit.proto.SetRecordingDeviceResponse", + () => [ + { no: 1, name: "error", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, + ], +); + +/** + * Set the playout device (speaker/headphones). + * + * Call this before connecting to select which speaker to use for audio output. + * Device indices are 0-based and must be less than the playout device count. + * + * @generated from message livekit.proto.SetPlayoutDeviceRequest + */ +const SetPlayoutDeviceRequest = /*@__PURE__*/ proto2.makeMessageType( + "livekit.proto.SetPlayoutDeviceRequest", + () => [ + { no: 1, name: "platform_audio_handle", kind: "scalar", T: 4 /* ScalarType.UINT64 */, req: true }, + { no: 2, name: "index", kind: "scalar", T: 13 /* ScalarType.UINT32 */, req: true }, + ], +); + +/** + * @generated from message livekit.proto.SetPlayoutDeviceResponse + */ +const SetPlayoutDeviceResponse = /*@__PURE__*/ proto2.makeMessageType( + "livekit.proto.SetPlayoutDeviceResponse", + () => [ + { no: 1, name: "error", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, + ], +); + exports.SoxResamplerDataType = SoxResamplerDataType; exports.SoxQualityRecipe = SoxQualityRecipe; @@ -665,3 +812,14 @@ exports.SoxResamplerInfo = SoxResamplerInfo; exports.OwnedSoxResampler = OwnedSoxResampler; exports.LoadAudioFilterPluginRequest = LoadAudioFilterPluginRequest; exports.LoadAudioFilterPluginResponse = LoadAudioFilterPluginResponse; +exports.AudioDeviceInfo = AudioDeviceInfo; +exports.PlatformAudioInfo = PlatformAudioInfo; +exports.OwnedPlatformAudio = OwnedPlatformAudio; +exports.NewPlatformAudioRequest = NewPlatformAudioRequest; +exports.NewPlatformAudioResponse = NewPlatformAudioResponse; +exports.GetAudioDevicesRequest = GetAudioDevicesRequest; +exports.GetAudioDevicesResponse = GetAudioDevicesResponse; +exports.SetRecordingDeviceRequest = SetRecordingDeviceRequest; +exports.SetRecordingDeviceResponse = SetRecordingDeviceResponse; +exports.SetPlayoutDeviceRequest = SetPlayoutDeviceRequest; +exports.SetPlayoutDeviceResponse = SetPlayoutDeviceResponse; diff --git a/livekit-ffi-node-bindings/proto/ffi_pb.d.ts b/livekit-ffi-node-bindings/proto/ffi_pb.d.ts index 05d98054d..4283006ab 100644 --- a/livekit-ffi-node-bindings/proto/ffi_pb.d.ts +++ b/livekit-ffi-node-bindings/proto/ffi_pb.d.ts @@ -22,7 +22,7 @@ import { Message, proto2 } from "@bufbuild/protobuf"; import type { ConnectCallback, ConnectRequest, ConnectResponse, DisconnectCallback, DisconnectRequest, DisconnectResponse, EditChatMessageRequest, GetSessionStatsCallback, GetSessionStatsRequest, GetSessionStatsResponse, PublishDataCallback, PublishDataRequest, PublishDataResponse, PublishSipDtmfCallback, PublishSipDtmfRequest, PublishSipDtmfResponse, PublishTrackCallback, PublishTrackRequest, PublishTrackResponse, PublishTranscriptionCallback, PublishTranscriptionRequest, PublishTranscriptionResponse, RoomEvent, SendChatMessageCallback, SendChatMessageRequest, SendChatMessageResponse, SendStreamChunkCallback, SendStreamChunkRequest, SendStreamChunkResponse, SendStreamHeaderCallback, SendStreamHeaderRequest, SendStreamHeaderResponse, SendStreamTrailerCallback, SendStreamTrailerRequest, SendStreamTrailerResponse, SetDataChannelBufferedAmountLowThresholdRequest, SetDataChannelBufferedAmountLowThresholdResponse, SetLocalAttributesCallback, SetLocalAttributesRequest, SetLocalAttributesResponse, SetLocalMetadataCallback, SetLocalMetadataRequest, SetLocalMetadataResponse, SetLocalNameCallback, SetLocalNameRequest, SetLocalNameResponse, SetSubscribedRequest, SetSubscribedResponse, UnpublishTrackCallback, UnpublishTrackRequest, UnpublishTrackResponse } from "./room_pb.js"; import type { CreateAudioTrackRequest, CreateAudioTrackResponse, CreateVideoTrackRequest, CreateVideoTrackResponse, EnableRemoteTrackRequest, EnableRemoteTrackResponse, GetStatsCallback, GetStatsRequest, GetStatsResponse, LocalTrackMuteRequest, LocalTrackMuteResponse, SetTrackSubscriptionPermissionsRequest, SetTrackSubscriptionPermissionsResponse, TrackEvent } from "./track_pb.js"; import type { CaptureVideoFrameRequest, CaptureVideoFrameResponse, NewVideoSourceRequest, NewVideoSourceResponse, NewVideoStreamRequest, NewVideoStreamResponse, VideoConvertRequest, VideoConvertResponse, VideoStreamEvent, VideoStreamFromParticipantRequest, VideoStreamFromParticipantResponse } from "./video_frame_pb.js"; -import type { ApmProcessReverseStreamRequest, ApmProcessReverseStreamResponse, ApmProcessStreamRequest, ApmProcessStreamResponse, ApmSetStreamDelayRequest, ApmSetStreamDelayResponse, AudioStreamEvent, AudioStreamFromParticipantRequest, AudioStreamFromParticipantResponse, CaptureAudioFrameCallback, CaptureAudioFrameRequest, CaptureAudioFrameResponse, ClearAudioBufferRequest, ClearAudioBufferResponse, FlushSoxResamplerRequest, FlushSoxResamplerResponse, LoadAudioFilterPluginRequest, LoadAudioFilterPluginResponse, NewApmRequest, NewApmResponse, NewAudioResamplerRequest, NewAudioResamplerResponse, NewAudioSourceRequest, NewAudioSourceResponse, NewAudioStreamRequest, NewAudioStreamResponse, NewSoxResamplerRequest, NewSoxResamplerResponse, PushSoxResamplerRequest, PushSoxResamplerResponse, RemixAndResampleRequest, RemixAndResampleResponse } from "./audio_frame_pb.js"; +import type { ApmProcessReverseStreamRequest, ApmProcessReverseStreamResponse, ApmProcessStreamRequest, ApmProcessStreamResponse, ApmSetStreamDelayRequest, ApmSetStreamDelayResponse, AudioStreamEvent, AudioStreamFromParticipantRequest, AudioStreamFromParticipantResponse, CaptureAudioFrameCallback, CaptureAudioFrameRequest, CaptureAudioFrameResponse, ClearAudioBufferRequest, ClearAudioBufferResponse, FlushSoxResamplerRequest, FlushSoxResamplerResponse, GetAudioDevicesRequest, GetAudioDevicesResponse, LoadAudioFilterPluginRequest, LoadAudioFilterPluginResponse, NewApmRequest, NewApmResponse, NewAudioResamplerRequest, NewAudioResamplerResponse, NewAudioSourceRequest, NewAudioSourceResponse, NewAudioStreamRequest, NewAudioStreamResponse, NewPlatformAudioRequest, NewPlatformAudioResponse, NewSoxResamplerRequest, NewSoxResamplerResponse, PushSoxResamplerRequest, PushSoxResamplerResponse, RemixAndResampleRequest, RemixAndResampleResponse, SetPlayoutDeviceRequest, SetPlayoutDeviceResponse, SetRecordingDeviceRequest, SetRecordingDeviceResponse } from "./audio_frame_pb.js"; import type { E2eeRequest, E2eeResponse } from "./e2ee_pb.js"; import type { PerformRpcCallback, PerformRpcRequest, PerformRpcResponse, RegisterRpcMethodRequest, RegisterRpcMethodResponse, RpcMethodInvocationEvent, RpcMethodInvocationResponseRequest, RpcMethodInvocationResponseResponse, UnregisterRpcMethodRequest, UnregisterRpcMethodResponse } from "./rpc_pb.js"; import type { EnableRemoteTrackPublicationRequest, EnableRemoteTrackPublicationResponse, SetRemoteTrackPublicationQualityRequest, SetRemoteTrackPublicationQualityResponse, UpdateRemoteTrackPublicationDimensionRequest, UpdateRemoteTrackPublicationDimensionResponse } from "./track_publication_pb.js"; @@ -537,6 +537,32 @@ export declare class FfiRequest extends Message { */ value: DataTrackStreamReadRequest; case: "dataTrackStreamRead"; + } | { + /** + * Platform Audio (ADM) + * + * @generated from field: livekit.proto.NewPlatformAudioRequest new_platform_audio = 76; + */ + value: NewPlatformAudioRequest; + case: "newPlatformAudio"; + } | { + /** + * @generated from field: livekit.proto.GetAudioDevicesRequest get_audio_devices = 77; + */ + value: GetAudioDevicesRequest; + case: "getAudioDevices"; + } | { + /** + * @generated from field: livekit.proto.SetRecordingDeviceRequest set_recording_device = 78; + */ + value: SetRecordingDeviceRequest; + case: "setRecordingDevice"; + } | { + /** + * @generated from field: livekit.proto.SetPlayoutDeviceRequest set_playout_device = 79; + */ + value: SetPlayoutDeviceRequest; + case: "setPlayoutDevice"; } | { case: undefined; value?: undefined }; constructor(data?: PartialMessage); @@ -1025,6 +1051,32 @@ export declare class FfiResponse extends Message { */ value: DataTrackStreamReadResponse; case: "dataTrackStreamRead"; + } | { + /** + * Platform Audio (ADM) + * + * @generated from field: livekit.proto.NewPlatformAudioResponse new_platform_audio = 75; + */ + value: NewPlatformAudioResponse; + case: "newPlatformAudio"; + } | { + /** + * @generated from field: livekit.proto.GetAudioDevicesResponse get_audio_devices = 76; + */ + value: GetAudioDevicesResponse; + case: "getAudioDevices"; + } | { + /** + * @generated from field: livekit.proto.SetRecordingDeviceResponse set_recording_device = 77; + */ + value: SetRecordingDeviceResponse; + case: "setRecordingDevice"; + } | { + /** + * @generated from field: livekit.proto.SetPlayoutDeviceResponse set_playout_device = 78; + */ + value: SetPlayoutDeviceResponse; + case: "setPlayoutDevice"; } | { case: undefined; value?: undefined }; constructor(data?: PartialMessage); diff --git a/livekit-ffi-node-bindings/proto/ffi_pb.js b/livekit-ffi-node-bindings/proto/ffi_pb.js index 22727f9b7..d50bc8383 100644 --- a/livekit-ffi-node-bindings/proto/ffi_pb.js +++ b/livekit-ffi-node-bindings/proto/ffi_pb.js @@ -24,7 +24,7 @@ const { proto2 } = require("@bufbuild/protobuf"); const { ConnectCallback, ConnectRequest, ConnectResponse, DisconnectCallback, DisconnectRequest, DisconnectResponse, EditChatMessageRequest, GetSessionStatsCallback, GetSessionStatsRequest, GetSessionStatsResponse, PublishDataCallback, PublishDataRequest, PublishDataResponse, PublishSipDtmfCallback, PublishSipDtmfRequest, PublishSipDtmfResponse, PublishTrackCallback, PublishTrackRequest, PublishTrackResponse, PublishTranscriptionCallback, PublishTranscriptionRequest, PublishTranscriptionResponse, RoomEvent, SendChatMessageCallback, SendChatMessageRequest, SendChatMessageResponse, SendStreamChunkCallback, SendStreamChunkRequest, SendStreamChunkResponse, SendStreamHeaderCallback, SendStreamHeaderRequest, SendStreamHeaderResponse, SendStreamTrailerCallback, SendStreamTrailerRequest, SendStreamTrailerResponse, SetDataChannelBufferedAmountLowThresholdRequest, SetDataChannelBufferedAmountLowThresholdResponse, SetLocalAttributesCallback, SetLocalAttributesRequest, SetLocalAttributesResponse, SetLocalMetadataCallback, SetLocalMetadataRequest, SetLocalMetadataResponse, SetLocalNameCallback, SetLocalNameRequest, SetLocalNameResponse, SetSubscribedRequest, SetSubscribedResponse, UnpublishTrackCallback, UnpublishTrackRequest, UnpublishTrackResponse } = require("./room_pb.js"); const { CreateAudioTrackRequest, CreateAudioTrackResponse, CreateVideoTrackRequest, CreateVideoTrackResponse, EnableRemoteTrackRequest, EnableRemoteTrackResponse, GetStatsCallback, GetStatsRequest, GetStatsResponse, LocalTrackMuteRequest, LocalTrackMuteResponse, SetTrackSubscriptionPermissionsRequest, SetTrackSubscriptionPermissionsResponse, TrackEvent } = require("./track_pb.js"); const { CaptureVideoFrameRequest, CaptureVideoFrameResponse, NewVideoSourceRequest, NewVideoSourceResponse, NewVideoStreamRequest, NewVideoStreamResponse, VideoConvertRequest, VideoConvertResponse, VideoStreamEvent, VideoStreamFromParticipantRequest, VideoStreamFromParticipantResponse } = require("./video_frame_pb.js"); -const { ApmProcessReverseStreamRequest, ApmProcessReverseStreamResponse, ApmProcessStreamRequest, ApmProcessStreamResponse, ApmSetStreamDelayRequest, ApmSetStreamDelayResponse, AudioStreamEvent, AudioStreamFromParticipantRequest, AudioStreamFromParticipantResponse, CaptureAudioFrameCallback, CaptureAudioFrameRequest, CaptureAudioFrameResponse, ClearAudioBufferRequest, ClearAudioBufferResponse, FlushSoxResamplerRequest, FlushSoxResamplerResponse, LoadAudioFilterPluginRequest, LoadAudioFilterPluginResponse, NewApmRequest, NewApmResponse, NewAudioResamplerRequest, NewAudioResamplerResponse, NewAudioSourceRequest, NewAudioSourceResponse, NewAudioStreamRequest, NewAudioStreamResponse, NewSoxResamplerRequest, NewSoxResamplerResponse, PushSoxResamplerRequest, PushSoxResamplerResponse, RemixAndResampleRequest, RemixAndResampleResponse } = require("./audio_frame_pb.js"); +const { ApmProcessReverseStreamRequest, ApmProcessReverseStreamResponse, ApmProcessStreamRequest, ApmProcessStreamResponse, ApmSetStreamDelayRequest, ApmSetStreamDelayResponse, AudioStreamEvent, AudioStreamFromParticipantRequest, AudioStreamFromParticipantResponse, CaptureAudioFrameCallback, CaptureAudioFrameRequest, CaptureAudioFrameResponse, ClearAudioBufferRequest, ClearAudioBufferResponse, FlushSoxResamplerRequest, FlushSoxResamplerResponse, GetAudioDevicesRequest, GetAudioDevicesResponse, LoadAudioFilterPluginRequest, LoadAudioFilterPluginResponse, NewApmRequest, NewApmResponse, NewAudioResamplerRequest, NewAudioResamplerResponse, NewAudioSourceRequest, NewAudioSourceResponse, NewAudioStreamRequest, NewAudioStreamResponse, NewPlatformAudioRequest, NewPlatformAudioResponse, NewSoxResamplerRequest, NewSoxResamplerResponse, PushSoxResamplerRequest, PushSoxResamplerResponse, RemixAndResampleRequest, RemixAndResampleResponse, SetPlayoutDeviceRequest, SetPlayoutDeviceResponse, SetRecordingDeviceRequest, SetRecordingDeviceResponse } = require("./audio_frame_pb.js"); const { E2eeRequest, E2eeResponse } = require("./e2ee_pb.js"); const { PerformRpcCallback, PerformRpcRequest, PerformRpcResponse, RegisterRpcMethodRequest, RegisterRpcMethodResponse, RpcMethodInvocationEvent, RpcMethodInvocationResponseRequest, RpcMethodInvocationResponseResponse, UnregisterRpcMethodRequest, UnregisterRpcMethodResponse } = require("./rpc_pb.js"); const { EnableRemoteTrackPublicationRequest, EnableRemoteTrackPublicationResponse, SetRemoteTrackPublicationQualityRequest, SetRemoteTrackPublicationQualityResponse, UpdateRemoteTrackPublicationDimensionRequest, UpdateRemoteTrackPublicationDimensionResponse } = require("./track_publication_pb.js"); @@ -128,6 +128,10 @@ const FfiRequest = /*@__PURE__*/ proto2.makeMessageType( { no: 73, name: "subscribe_data_track", kind: "message", T: SubscribeDataTrackRequest, oneof: "message" }, { no: 74, name: "remote_data_track_is_published", kind: "message", T: RemoteDataTrackIsPublishedRequest, oneof: "message" }, { no: 75, name: "data_track_stream_read", kind: "message", T: DataTrackStreamReadRequest, oneof: "message" }, + { no: 76, name: "new_platform_audio", kind: "message", T: NewPlatformAudioRequest, oneof: "message" }, + { no: 77, name: "get_audio_devices", kind: "message", T: GetAudioDevicesRequest, oneof: "message" }, + { no: 78, name: "set_recording_device", kind: "message", T: SetRecordingDeviceRequest, oneof: "message" }, + { no: 79, name: "set_playout_device", kind: "message", T: SetPlayoutDeviceRequest, oneof: "message" }, ], ); @@ -212,6 +216,10 @@ const FfiResponse = /*@__PURE__*/ proto2.makeMessageType( { no: 72, name: "subscribe_data_track", kind: "message", T: SubscribeDataTrackResponse, oneof: "message" }, { no: 73, name: "remote_data_track_is_published", kind: "message", T: RemoteDataTrackIsPublishedResponse, oneof: "message" }, { no: 74, name: "data_track_stream_read", kind: "message", T: DataTrackStreamReadResponse, oneof: "message" }, + { no: 75, name: "new_platform_audio", kind: "message", T: NewPlatformAudioResponse, oneof: "message" }, + { no: 76, name: "get_audio_devices", kind: "message", T: GetAudioDevicesResponse, oneof: "message" }, + { no: 77, name: "set_recording_device", kind: "message", T: SetRecordingDeviceResponse, oneof: "message" }, + { no: 78, name: "set_playout_device", kind: "message", T: SetPlayoutDeviceResponse, oneof: "message" }, ], ); diff --git a/livekit-ffi/protocol/audio_frame.proto b/livekit-ffi/protocol/audio_frame.proto index 99baf3678..53cea0591 100644 --- a/livekit-ffi/protocol/audio_frame.proto +++ b/livekit-ffi/protocol/audio_frame.proto @@ -73,6 +73,9 @@ message NewAudioSourceRequest { required uint32 sample_rate = 3; required uint32 num_channels = 4; optional uint32 queue_size_ms = 5; + // For AudioSourcePlatform: the PlatformAudio handle to configure audio processing on. + // If provided with options, audio processing will be configured on the PlatformAudio. + optional uint64 platform_audio_handle = 6; } message NewAudioSourceResponse { required OwnedAudioSource source = 1; } @@ -283,10 +286,17 @@ message AudioSourceOptions { required bool echo_cancellation = 1; required bool noise_suppression = 2; required bool auto_gain_control = 3; + // Prefer hardware audio processing (e.g., iOS VPIO). Lower latency. + // Only applies to AudioSourcePlatform. Default: true. + optional bool prefer_hardware = 4; } enum AudioSourceType { + // Push-based audio source - manually capture frames via CaptureAudioFrameRequest AUDIO_SOURCE_NATIVE = 0; + // Platform ADM-based audio source - captures from microphone automatically + // Requires PlatformAudio to be created first to enable ADM recording + AUDIO_SOURCE_PLATFORM = 1; } message AudioSourceInfo { @@ -340,3 +350,117 @@ message LoadAudioFilterPluginRequest { message LoadAudioFilterPluginResponse { optional string error = 1; } + +// +// PlatformAudio - Platform audio device management via WebRTC's ADM +// +// PlatformAudio provides access to the platform's audio devices (microphones and +// speakers) via WebRTC's Audio Device Module (ADM). Use it to: +// +// - Capture audio from the microphone for publishing +// - Play received audio through the speakers +// - Enumerate and select audio devices +// +// # Usage +// +// 1. Create a PlatformAudio handle with NewPlatformAudioRequest +// 2. Enumerate devices with GetAudioDevicesRequest (optional) +// 3. Select devices with SetRecordingDeviceRequest/SetPlayoutDeviceRequest (optional) +// 4. Create an audio track using AudioSourcePlatform type +// 5. When done, drop the handle (the ADM is disabled when all handles are released) +// + +// Information about an audio device. +message AudioDeviceInfo { + // Device index (0-based). + required uint32 index = 1; + // Device name as reported by the operating system. + required string name = 2; +} + +// Information about a PlatformAudio instance. +message PlatformAudioInfo { + // Number of available recording (microphone) devices. + required int32 recording_device_count = 1; + // Number of available playout (speaker) devices. + required int32 playout_device_count = 2; +} + +// Owned PlatformAudio handle with info. +message OwnedPlatformAudio { + required FfiOwnedHandle handle = 1; + required PlatformAudioInfo info = 2; +} + +// Create a new PlatformAudio instance. +// +// This enables the platform ADM for microphone capture and speaker playout. +// If another PlatformAudio instance exists, this reuses the same underlying ADM. +// +// The returned handle must be kept alive while platform audio is needed. +// When all handles are released, the ADM is automatically disabled. +message NewPlatformAudioRequest {} + +message NewPlatformAudioResponse { + oneof message { + // The PlatformAudio handle on success. + OwnedPlatformAudio platform_audio = 1; + // Error message if creation failed. + string error = 2; + } +} + +// Get available audio devices. +// +// Returns lists of available recording (microphone) and playout (speaker) devices. +message GetAudioDevicesRequest { + // The PlatformAudio handle. + required uint64 platform_audio_handle = 1; +} + +message GetAudioDevicesResponse { + // Available playout devices (speakers/headphones). + repeated AudioDeviceInfo playout_devices = 1; + // Available recording devices (microphones). + repeated AudioDeviceInfo recording_devices = 2; + // Error message if enumeration failed, empty/absent on success. + optional string error = 3; +} + +// Set the recording device (microphone). +// +// Call this before creating audio tracks to select which microphone to use. +// Device indices are 0-based and must be less than the recording device count. +message SetRecordingDeviceRequest { + // The PlatformAudio handle. + required uint64 platform_audio_handle = 1; + // Device index from GetAudioDevicesResponse.recording_devices. + required uint32 index = 2; +} + +message SetRecordingDeviceResponse { + // Error message if the operation failed: + // - "Invalid device index" if index >= recording device count + // - Other platform-specific errors + // Empty/absent on success. + optional string error = 1; +} + +// Set the playout device (speaker/headphones). +// +// Call this before connecting to select which speaker to use for audio output. +// Device indices are 0-based and must be less than the playout device count. +message SetPlayoutDeviceRequest { + // The PlatformAudio handle. + required uint64 platform_audio_handle = 1; + // Device index from GetAudioDevicesResponse.playout_devices. + required uint32 index = 2; +} + +message SetPlayoutDeviceResponse { + // Error message if the operation failed: + // - "Invalid device index" if index >= playout device count + // - Other platform-specific errors + // Empty/absent on success. + optional string error = 1; +} diff --git a/livekit-ffi/protocol/ffi.proto b/livekit-ffi/protocol/ffi.proto index b27a7b865..c734e13b0 100644 --- a/livekit-ffi/protocol/ffi.proto +++ b/livekit-ffi/protocol/ffi.proto @@ -164,7 +164,13 @@ message FfiRequest { RemoteDataTrackIsPublishedRequest remote_data_track_is_published = 74; DataTrackStreamReadRequest data_track_stream_read = 75; - // NEXT_ID: 76 + // Platform Audio (ADM) + NewPlatformAudioRequest new_platform_audio = 76; + GetAudioDevicesRequest get_audio_devices = 77; + SetRecordingDeviceRequest set_recording_device = 78; + SetPlayoutDeviceRequest set_playout_device = 79; + + // NEXT_ID: 80 } } @@ -274,7 +280,13 @@ message FfiResponse { RemoteDataTrackIsPublishedResponse remote_data_track_is_published = 73; DataTrackStreamReadResponse data_track_stream_read = 74; - // NEXT_ID: 75 + // Platform Audio (ADM) + NewPlatformAudioResponse new_platform_audio = 75; + GetAudioDevicesResponse get_audio_devices = 76; + SetRecordingDeviceResponse set_recording_device = 77; + SetPlayoutDeviceResponse set_playout_device = 78; + + // NEXT_ID: 79 } } diff --git a/livekit-ffi/src/server/audio_source.rs b/livekit-ffi/src/server/audio_source.rs index 47e33c6f2..e5c8c49b1 100644 --- a/livekit-ffi/src/server/audio_source.rs +++ b/livekit-ffi/src/server/audio_source.rs @@ -47,6 +47,37 @@ impl FfiAudioSource { ); RtcAudioSource::Native(audio_source) } + #[cfg(not(target_arch = "wasm32"))] + proto::AudioSourceType::AudioSourcePlatform => { + // Platform ADM-based source - captures from microphone automatically + // PlatformAudio must be created first to enable ADM recording + + // If options and platform_audio_handle are provided, configure audio processing + if let (Some(ref options), Some(handle)) = + (&new_source.options, new_source.platform_audio_handle) + { + if let Ok(ffi_audio) = + server.retrieve_handle::(handle) + { + let processing_options = livekit::AudioProcessingOptions { + echo_cancellation: options.echo_cancellation, + noise_suppression: options.noise_suppression, + auto_gain_control: options.auto_gain_control, + prefer_hardware_processing: options.prefer_hardware.unwrap_or(true), + }; + if let Err(e) = + ffi_audio.audio.configure_audio_processing(processing_options) + { + log::warn!( + "Failed to configure audio processing for platform source: {}", + e + ); + } + } + } + + RtcAudioSource::Device + } _ => return Err(FfiError::InvalidRequest("unsupported audio source type".into())), }; diff --git a/livekit-ffi/src/server/requests.rs b/livekit-ffi/src/server/requests.rs index e27a54168..34cb029bf 100644 --- a/livekit-ffi/src/server/requests.rs +++ b/livekit-ffi/src/server/requests.rs @@ -1367,7 +1367,303 @@ pub fn handle_request( on_remote_data_track_is_published(server, req)?.into() } Request::DataTrackStreamRead(req) => on_data_track_stream_read(server, req)?.into(), + // Platform Audio + Request::NewPlatformAudio(req) => on_new_platform_audio(server, req)?.into(), + Request::GetAudioDevices(req) => on_get_audio_devices(server, req)?.into(), + Request::SetRecordingDevice(req) => on_set_recording_device(server, req)?.into(), + Request::SetPlayoutDevice(req) => on_set_playout_device(server, req)?.into(), }); Ok(res) } + +// ==================== Platform Audio ==================== + +/// FFI wrapper for PlatformAudio handle. +pub struct FfiPlatformAudio { + pub audio: PlatformAudio, +} + +impl super::FfiHandle for FfiPlatformAudio {} + +fn on_new_platform_audio( + server: &'static FfiServer, + _req: proto::NewPlatformAudioRequest, +) -> FfiResult { + match PlatformAudio::new() { + Ok(audio) => { + let handle_id = server.next_id(); + let info = proto::PlatformAudioInfo { + recording_device_count: audio.recording_devices() as i32, + playout_device_count: audio.playout_devices() as i32, + }; + + server.store_handle(handle_id, FfiPlatformAudio { audio }); + + Ok(proto::NewPlatformAudioResponse { + message: Some(proto::new_platform_audio_response::Message::PlatformAudio( + proto::OwnedPlatformAudio { + handle: proto::FfiOwnedHandle { id: handle_id }, + info, + }, + )), + }) + } + Err(e) => Ok(proto::NewPlatformAudioResponse { + message: Some(proto::new_platform_audio_response::Message::Error(e.to_string())), + }), + } +} + +fn on_get_audio_devices( + server: &'static FfiServer, + req: proto::GetAudioDevicesRequest, +) -> FfiResult { + let ffi_audio = server.retrieve_handle::(req.platform_audio_handle)?; + let audio = &ffi_audio.audio; + + let mut playout_devices = vec![]; + let mut recording_devices = vec![]; + + // Enumerate playout devices + let playout_count = audio.playout_devices(); + for i in 0..playout_count as u32 { + playout_devices + .push(proto::AudioDeviceInfo { index: i, name: audio.playout_device_name(i as u16) }); + } + + // Enumerate recording devices + let recording_count = audio.recording_devices(); + for i in 0..recording_count as u32 { + recording_devices + .push(proto::AudioDeviceInfo { index: i, name: audio.recording_device_name(i as u16) }); + } + + Ok(proto::GetAudioDevicesResponse { playout_devices, recording_devices, error: None }) +} + +fn on_set_recording_device( + server: &'static FfiServer, + req: proto::SetRecordingDeviceRequest, +) -> FfiResult { + let ffi_audio = server.retrieve_handle::(req.platform_audio_handle)?; + + match ffi_audio.audio.set_recording_device(req.index as u16) { + Ok(()) => Ok(proto::SetRecordingDeviceResponse { error: None }), + Err(e) => Ok(proto::SetRecordingDeviceResponse { error: Some(e.to_string()) }), + } +} + +fn on_set_playout_device( + server: &'static FfiServer, + req: proto::SetPlayoutDeviceRequest, +) -> FfiResult { + let ffi_audio = server.retrieve_handle::(req.platform_audio_handle)?; + + match ffi_audio.audio.set_playout_device(req.index as u16) { + Ok(()) => Ok(proto::SetPlayoutDeviceResponse { error: None }), + Err(e) => Ok(proto::SetPlayoutDeviceResponse { error: Some(e.to_string()) }), + } +} + +// ============================================================================= +// Tests +// ============================================================================= + +#[cfg(test)] +mod tests { + use super::*; + use crate::FFI_SERVER; + + /// Helper to get a static reference to FFI_SERVER for tests + fn server() -> &'static FfiServer { + &FFI_SERVER + } + + #[test] + fn test_new_platform_audio() { + let req = proto::NewPlatformAudioRequest {}; + let res = on_new_platform_audio(server(), req).unwrap(); + + match res.message { + Some(proto::new_platform_audio_response::Message::PlatformAudio(audio)) => { + // Verify we got a valid handle + assert!(audio.handle.id > 0); + + // Verify device counts are reasonable (>= 0) + assert!(audio.info.recording_device_count >= 0); + assert!(audio.info.playout_device_count >= 0); + + println!( + "PlatformAudio created: handle={}, recording_devices={}, playout_devices={}", + audio.handle.id, + audio.info.recording_device_count, + audio.info.playout_device_count + ); + + // Clean up - drop the handle + server().drop_handle(audio.handle.id); + } + Some(proto::new_platform_audio_response::Message::Error(e)) => { + panic!("Failed to create PlatformAudio: {}", e); + } + None => { + panic!("Empty response"); + } + } + } + + #[test] + fn test_get_audio_devices() { + // First create a PlatformAudio handle + let create_req = proto::NewPlatformAudioRequest {}; + let create_res = on_new_platform_audio(server(), create_req).unwrap(); + + let handle_id = match create_res.message { + Some(proto::new_platform_audio_response::Message::PlatformAudio(audio)) => { + audio.handle.id + } + _ => panic!("Failed to create PlatformAudio"), + }; + + // Get audio devices + let req = proto::GetAudioDevicesRequest { platform_audio_handle: handle_id }; + let res = on_get_audio_devices(server(), req).unwrap(); + + assert!(res.error.is_none(), "Error: {:?}", res.error); + + println!("Recording devices:"); + for device in &res.recording_devices { + println!(" [{}] {}", device.index, device.name); + } + + println!("Playout devices:"); + for device in &res.playout_devices { + println!(" [{}] {}", device.index, device.name); + } + + // Clean up + server().drop_handle(handle_id); + } + + #[test] + fn test_set_recording_device() { + // Create PlatformAudio handle + let create_req = proto::NewPlatformAudioRequest {}; + let create_res = on_new_platform_audio(server(), create_req).unwrap(); + + let (handle_id, recording_count) = match create_res.message { + Some(proto::new_platform_audio_response::Message::PlatformAudio(audio)) => { + (audio.handle.id, audio.info.recording_device_count) + } + _ => panic!("Failed to create PlatformAudio"), + }; + + // Set recording device if available + if recording_count > 0 { + let req = + proto::SetRecordingDeviceRequest { platform_audio_handle: handle_id, index: 0 }; + let res = on_set_recording_device(server(), req).unwrap(); + assert!(res.error.is_none(), "Error: {:?}", res.error); + println!("Set recording device to index 0"); + } + + // Test invalid device index + let req = + proto::SetRecordingDeviceRequest { platform_audio_handle: handle_id, index: 9999 }; + let res = on_set_recording_device(server(), req).unwrap(); + assert!(res.error.is_some(), "Should fail with invalid index"); + println!("Invalid index correctly rejected: {:?}", res.error); + + // Clean up + server().drop_handle(handle_id); + } + + #[test] + fn test_set_playout_device() { + // Create PlatformAudio handle + let create_req = proto::NewPlatformAudioRequest {}; + let create_res = on_new_platform_audio(server(), create_req).unwrap(); + + let (handle_id, playout_count) = match create_res.message { + Some(proto::new_platform_audio_response::Message::PlatformAudio(audio)) => { + (audio.handle.id, audio.info.playout_device_count) + } + _ => panic!("Failed to create PlatformAudio"), + }; + + // Set playout device if available + if playout_count > 0 { + let req = proto::SetPlayoutDeviceRequest { platform_audio_handle: handle_id, index: 0 }; + let res = on_set_playout_device(server(), req).unwrap(); + assert!(res.error.is_none(), "Error: {:?}", res.error); + println!("Set playout device to index 0"); + } + + // Test invalid device index + let req = proto::SetPlayoutDeviceRequest { platform_audio_handle: handle_id, index: 9999 }; + let res = on_set_playout_device(server(), req).unwrap(); + assert!(res.error.is_some(), "Should fail with invalid index"); + println!("Invalid index correctly rejected: {:?}", res.error); + + // Clean up + server().drop_handle(handle_id); + } + + #[test] + fn test_platform_audio_handle_lifecycle() { + // Create first handle + let req1 = proto::NewPlatformAudioRequest {}; + let res1 = on_new_platform_audio(server(), req1).unwrap(); + let handle1 = match res1.message { + Some(proto::new_platform_audio_response::Message::PlatformAudio(audio)) => { + audio.handle.id + } + _ => panic!("Failed to create first PlatformAudio"), + }; + println!("Created handle 1: {}", handle1); + + // Create second handle (should share ADM) + let req2 = proto::NewPlatformAudioRequest {}; + let res2 = on_new_platform_audio(server(), req2).unwrap(); + let handle2 = match res2.message { + Some(proto::new_platform_audio_response::Message::PlatformAudio(audio)) => { + audio.handle.id + } + _ => panic!("Failed to create second PlatformAudio"), + }; + println!("Created handle 2: {}", handle2); + + // Both handles should be different + assert_ne!(handle1, handle2); + + // Drop first handle + assert!(server().drop_handle(handle1)); + println!("Dropped handle 1"); + + // Second handle should still work + let req = proto::GetAudioDevicesRequest { platform_audio_handle: handle2 }; + let res = on_get_audio_devices(server(), req).unwrap(); + assert!(res.error.is_none()); + println!("Handle 2 still works after handle 1 dropped"); + + // Drop second handle + assert!(server().drop_handle(handle2)); + println!("Dropped handle 2"); + + // Trying to use dropped handle should fail + let req = proto::GetAudioDevicesRequest { platform_audio_handle: handle2 }; + let res = on_get_audio_devices(server(), req); + assert!(res.is_err()); + println!("Dropped handle correctly rejected"); + } + + #[test] + fn test_invalid_handle() { + // Try to get devices with invalid handle + let req = proto::GetAudioDevicesRequest { platform_audio_handle: 99999 }; + let res = on_get_audio_devices(server(), req); + assert!(res.is_err()); + println!("Invalid handle correctly rejected"); + } +} diff --git a/livekit/Cargo.toml b/livekit/Cargo.toml index d3d3441b0..dc3d70dd3 100644 --- a/livekit/Cargo.toml +++ b/livekit/Cargo.toml @@ -51,3 +51,4 @@ base64 = "0.22" anyhow = "1.0.99" test-log = "0.2.18" test-case = "3.3" +serial_test = "3.0" diff --git a/livekit/src/lib.rs b/livekit/src/lib.rs index 00abe5de8..6dd339c76 100644 --- a/livekit/src/lib.rs +++ b/livekit/src/lib.rs @@ -15,7 +15,7 @@ mod plugin; pub mod proto; mod room; -mod rtc_engine; +pub mod rtc_engine; pub mod webrtc { pub use libwebrtc::*; @@ -26,6 +26,12 @@ pub use room::*; /// `use livekit::prelude::*;` to import livekit types pub mod prelude; +// Platform Audio Device Module (ADM) management +#[cfg(not(target_arch = "wasm32"))] +mod platform_audio; +#[cfg(not(target_arch = "wasm32"))] +pub use platform_audio::*; + #[cfg(feature = "dispatcher")] pub mod dispatcher { pub use livekit_runtime::set_dispatcher; diff --git a/livekit/src/platform_audio/error.rs b/livekit/src/platform_audio/error.rs new file mode 100644 index 000000000..23026a968 --- /dev/null +++ b/livekit/src/platform_audio/error.rs @@ -0,0 +1,126 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Error types for platform audio operations. + +use std::fmt; + +/// Errors that can occur during audio operations. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AudioError { + /// Platform ADM could not be initialized. + /// + /// This can happen if: + /// - No audio devices are available + /// - Audio permissions are not granted + /// - Platform audio subsystem is unavailable + PlatformInitFailed, + + /// The specified device index is invalid. + /// + /// Device indices are 0-based and must be less than the device count. + InvalidDeviceIndex, + + /// An audio operation failed. + OperationFailed(String), +} + +impl fmt::Display for AudioError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + AudioError::PlatformInitFailed => { + write!(f, "Failed to initialize platform audio") + } + AudioError::InvalidDeviceIndex => write!(f, "Invalid device index"), + AudioError::OperationFailed(msg) => write!(f, "Audio operation failed: {}", msg), + } + } +} + +impl std::error::Error for AudioError {} + +/// Result type for audio operations. +pub type AudioResult = Result; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn audio_error_display() { + let err = AudioError::PlatformInitFailed; + let msg = format!("{}", err); + assert!(msg.contains("platform audio")); + + let err = AudioError::InvalidDeviceIndex; + let msg = format!("{}", err); + assert!(msg.contains("Invalid device index")); + + let err = AudioError::OperationFailed("test message".to_string()); + let msg = format!("{}", err); + assert!(msg.contains("test message")); + } + + #[test] + fn audio_error_debug() { + let err = AudioError::PlatformInitFailed; + let debug_str = format!("{:?}", err); + assert!(debug_str.contains("PlatformInitFailed")); + + let err = AudioError::InvalidDeviceIndex; + let debug_str = format!("{:?}", err); + assert!(debug_str.contains("InvalidDeviceIndex")); + } + + #[test] + fn audio_error_equality() { + assert_eq!(AudioError::PlatformInitFailed, AudioError::PlatformInitFailed); + assert_eq!(AudioError::InvalidDeviceIndex, AudioError::InvalidDeviceIndex); + assert_eq!( + AudioError::OperationFailed("a".to_string()), + AudioError::OperationFailed("a".to_string()) + ); + assert_ne!( + AudioError::OperationFailed("a".to_string()), + AudioError::OperationFailed("b".to_string()) + ); + } + + #[test] + fn audio_error_clone() { + let err = AudioError::OperationFailed("test".to_string()); + let cloned = err.clone(); + assert_eq!(err, cloned); + } + + #[test] + fn audio_error_is_std_error() { + let err: Box = Box::new(AudioError::InvalidDeviceIndex); + assert!(err.to_string().contains("Invalid device index")); + } + + #[test] + fn audio_result_ok() { + let result: AudioResult = Ok(42); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), 42); + } + + #[test] + fn audio_result_err() { + let result: AudioResult = Err(AudioError::InvalidDeviceIndex); + assert!(result.is_err()); + assert_eq!(result.unwrap_err(), AudioError::InvalidDeviceIndex); + } +} diff --git a/livekit/src/platform_audio/mod.rs b/livekit/src/platform_audio/mod.rs new file mode 100644 index 000000000..195f7863f --- /dev/null +++ b/livekit/src/platform_audio/mod.rs @@ -0,0 +1,932 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Platform audio device management for the LiveKit SDK. +//! +//! This module provides [`PlatformAudio`] for accessing platform audio devices +//! (microphones and speakers) via WebRTC's Audio Device Module (ADM). +//! +//! # Overview +//! +//! The SDK supports two ways to handle audio: +//! +//! - **Manual audio** (default): Use [`NativeAudioSource`] to push audio frames manually. +//! Suitable for agents, TTS, file streaming, or testing. +//! +//! - **Platform audio**: Use [`PlatformAudio`] to capture from microphone and play +//! to speakers automatically. Suitable for VoIP applications. +//! +//! # Using Platform Audio +//! +//! ```rust,ignore +//! use livekit::prelude::*; +//! +//! // Create PlatformAudio instance (enables platform ADM) +//! let audio = PlatformAudio::new()?; +//! +//! // Enumerate devices +//! for i in 0..audio.recording_devices() as u16 { +//! println!("Mic {}: {}", i, audio.recording_device_name(i)); +//! } +//! +//! // Select a device +//! audio.set_recording_device(0)?; +//! +//! // Create and publish audio track +//! let track = LocalAudioTrack::create_audio_track("mic", audio.rtc_source()); +//! room.local_participant().publish_track(LocalTrack::Audio(track), opts).await?; +//! +//! // When audio is dropped, platform ADM is automatically disabled +//! ``` +//! +//! # Combining with NativeAudioSource +//! +//! You can use both platform audio and manual audio simultaneously: +//! +//! ```rust,ignore +//! use livekit::prelude::*; +//! use livekit::webrtc::audio_source::native::NativeAudioSource; +//! +//! // Track A: Microphone via platform audio +//! let mic = PlatformAudio::new()?; +//! let mic_track = LocalAudioTrack::create_audio_track("mic", mic.rtc_source()); +//! +//! // Track B: Screen capture via manual pushing +//! let screen_source = NativeAudioSource::new(opts, 48000, 2, 100); +//! let screen_track = LocalAudioTrack::create_audio_track( +//! "screen", +//! RtcAudioSource::Native(screen_source), +//! ); +//! +//! // Publish both +//! room.local_participant().publish_track(LocalTrack::Audio(mic_track), opts).await?; +//! room.local_participant().publish_track(LocalTrack::Audio(screen_track), opts).await?; +//! ``` +//! +//! # Reference Counting +//! +//! Multiple [`PlatformAudio`] instances share the same underlying ADM: +//! +//! ```rust,ignore +//! let audio1 = PlatformAudio::new()?; // Enables ADM +//! let audio2 = PlatformAudio::new()?; // Reuses same ADM +//! let audio3 = audio1.clone(); // Shares same ADM +//! +//! drop(audio1); +//! drop(audio2); +//! // ADM still active (audio3 holds reference) +//! +//! drop(audio3); +//! // ADM now disabled +//! ``` +//! +//! # Platform-Specific Notes +//! +//! - **iOS**: Creates a VPIO (Voice Processing IO) AudioUnit. Only one VPIO +//! can exist per process. Drop all `PlatformAudio` instances to release it. +//! - **macOS**: Uses CoreAudio. +//! - **Windows**: Uses WASAPI. +//! - **Linux**: Uses PulseAudio or ALSA. +//! +//! [`NativeAudioSource`]: crate::webrtc::audio_source::native::NativeAudioSource + +mod error; +mod processing; + +pub use error::{AudioError, AudioResult}; +pub use processing::{AudioProcessingOptions, AudioProcessingType}; + +// Re-export RtcAudioSource for convenience +pub use libwebrtc::audio_source::RtcAudioSource; + +use std::fmt; +use std::sync::{Arc, Weak}; + +use lazy_static::lazy_static; +use parking_lot::Mutex; + +use crate::rtc_engine::lk_runtime::LkRuntime; + +// ============================================================================= +// PlatformAudio - Reference-counted platform audio device management +// ============================================================================= + +lazy_static! { + /// Weak reference to the shared Platform ADM handle. + /// When all strong references are dropped, the ADM is automatically disabled. + static ref PLATFORM_ADM_HANDLE: Mutex> = Mutex::new(Weak::new()); +} + +/// Internal handle for platform audio. +/// +/// This is a marker type that tracks PlatformAudio usage. The Platform ADM +/// is always enabled and does not need to be disabled when dropped. +struct PlatformAdmHandle { + runtime: Arc, +} + +impl Drop for PlatformAdmHandle { + fn drop(&mut self) { + log::debug!("PlatformAdmHandle dropped"); + // Platform ADM is always enabled, no cleanup needed + } +} + +/// Platform audio device management for microphone capture and speaker playout. +/// +/// `PlatformAudio` provides access to the platform's audio devices via WebRTC's +/// Audio Device Module (ADM). Use it to: +/// +/// - Enumerate available microphones and speakers +/// - Select which devices to use +/// - Create audio tracks that capture from the microphone +/// +/// # Creating a PlatformAudio Instance +/// +/// ```rust,ignore +/// use livekit::PlatformAudio; +/// +/// let audio = PlatformAudio::new()?; +/// ``` +/// +/// This enables the platform ADM. If an instance already exists, the new +/// instance shares the same underlying ADM. +/// +/// # Device Enumeration +/// +/// ```rust,ignore +/// // List microphones +/// for i in 0..audio.recording_devices() as u16 { +/// println!("Mic {}: {}", i, audio.recording_device_name(i)); +/// } +/// +/// // List speakers +/// for i in 0..audio.playout_devices() as u16 { +/// println!("Speaker {}: {}", i, audio.playout_device_name(i)); +/// } +/// ``` +/// +/// # Device Selection +/// +/// ```rust,ignore +/// // Select microphone by index +/// audio.set_recording_device(0)?; +/// +/// // Select speaker by index +/// audio.set_playout_device(0)?; +/// +/// // Hot-swap devices during active session +/// audio.switch_recording_device(1)?; +/// audio.switch_playout_device(1)?; +/// ``` +/// +/// # Creating Audio Tracks +/// +/// ```rust,ignore +/// use livekit::prelude::*; +/// +/// let audio = PlatformAudio::new()?; +/// let track = LocalAudioTrack::create_audio_track("microphone", audio.rtc_source()); +/// +/// room.local_participant() +/// .publish_track(LocalTrack::Audio(track), opts) +/// .await?; +/// ``` +/// +/// # Lifecycle Management +/// +/// `PlatformAudio` uses reference counting. Multiple instances share the same +/// underlying ADM, and the ADM is automatically disabled when all instances +/// are dropped. +/// +/// ```rust,ignore +/// let audio1 = PlatformAudio::new()?; // Enables ADM +/// let audio2 = PlatformAudio::new()?; // Shares ADM (ref_count = 2) +/// let audio3 = audio1.clone(); // Shares ADM (ref_count = 3) +/// +/// drop(audio1); // ref_count = 2, ADM still active +/// drop(audio2); // ref_count = 1, ADM still active +/// drop(audio3); // ref_count = 0, ADM disabled +/// ``` +/// +/// You can also explicitly release: +/// +/// ```rust,ignore +/// audio.release(); // Equivalent to drop(audio) +/// ``` +/// +/// # Platform-Specific Notes +/// +/// - **iOS**: Creates a VPIO AudioUnit (exclusive microphone access). +/// Drop all instances to allow other audio frameworks to use the mic. +/// - **macOS**: Uses CoreAudio for device management. +/// - **Windows**: Uses WASAPI for device management. +/// - **Linux**: Uses PulseAudio or ALSA. +#[derive(Clone)] +pub struct PlatformAudio { + /// Shared ownership of the Platform ADM handle. + /// When the last clone is dropped, the ADM is disabled. + handle: Arc, +} + +impl PlatformAudio { + /// Creates a new `PlatformAudio` instance. + /// + /// Platform ADM is always available and initialized at startup. + /// If another `PlatformAudio` instance exists, this reuses the same handle. + /// + /// # Errors + /// + /// Returns [`AudioError::PlatformInitFailed`] if no audio devices are available. + /// + /// # Example + /// + /// ```rust,ignore + /// use livekit::PlatformAudio; + /// + /// let audio = PlatformAudio::new()?; + /// println!("Found {} microphones", audio.recording_devices()); + /// ``` + pub fn new() -> AudioResult { + let mut handle_ref = PLATFORM_ADM_HANDLE.lock(); + + // Try to reuse existing handle + if let Some(handle) = handle_ref.upgrade() { + log::debug!("PlatformAudio: reusing existing handle"); + return Ok(Self { handle }); + } + + // Create new handle (Platform ADM is always enabled at startup) + log::debug!("PlatformAudio: creating new handle"); + let runtime = LkRuntime::instance(); + + // Enable ADM recording since PlatformAudio needs microphone access + // Recording is disabled by default to prevent interference with NativeAudioSource + runtime.set_adm_recording_enabled(true); + log::info!("PlatformAudio: enabled ADM recording for microphone capture"); + + // Verify Platform ADM is working by checking device count + let recording_count = runtime.recording_devices(); + let playout_count = runtime.playout_devices(); + log::info!( + "PlatformAudio: {} recording devices, {} playout devices", + recording_count, + playout_count + ); + + let handle = Arc::new(PlatformAdmHandle { runtime }); + *handle_ref = Arc::downgrade(&handle); + + let audio = Self { handle }; + + // Configure audio processing with hardware preferred and all options enabled by default + // This provides the best audio quality with echo cancellation, noise suppression, and AGC + let options = AudioProcessingOptions { + echo_cancellation: true, + noise_suppression: true, + auto_gain_control: true, + prefer_hardware_processing: true, + }; + if let Err(e) = audio.configure_audio_processing(options) { + log::warn!("PlatformAudio: failed to configure audio processing: {}", e); + } + + Ok(audio) + } + + // ========================================================================= + // Audio Source + // ========================================================================= + + /// Returns the [`RtcAudioSource`] to use when creating audio tracks. + /// + /// This returns `RtcAudioSource::Device`, which tells the track to capture + /// audio from the platform's selected recording device (microphone). + /// + /// # Example + /// + /// ```rust,ignore + /// use livekit::prelude::*; + /// + /// let audio = PlatformAudio::new()?; + /// let track = LocalAudioTrack::create_audio_track("mic", audio.rtc_source()); + /// ``` + pub fn rtc_source(&self) -> RtcAudioSource { + RtcAudioSource::Device + } + + // ========================================================================= + // Device Enumeration + // ========================================================================= + + /// Returns the number of available recording (microphone) devices. + /// + /// # Example + /// + /// ```rust,ignore + /// let audio = PlatformAudio::new()?; + /// println!("Found {} microphones", audio.recording_devices()); + /// ``` + pub fn recording_devices(&self) -> i16 { + self.handle.runtime.recording_devices() + } + + /// Returns the number of available playout (speaker) devices. + /// + /// # Example + /// + /// ```rust,ignore + /// let audio = PlatformAudio::new()?; + /// println!("Found {} speakers", audio.playout_devices()); + /// ``` + pub fn playout_devices(&self) -> i16 { + self.handle.runtime.playout_devices() + } + + /// Returns the name of a recording device by index. + /// + /// # Arguments + /// + /// * `index` - Device index (0-based, must be < `recording_devices()`) + /// + /// # Example + /// + /// ```rust,ignore + /// let audio = PlatformAudio::new()?; + /// for i in 0..audio.recording_devices() as u16 { + /// println!("Mic {}: {}", i, audio.recording_device_name(i)); + /// } + /// ``` + pub fn recording_device_name(&self, index: u16) -> String { + self.handle.runtime.recording_device_name(index) + } + + /// Returns the name of a playout device by index. + /// + /// # Arguments + /// + /// * `index` - Device index (0-based, must be < `playout_devices()`) + /// + /// # Example + /// + /// ```rust,ignore + /// let audio = PlatformAudio::new()?; + /// for i in 0..audio.playout_devices() as u16 { + /// println!("Speaker {}: {}", i, audio.playout_device_name(i)); + /// } + /// ``` + pub fn playout_device_name(&self, index: u16) -> String { + self.handle.runtime.playout_device_name(index) + } + + // ========================================================================= + // Device Selection + // ========================================================================= + + /// Selects a recording (microphone) device by index. + /// + /// Call this before creating audio tracks to select which microphone to use. + /// + /// # Arguments + /// + /// * `index` - Device index (0-based, must be < `recording_devices()`) + /// + /// # Errors + /// + /// - [`AudioError::InvalidDeviceIndex`] if index is out of range + /// - [`AudioError::OperationFailed`] if device selection fails + /// + /// # Example + /// + /// ```rust,ignore + /// let audio = PlatformAudio::new()?; + /// audio.set_recording_device(0)?; // Select first microphone + /// ``` + pub fn set_recording_device(&self, index: u16) -> AudioResult<()> { + let count = self.recording_devices(); + if index >= count as u16 { + return Err(AudioError::InvalidDeviceIndex); + } + + let result = self.handle.runtime.set_recording_device(index); + if result == 0 { + Ok(()) + } else { + Err(AudioError::OperationFailed(format!("set_recording_device returned {}", result))) + } + } + + /// Selects a playout (speaker) device by index. + /// + /// Call this before connecting to select which speaker to use for audio output. + /// + /// # Arguments + /// + /// * `index` - Device index (0-based, must be < `playout_devices()`) + /// + /// # Errors + /// + /// - [`AudioError::InvalidDeviceIndex`] if index is out of range + /// - [`AudioError::OperationFailed`] if device selection fails + /// + /// # Example + /// + /// ```rust,ignore + /// let audio = PlatformAudio::new()?; + /// audio.set_playout_device(0)?; // Select first speaker + /// ``` + pub fn set_playout_device(&self, index: u16) -> AudioResult<()> { + let count = self.playout_devices(); + if index >= count as u16 { + return Err(AudioError::InvalidDeviceIndex); + } + + let runtime = &self.handle.runtime; + let result = runtime.set_playout_device(index); + if result != 0 { + return Err(AudioError::OperationFailed(format!( + "set_playout_device returned {}", + result + ))); + } + + // On iOS, we need to explicitly initialize and start playout. + // On desktop platforms, WebRTC auto-starts playout when remote audio arrives, + // but on iOS with VPIO this doesn't happen automatically. + #[cfg(target_os = "ios")] + { + let init_result = runtime.init_playout(); + if init_result != 0 { + log::warn!("set_playout_device: init_playout returned {}", init_result); + } else { + let start_result = runtime.start_playout(); + if start_result != 0 { + log::warn!("set_playout_device: start_playout returned {}", start_result); + } else { + log::info!("set_playout_device: playout initialized and started for device {}", index); + } + } + } + + Ok(()) + } + + /// Switches the recording device while audio is active (hot-swap). + /// + /// Unlike [`set_recording_device`], this method handles the stop/change/restart + /// sequence required when recording is already active. + /// + /// # Arguments + /// + /// * `index` - Device index (0-based, must be < `recording_devices()`) + /// + /// # Errors + /// + /// - [`AudioError::InvalidDeviceIndex`] if index is out of range + /// - [`AudioError::OperationFailed`] if any step fails + /// + /// # Example + /// + /// ```rust,ignore + /// // During an active call, switch to a different microphone + /// audio.switch_recording_device(1)?; + /// ``` + /// + /// [`set_recording_device`]: Self::set_recording_device + pub fn switch_recording_device(&self, index: u16) -> AudioResult<()> { + let count = self.recording_devices(); + if index >= count as u16 { + return Err(AudioError::InvalidDeviceIndex); + } + + let runtime = &self.handle.runtime; + let was_initialized = runtime.recording_is_initialized(); + + if was_initialized { + let result = runtime.stop_recording(); + if result != 0 { + return Err(AudioError::OperationFailed(format!( + "stop_recording returned {}", + result + ))); + } + } + + let result = runtime.set_recording_device(index); + if result != 0 { + return Err(AudioError::OperationFailed(format!( + "set_recording_device returned {}", + result + ))); + } + + if was_initialized { + let result = runtime.init_recording(); + if result != 0 { + return Err(AudioError::OperationFailed(format!( + "init_recording returned {}", + result + ))); + } + + let result = runtime.start_recording(); + if result != 0 { + return Err(AudioError::OperationFailed(format!( + "start_recording returned {}", + result + ))); + } + } + + Ok(()) + } + + /// Switches the playout device while audio is active (hot-swap). + /// + /// Unlike [`set_playout_device`], this method handles the stop/change/restart + /// sequence required when playout is already active. + /// + /// # Arguments + /// + /// * `index` - Device index (0-based, must be < `playout_devices()`) + /// + /// # Errors + /// + /// - [`AudioError::InvalidDeviceIndex`] if index is out of range + /// - [`AudioError::OperationFailed`] if any step fails + /// + /// # Example + /// + /// ```rust,ignore + /// // During an active call, switch to a different speaker + /// audio.switch_playout_device(1)?; + /// ``` + /// + /// [`set_playout_device`]: Self::set_playout_device + pub fn switch_playout_device(&self, index: u16) -> AudioResult<()> { + let count = self.playout_devices(); + if index >= count as u16 { + return Err(AudioError::InvalidDeviceIndex); + } + + let runtime = &self.handle.runtime; + let was_initialized = runtime.playout_is_initialized(); + + if was_initialized { + let result = runtime.stop_playout(); + if result != 0 { + return Err(AudioError::OperationFailed(format!( + "stop_playout returned {}", + result + ))); + } + } + + let result = runtime.set_playout_device(index); + if result != 0 { + return Err(AudioError::OperationFailed(format!( + "set_playout_device returned {}", + result + ))); + } + + if was_initialized { + let result = runtime.init_playout(); + if result != 0 { + return Err(AudioError::OperationFailed(format!( + "init_playout returned {}", + result + ))); + } + + let result = runtime.start_playout(); + if result != 0 { + return Err(AudioError::OperationFailed(format!( + "start_playout returned {}", + result + ))); + } + } + + Ok(()) + } + + // ========================================================================= + // Lifecycle Management + // ========================================================================= + + /// Returns the number of active references to the platform ADM. + /// + /// This includes all `PlatformAudio` instances sharing the same ADM. + /// + /// # Example + /// + /// ```rust,ignore + /// let audio1 = PlatformAudio::new()?; + /// assert_eq!(audio1.ref_count(), 1); + /// + /// let audio2 = audio1.clone(); + /// assert_eq!(audio1.ref_count(), 2); + /// ``` + pub fn ref_count(&self) -> usize { + Arc::strong_count(&self.handle) + } + + /// Explicitly releases this instance's reference to the platform ADM. + /// + /// This is equivalent to `drop(self)`. If this is the last reference, + /// the platform ADM is disabled and hardware resources are released. + /// + /// # Example + /// + /// ```rust,ignore + /// let audio = PlatformAudio::new()?; + /// // ... use audio ... + /// audio.release(); // ADM disabled if this was the last reference + /// ``` + pub fn release(self) { + drop(self); + } + + // ========================================================================= + // Audio Processing (AEC, AGC, NS) + // ========================================================================= + + /// Checks if hardware echo cancellation is available on this device. + /// + /// # Platform Behavior + /// + /// - **iOS**: Returns `true` (VPIO provides hardware AEC) + /// - **Android**: Returns `true` on devices with hardware AEC support + /// - **Desktop**: Returns `false` (hardware AEC not available) + /// + /// # Example + /// + /// ```rust,ignore + /// let audio = PlatformAudio::new()?; + /// if audio.is_hardware_aec_available() { + /// println!("Hardware AEC is available"); + /// } + /// ``` + pub fn is_hardware_aec_available(&self) -> bool { + self.handle.runtime.builtin_aec_is_available() + } + + /// Checks if hardware automatic gain control is available on this device. + /// + /// # Platform Behavior + /// + /// - **iOS**: Returns `true` (VPIO provides hardware AGC) + /// - **Android**: Returns `true` on devices with hardware AGC support + /// - **Desktop**: Returns `false` (hardware AGC not available) + pub fn is_hardware_agc_available(&self) -> bool { + self.handle.runtime.builtin_agc_is_available() + } + + /// Checks if hardware noise suppression is available on this device. + /// + /// # Platform Behavior + /// + /// - **iOS**: Returns `true` (VPIO provides hardware NS) + /// - **Android**: Returns `true` on devices with hardware NS support + /// - **Desktop**: Returns `false` (hardware NS not available) + pub fn is_hardware_ns_available(&self) -> bool { + self.handle.runtime.builtin_ns_is_available() + } + + /// Gets the type of echo cancellation currently active. + /// + /// # Returns + /// + /// - [`AudioProcessingType::Hardware`] if hardware AEC is available and enabled + /// - [`AudioProcessingType::Software`] if using WebRTC's software AEC + /// - [`AudioProcessingType::None`] if AEC is disabled + /// + /// # Example + /// + /// ```rust,ignore + /// let audio = PlatformAudio::new()?; + /// match audio.active_aec_type() { + /// AudioProcessingType::Hardware => println!("Using hardware AEC"), + /// AudioProcessingType::Software => println!("Using software AEC"), + /// AudioProcessingType::None => println!("AEC disabled"), + /// } + /// ``` + pub fn active_aec_type(&self) -> AudioProcessingType { + if self.is_hardware_aec_available() { + AudioProcessingType::Hardware + } else { + AudioProcessingType::Software + } + } + + /// Gets the type of automatic gain control currently active. + pub fn active_agc_type(&self) -> AudioProcessingType { + if self.is_hardware_agc_available() { + AudioProcessingType::Hardware + } else { + AudioProcessingType::Software + } + } + + /// Gets the type of noise suppression currently active. + pub fn active_ns_type(&self) -> AudioProcessingType { + if self.is_hardware_ns_available() { + AudioProcessingType::Hardware + } else { + AudioProcessingType::Software + } + } + + /// Configures audio processing with the given options. + /// + /// This method configures echo cancellation, noise suppression, and + /// automatic gain control based on the provided options. + /// + /// # Platform Behavior + /// + /// - **iOS**: `prefer_hardware_processing` is ignored (always uses VPIO) + /// - **Android**: When `prefer_hardware_processing` is `false`, hardware + /// effects are disabled and WebRTC's software APM is used instead + /// - **Desktop**: `prefer_hardware_processing` is ignored (hardware not available) + /// + /// # Example + /// + /// ```rust,ignore + /// use livekit::{PlatformAudio, AudioProcessingOptions}; + /// + /// let audio = PlatformAudio::new()?; + /// + /// // Use defaults (software processing recommended) + /// audio.configure_audio_processing(AudioProcessingOptions::default())?; + /// + /// // Disable echo cancellation + /// audio.configure_audio_processing(AudioProcessingOptions { + /// echo_cancellation: false, + /// ..Default::default() + /// })?; + /// ``` + pub fn configure_audio_processing(&self, options: AudioProcessingOptions) -> AudioResult<()> { + let runtime = &self.handle.runtime; + + // Configure hardware vs software processing preference + // When prefer_hardware_processing is false, we disable hardware effects + // to force WebRTC to use its software APM instead + let use_hardware = options.prefer_hardware_processing; + + // Enable/disable hardware AEC + // Note: When hardware is disabled, WebRTC automatically falls back to software + if runtime.builtin_aec_is_available() { + let enable_hw = use_hardware && options.echo_cancellation; + let result = runtime.enable_builtin_aec(enable_hw); + if result != 0 { + log::warn!("enable_builtin_aec({}) returned {}", enable_hw, result); + } + } + + // Enable/disable hardware AGC + if runtime.builtin_agc_is_available() { + let enable_hw = use_hardware && options.auto_gain_control; + let result = runtime.enable_builtin_agc(enable_hw); + if result != 0 { + log::warn!("enable_builtin_agc({}) returned {}", enable_hw, result); + } + } + + // Enable/disable hardware NS + if runtime.builtin_ns_is_available() { + let enable_hw = use_hardware && options.noise_suppression; + let result = runtime.enable_builtin_ns(enable_hw); + if result != 0 { + log::warn!("enable_builtin_ns({}) returned {}", enable_hw, result); + } + } + + log::info!( + "Audio processing configured: AEC={}, AGC={}, NS={}, prefer_hw={}", + options.echo_cancellation, + options.auto_gain_control, + options.noise_suppression, + options.prefer_hardware_processing + ); + + Ok(()) + } + + /// Enables or disables echo cancellation. + /// + /// This is a convenience method equivalent to calling `configure_audio_processing` + /// with only the `echo_cancellation` field changed. + /// + /// # Arguments + /// + /// * `enable` - `true` to enable AEC, `false` to disable + /// * `prefer_hardware` - `true` to prefer hardware AEC on supported devices + pub fn set_echo_cancellation(&self, enable: bool, prefer_hardware: bool) -> AudioResult<()> { + if self.is_hardware_aec_available() { + let enable_hw = enable && prefer_hardware; + let result = self.handle.runtime.enable_builtin_aec(enable_hw); + if result != 0 { + return Err(AudioError::OperationFailed(format!( + "enable_builtin_aec returned {}", + result + ))); + } + } + Ok(()) + } + + /// Enables or disables automatic gain control. + /// + /// # Arguments + /// + /// * `enable` - `true` to enable AGC, `false` to disable + /// * `prefer_hardware` - `true` to prefer hardware AGC on supported devices + pub fn set_auto_gain_control(&self, enable: bool, prefer_hardware: bool) -> AudioResult<()> { + if self.is_hardware_agc_available() { + let enable_hw = enable && prefer_hardware; + let result = self.handle.runtime.enable_builtin_agc(enable_hw); + if result != 0 { + return Err(AudioError::OperationFailed(format!( + "enable_builtin_agc returned {}", + result + ))); + } + } + Ok(()) + } + + /// Enables or disables noise suppression. + /// + /// # Arguments + /// + /// * `enable` - `true` to enable NS, `false` to disable + /// * `prefer_hardware` - `true` to prefer hardware NS on supported devices + pub fn set_noise_suppression(&self, enable: bool, prefer_hardware: bool) -> AudioResult<()> { + if self.is_hardware_ns_available() { + let enable_hw = enable && prefer_hardware; + let result = self.handle.runtime.enable_builtin_ns(enable_hw); + if result != 0 { + return Err(AudioError::OperationFailed(format!( + "enable_builtin_ns returned {}", + result + ))); + } + } + Ok(()) + } +} + +impl fmt::Debug for PlatformAudio { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PlatformAudio") + .field("ref_count", &self.ref_count()) + .field("recording_devices", &self.recording_devices()) + .field("playout_devices", &self.playout_devices()) + .finish() + } +} + +/// Resets the platform audio handle references. +/// +/// This drops all references to the platform audio handle, allowing +/// a fresh `PlatformAudio` instance to be created. The Platform ADM +/// itself remains active. +/// +/// # Example +/// +/// ```rust,ignore +/// use livekit::{PlatformAudio, reset_platform_audio}; +/// +/// let audio = PlatformAudio::new()?; +/// // ... use audio ... +/// +/// // Reset handle references +/// reset_platform_audio(); +/// ``` +pub fn reset_platform_audio() { + let mut handle_ref = PLATFORM_ADM_HANDLE.lock(); + *handle_ref = Weak::new(); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rtc_audio_source_device_variant() { + let source = RtcAudioSource::Device; + assert!(matches!(source, RtcAudioSource::Device)); + } +} diff --git a/livekit/src/platform_audio/processing.rs b/livekit/src/platform_audio/processing.rs new file mode 100644 index 000000000..033cee6b5 --- /dev/null +++ b/livekit/src/platform_audio/processing.rs @@ -0,0 +1,113 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Audio processing configuration types (AEC, AGC, NS). + +/// The type of audio processing being used. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AudioProcessingType { + /// Hardware audio processing (iOS VPIO, Android hardware effects). + Hardware, + /// Software audio processing (WebRTC's built-in APM). + Software, + /// Audio processing is not available or disabled. + None, +} + +impl Default for AudioProcessingType { + fn default() -> Self { + Self::Software + } +} + +/// Configuration options for audio processing (AEC, AGC, NS). +/// +/// # Platform Behavior +/// +/// - **iOS**: Hardware processing via VPIO is always used. `prefer_hardware_processing` +/// is ignored since iOS provides excellent hardware AEC/AGC/NS. +/// +/// - **Android**: When `prefer_hardware_processing` is `true`, hardware effects are +/// used if available. However, hardware AEC is unreliable on many Android devices, +/// so the default is `false` (software processing). +/// +/// - **Desktop** (macOS, Windows, Linux): Hardware processing is not available. +/// WebRTC's software Audio Processing Module (APM) is always used. +/// +/// # Example +/// +/// ```rust,ignore +/// use livekit::AudioProcessingOptions; +/// +/// // Use defaults (software processing, all effects enabled) +/// let opts = AudioProcessingOptions::default(); +/// +/// // Disable echo cancellation +/// let opts = AudioProcessingOptions { +/// echo_cancellation: false, +/// ..Default::default() +/// }; +/// +/// // Try hardware processing on Android (use with caution) +/// let opts = AudioProcessingOptions { +/// prefer_hardware_processing: true, +/// ..Default::default() +/// }; +/// ``` +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AudioProcessingOptions { + /// Enable echo cancellation. + /// + /// Echo cancellation removes acoustic echo from the microphone signal, + /// which occurs when the speaker output is picked up by the microphone. + /// + /// Default: `true` + pub echo_cancellation: bool, + + /// Enable noise suppression. + /// + /// Noise suppression reduces background noise in the microphone signal. + /// + /// Default: `true` + pub noise_suppression: bool, + + /// Enable automatic gain control. + /// + /// AGC automatically adjusts the microphone volume to maintain + /// consistent audio levels. + /// + /// Default: `true` + pub auto_gain_control: bool, + + /// Prefer hardware audio processing when available. + /// + /// - **iOS**: Ignored (always uses VPIO hardware) + /// - **Android**: When `true`, uses hardware effects if available. + /// Default is `false` because hardware AEC is unreliable on many devices. + /// - **Desktop**: Ignored (hardware not available) + /// + /// Default: `false` (use reliable software processing) + pub prefer_hardware_processing: bool, +} + +impl Default for AudioProcessingOptions { + fn default() -> Self { + Self { + echo_cancellation: true, + noise_suppression: true, + auto_gain_control: true, + prefer_hardware_processing: false, + } + } +} diff --git a/livekit/src/prelude.rs b/livekit/src/prelude.rs index 425c05aad..86b42dfe4 100644 --- a/livekit/src/prelude.rs +++ b/livekit/src/prelude.rs @@ -33,3 +33,10 @@ pub use crate::{ ConnectionState, DataPacket, DataPacketKind, Room, RoomError, RoomEvent, RoomOptions, RoomResult, RoomSdkOptions, SipDTMF, Transcription, TranscriptionSegment, }; + +// Platform audio device management (native platforms only) +#[cfg(not(target_arch = "wasm32"))] +pub use crate::platform_audio::{ + AudioError, AudioProcessingOptions, AudioProcessingType, AudioResult, PlatformAudio, + RtcAudioSource, +}; diff --git a/livekit/src/room/track/local_audio_track.rs b/livekit/src/room/track/local_audio_track.rs index 7ac31adb4..1fecdca7f 100644 --- a/livekit/src/room/track/local_audio_track.rs +++ b/livekit/src/room/track/local_audio_track.rs @@ -59,6 +59,16 @@ impl LocalAudioTrack { .pc_factory() .create_audio_track(&libwebrtc::native::create_random_uuid(), native_source) } + #[cfg(not(target_arch = "wasm32"))] + RtcAudioSource::Device => { + // Create an audio track that uses the Platform ADM for capture. + // Use PlatformAudio::new() to enable platform audio before creating this track. + use libwebrtc::peer_connection_factory::native::PeerConnectionFactoryExt; + LkRuntime::instance() + .pc_factory() + .create_device_audio_track(&libwebrtc::native::create_random_uuid()) + } + #[allow(unreachable_patterns)] _ => panic!("unsupported audio source"), }; Self::new(name.to_string(), rtc_track, source) diff --git a/livekit/src/rtc_engine/lk_runtime.rs b/livekit/src/rtc_engine/lk_runtime.rs index fe5318e84..1bb43a9b1 100644 --- a/livekit/src/rtc_engine/lk_runtime.rs +++ b/livekit/src/rtc_engine/lk_runtime.rs @@ -21,6 +21,9 @@ use lazy_static::lazy_static; use libwebrtc::prelude::*; use parking_lot::Mutex; +#[cfg(not(target_arch = "wasm32"))] +use libwebrtc::peer_connection_factory::native::PeerConnectionFactoryExt; + lazy_static! { static ref LK_RUNTIME: Mutex> = Mutex::new(Weak::new()); } @@ -51,6 +54,146 @@ impl LkRuntime { pub fn pc_factory(&self) -> &PeerConnectionFactory { &self.pc_factory } + + // ===== Device Management Methods ===== + + /// Get the number of playout (output) devices + #[cfg(not(target_arch = "wasm32"))] + pub fn playout_devices(&self) -> i16 { + self.pc_factory.playout_devices() + } + + /// Get the number of recording (input) devices + #[cfg(not(target_arch = "wasm32"))] + pub fn recording_devices(&self) -> i16 { + self.pc_factory.recording_devices() + } + + /// Get the name of a playout device by index + #[cfg(not(target_arch = "wasm32"))] + pub fn playout_device_name(&self, index: u16) -> String { + self.pc_factory.playout_device_name(index) + } + + /// Get the name of a recording device by index + #[cfg(not(target_arch = "wasm32"))] + pub fn recording_device_name(&self, index: u16) -> String { + self.pc_factory.recording_device_name(index) + } + + /// Set the playout device by index + #[cfg(not(target_arch = "wasm32"))] + pub fn set_playout_device(&self, index: u16) -> i32 { + self.pc_factory.set_playout_device(index) + } + + /// Set the recording device by index + #[cfg(not(target_arch = "wasm32"))] + pub fn set_recording_device(&self, index: u16) -> i32 { + self.pc_factory.set_recording_device(index) + } + + /// Stop recording (clears initialized state, allowing device switch) + #[cfg(not(target_arch = "wasm32"))] + pub fn stop_recording(&self) -> i32 { + self.pc_factory.stop_recording() + } + + /// Initialize recording + #[cfg(not(target_arch = "wasm32"))] + pub fn init_recording(&self) -> i32 { + self.pc_factory.init_recording() + } + + /// Start recording + #[cfg(not(target_arch = "wasm32"))] + pub fn start_recording(&self) -> i32 { + self.pc_factory.start_recording() + } + + /// Check if recording is initialized + #[cfg(not(target_arch = "wasm32"))] + pub fn recording_is_initialized(&self) -> bool { + self.pc_factory.recording_is_initialized() + } + + /// Stop playout (clears initialized state, allowing device switch) + #[cfg(not(target_arch = "wasm32"))] + pub fn stop_playout(&self) -> i32 { + self.pc_factory.stop_playout() + } + + /// Initialize playout + #[cfg(not(target_arch = "wasm32"))] + pub fn init_playout(&self) -> i32 { + self.pc_factory.init_playout() + } + + /// Start playout + #[cfg(not(target_arch = "wasm32"))] + pub fn start_playout(&self) -> i32 { + self.pc_factory.start_playout() + } + + /// Check if playout is initialized + #[cfg(not(target_arch = "wasm32"))] + pub fn playout_is_initialized(&self) -> bool { + self.pc_factory.playout_is_initialized() + } + + // ===== Built-in Audio Processing Methods ===== + + /// Check if built-in (hardware) AEC is available + #[cfg(not(target_arch = "wasm32"))] + pub fn builtin_aec_is_available(&self) -> bool { + self.pc_factory.builtin_aec_is_available() + } + + /// Check if built-in (hardware) AGC is available + #[cfg(not(target_arch = "wasm32"))] + pub fn builtin_agc_is_available(&self) -> bool { + self.pc_factory.builtin_agc_is_available() + } + + /// Check if built-in (hardware) NS is available + #[cfg(not(target_arch = "wasm32"))] + pub fn builtin_ns_is_available(&self) -> bool { + self.pc_factory.builtin_ns_is_available() + } + + /// Enable or disable built-in (hardware) AEC + #[cfg(not(target_arch = "wasm32"))] + pub fn enable_builtin_aec(&self, enable: bool) -> i32 { + self.pc_factory.enable_builtin_aec(enable) + } + + /// Enable or disable built-in (hardware) AGC + #[cfg(not(target_arch = "wasm32"))] + pub fn enable_builtin_agc(&self, enable: bool) -> i32 { + self.pc_factory.enable_builtin_agc(enable) + } + + /// Enable or disable built-in (hardware) NS + #[cfg(not(target_arch = "wasm32"))] + pub fn enable_builtin_ns(&self, enable: bool) -> i32 { + self.pc_factory.enable_builtin_ns(enable) + } + + /// Control whether ADM recording (microphone) is enabled. + /// + /// When disabled, WebRTC's calls to InitRecording/StartRecording will be no-ops. + /// Use this when only using NativeAudioSource (no microphone capture needed). + /// This prevents the microphone from interfering with the audio pipeline. + #[cfg(not(target_arch = "wasm32"))] + pub fn set_adm_recording_enabled(&self, enabled: bool) { + self.pc_factory.set_adm_recording_enabled(enabled) + } + + /// Check if ADM recording (microphone) is enabled. + #[cfg(not(target_arch = "wasm32"))] + pub fn adm_recording_enabled(&self) -> bool { + self.pc_factory.adm_recording_enabled() + } } impl Drop for LkRuntime { diff --git a/livekit/src/rtc_engine/peer_transport.rs b/livekit/src/rtc_engine/peer_transport.rs index da1ac2821..8c12fea0f 100644 --- a/livekit/src/rtc_engine/peer_transport.rs +++ b/livekit/src/rtc_engine/peer_transport.rs @@ -353,6 +353,13 @@ impl PeerTransport { let mut offer = self.peer_connection.create_offer(options).await?; let mut sdp = offer.to_string(); + // Log audio m-lines to debug sample rate issues + for line in sdp.lines() { + if line.starts_with("m=audio") || line.contains("opus") || line.contains("a=rtpmap") { + log::info!("SDP audio: {}", line); + } + } + if inner.single_pc_mode { // Fix inactive media m-lines to recvonly for single PC mode. // WebRTC can generate a=inactive even when transceivers are recvonly. diff --git a/livekit/tests/platform_audio_test.rs b/livekit/tests/platform_audio_test.rs new file mode 100644 index 000000000..33e1a2869 --- /dev/null +++ b/livekit/tests/platform_audio_test.rs @@ -0,0 +1,1061 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Tests for PlatformAudio functionality. +//! +//! These tests verify: +//! - PlatformAudio creation and reference counting +//! - Device enumeration and selection +//! - Audio processing configuration (AEC, AGC, NS) +//! - Integration with room connections +//! - Combining PlatformAudio with NativeAudioSource + +mod common; + +use std::time::Duration; + +use anyhow::{anyhow, Result}; +#[cfg(feature = "__lk-e2e-test")] +use livekit::options::TrackPublishOptions; +use livekit::{prelude::*, AudioError, AudioResult, PlatformAudio, RtcAudioSource}; +use serial_test::serial; +use tokio::time::timeout; + +#[cfg(feature = "__lk-e2e-test")] +use common::test_rooms; + +// ============================================================================= +// Unit Tests (no E2E feature required) +// ============================================================================= + +#[test] +fn test_audio_error_display() { + let err = AudioError::PlatformInitFailed; + let msg = format!("{}", err); + assert!(msg.contains("platform audio")); + + let err = AudioError::InvalidDeviceIndex; + let msg = format!("{}", err); + assert!(msg.contains("Invalid device index")); + + let err = AudioError::OperationFailed("test message".to_string()); + let msg = format!("{}", err); + assert!(msg.contains("test message")); +} + +#[test] +fn test_audio_error_equality() { + assert_eq!(AudioError::PlatformInitFailed, AudioError::PlatformInitFailed); + assert_eq!(AudioError::InvalidDeviceIndex, AudioError::InvalidDeviceIndex); + assert_eq!( + AudioError::OperationFailed("a".to_string()), + AudioError::OperationFailed("a".to_string()) + ); + assert_ne!( + AudioError::OperationFailed("a".to_string()), + AudioError::OperationFailed("b".to_string()) + ); +} + +#[test] +fn test_audio_error_clone() { + let err = AudioError::OperationFailed("test".to_string()); + let cloned = err.clone(); + assert_eq!(err, cloned); +} + +#[test] +fn test_audio_result_ok() { + let result: AudioResult = Ok(42); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), 42); +} + +#[test] +fn test_audio_result_err() { + let result: AudioResult = Err(AudioError::InvalidDeviceIndex); + assert!(result.is_err()); + assert_eq!(result.unwrap_err(), AudioError::InvalidDeviceIndex); +} + +#[test] +fn test_rtc_audio_source_device() { + let source = RtcAudioSource::Device; + assert!(matches!(source, RtcAudioSource::Device)); +} + +#[test] +fn test_audio_processing_options_default() { + use livekit::AudioProcessingOptions; + + let opts = AudioProcessingOptions::default(); + assert!(opts.echo_cancellation); + assert!(opts.noise_suppression); + assert!(opts.auto_gain_control); + assert!(!opts.prefer_hardware_processing); // Default to software (more reliable) +} + +#[test] +fn test_audio_processing_options_custom() { + use livekit::AudioProcessingOptions; + + let opts = AudioProcessingOptions { + echo_cancellation: false, + noise_suppression: true, + auto_gain_control: false, + prefer_hardware_processing: true, + }; + assert!(!opts.echo_cancellation); + assert!(opts.noise_suppression); + assert!(!opts.auto_gain_control); + assert!(opts.prefer_hardware_processing); +} + +#[test] +fn test_audio_processing_type_default() { + use livekit::AudioProcessingType; + + let atype = AudioProcessingType::default(); + assert_eq!(atype, AudioProcessingType::Software); +} + +#[test] +fn test_audio_processing_type_variants() { + use livekit::AudioProcessingType; + + let hw = AudioProcessingType::Hardware; + let sw = AudioProcessingType::Software; + let none = AudioProcessingType::None; + + assert_ne!(hw, sw); + assert_ne!(sw, none); + assert_ne!(hw, none); + + // Test Debug + assert!(format!("{:?}", hw).contains("Hardware")); + assert!(format!("{:?}", sw).contains("Software")); + assert!(format!("{:?}", none).contains("None")); +} + +#[test] +fn test_audio_processing_options_clone() { + use livekit::AudioProcessingOptions; + + let opts = AudioProcessingOptions { + echo_cancellation: false, + noise_suppression: true, + auto_gain_control: false, + prefer_hardware_processing: true, + }; + let cloned = opts.clone(); + assert_eq!(opts, cloned); +} + +// ============================================================================= +// Standalone Tests (no E2E feature required, but require audio hardware) +// ============================================================================= + +/// Test PlatformAudio creation and basic functionality. +/// This test doesn't require a room connection. +#[test_log::test(tokio::test)] +#[serial] +async fn test_platform_audio_standalone_creation() -> Result<()> { + use livekit::reset_platform_audio; + + // Ensure clean state + reset_platform_audio(); + + // Create PlatformAudio + let audio = PlatformAudio::new()?; + log::info!("PlatformAudio created successfully"); + + // Check ref count + assert_eq!(audio.ref_count(), 1); + log::info!("Initial ref_count: {}", audio.ref_count()); + + // Check source type + assert!(matches!(audio.rtc_source(), RtcAudioSource::Device)); + log::info!("rtc_source() returns Device variant"); + + // Enumerate devices + let recording_count = audio.recording_devices(); + let playout_count = audio.playout_devices(); + log::info!("Found {} recording devices, {} playout devices", recording_count, playout_count); + + // List recording devices + for i in 0..recording_count as u16 { + let name = audio.recording_device_name(i); + log::info!(" Recording device {}: {}", i, name); + } + + // List playout devices + for i in 0..playout_count as u16 { + let name = audio.playout_device_name(i); + log::info!(" Playout device {}: {}", i, name); + } + + // Test Debug trait + let debug_str = format!("{:?}", audio); + log::info!("Debug output: {}", debug_str); + assert!(debug_str.contains("PlatformAudio")); + + // Cleanup + drop(audio); + log::info!("PlatformAudio dropped successfully"); + + Ok(()) +} + +/// Test PlatformAudio reference counting without room connection. +#[test_log::test(tokio::test)] +#[serial] +async fn test_platform_audio_standalone_ref_counting() -> Result<()> { + use livekit::reset_platform_audio; + + reset_platform_audio(); + + // Create first instance + let audio1 = PlatformAudio::new()?; + assert_eq!(audio1.ref_count(), 1); + log::info!("Created audio1, ref_count: {}", audio1.ref_count()); + + // Create second instance - should share ADM + let audio2 = PlatformAudio::new()?; + assert_eq!(audio1.ref_count(), 2); + assert_eq!(audio2.ref_count(), 2); + log::info!("Created audio2, ref_count: {}", audio1.ref_count()); + + // Clone - should increase ref count + let audio3 = audio1.clone(); + assert_eq!(audio1.ref_count(), 3); + assert_eq!(audio2.ref_count(), 3); + assert_eq!(audio3.ref_count(), 3); + log::info!("Cloned audio1 to audio3, ref_count: {}", audio1.ref_count()); + + // Drop one + drop(audio2); + assert_eq!(audio1.ref_count(), 2); + log::info!("Dropped audio2, ref_count: {}", audio1.ref_count()); + + // Drop all + drop(audio1); + assert_eq!(audio3.ref_count(), 1); + log::info!("Dropped audio1, audio3 ref_count: {}", audio3.ref_count()); + + drop(audio3); + log::info!("Dropped audio3, all references released"); + + Ok(()) +} + +/// Test device selection without room connection. +#[test_log::test(tokio::test)] +#[serial] +async fn test_platform_audio_standalone_device_selection() -> Result<()> { + use livekit::reset_platform_audio; + + reset_platform_audio(); + + let audio = PlatformAudio::new()?; + + let recording_count = audio.recording_devices(); + let playout_count = audio.playout_devices(); + + // Select recording device if available + if recording_count > 0 { + audio.set_recording_device(0)?; + log::info!("Selected recording device 0: {}", audio.recording_device_name(0)); + } else { + log::info!("No recording devices available"); + } + + // Select playout device if available + if playout_count > 0 { + audio.set_playout_device(0)?; + log::info!("Selected playout device 0: {}", audio.playout_device_name(0)); + } else { + log::info!("No playout devices available"); + } + + // Test invalid device index + let result = audio.set_recording_device(9999); + assert!(matches!(result, Err(AudioError::InvalidDeviceIndex))); + log::info!("Invalid recording device index correctly rejected"); + + let result = audio.set_playout_device(9999); + assert!(matches!(result, Err(AudioError::InvalidDeviceIndex))); + log::info!("Invalid playout device index correctly rejected"); + + drop(audio); + Ok(()) +} + +/// Test audio processing configuration without room connection. +#[test_log::test(tokio::test)] +#[serial] +async fn test_platform_audio_standalone_processing_config() -> Result<()> { + use livekit::reset_platform_audio; + use livekit::AudioProcessingOptions; + use livekit::AudioProcessingType; + + reset_platform_audio(); + + let audio = PlatformAudio::new()?; + + // Query hardware availability + let hw_aec = audio.is_hardware_aec_available(); + let hw_agc = audio.is_hardware_agc_available(); + let hw_ns = audio.is_hardware_ns_available(); + log::info!("Hardware availability: AEC={}, AGC={}, NS={}", hw_aec, hw_agc, hw_ns); + + // Query active processing types + let aec_type = audio.active_aec_type(); + let agc_type = audio.active_agc_type(); + let ns_type = audio.active_ns_type(); + log::info!("Active processing: AEC={:?}, AGC={:?}, NS={:?}", aec_type, agc_type, ns_type); + + // Verify consistency + if hw_aec { + assert_eq!(aec_type, AudioProcessingType::Hardware); + } else { + assert_eq!(aec_type, AudioProcessingType::Software); + } + + // Configure with default options + audio.configure_audio_processing(AudioProcessingOptions::default())?; + log::info!("Configured with default options"); + + // Configure with custom options + audio.configure_audio_processing(AudioProcessingOptions { + echo_cancellation: true, + noise_suppression: false, + auto_gain_control: true, + prefer_hardware_processing: false, + })?; + log::info!("Configured with custom options"); + + // Test individual controls + audio.set_echo_cancellation(true, false)?; + log::info!("Set AEC: enabled, prefer software"); + + audio.set_auto_gain_control(true, false)?; + log::info!("Set AGC: enabled, prefer software"); + + audio.set_noise_suppression(true, false)?; + log::info!("Set NS: enabled, prefer software"); + + drop(audio); + Ok(()) +} + +/// Test reset_platform_audio function. +#[test_log::test(tokio::test)] +#[serial] +async fn test_platform_audio_standalone_reset() -> Result<()> { + use livekit::reset_platform_audio; + + reset_platform_audio(); + + let audio1 = PlatformAudio::new()?; + assert_eq!(audio1.ref_count(), 1); + log::info!("Created audio1, ref_count: {}", audio1.ref_count()); + + // Force reset (drops internal weak reference, but audio1 still holds strong ref) + reset_platform_audio(); + log::info!("Called reset_platform_audio()"); + + // audio1 still holds a strong reference, so ref_count is still 1 + assert_eq!(audio1.ref_count(), 1); + log::info!("audio1 ref_count after reset: {}", audio1.ref_count()); + + // Create new instance after reset + // Since weak reference was cleared, audio2 creates a NEW handle + // audio1 and audio2 now have SEPARATE handles (not shared) + let audio2 = PlatformAudio::new()?; + assert_eq!(audio2.ref_count(), 1); // New separate handle + assert_eq!(audio1.ref_count(), 1); // Original handle unchanged + log::info!("Created audio2 after reset, audio2.ref_count: {}", audio2.ref_count()); + log::info!("audio1.ref_count still: {}", audio1.ref_count()); + + // Now if we create audio3, it should share with audio2 (the new handle) + let audio3 = PlatformAudio::new()?; + assert_eq!(audio2.ref_count(), 2); // Shares with audio3 + assert_eq!(audio3.ref_count(), 2); + assert_eq!(audio1.ref_count(), 1); // Still separate + log::info!( + "Created audio3, audio2/3 ref_count: {}, audio1 ref_count: {}", + audio2.ref_count(), + audio1.ref_count() + ); + + drop(audio1); + drop(audio2); + drop(audio3); + + log::info!("reset_platform_audio test completed"); + Ok(()) +} + +/// Test PlatformAudio lifecycle - create, use, destroy. +#[test_log::test(tokio::test)] +#[serial] +async fn test_platform_audio_standalone_lifecycle() -> Result<()> { + use livekit::reset_platform_audio; + + reset_platform_audio(); + + // Phase 1: Create and configure + log::info!("=== Phase 1: Create and Configure ==="); + let audio = PlatformAudio::new()?; + log::info!("Created PlatformAudio"); + + let recording_count = audio.recording_devices(); + let playout_count = audio.playout_devices(); + log::info!("Devices: {} recording, {} playout", recording_count, playout_count); + + if recording_count > 0 { + audio.set_recording_device(0)?; + log::info!("Set recording device to 0"); + } + if playout_count > 0 { + audio.set_playout_device(0)?; + log::info!("Set playout device to 0"); + } + + // Phase 2: Get audio source (simulating track creation) + log::info!("=== Phase 2: Get Audio Source ==="); + let source = audio.rtc_source(); + assert!(matches!(source, RtcAudioSource::Device)); + log::info!("Got RtcAudioSource::Device for track creation"); + + // Phase 3: Simulate some activity + log::info!("=== Phase 3: Simulated Activity ==="); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + log::info!("Simulated 100ms of activity"); + + // Phase 4: Cleanup + log::info!("=== Phase 4: Cleanup ==="); + audio.release(); + log::info!("Called release(), PlatformAudio destroyed"); + + // Phase 5: Verify we can create again + log::info!("=== Phase 5: Verify Re-creation ==="); + let audio2 = PlatformAudio::new()?; + assert_eq!(audio2.ref_count(), 1); + log::info!("Created new PlatformAudio successfully"); + drop(audio2); + + log::info!("Lifecycle test completed successfully"); + Ok(()) +} + +// ============================================================================= +// E2E Tests (require __lk-e2e-test feature) +// ============================================================================= + +/// Test PlatformAudio creation. +#[cfg(feature = "__lk-e2e-test")] +#[test_log::test(tokio::test)] +#[serial] +async fn test_platform_audio_creation() -> Result<()> { + use livekit::reset_platform_audio; + + // Ensure clean state + reset_platform_audio(); + + // Create PlatformAudio + let audio = PlatformAudio::new()?; + assert_eq!(audio.ref_count(), 1); + assert!(matches!(audio.rtc_source(), RtcAudioSource::Device)); + + let recording_count = audio.recording_devices(); + let playout_count = audio.playout_devices(); + log::info!( + "PlatformAudio: {} recording devices, {} playout devices", + recording_count, + playout_count + ); + + // Cleanup + drop(audio); + Ok(()) +} + +/// Test PlatformAudio reference counting. +#[cfg(feature = "__lk-e2e-test")] +#[test_log::test(tokio::test)] +#[serial] +async fn test_platform_audio_ref_counting() -> Result<()> { + use livekit::reset_platform_audio; + + reset_platform_audio(); + + // Create first instance + let audio1 = PlatformAudio::new()?; + assert_eq!(audio1.ref_count(), 1); + + // Create second instance - should share ADM + let audio2 = PlatformAudio::new()?; + assert_eq!(audio1.ref_count(), 2); + assert_eq!(audio2.ref_count(), 2); + + // Clone - should increase ref count + let audio3 = audio1.clone(); + assert_eq!(audio1.ref_count(), 3); + + // Drop one + drop(audio2); + assert_eq!(audio1.ref_count(), 2); + + // Drop all + drop(audio1); + drop(audio3); + + log::info!("Reference counting works correctly"); + Ok(()) +} + +/// Test device enumeration. +#[cfg(feature = "__lk-e2e-test")] +#[test_log::test(tokio::test)] +#[serial] +async fn test_platform_audio_device_enumeration() -> Result<()> { + use livekit::reset_platform_audio; + + reset_platform_audio(); + + let audio = PlatformAudio::new()?; + + // Enumerate recording devices + let recording_count = audio.recording_devices(); + log::info!("Recording devices: {}", recording_count); + for i in 0..recording_count as u16 { + let name = audio.recording_device_name(i); + log::info!(" Mic {}: {}", i, name); + } + + // Enumerate playout devices + let playout_count = audio.playout_devices(); + log::info!("Playout devices: {}", playout_count); + for i in 0..playout_count as u16 { + let name = audio.playout_device_name(i); + log::info!(" Speaker {}: {}", i, name); + } + + drop(audio); + Ok(()) +} + +/// Test device selection. +#[cfg(feature = "__lk-e2e-test")] +#[test_log::test(tokio::test)] +#[serial] +async fn test_platform_audio_device_selection() -> Result<()> { + use livekit::reset_platform_audio; + + reset_platform_audio(); + + let audio = PlatformAudio::new()?; + + let recording_count = audio.recording_devices(); + let playout_count = audio.playout_devices(); + + // Select recording device if available + if recording_count > 0 { + audio.set_recording_device(0)?; + log::info!("Selected recording device 0"); + } + + // Select playout device if available + if playout_count > 0 { + audio.set_playout_device(0)?; + log::info!("Selected playout device 0"); + } + + // Test invalid device index + let result = audio.set_recording_device(9999); + assert!(matches!(result, Err(AudioError::InvalidDeviceIndex))); + + drop(audio); + Ok(()) +} + +/// Test explicit release. +#[cfg(feature = "__lk-e2e-test")] +#[test_log::test(tokio::test)] +#[serial] +async fn test_platform_audio_release() -> Result<()> { + use livekit::reset_platform_audio; + + reset_platform_audio(); + + let audio = PlatformAudio::new()?; + assert_eq!(audio.ref_count(), 1); + + // Explicit release + audio.release(); + + log::info!("Explicit release works"); + Ok(()) +} + +/// Test combining PlatformAudio with NativeAudioSource. +#[cfg(feature = "__lk-e2e-test")] +#[test_log::test(tokio::test)] +#[serial] +async fn test_platform_audio_with_native_source() -> Result<()> { + use livekit::reset_platform_audio; + use livekit::webrtc::audio_source::native::NativeAudioSource; + use livekit::webrtc::audio_source::AudioSourceOptions; + + reset_platform_audio(); + + // Create PlatformAudio for microphone + let mic = PlatformAudio::new()?; + log::info!("Created PlatformAudio with {} mics", mic.recording_devices()); + + // Create NativeAudioSource for screen capture / TTS + let screen_source = NativeAudioSource::new(AudioSourceOptions::default(), 48000, 2, 100); + + // Both can coexist + let mic_source = mic.rtc_source(); + let screen_rtc_source = RtcAudioSource::Native(screen_source.clone()); + + assert!(matches!(mic_source, RtcAudioSource::Device)); + assert!(matches!(screen_rtc_source, RtcAudioSource::Native(_))); + + log::info!("PlatformAudio and NativeAudioSource can coexist"); + + drop(mic); + Ok(()) +} + +/// Test PlatformAudio with room connection. +#[cfg(feature = "__lk-e2e-test")] +#[test_log::test(tokio::test)] +#[serial] +async fn test_platform_audio_room_connection() -> Result<()> { + use livekit::reset_platform_audio; + + reset_platform_audio(); + + let audio = PlatformAudio::new()?; + let recording_count = audio.recording_devices(); + + log::info!("Connecting to room with {} recording devices", recording_count); + + // Connect to room + let mut rooms = test_rooms(1).await?; + let (room, _events) = rooms.pop().unwrap(); + + assert_eq!(room.connection_state(), ConnectionState::Connected); + + // Publish track if microphone available + if recording_count > 0 { + audio.set_recording_device(0)?; + + let track = LocalAudioTrack::create_audio_track("microphone", audio.rtc_source()); + room.local_participant() + .publish_track( + LocalTrack::Audio(track), + TrackPublishOptions { source: TrackSource::Microphone, ..Default::default() }, + ) + .await?; + + log::info!("Published audio track using PlatformAudio"); + + // Verify track is published + let publications = room.local_participant().track_publications(); + assert!( + publications.values().any(|p| p.source() == TrackSource::Microphone), + "Microphone track should be published" + ); + } else { + log::info!("Skipping track publish - no microphone available"); + } + + // Disconnect + room.close().await?; + + // Drop PlatformAudio to release hardware + drop(audio); + + log::info!("Room connection test completed"); + Ok(()) +} + +/// Test two participants with PlatformAudio. +#[cfg(feature = "__lk-e2e-test")] +#[test_log::test(tokio::test)] +#[serial] +async fn test_platform_audio_two_participants() -> Result<()> { + use livekit::reset_platform_audio; + + reset_platform_audio(); + + let audio = PlatformAudio::new()?; + let recording_count = audio.recording_devices(); + + if recording_count == 0 { + log::info!("Skipping two participants test - no microphone available"); + drop(audio); + return Ok(()); + } + + audio.set_recording_device(0)?; + + // Connect two participants + let mut rooms = test_rooms(2).await?; + let (pub_room, _) = rooms.pop().unwrap(); + let (sub_room, mut sub_events) = rooms.pop().unwrap(); + + // Publisher creates and publishes audio track + let track = LocalAudioTrack::create_audio_track("microphone", audio.rtc_source()); + pub_room + .local_participant() + .publish_track( + LocalTrack::Audio(track), + TrackPublishOptions { source: TrackSource::Microphone, ..Default::default() }, + ) + .await?; + + log::info!("Publisher published audio track"); + + // Subscriber should receive the track + let wait_for_track = async { + while let Some(event) = sub_events.recv().await { + if let RoomEvent::TrackSubscribed { track: _, publication, participant } = event { + log::info!( + "Subscriber received track from {} ({:?})", + participant.identity(), + publication.source() + ); + assert_eq!(publication.source(), TrackSource::Microphone); + return Ok(()); + } + } + Err(anyhow!("Never received track subscription")) + }; + + timeout(Duration::from_secs(10), wait_for_track).await??; + + // Clean up + pub_room.close().await?; + sub_room.close().await?; + drop(audio); + + Ok(()) +} + +/// Test device hot-switching during active session. +#[cfg(feature = "__lk-e2e-test")] +#[test_log::test(tokio::test)] +#[serial] +async fn test_platform_audio_device_switching() -> Result<()> { + use livekit::reset_platform_audio; + + reset_platform_audio(); + + let audio = PlatformAudio::new()?; + let recording_count = audio.recording_devices() as u16; + let playout_count = audio.playout_devices() as u16; + + log::info!( + "Device switching test: {} recording, {} playout devices", + recording_count, + playout_count + ); + + // Need at least 2 devices to test switching + let can_switch_recording = recording_count >= 2; + let can_switch_playout = playout_count >= 2; + + if !can_switch_recording && !can_switch_playout { + log::info!("Skipping device switching test (need at least 2 devices)"); + drop(audio); + return Ok(()); + } + + // Connect to room + let mut rooms = test_rooms(1).await?; + let (room, _events) = rooms.pop().unwrap(); + + // Publish track if microphone available + if recording_count > 0 { + let track = LocalAudioTrack::create_audio_track("microphone", audio.rtc_source()); + room.local_participant() + .publish_track( + LocalTrack::Audio(track), + TrackPublishOptions { source: TrackSource::Microphone, ..Default::default() }, + ) + .await?; + } + + // Switch recording devices + if can_switch_recording { + log::info!("Switching recording device 0 -> 1"); + audio.switch_recording_device(1)?; + tokio::time::sleep(Duration::from_millis(100)).await; + + log::info!("Switching recording device 1 -> 0"); + audio.switch_recording_device(0)?; + } + + // Switch playout devices + if can_switch_playout { + log::info!("Switching playout device 0 -> 1"); + audio.switch_playout_device(1)?; + tokio::time::sleep(Duration::from_millis(100)).await; + + log::info!("Switching playout device 1 -> 0"); + audio.switch_playout_device(0)?; + } + + // Clean up + room.close().await?; + drop(audio); + + Ok(()) +} + +/// Test reset_platform_audio function. +#[cfg(feature = "__lk-e2e-test")] +#[test_log::test(tokio::test)] +#[serial] +async fn test_reset_platform_audio() -> Result<()> { + use livekit::reset_platform_audio; + + reset_platform_audio(); + + let audio = PlatformAudio::new()?; + assert!(audio.recording_devices() >= 0 || audio.playout_devices() >= 0); + + // Force reset + reset_platform_audio(); + + // Can create new instance after reset + let audio2 = PlatformAudio::new()?; + assert_eq!(audio2.ref_count(), 1); + + drop(audio); + drop(audio2); + + log::info!("reset_platform_audio works correctly"); + Ok(()) +} + +/// Test hardware audio processing availability queries. +#[cfg(feature = "__lk-e2e-test")] +#[test_log::test(tokio::test)] +#[serial] +async fn test_platform_audio_hardware_availability() -> Result<()> { + use livekit::reset_platform_audio; + use livekit::AudioProcessingType; + + reset_platform_audio(); + + let audio = PlatformAudio::new()?; + + // Query hardware availability + let hw_aec = audio.is_hardware_aec_available(); + let hw_agc = audio.is_hardware_agc_available(); + let hw_ns = audio.is_hardware_ns_available(); + + log::info!( + "Hardware audio processing availability: AEC={}, AGC={}, NS={}", + hw_aec, + hw_agc, + hw_ns + ); + + // On desktop (macOS, Windows, Linux), hardware is typically not available + // On iOS, hardware is always available + // On Android, it depends on the device + + // Query active types + let aec_type = audio.active_aec_type(); + let agc_type = audio.active_agc_type(); + let ns_type = audio.active_ns_type(); + + log::info!("Active audio processing: AEC={:?}, AGC={:?}, NS={:?}", aec_type, agc_type, ns_type); + + // Verify consistency: if hardware is available, active type should be Hardware + if hw_aec { + assert_eq!(aec_type, AudioProcessingType::Hardware); + } else { + assert_eq!(aec_type, AudioProcessingType::Software); + } + + drop(audio); + Ok(()) +} + +/// Test audio processing configuration. +#[cfg(feature = "__lk-e2e-test")] +#[test_log::test(tokio::test)] +#[serial] +async fn test_platform_audio_configure_processing() -> Result<()> { + use livekit::reset_platform_audio; + use livekit::AudioProcessingOptions; + + reset_platform_audio(); + + let audio = PlatformAudio::new()?; + + // Configure with defaults + audio.configure_audio_processing(AudioProcessingOptions::default())?; + log::info!("Configured with default options"); + + // Configure with custom options + let custom_opts = AudioProcessingOptions { + echo_cancellation: true, + noise_suppression: true, + auto_gain_control: true, + prefer_hardware_processing: false, + }; + audio.configure_audio_processing(custom_opts)?; + log::info!("Configured with custom options (software preferred)"); + + // Configure with hardware preference + let hw_opts = AudioProcessingOptions { + echo_cancellation: true, + noise_suppression: true, + auto_gain_control: true, + prefer_hardware_processing: true, + }; + audio.configure_audio_processing(hw_opts)?; + log::info!("Configured with hardware preference"); + + // Disable some features + let minimal_opts = AudioProcessingOptions { + echo_cancellation: false, + noise_suppression: true, + auto_gain_control: false, + prefer_hardware_processing: false, + }; + audio.configure_audio_processing(minimal_opts)?; + log::info!("Configured with minimal options"); + + drop(audio); + Ok(()) +} + +/// Test individual audio processing control methods. +#[cfg(feature = "__lk-e2e-test")] +#[test_log::test(tokio::test)] +#[serial] +async fn test_platform_audio_individual_controls() -> Result<()> { + use livekit::reset_platform_audio; + + reset_platform_audio(); + + let audio = PlatformAudio::new()?; + + // Test echo cancellation control + audio.set_echo_cancellation(true, false)?; + log::info!("AEC enabled (software)"); + + audio.set_echo_cancellation(true, true)?; + log::info!("AEC enabled (hardware preferred)"); + + audio.set_echo_cancellation(false, false)?; + log::info!("AEC disabled"); + + // Test auto gain control + audio.set_auto_gain_control(true, false)?; + log::info!("AGC enabled (software)"); + + audio.set_auto_gain_control(false, false)?; + log::info!("AGC disabled"); + + // Test noise suppression + audio.set_noise_suppression(true, false)?; + log::info!("NS enabled (software)"); + + audio.set_noise_suppression(false, false)?; + log::info!("NS disabled"); + + drop(audio); + Ok(()) +} + +/// Test audio processing with room connection. +#[cfg(feature = "__lk-e2e-test")] +#[test_log::test(tokio::test)] +#[serial] +async fn test_platform_audio_processing_with_room() -> Result<()> { + use livekit::reset_platform_audio; + use livekit::AudioProcessingOptions; + + reset_platform_audio(); + + let audio = PlatformAudio::new()?; + let recording_count = audio.recording_devices(); + + if recording_count == 0 { + log::info!("Skipping test - no microphone available"); + drop(audio); + return Ok(()); + } + + // Configure audio processing before connecting + audio.configure_audio_processing(AudioProcessingOptions { + echo_cancellation: true, + noise_suppression: true, + auto_gain_control: true, + prefer_hardware_processing: false, // Use reliable software processing + })?; + + log::info!("Audio processing configured, connecting to room..."); + + // Connect to room + let mut rooms = test_rooms(1).await?; + let (room, _events) = rooms.pop().unwrap(); + + audio.set_recording_device(0)?; + + // Publish track + let track = LocalAudioTrack::create_audio_track("microphone", audio.rtc_source()); + room.local_participant() + .publish_track( + LocalTrack::Audio(track), + TrackPublishOptions { source: TrackSource::Microphone, ..Default::default() }, + ) + .await?; + + log::info!("Published audio track with configured processing"); + + // Verify track is published + let publications = room.local_participant().track_publications(); + assert!( + publications.values().any(|p| p.source() == TrackSource::Microphone), + "Microphone track should be published" + ); + + // Reconfigure audio processing while connected + audio.configure_audio_processing(AudioProcessingOptions { + echo_cancellation: true, + noise_suppression: false, // Disable NS + auto_gain_control: true, + prefer_hardware_processing: false, + })?; + log::info!("Reconfigured audio processing while connected"); + + // Clean up + room.close().await?; + drop(audio); + + log::info!("Audio processing with room test completed"); + Ok(()) +} diff --git a/webrtc-sys/build.rs b/webrtc-sys/build.rs index 4d779777e..b5bb2c2e6 100644 --- a/webrtc-sys/build.rs +++ b/webrtc-sys/build.rs @@ -83,6 +83,7 @@ fn main() { "src/video_encoder_factory.cpp", "src/video_decoder_factory.cpp", "src/audio_device.cpp", + "src/adm_proxy.cpp", "src/audio_resampler.cpp", "src/frame_cryptor.cpp", "src/global_task_queue.cpp", diff --git a/webrtc-sys/include/livekit/adm_proxy.h b/webrtc-sys/include/livekit/adm_proxy.h new file mode 100644 index 000000000..5f84379b2 --- /dev/null +++ b/webrtc-sys/include/livekit/adm_proxy.h @@ -0,0 +1,149 @@ +/* + * Copyright 2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "api/environment/environment.h" +#include "api/scoped_refptr.h" +#include "modules/audio_device/include/audio_device.h" +#include "modules/audio_device/include/audio_device_defines.h" + +namespace webrtc { +class Thread; +} // namespace webrtc + +namespace livekit_ffi { + +// ADM Proxy that wraps WebRTC's platform-specific AudioDeviceModule. +// +// This proxy provides control over microphone recording - when recording is +// disabled, InitRecording/StartRecording return success but do nothing. +// This is useful when only using NativeAudioSource (no microphone capture needed). +class AdmProxy : public webrtc::AudioDeviceModule { + public: + explicit AdmProxy(const webrtc::Environment& env, + webrtc::Thread* worker_thread); + ~AdmProxy() override; + + // Check if platform ADM was successfully initialized + bool is_initialized() const; + + // Control whether recording (microphone) is enabled. + // When disabled, InitRecording/StartRecording will return success but do nothing. + // This is useful when only using NativeAudioSource (no microphone capture needed). + void set_recording_enabled(bool enabled); + bool recording_enabled() const; + + // AudioDeviceModule interface implementation + int32_t ActiveAudioLayer(AudioLayer* audioLayer) const override; + int32_t RegisterAudioCallback(webrtc::AudioTransport* transport) override; + + int32_t Init() override; + int32_t Terminate() override; + bool Initialized() const override; + + int16_t PlayoutDevices() override; + int16_t RecordingDevices() override; + int32_t PlayoutDeviceName(uint16_t index, + char name[webrtc::kAdmMaxDeviceNameSize], + char guid[webrtc::kAdmMaxGuidSize]) override; + int32_t RecordingDeviceName(uint16_t index, + char name[webrtc::kAdmMaxDeviceNameSize], + char guid[webrtc::kAdmMaxGuidSize]) override; + + int32_t SetPlayoutDevice(uint16_t index) override; + int32_t SetPlayoutDevice(WindowsDeviceType device) override; + int32_t SetRecordingDevice(uint16_t index) override; + int32_t SetRecordingDevice(WindowsDeviceType device) override; + + int32_t PlayoutIsAvailable(bool* available) override; + int32_t InitPlayout() override; + bool PlayoutIsInitialized() const override; + int32_t RecordingIsAvailable(bool* available) override; + int32_t InitRecording() override; + bool RecordingIsInitialized() const override; + + int32_t StartPlayout() override; + int32_t StopPlayout() override; + bool Playing() const override; + int32_t StartRecording() override; + int32_t StopRecording() override; + bool Recording() const override; + + int32_t InitSpeaker() override; + bool SpeakerIsInitialized() const override; + int32_t InitMicrophone() override; + bool MicrophoneIsInitialized() const override; + + int32_t SpeakerVolumeIsAvailable(bool* available) override; + int32_t SetSpeakerVolume(uint32_t volume) override; + int32_t SpeakerVolume(uint32_t* volume) const override; + int32_t MaxSpeakerVolume(uint32_t* maxVolume) const override; + int32_t MinSpeakerVolume(uint32_t* minVolume) const override; + + int32_t MicrophoneVolumeIsAvailable(bool* available) override; + int32_t SetMicrophoneVolume(uint32_t volume) override; + int32_t MicrophoneVolume(uint32_t* volume) const override; + int32_t MaxMicrophoneVolume(uint32_t* maxVolume) const override; + int32_t MinMicrophoneVolume(uint32_t* minVolume) const override; + + int32_t SpeakerMuteIsAvailable(bool* available) override; + int32_t SetSpeakerMute(bool enable) override; + int32_t SpeakerMute(bool* enabled) const override; + + int32_t MicrophoneMuteIsAvailable(bool* available) override; + int32_t SetMicrophoneMute(bool enable) override; + int32_t MicrophoneMute(bool* enabled) const override; + + int32_t StereoPlayoutIsAvailable(bool* available) const override; + int32_t SetStereoPlayout(bool enable) override; + int32_t StereoPlayout(bool* enabled) const override; + int32_t StereoRecordingIsAvailable(bool* available) const override; + int32_t SetStereoRecording(bool enable) override; + int32_t StereoRecording(bool* enabled) const override; + + int32_t PlayoutDelay(uint16_t* delayMS) const override; + + bool BuiltInAECIsAvailable() const override; + bool BuiltInAGCIsAvailable() const override; + bool BuiltInNSIsAvailable() const override; + + int32_t EnableBuiltInAEC(bool enable) override; + int32_t EnableBuiltInAGC(bool enable) override; + int32_t EnableBuiltInNS(bool enable) override; + +#if defined(WEBRTC_IOS) + int GetPlayoutAudioParameters(webrtc::AudioParameters* params) const override; + int GetRecordAudioParameters(webrtc::AudioParameters* params) const override; +#endif + + int32_t SetObserver(webrtc::AudioDeviceObserver* observer) override; + + private: + const webrtc::Environment& env_; + webrtc::Thread* worker_thread_; + + // The underlying platform ADM + webrtc::scoped_refptr platform_adm_; + bool adm_initialized_ = false; + + // When false, recording (microphone) operations are no-ops. + // Defaults to FALSE - microphone is opt-in, not opt-out. + // Call set_recording_enabled(true) when using PlatformAudio for microphone capture. + bool recording_enabled_ = false; +}; + +} // namespace livekit_ffi diff --git a/webrtc-sys/include/livekit/audio_track.h b/webrtc-sys/include/livekit/audio_track.h index 5f1d37c9f..ba4afe90c 100644 --- a/webrtc-sys/include/livekit/audio_track.h +++ b/webrtc-sys/include/livekit/audio_track.h @@ -126,6 +126,10 @@ class AudioTrackSource { void clear_buffer(); + // Override to indicate this is an external audio source. + // This prevents AudioState from sending device audio to streams using this source. + bool is_external_source() const override { return true; } + private: mutable webrtc::Mutex mutex_; std::unique_ptr audio_queue_; diff --git a/webrtc-sys/include/livekit/peer_connection_factory.h b/webrtc-sys/include/livekit/peer_connection_factory.h index 0e77dbadb..56dd0e76c 100644 --- a/webrtc-sys/include/livekit/peer_connection_factory.h +++ b/webrtc-sys/include/livekit/peer_connection_factory.h @@ -20,7 +20,7 @@ #include "api/peer_connection_interface.h" #include "api/scoped_refptr.h" #include "api/task_queue/task_queue_factory.h" -#include "livekit/audio_device.h" +#include "livekit/adm_proxy.h" #include "media_stream.h" #include "rtp_parameters.h" #include "rust/cxx.h" @@ -57,15 +57,57 @@ class PeerConnectionFactory { rust::String label, std::shared_ptr source) const; + // Create an audio track that uses the ADM for capture (microphone) + // This creates a track that captures from the selected recording device + std::shared_ptr create_device_audio_track( + rust::String label) const; + RtpCapabilities rtp_sender_capabilities(MediaType type) const; RtpCapabilities rtp_receiver_capabilities(MediaType type) const; std::shared_ptr rtc_runtime() const { return rtc_runtime_; } + // Device enumeration + int16_t playout_devices() const; + int16_t recording_devices() const; + rust::String playout_device_name(uint16_t index) const; + rust::String recording_device_name(uint16_t index) const; + + // Device selection + int32_t set_playout_device(uint16_t index) const; + int32_t set_recording_device(uint16_t index) const; + + // Recording control (for device switching while active) + int32_t stop_recording() const; + int32_t init_recording() const; + int32_t start_recording() const; + bool recording_is_initialized() const; + + // Playout control (for device switching while active) + int32_t stop_playout() const; + int32_t init_playout() const; + int32_t start_playout() const; + bool playout_is_initialized() const; + + // Built-in audio processing (hardware AEC/AGC/NS) + // These are only available on iOS and some Android devices + bool builtin_aec_is_available() const; + bool builtin_agc_is_available() const; + bool builtin_ns_is_available() const; + int32_t enable_builtin_aec(bool enable) const; + int32_t enable_builtin_agc(bool enable) const; + int32_t enable_builtin_ns(bool enable) const; + + // Control whether ADM recording (microphone) is enabled. + // When disabled, WebRTC's calls to InitRecording/StartRecording will be no-ops. + // Use this when only using NativeAudioSource (no microphone capture needed). + void set_adm_recording_enabled(bool enabled) const; + bool adm_recording_enabled() const; + private: std::shared_ptr rtc_runtime_; - webrtc::scoped_refptr audio_device_; + webrtc::scoped_refptr adm_proxy_; webrtc::scoped_refptr peer_factory_; webrtc::Environment env_; }; diff --git a/webrtc-sys/libwebrtc/build_android.sh b/webrtc-sys/libwebrtc/build_android.sh index bd346f528..c89a2eaf3 100755 --- a/webrtc-sys/libwebrtc/build_android.sh +++ b/webrtc-sys/libwebrtc/build_android.sh @@ -71,6 +71,7 @@ cd src git apply "$COMMAND_DIR/patches/ssl_verify_callback_with_native_handle.patch" -v --ignore-space-change --ignore-whitespace --whitespace=nowarn git apply "$COMMAND_DIR/patches/add_deps.patch" -v --ignore-space-change --ignore-whitespace --whitespace=nowarn git apply "$COMMAND_DIR/patches/android_use_libunwind.patch" -v --ignore-space-change --ignore-whitespace --whitespace=nowarn +git apply "$COMMAND_DIR/patches/external_audio_source.patch" -v --ignore-space-change --ignore-whitespace --whitespace=nowarn # livekit prefixed jni git apply "$COMMAND_DIR/patches/jni_prefix.patch" -v --ignore-space-change --ignore-whitespace --whitespace=nowarn @@ -105,7 +106,6 @@ args="is_debug=$debug \ enable_iterator_debugging=false \ android_package_prefix=\"livekit\" \ use_custom_libcxx=false \ - use_clang_modules=false \ use_rtti=true" if [ "$debug" = "true" ]; then @@ -125,13 +125,15 @@ autoninja -C "$OUTPUT_DIR" :default \ # don't include nasm ar -rc "$ARTIFACTS_DIR/lib/libwebrtc.a" `find "$OUTPUT_DIR/obj" -name '*.o' -not -path "*/third_party/nasm/*"` -python3 "./src/tools_webrtc/libs/generate_licenses.py" \ - --target :default "$OUTPUT_DIR" "$OUTPUT_DIR" +# License generation is optional - may fail with some Python versions +# Use vpython3 from depot_tools for consistent Python version +vpython3 "./src/tools_webrtc/libs/generate_licenses.py" \ + --target :default "$OUTPUT_DIR" "$OUTPUT_DIR" || echo "Warning: License generation failed (non-critical)" cp "$OUTPUT_DIR/obj/webrtc.ninja" "$ARTIFACTS_DIR" cp "$OUTPUT_DIR/libjingle_peerconnection_so.so" "$ARTIFACTS_DIR/lib" cp "$OUTPUT_DIR/args.gn" "$ARTIFACTS_DIR" -cp "$OUTPUT_DIR/LICENSE.md" "$ARTIFACTS_DIR" +cp "$OUTPUT_DIR/LICENSE.md" "$ARTIFACTS_DIR" 2>/dev/null || echo "Warning: LICENSE.md not found (non-critical)" mkdir -p "$COMMAND_DIR/prefixed-jni/libs" cp "$OUTPUT_DIR/lib.java/sdk/android/libwebrtc.jar" "$COMMAND_DIR/prefixed-jni/libs/classes.jar" diff --git a/webrtc-sys/libwebrtc/build_ios.sh b/webrtc-sys/libwebrtc/build_ios.sh index f4a7d1167..e891a3683 100755 --- a/webrtc-sys/libwebrtc/build_ios.sh +++ b/webrtc-sys/libwebrtc/build_ios.sh @@ -78,9 +78,22 @@ then fi cd src + +# Apply patches only if not already applied (check with --reverse --check) +apply_patch_if_needed() { + local patch="$1" + if git apply --reverse --check "$patch" 2>/dev/null; then + echo "Patch already applied: $(basename "$patch")" + else + echo "Applying patch: $(basename "$patch")" + git apply "$patch" -v --ignore-space-change --ignore-whitespace --whitespace=nowarn || true + fi +} + # git apply "$COMMAND_DIR/patches/add_licenses.patch" -v --ignore-space-change --ignore-whitespace --whitespace=nowarn -git apply "$COMMAND_DIR/patches/ssl_verify_callback_with_native_handle.patch" -v --ignore-space-change --ignore-whitespace --whitespace=nowarn -git apply "$COMMAND_DIR/patches/add_deps.patch" -v --ignore-space-change --ignore-whitespace --whitespace=nowarn +apply_patch_if_needed "$COMMAND_DIR/patches/ssl_verify_callback_with_native_handle.patch" +apply_patch_if_needed "$COMMAND_DIR/patches/add_deps.patch" +apply_patch_if_needed "$COMMAND_DIR/patches/external_audio_source.patch" cd .. @@ -111,7 +124,6 @@ gn gen "$OUTPUT_DIR" --root="src" \ rtc_enable_objc_symbol_export=false \ rtc_use_h264=false \ use_custom_libcxx=false \ - use_clang_modules=false \ clang_use_chrome_plugins=false \ use_rtti=true \ use_lld=false" @@ -130,13 +142,14 @@ ninja -C "$OUTPUT_DIR" :default \ # don't include nasm ar -rc "$ARTIFACTS_DIR/lib/libwebrtc.a" `find "$OUTPUT_DIR/obj" -name '*.o' -not -path "*/third_party/nasm/*"` -python3 "./src/tools_webrtc/libs/generate_licenses.py" \ - --target :webrtc "$OUTPUT_DIR" "$OUTPUT_DIR" +# License generation is optional - may fail with some Python versions +# Use vpython3 from depot_tools for consistent Python version +vpython3 "./src/tools_webrtc/libs/generate_licenses.py" \ + --target :webrtc "$OUTPUT_DIR" "$OUTPUT_DIR" || echo "Warning: License generation failed (non-critical)" cp "$OUTPUT_DIR/obj/webrtc.ninja" "$ARTIFACTS_DIR" -cp "$OUTPUT_DIR/obj/modules/desktop_capture/desktop_capture.ninja" "$ARTIFACTS_DIR" +cp "$OUTPUT_DIR/obj/modules/desktop_capture/desktop_capture.ninja" "$ARTIFACTS_DIR" 2>/dev/null || true cp "$OUTPUT_DIR/args.gn" "$ARTIFACTS_DIR" -cp "$OUTPUT_DIR/LICENSE.md" "$ARTIFACTS_DIR" cd src find . -name "*.h" -print | cpio -pd "$ARTIFACTS_DIR/include" diff --git a/webrtc-sys/libwebrtc/build_linux.sh b/webrtc-sys/libwebrtc/build_linux.sh index 926e21bef..f6b0f112b 100755 --- a/webrtc-sys/libwebrtc/build_linux.sh +++ b/webrtc-sys/libwebrtc/build_linux.sh @@ -71,6 +71,7 @@ git apply "$COMMAND_DIR/patches/add_licenses.patch" -v --ignore-space-change --i git apply "$COMMAND_DIR/patches/ssl_verify_callback_with_native_handle.patch" -v --ignore-space-change --ignore-whitespace --whitespace=nowarn git apply "$COMMAND_DIR/patches/add_deps.patch" -v --ignore-space-change --ignore-whitespace --whitespace=nowarn git apply "$COMMAND_DIR/patches/fix_desktop_capture_compile.patch" -v --ignore-space-change --ignore-whitespace --whitespace=nowarn +git apply "$COMMAND_DIR/patches/external_audio_source.patch" -v --ignore-space-change --ignore-whitespace --whitespace=nowarn # Disable CREL (compact relocations). Chromium's build enables experimental # CREL via -Wa,--crel which causes segfaults on aarch64-linux (and is known @@ -105,7 +106,6 @@ args="is_debug=$debug \ treat_warnings_as_errors=false \ use_llvm_libatomic=false \ use_custom_libcxx=false \ - use_clang_modules=false \ use_custom_libcxx_for_host=false \ rtc_include_tests=false \ rtc_build_tools=false \ @@ -134,13 +134,15 @@ ninja -C "$OUTPUT_DIR" :default ar -rc "$ARTIFACTS_DIR/lib/libwebrtc.a" `find "$OUTPUT_DIR/obj" -name '*.o' -not -path "*/third_party/nasm/*"` src/third_party/llvm-build/Release+Asserts/bin/llvm-objcopy --redefine-syms="$COMMAND_DIR/boringssl_prefix_symbols.txt" "$ARTIFACTS_DIR/lib/libwebrtc.a" -python3 "./src/tools_webrtc/libs/generate_licenses.py" \ - --target :default "$OUTPUT_DIR" "$OUTPUT_DIR" +# License generation is optional - may fail with some Python versions +# Use vpython3 from depot_tools for consistent Python version +vpython3 "./src/tools_webrtc/libs/generate_licenses.py" \ + --target :default "$OUTPUT_DIR" "$OUTPUT_DIR" || echo "Warning: License generation failed (non-critical)" cp "$OUTPUT_DIR/obj/webrtc.ninja" "$ARTIFACTS_DIR" cp "$OUTPUT_DIR/obj/modules/desktop_capture/desktop_capture.ninja" "$ARTIFACTS_DIR" cp "$OUTPUT_DIR/args.gn" "$ARTIFACTS_DIR" -cp "$OUTPUT_DIR/LICENSE.md" "$ARTIFACTS_DIR" +cp "$OUTPUT_DIR/LICENSE.md" "$ARTIFACTS_DIR" 2>/dev/null || echo "Warning: LICENSE.md not found (non-critical)" cd src find . -name "*.h" -print | cpio -pd "$ARTIFACTS_DIR/include" diff --git a/webrtc-sys/libwebrtc/build_macos.sh b/webrtc-sys/libwebrtc/build_macos.sh index 98cf0ce8f..46120ae6b 100755 --- a/webrtc-sys/libwebrtc/build_macos.sh +++ b/webrtc-sys/libwebrtc/build_macos.sh @@ -67,9 +67,22 @@ then fi cd src -git apply "$COMMAND_DIR/patches/add_licenses.patch" -v --ignore-space-change --ignore-whitespace --whitespace=nowarn -git apply "$COMMAND_DIR/patches/ssl_verify_callback_with_native_handle.patch" -v --ignore-space-change --ignore-whitespace --whitespace=nowarn -git apply "$COMMAND_DIR/patches/add_deps.patch" -v --ignore-space-change --ignore-whitespace --whitespace=nowarn + +# Apply patches only if not already applied (check with --reverse --check) +apply_patch_if_needed() { + local patch="$1" + if git apply --reverse --check "$patch" 2>/dev/null; then + echo "Patch already applied: $(basename "$patch")" + else + echo "Applying patch: $(basename "$patch")" + git apply "$patch" -v --ignore-space-change --ignore-whitespace --whitespace=nowarn || true + fi +} + +apply_patch_if_needed "$COMMAND_DIR/patches/add_licenses.patch" +apply_patch_if_needed "$COMMAND_DIR/patches/ssl_verify_callback_with_native_handle.patch" +apply_patch_if_needed "$COMMAND_DIR/patches/add_deps.patch" +apply_patch_if_needed "$COMMAND_DIR/patches/external_audio_source.patch" cd .. @@ -103,10 +116,10 @@ gn gen "$OUTPUT_DIR" --root="src" \ rtc_use_h264=true \ rtc_use_h265=true \ use_custom_libcxx=false \ - use_clang_modules=false \ clang_use_chrome_plugins=false \ use_rtti=true \ - use_lld=false" + use_lld=false \ + rtc_include_internal_audio_device=true" # build static library ninja -C "$OUTPUT_DIR" :default \ @@ -117,19 +130,22 @@ ninja -C "$OUTPUT_DIR" :default \ pc:peer_connection \ sdk:videocapture_objc \ sdk:mac_framework_objc \ - desktop_capture_objc + desktop_capture_objc \ + modules/audio_device:audio_device # make libwebrtc.a # don't include nasm ar -rc "$ARTIFACTS_DIR/lib/libwebrtc.a" `find "$OUTPUT_DIR/obj" -name '*.o' -not -path "*/third_party/nasm/*"` -python3 "./src/tools_webrtc/libs/generate_licenses.py" \ - --target :webrtc "$OUTPUT_DIR" "$OUTPUT_DIR" +# License generation is optional - may fail with some Python versions +# Use vpython3 from depot_tools for consistent Python version +vpython3 "./src/tools_webrtc/libs/generate_licenses.py" \ + --target :webrtc "$OUTPUT_DIR" "$OUTPUT_DIR" || echo "Warning: License generation failed (non-critical)" cp "$OUTPUT_DIR/obj/webrtc.ninja" "$ARTIFACTS_DIR" -cp "$OUTPUT_DIR/obj/modules/desktop_capture/desktop_capture.ninja" "$ARTIFACTS_DIR" +cp "$OUTPUT_DIR/obj/modules/desktop_capture/desktop_capture.ninja" "$ARTIFACTS_DIR" 2>/dev/null || true cp "$OUTPUT_DIR/args.gn" "$ARTIFACTS_DIR" -cp "$OUTPUT_DIR/LICENSE.md" "$ARTIFACTS_DIR" +cp "$OUTPUT_DIR/LICENSE.md" "$ARTIFACTS_DIR" 2>/dev/null || echo "Warning: LICENSE.md not found (non-critical)" cd src find . -name "*.h" -print | cpio -pd "$ARTIFACTS_DIR/include" diff --git a/webrtc-sys/libwebrtc/build_windows.cmd b/webrtc-sys/libwebrtc/build_windows.cmd index ad247a395..b4c3b5f15 100644 --- a/webrtc-sys/libwebrtc/build_windows.cmd +++ b/webrtc-sys/libwebrtc/build_windows.cmd @@ -54,6 +54,7 @@ call git apply "%COMMAND_DIR%/patches/add_licenses.patch" -v --ignore-space-chan call git apply "%COMMAND_DIR%/patches/add_deps.patch" -v --ignore-space-change --ignore-whitespace --whitespace=nowarn call git apply "%COMMAND_DIR%/patches/windows_silence_warnings.patch" -v --ignore-space-change --ignore-whitespace --whitespace=nowarn call git apply "%COMMAND_DIR%/patches/ssl_verify_callback_with_native_handle.patch" -v --ignore-space-change --ignore-whitespace --whitespace=nowarn +call git apply "%COMMAND_DIR%/patches/external_audio_source.patch" -v --ignore-space-change --ignore-whitespace --whitespace=nowarn copy ".vpython3" "..\" diff --git a/webrtc-sys/libwebrtc/patches/external_audio_source.patch b/webrtc-sys/libwebrtc/patches/external_audio_source.patch new file mode 100644 index 000000000..85263299f --- /dev/null +++ b/webrtc-sys/libwebrtc/patches/external_audio_source.patch @@ -0,0 +1,99 @@ +diff --git a/api/media_stream_interface.h b/api/media_stream_interface.h +index fb1cc4e58e..85062ba60e 100644 +--- a/api/media_stream_interface.h ++++ b/api/media_stream_interface.h +@@ -267,6 +267,11 @@ class RTC_EXPORT AudioSourceInterface : public MediaSourceInterface { + // (for some of the settings this approach is broken, e.g. setting + // audio network adaptation on the source is the wrong layer of abstraction). + virtual const AudioOptions options() const; ++ ++ // Returns true if this source delivers audio externally (via AddSink), ++ // bypassing the ADM/AudioState audio distribution path. ++ // When true, AudioSendStream should not register with AudioState. ++ virtual bool is_external_source() const { return false; } + }; + + // Interface of the audio processor used by the audio track to collect +diff --git a/media/base/audio_source.h b/media/base/audio_source.h +index 04a7d19dfa..9f513f3c75 100644 +--- a/media/base/audio_source.h ++++ b/media/base/audio_source.h +@@ -49,6 +49,10 @@ class AudioSource { + // to the source at a time. + virtual void SetSink(Sink* sink) = 0; + ++ // Returns true if this source delivers audio externally (bypassing ADM). ++ // When true, AudioSendStream should not register with AudioState. ++ virtual bool is_external_source() const { return false; } ++ + protected: + virtual ~AudioSource() {} + }; +diff --git a/call/audio_send_stream.h b/call/audio_send_stream.h +index 84341b5cb1..9359777bc9 100644 +--- a/call/audio_send_stream.h ++++ b/call/audio_send_stream.h +@@ -178,6 +178,12 @@ class AudioSendStream : public AudioSender { + // An optional frame transformer used by insertable streams to transform + // encoded frames. + scoped_refptr frame_transformer; ++ ++ // When true, this stream uses an external audio source (not ADM). ++ // AudioState will NOT send device-captured audio to this stream. ++ // Audio is delivered directly via the source's AddSink mechanism. ++ // This prevents mixing of device audio with externally-sourced audio. ++ bool external_source = false; + }; + + virtual ~AudioSendStream() = default; +diff --git a/audio/audio_send_stream.cc b/audio/audio_send_stream.cc +index 76156ce830..10b59d3ff6 100644 +--- a/audio/audio_send_stream.cc ++++ b/audio/audio_send_stream.cc +@@ -373,8 +373,13 @@ void AudioSendStream::Start() { + } + channel_send_->StartSend(); + sending_ = true; +- audio_state()->AddSendingStream(this, encoder_sample_rate_hz_, +- encoder_num_channels_); ++ // Only register with AudioState if not using external source. ++ // External sources (like NativeAudioSource) deliver audio directly via AddSink, ++ // so we don't want AudioState to also send device audio to this stream. ++ if (!config_.external_source) { ++ audio_state()->AddSendingStream(this, encoder_sample_rate_hz_, ++ encoder_num_channels_); ++ } + } + + void AudioSendStream::Stop() { +@@ -386,7 +391,10 @@ void AudioSendStream::Stop() { + RemoveBitrateObserver(); + channel_send_->StopSend(); + sending_ = false; +- audio_state()->RemoveSendingStream(this); ++ // Only unregister if we registered (when not using external source). ++ if (!config_.external_source) { ++ audio_state()->RemoveSendingStream(this); ++ } + } + + void AudioSendStream::SendAudioData(std::unique_ptr audio_frame) { +diff --git a/media/engine/webrtc_voice_engine.cc b/media/engine/webrtc_voice_engine.cc +index 762f9d584c..4ce07ddc9d 100644 +--- a/media/engine/webrtc_voice_engine.cc ++++ b/media/engine/webrtc_voice_engine.cc +@@ -1017,6 +1017,14 @@ class WebRtcVoiceSendChannel::WebRtcAudioSendStream : public AudioSource::Sink { + RTC_DCHECK(source_ == source); + return; + } ++ ++ // Check if this is an external audio source (delivers audio via AddSink). ++ // If so, mark the config so AudioState doesn't send device audio to this ++ // stream. This must be done before UpdateSendState() calls Start(). ++ if (source->is_external_source() && !config_.external_source) { ++ config_.external_source = true; ++ stream_->Reconfigure(config_, nullptr); ++ } + source->SetSink(this); + source_ = source; + UpdateSendState(); diff --git a/webrtc-sys/src/adm_proxy.cpp b/webrtc-sys/src/adm_proxy.cpp new file mode 100644 index 000000000..c10345a4c --- /dev/null +++ b/webrtc-sys/src/adm_proxy.cpp @@ -0,0 +1,425 @@ +/* + * Copyright 2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "livekit/adm_proxy.h" + +#include "api/audio/audio_device.h" +#include "api/audio/create_audio_device_module.h" +#include "rtc_base/logging.h" +#include "rtc_base/thread.h" + +namespace livekit_ffi { + +AdmProxy::AdmProxy(const webrtc::Environment& env, webrtc::Thread* worker_thread) + : env_(env), worker_thread_(worker_thread) { + // Create the platform ADM + platform_adm_ = webrtc::CreateAudioDeviceModule( + env_, webrtc::AudioDeviceModule::kPlatformDefaultAudio); + + if (!platform_adm_) { + RTC_LOG(LS_ERROR) << "AdmProxy: Failed to create Platform ADM"; + return; + } + + // Initialize the platform ADM + int32_t init_result = platform_adm_->Init(); + if (init_result != 0) { + RTC_LOG(LS_ERROR) << "AdmProxy: Failed to initialize Platform ADM, error=" << init_result; + platform_adm_ = nullptr; + return; + } + + adm_initialized_ = true; + RTC_LOG(LS_INFO) << "AdmProxy: Platform ADM initialized, " + << platform_adm_->RecordingDevices() << " recording devices, " + << platform_adm_->PlayoutDevices() << " playout devices"; +} + +AdmProxy::~AdmProxy() { + RTC_LOG(LS_VERBOSE) << "AdmProxy::~AdmProxy()"; + if (platform_adm_) { + platform_adm_->Terminate(); + platform_adm_ = nullptr; + } +} + +bool AdmProxy::is_initialized() const { + return adm_initialized_; +} + +void AdmProxy::set_recording_enabled(bool enabled) { + RTC_LOG(LS_INFO) << "AdmProxy::set_recording_enabled(" << enabled << ")"; + recording_enabled_ = enabled; +} + +bool AdmProxy::recording_enabled() const { + return recording_enabled_; +} + +// AudioDeviceModule interface - delegate all calls to platform_adm_ + +int32_t AdmProxy::ActiveAudioLayer(AudioLayer* audioLayer) const { + if (!platform_adm_) { + *audioLayer = AudioLayer::kDummyAudio; + return 0; + } + return platform_adm_->ActiveAudioLayer(audioLayer); +} + +int32_t AdmProxy::RegisterAudioCallback(webrtc::AudioTransport* transport) { + if (!platform_adm_) return -1; + return platform_adm_->RegisterAudioCallback(transport); +} + +int32_t AdmProxy::Init() { + // Already initialized in constructor + if (!platform_adm_) return -1; + return 0; +} + +int32_t AdmProxy::Terminate() { + if (!platform_adm_) return 0; + adm_initialized_ = false; + return platform_adm_->Terminate(); +} + +bool AdmProxy::Initialized() const { + if (!platform_adm_) return false; + return platform_adm_->Initialized(); +} + +int16_t AdmProxy::PlayoutDevices() { + if (!platform_adm_) return 0; + return platform_adm_->PlayoutDevices(); +} + +int16_t AdmProxy::RecordingDevices() { + if (!platform_adm_) return 0; + return platform_adm_->RecordingDevices(); +} + +int32_t AdmProxy::PlayoutDeviceName(uint16_t index, + char name[webrtc::kAdmMaxDeviceNameSize], + char guid[webrtc::kAdmMaxGuidSize]) { + if (!platform_adm_) return -1; + return platform_adm_->PlayoutDeviceName(index, name, guid); +} + +int32_t AdmProxy::RecordingDeviceName(uint16_t index, + char name[webrtc::kAdmMaxDeviceNameSize], + char guid[webrtc::kAdmMaxGuidSize]) { + if (!platform_adm_) return -1; + return platform_adm_->RecordingDeviceName(index, name, guid); +} + +int32_t AdmProxy::SetPlayoutDevice(uint16_t index) { + if (!platform_adm_) return -1; + return platform_adm_->SetPlayoutDevice(index); +} + +int32_t AdmProxy::SetPlayoutDevice(WindowsDeviceType device) { + if (!platform_adm_) return -1; + return platform_adm_->SetPlayoutDevice(device); +} + +int32_t AdmProxy::SetRecordingDevice(uint16_t index) { + if (!platform_adm_) return -1; + return platform_adm_->SetRecordingDevice(index); +} + +int32_t AdmProxy::SetRecordingDevice(WindowsDeviceType device) { + if (!platform_adm_) return -1; + return platform_adm_->SetRecordingDevice(device); +} + +int32_t AdmProxy::PlayoutIsAvailable(bool* available) { + if (!platform_adm_) { + *available = false; + return 0; + } + return platform_adm_->PlayoutIsAvailable(available); +} + +int32_t AdmProxy::InitPlayout() { + if (!platform_adm_) return -1; + return platform_adm_->InitPlayout(); +} + +bool AdmProxy::PlayoutIsInitialized() const { + if (!platform_adm_) return false; + return platform_adm_->PlayoutIsInitialized(); +} + +int32_t AdmProxy::RecordingIsAvailable(bool* available) { + if (!platform_adm_) { + *available = false; + return 0; + } + return platform_adm_->RecordingIsAvailable(available); +} + +int32_t AdmProxy::InitRecording() { + if (!platform_adm_) return -1; + if (!recording_enabled_) { + return 0; // Return success but don't actually initialize + } + return platform_adm_->InitRecording(); +} + +bool AdmProxy::RecordingIsInitialized() const { + if (!platform_adm_) return false; + if (!recording_enabled_) return false; + return platform_adm_->RecordingIsInitialized(); +} + +int32_t AdmProxy::StartPlayout() { + if (!platform_adm_) return -1; + return platform_adm_->StartPlayout(); +} + +int32_t AdmProxy::StopPlayout() { + if (!platform_adm_) return 0; + return platform_adm_->StopPlayout(); +} + +bool AdmProxy::Playing() const { + if (!platform_adm_) return false; + return platform_adm_->Playing(); +} + +int32_t AdmProxy::StartRecording() { + if (!platform_adm_) return -1; + if (!recording_enabled_) { + return 0; // Return success but don't actually start + } + return platform_adm_->StartRecording(); +} + +int32_t AdmProxy::StopRecording() { + if (!platform_adm_) return 0; + return platform_adm_->StopRecording(); +} + +bool AdmProxy::Recording() const { + if (!platform_adm_) return false; + if (!recording_enabled_) return false; + return platform_adm_->Recording(); +} + +int32_t AdmProxy::InitSpeaker() { + if (!platform_adm_) return -1; + return platform_adm_->InitSpeaker(); +} + +bool AdmProxy::SpeakerIsInitialized() const { + if (!platform_adm_) return false; + return platform_adm_->SpeakerIsInitialized(); +} + +int32_t AdmProxy::InitMicrophone() { + if (!platform_adm_) return -1; + return platform_adm_->InitMicrophone(); +} + +bool AdmProxy::MicrophoneIsInitialized() const { + if (!platform_adm_) return false; + return platform_adm_->MicrophoneIsInitialized(); +} + +int32_t AdmProxy::SpeakerVolumeIsAvailable(bool* available) { + if (!platform_adm_) { + *available = false; + return 0; + } + return platform_adm_->SpeakerVolumeIsAvailable(available); +} + +int32_t AdmProxy::SetSpeakerVolume(uint32_t volume) { + if (!platform_adm_) return -1; + return platform_adm_->SetSpeakerVolume(volume); +} + +int32_t AdmProxy::SpeakerVolume(uint32_t* volume) const { + if (!platform_adm_) return -1; + return platform_adm_->SpeakerVolume(volume); +} + +int32_t AdmProxy::MaxSpeakerVolume(uint32_t* maxVolume) const { + if (!platform_adm_) return -1; + return platform_adm_->MaxSpeakerVolume(maxVolume); +} + +int32_t AdmProxy::MinSpeakerVolume(uint32_t* minVolume) const { + if (!platform_adm_) return -1; + return platform_adm_->MinSpeakerVolume(minVolume); +} + +int32_t AdmProxy::MicrophoneVolumeIsAvailable(bool* available) { + if (!platform_adm_) { + *available = false; + return 0; + } + return platform_adm_->MicrophoneVolumeIsAvailable(available); +} + +int32_t AdmProxy::SetMicrophoneVolume(uint32_t volume) { + if (!platform_adm_) return -1; + return platform_adm_->SetMicrophoneVolume(volume); +} + +int32_t AdmProxy::MicrophoneVolume(uint32_t* volume) const { + if (!platform_adm_) return -1; + return platform_adm_->MicrophoneVolume(volume); +} + +int32_t AdmProxy::MaxMicrophoneVolume(uint32_t* maxVolume) const { + if (!platform_adm_) return -1; + return platform_adm_->MaxMicrophoneVolume(maxVolume); +} + +int32_t AdmProxy::MinMicrophoneVolume(uint32_t* minVolume) const { + if (!platform_adm_) return -1; + return platform_adm_->MinMicrophoneVolume(minVolume); +} + +int32_t AdmProxy::SpeakerMuteIsAvailable(bool* available) { + if (!platform_adm_) { + *available = false; + return 0; + } + return platform_adm_->SpeakerMuteIsAvailable(available); +} + +int32_t AdmProxy::SetSpeakerMute(bool enable) { + if (!platform_adm_) return -1; + return platform_adm_->SetSpeakerMute(enable); +} + +int32_t AdmProxy::SpeakerMute(bool* enabled) const { + if (!platform_adm_) return -1; + return platform_adm_->SpeakerMute(enabled); +} + +int32_t AdmProxy::MicrophoneMuteIsAvailable(bool* available) { + if (!platform_adm_) { + *available = false; + return 0; + } + return platform_adm_->MicrophoneMuteIsAvailable(available); +} + +int32_t AdmProxy::SetMicrophoneMute(bool enable) { + if (!platform_adm_) return -1; + return platform_adm_->SetMicrophoneMute(enable); +} + +int32_t AdmProxy::MicrophoneMute(bool* enabled) const { + if (!platform_adm_) return -1; + return platform_adm_->MicrophoneMute(enabled); +} + +int32_t AdmProxy::StereoPlayoutIsAvailable(bool* available) const { + if (!platform_adm_) { + *available = false; + return 0; + } + return platform_adm_->StereoPlayoutIsAvailable(available); +} + +int32_t AdmProxy::SetStereoPlayout(bool enable) { + if (!platform_adm_) return -1; + return platform_adm_->SetStereoPlayout(enable); +} + +int32_t AdmProxy::StereoPlayout(bool* enabled) const { + if (!platform_adm_) return -1; + return platform_adm_->StereoPlayout(enabled); +} + +int32_t AdmProxy::StereoRecordingIsAvailable(bool* available) const { + if (!platform_adm_) { + *available = false; + return 0; + } + return platform_adm_->StereoRecordingIsAvailable(available); +} + +int32_t AdmProxy::SetStereoRecording(bool enable) { + if (!platform_adm_) return -1; + return platform_adm_->SetStereoRecording(enable); +} + +int32_t AdmProxy::StereoRecording(bool* enabled) const { + if (!platform_adm_) return -1; + return platform_adm_->StereoRecording(enabled); +} + +int32_t AdmProxy::PlayoutDelay(uint16_t* delayMS) const { + if (!platform_adm_) { + *delayMS = 0; + return 0; + } + return platform_adm_->PlayoutDelay(delayMS); +} + +bool AdmProxy::BuiltInAECIsAvailable() const { + if (!platform_adm_) return false; + return platform_adm_->BuiltInAECIsAvailable(); +} + +bool AdmProxy::BuiltInAGCIsAvailable() const { + if (!platform_adm_) return false; + return platform_adm_->BuiltInAGCIsAvailable(); +} + +bool AdmProxy::BuiltInNSIsAvailable() const { + if (!platform_adm_) return false; + return platform_adm_->BuiltInNSIsAvailable(); +} + +int32_t AdmProxy::EnableBuiltInAEC(bool enable) { + if (!platform_adm_) return -1; + return platform_adm_->EnableBuiltInAEC(enable); +} + +int32_t AdmProxy::EnableBuiltInAGC(bool enable) { + if (!platform_adm_) return -1; + return platform_adm_->EnableBuiltInAGC(enable); +} + +int32_t AdmProxy::EnableBuiltInNS(bool enable) { + if (!platform_adm_) return -1; + return platform_adm_->EnableBuiltInNS(enable); +} + +#if defined(WEBRTC_IOS) +int AdmProxy::GetPlayoutAudioParameters(webrtc::AudioParameters* params) const { + if (!platform_adm_) return -1; + return platform_adm_->GetPlayoutAudioParameters(params); +} + +int AdmProxy::GetRecordAudioParameters(webrtc::AudioParameters* params) const { + if (!platform_adm_) return -1; + return platform_adm_->GetRecordAudioParameters(params); +} +#endif + +int32_t AdmProxy::SetObserver(webrtc::AudioDeviceObserver* observer) { + if (!platform_adm_) return -1; + return platform_adm_->SetObserver(observer); +} + +} // namespace livekit_ffi diff --git a/webrtc-sys/src/audio_track.cpp b/webrtc-sys/src/audio_track.cpp index f5bb47a4e..1738515fb 100644 --- a/webrtc-sys/src/audio_track.cpp +++ b/webrtc-sys/src/audio_track.cpp @@ -32,7 +32,6 @@ #include "rtc_base/logging.h" #include "rtc_base/ref_counted_object.h" #include "rtc_base/synchronization/mutex.h" -#include "rtc_base/time_utils.h" #include "rust/cxx.h" #include "webrtc-sys/src/audio_track.rs.h" @@ -220,7 +219,7 @@ bool AudioTrackSource::InternalSource::capture_frame( } } else { - // capture directly when the queue buffer is 0 (frame size must be 10ms) + // Fast path: capture directly when the queue buffer is 0 (frame size must be 10ms) for (auto sink : sinks_) sink->OnData(data.data(), sizeof(int16_t) * 8, sample_rate, number_of_channels, number_of_frames); diff --git a/webrtc-sys/src/peer_connection_factory.cpp b/webrtc-sys/src/peer_connection_factory.cpp index a0e27a0a2..50b068a46 100644 --- a/webrtc-sys/src/peer_connection_factory.cpp +++ b/webrtc-sys/src/peer_connection_factory.cpp @@ -31,7 +31,9 @@ #include "api/task_queue/default_task_queue_factory.h" #include "api/video_codecs/builtin_video_decoder_factory.h" #include "api/video_codecs/builtin_video_encoder_factory.h" -#include "livekit/audio_device.h" +#include "api/audio/audio_device.h" +#include "api/audio_options.h" +#include "livekit/adm_proxy.h" #include "livekit/audio_track.h" #include "livekit/peer_connection.h" #include "livekit/rtc_error.h" @@ -51,22 +53,20 @@ PeerConnectionFactory::PeerConnectionFactory( std::shared_ptr rtc_runtime) : rtc_runtime_(rtc_runtime), env_(webrtc::EnvironmentFactory().Create()) { - RTC_LOG(LS_VERBOSE) << "PeerConnectionFactory::PeerConnectionFactory()"; - webrtc::PeerConnectionFactoryDependencies dependencies; dependencies.network_thread = rtc_runtime_->network_thread(); dependencies.worker_thread = rtc_runtime_->worker_thread(); dependencies.signaling_thread = rtc_runtime_->signaling_thread(); dependencies.socket_factory = rtc_runtime_->network_thread()->socketserver(); dependencies.event_log_factory = std::make_unique(); - // TODO: - // dependencies.trials = std::make_unique(); - audio_device_ = rtc_runtime_->worker_thread()->BlockingCall([&] { - return webrtc::make_ref_counted(env_); + // Create AdmProxy - it creates and initializes Platform ADM internally + adm_proxy_ = rtc_runtime_->worker_thread()->BlockingCall([&] { + return webrtc::make_ref_counted( + env_, rtc_runtime_->worker_thread()); }); - dependencies.adm = audio_device_; + dependencies.adm = adm_proxy_; dependencies.video_encoder_factory = std::move(std::make_unique()); @@ -91,7 +91,7 @@ PeerConnectionFactory::~PeerConnectionFactory() { peer_factory_ = nullptr; rtc_runtime_->worker_thread()->BlockingCall( - [this] { audio_device_ = nullptr; }); + [this] { adm_proxy_ = nullptr; }); } std::shared_ptr PeerConnectionFactory::create_peer_connection( @@ -124,6 +124,27 @@ std::shared_ptr PeerConnectionFactory::create_audio_track( peer_factory_->CreateAudioTrack(label.c_str(), source->get().get()))); } +std::shared_ptr PeerConnectionFactory::create_device_audio_track( + rust::String label) const { + // Create an audio source that uses the ADM for capture + webrtc::AudioOptions audio_options; + audio_options.echo_cancellation = true; + audio_options.auto_gain_control = true; + audio_options.noise_suppression = true; + + webrtc::scoped_refptr audio_source = + peer_factory_->CreateAudioSource(audio_options); + + if (!audio_source) { + RTC_LOG(LS_ERROR) << "Failed to create device audio source"; + return nullptr; + } + + return std::static_pointer_cast( + rtc_runtime_->get_or_create_media_stream_track( + peer_factory_->CreateAudioTrack(label.c_str(), audio_source.get()))); +} + RtpCapabilities PeerConnectionFactory::rtp_sender_capabilities( MediaType type) const { return to_rust_rtp_capabilities(peer_factory_->GetRtpSenderCapabilities( @@ -136,6 +157,102 @@ RtpCapabilities PeerConnectionFactory::rtp_receiver_capabilities( static_cast(type))); } +// Device enumeration and management + +int16_t PeerConnectionFactory::playout_devices() const { + return adm_proxy_->PlayoutDevices(); +} + +int16_t PeerConnectionFactory::recording_devices() const { + return adm_proxy_->RecordingDevices(); +} + +rust::String PeerConnectionFactory::playout_device_name(uint16_t index) const { + char name[webrtc::kAdmMaxDeviceNameSize] = {0}; + char guid[webrtc::kAdmMaxGuidSize] = {0}; + adm_proxy_->PlayoutDeviceName(index, name, guid); + return rust::String(name); +} + +rust::String PeerConnectionFactory::recording_device_name(uint16_t index) const { + char name[webrtc::kAdmMaxDeviceNameSize] = {0}; + char guid[webrtc::kAdmMaxGuidSize] = {0}; + adm_proxy_->RecordingDeviceName(index, name, guid); + return rust::String(name); +} + +int32_t PeerConnectionFactory::set_playout_device(uint16_t index) const { + return adm_proxy_->SetPlayoutDevice(index); +} + +int32_t PeerConnectionFactory::set_recording_device(uint16_t index) const { + return adm_proxy_->SetRecordingDevice(index); +} + +int32_t PeerConnectionFactory::stop_recording() const { + return adm_proxy_->StopRecording(); +} + +int32_t PeerConnectionFactory::init_recording() const { + return adm_proxy_->InitRecording(); +} + +int32_t PeerConnectionFactory::start_recording() const { + return adm_proxy_->StartRecording(); +} + +bool PeerConnectionFactory::recording_is_initialized() const { + return adm_proxy_->RecordingIsInitialized(); +} + +int32_t PeerConnectionFactory::stop_playout() const { + return adm_proxy_->StopPlayout(); +} + +int32_t PeerConnectionFactory::init_playout() const { + return adm_proxy_->InitPlayout(); +} + +int32_t PeerConnectionFactory::start_playout() const { + return adm_proxy_->StartPlayout(); +} + +bool PeerConnectionFactory::playout_is_initialized() const { + return adm_proxy_->PlayoutIsInitialized(); +} + +bool PeerConnectionFactory::builtin_aec_is_available() const { + return adm_proxy_->BuiltInAECIsAvailable(); +} + +bool PeerConnectionFactory::builtin_agc_is_available() const { + return adm_proxy_->BuiltInAGCIsAvailable(); +} + +bool PeerConnectionFactory::builtin_ns_is_available() const { + return adm_proxy_->BuiltInNSIsAvailable(); +} + +int32_t PeerConnectionFactory::enable_builtin_aec(bool enable) const { + return adm_proxy_->EnableBuiltInAEC(enable); +} + +int32_t PeerConnectionFactory::enable_builtin_agc(bool enable) const { + return adm_proxy_->EnableBuiltInAGC(enable); +} + +int32_t PeerConnectionFactory::enable_builtin_ns(bool enable) const { + return adm_proxy_->EnableBuiltInNS(enable); +} + +void PeerConnectionFactory::set_adm_recording_enabled(bool enabled) const { + adm_proxy_->set_recording_enabled(enabled); +} + +bool PeerConnectionFactory::adm_recording_enabled() const { + return adm_proxy_->recording_enabled(); +} + std::shared_ptr create_peer_connection_factory() { return std::make_shared(RtcRuntime::create()); } diff --git a/webrtc-sys/src/peer_connection_factory.rs b/webrtc-sys/src/peer_connection_factory.rs index c18d8331c..179132cbc 100644 --- a/webrtc-sys/src/peer_connection_factory.rs +++ b/webrtc-sys/src/peer_connection_factory.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::Arc; +pub use cxx::SharedPtr; -use cxx::SharedPtr; +use std::sync::Arc; use crate::{ candidate::ffi::Candidate, data_channel::ffi::DataChannel, impl_thread_safety, @@ -107,6 +107,12 @@ pub mod ffi { source: SharedPtr, ) -> SharedPtr; + // Create an audio track that uses the ADM for capture (Platform ADM mode) + fn create_device_audio_track( + self: &PeerConnectionFactory, + label: String, + ) -> SharedPtr; + fn rtp_sender_capabilities( self: &PeerConnectionFactory, kind: MediaType, @@ -116,6 +122,43 @@ pub mod ffi { self: &PeerConnectionFactory, kind: MediaType, ) -> RtpCapabilities; + + // Device enumeration + fn playout_devices(self: &PeerConnectionFactory) -> i16; + fn recording_devices(self: &PeerConnectionFactory) -> i16; + fn playout_device_name(self: &PeerConnectionFactory, index: u16) -> String; + fn recording_device_name(self: &PeerConnectionFactory, index: u16) -> String; + + // Device selection + fn set_playout_device(self: &PeerConnectionFactory, index: u16) -> i32; + fn set_recording_device(self: &PeerConnectionFactory, index: u16) -> i32; + + // Recording control (for device switching while active) + fn stop_recording(self: &PeerConnectionFactory) -> i32; + fn init_recording(self: &PeerConnectionFactory) -> i32; + fn start_recording(self: &PeerConnectionFactory) -> i32; + fn recording_is_initialized(self: &PeerConnectionFactory) -> bool; + + // Playout control (for device switching while active) + fn stop_playout(self: &PeerConnectionFactory) -> i32; + fn init_playout(self: &PeerConnectionFactory) -> i32; + fn start_playout(self: &PeerConnectionFactory) -> i32; + fn playout_is_initialized(self: &PeerConnectionFactory) -> bool; + + // Built-in audio processing (hardware AEC/AGC/NS) + // These are only available on iOS and some Android devices + fn builtin_aec_is_available(self: &PeerConnectionFactory) -> bool; + fn builtin_agc_is_available(self: &PeerConnectionFactory) -> bool; + fn builtin_ns_is_available(self: &PeerConnectionFactory) -> bool; + fn enable_builtin_aec(self: &PeerConnectionFactory, enable: bool) -> i32; + fn enable_builtin_agc(self: &PeerConnectionFactory, enable: bool) -> i32; + fn enable_builtin_ns(self: &PeerConnectionFactory, enable: bool) -> i32; + + // Control whether ADM recording (microphone) is enabled. + // When disabled, InitRecording/StartRecording will be no-ops. + // Use this when only using NativeAudioSource (no microphone needed). + fn set_adm_recording_enabled(self: &PeerConnectionFactory, enabled: bool); + fn adm_recording_enabled(self: &PeerConnectionFactory) -> bool; } extern "Rust" {