From 9d5e87f25c3cd7562b93a2b874486bf283a82609 Mon Sep 17 00:00:00 2001 From: grohith327 Date: Tue, 28 Jul 2026 18:00:19 -0700 Subject: [PATCH 1/3] Enable core audio flow by default --- Sources/Fluid/Persistence/SettingsStore.swift | 12 ++------- .../HotkeyShortcutTests.swift | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/Sources/Fluid/Persistence/SettingsStore.swift b/Sources/Fluid/Persistence/SettingsStore.swift index 6a357658..500694d0 100644 --- a/Sources/Fluid/Persistence/SettingsStore.swift +++ b/Sources/Fluid/Persistence/SettingsStore.swift @@ -44,7 +44,6 @@ final class SettingsStore: ObservableObject { self.repairForcedOnboardingResetIfNeeded() self.migrateOverlayBottomOffsetTo50IfNeeded() self.migratePrivateAIContextDefaultTo4KIfNeeded() - self.disableExperimentalDirectAudioCaptureIfNeeded() self.refreshLaunchAtStartupStatus(clearError: true, logMismatch: false) } @@ -1781,11 +1780,11 @@ final class SettingsStore: ObservableObject { } } - /// Direct Core Audio capture is opt-in while the faster recording path is experimental. + /// Direct Core Audio capture defaults on while preserving stored user choices. var experimentalDirectAudioCaptureEnabled: Bool { get { let value = self.defaults.object(forKey: Keys.experimentalDirectAudioCaptureEnabled) - return value as? Bool ?? false + return value as? Bool ?? true } set { objectWillChange.send() @@ -1793,12 +1792,6 @@ final class SettingsStore: ObservableObject { } } - private func disableExperimentalDirectAudioCaptureIfNeeded() { - guard self.defaults.bool(forKey: Keys.experimentalDirectAudioCaptureForcedOff) == false else { return } - self.defaults.set(false, forKey: Keys.experimentalDirectAudioCaptureEnabled) - self.defaults.set(true, forKey: Keys.experimentalDirectAudioCaptureForcedOff) - } - var directAudioCaptureConsecutiveFailures: Int { get { self.defaults.integer(forKey: Keys.directAudioCaptureConsecutiveFailures) } set { self.defaults.set(max(0, newValue), forKey: Keys.directAudioCaptureConsecutiveFailures) } @@ -4928,7 +4921,6 @@ private extension SettingsStore { static let skipSilentRecordingsEnabled = "SkipSilentRecordingsEnabled" static let enableAIStreaming = "EnableAIStreaming" static let experimentalDirectAudioCaptureEnabled = "ExperimentalDirectAudioCaptureEnabled" - static let experimentalDirectAudioCaptureForcedOff = "ExperimentalDirectAudioCaptureForcedOff" static let directAudioCaptureConsecutiveFailures = "DirectAudioCaptureConsecutiveFailures" static let copyTranscriptionToClipboard = "CopyTranscriptionToClipboard" static let textInsertionMode = "TextInsertionMode" diff --git a/Tests/FluidDictationIntegrationTests/HotkeyShortcutTests.swift b/Tests/FluidDictationIntegrationTests/HotkeyShortcutTests.swift index 2fa39b99..26dbbd78 100644 --- a/Tests/FluidDictationIntegrationTests/HotkeyShortcutTests.swift +++ b/Tests/FluidDictationIntegrationTests/HotkeyShortcutTests.swift @@ -13,6 +13,7 @@ final class HotkeyShortcutTests: XCTestCase { private let microphoneSelectionModeKey = "MicrophoneSelectionMode" private let preferredInputDeviceUIDKey = "PreferredInputDeviceUID" private let systemInputDeviceUIDBeforeManualKey = "SystemInputDeviceUIDBeforeManual" + private let experimentalDirectAudioCaptureEnabledKey = "ExperimentalDirectAudioCaptureEnabled" @MainActor func testBottomOverlayRapidStopStartStopDoesNotDropFinalHide() async { @@ -167,6 +168,30 @@ final class HotkeyShortcutTests: XCTestCase { XCTAssertNil(decoded.settings.skipSilentRecordingsEnabled) } + func testFasterRecordingStartDefaultsToEnabledWhenUnset() { + self.withRestoredDefaults(keys: [self.experimentalDirectAudioCaptureEnabledKey]) { + UserDefaults.standard.removeObject(forKey: self.experimentalDirectAudioCaptureEnabledKey) + + XCTAssertTrue(SettingsStore.shared.experimentalDirectAudioCaptureEnabled) + } + } + + func testFasterRecordingStartPreservesStoredDisabledPreference() { + self.withRestoredDefaults(keys: [self.experimentalDirectAudioCaptureEnabledKey]) { + UserDefaults.standard.set(false, forKey: self.experimentalDirectAudioCaptureEnabledKey) + + XCTAssertFalse(SettingsStore.shared.experimentalDirectAudioCaptureEnabled) + } + } + + func testFasterRecordingStartPreservesStoredEnabledPreference() { + self.withRestoredDefaults(keys: [self.experimentalDirectAudioCaptureEnabledKey]) { + UserDefaults.standard.set(true, forKey: self.experimentalDirectAudioCaptureEnabledKey) + + XCTAssertTrue(SettingsStore.shared.experimentalDirectAudioCaptureEnabled) + } + } + func testLegacyKeyboardShortcutPayloadDefaultsToKeyboardKind() throws { let json = #"{"keyCode":61,"modifierFlagsRawValue":0}"# let data = try XCTUnwrap(json.data(using: .utf8)) From 9d597a55d61e776bc7cb0d5156f170bd601ec380 Mon Sep 17 00:00:00 2001 From: grohith327 Date: Tue, 28 Jul 2026 19:06:30 -0700 Subject: [PATCH 2/3] remove the duration mistmach check --- Sources/Fluid/Persistence/SettingsStore.swift | 6 - Sources/Fluid/Services/ASRService.swift | 295 +++++------------- .../Fluid/Services/NotificationService.swift | 72 ----- Sources/Fluid/UI/SettingsView.swift | 3 - .../HotkeyShortcutTests.swift | 31 -- 5 files changed, 70 insertions(+), 337 deletions(-) diff --git a/Sources/Fluid/Persistence/SettingsStore.swift b/Sources/Fluid/Persistence/SettingsStore.swift index 500694d0..8c91c842 100644 --- a/Sources/Fluid/Persistence/SettingsStore.swift +++ b/Sources/Fluid/Persistence/SettingsStore.swift @@ -1792,11 +1792,6 @@ final class SettingsStore: ObservableObject { } } - var directAudioCaptureConsecutiveFailures: Int { - get { self.defaults.integer(forKey: Keys.directAudioCaptureConsecutiveFailures) } - set { self.defaults.set(max(0, newValue), forKey: Keys.directAudioCaptureConsecutiveFailures) } - } - var copyTranscriptionToClipboard: Bool { get { self.defaults.bool(forKey: Keys.copyTranscriptionToClipboard) } set { self.defaults.set(newValue, forKey: Keys.copyTranscriptionToClipboard) } @@ -4921,7 +4916,6 @@ private extension SettingsStore { static let skipSilentRecordingsEnabled = "SkipSilentRecordingsEnabled" static let enableAIStreaming = "EnableAIStreaming" static let experimentalDirectAudioCaptureEnabled = "ExperimentalDirectAudioCaptureEnabled" - static let directAudioCaptureConsecutiveFailures = "DirectAudioCaptureConsecutiveFailures" static let copyTranscriptionToClipboard = "CopyTranscriptionToClipboard" static let textInsertionMode = "TextInsertionMode" static let autoUpdateCheckEnabled = "AutoUpdateCheckEnabled" diff --git a/Sources/Fluid/Services/ASRService.swift b/Sources/Fluid/Services/ASRService.swift index 96a4c052..55fe9a2d 100644 --- a/Sources/Fluid/Services/ASRService.swift +++ b/Sources/Fluid/Services/ASRService.swift @@ -171,19 +171,6 @@ final class ASRService: ObservableObject { ) } - nonisolated static func directCaptureDurationIsMismatched( - capturedMilliseconds: Int, - elapsedMilliseconds: Int - ) -> Bool { - guard elapsedMilliseconds >= 500 else { return false } - return capturedMilliseconds * 10 < elapsedMilliseconds * 7 || - capturedMilliseconds * 10 > elapsedMilliseconds * 13 - } - - nonisolated static func directCaptureShouldDisable(afterFailureCount failureCount: Int) -> Bool { - failureCount >= 3 - } - @Published var isRunning: Bool = false @Published var finalText: String = "" @Published var partialTranscription: String = "" @@ -719,7 +706,6 @@ final class ASRService: ObservableObject { private var directAudioInput: DirectCoreAudioInput? private var activeAudioCaptureBackend: AudioCaptureBackend = .none - private var isFallingBackFromDirectCapture = false private var hasPreparedAudioCapture: Bool { self.directAudioInput != nil || self.hasWarmAudioEngine @@ -817,7 +803,7 @@ final class ASRService: ObservableObject { DebugLogger.shared.debug("Audio engine cooled to stopped warm state (\(reason))", source: "ASRService") } - private func prewarmAudioEngineIfPossible( + private func prewarmConfiguredAudioCaptureIfPossible( reason: String, allowDuringRouteRecovery: Bool = false ) { @@ -839,10 +825,16 @@ final class ASRService: ObservableObject { } let startedAt = Date().timeIntervalSince1970 - if SettingsStore.shared.experimentalDirectAudioCaptureEnabled, - self.prepareDirectAudioInputIfPossible(reason: reason) - { - self.benchmarkLog("direct_audio_prewarm reason=\(reason) elapsedMs=\(self.elapsedMilliseconds(since: startedAt))") + if SettingsStore.shared.experimentalDirectAudioCaptureEnabled { + do { + _ = try self.prepareDirectAudioInput(reason: reason) + self.benchmarkLog("direct_audio_prewarm reason=\(reason) elapsedMs=\(self.elapsedMilliseconds(since: startedAt))") + } catch { + DebugLogger.shared.warning( + "Direct Core Audio prewarm failed: \(error.localizedDescription)", + source: "ASRService" + ) + } return } @@ -862,97 +854,91 @@ final class ASRService: ObservableObject { /// Prepares the direct device callback without starting hardware IO. This /// keeps the default idle state privacy-friendly while removing device and /// ring allocation from the hotkey path. - @discardableResult - private func prepareDirectAudioInputIfPossible(reason: String) -> Bool { - guard self.micStatus == .authorized else { return false } + private func prepareDirectAudioInput(reason: String) throws -> DirectCoreAudioInput { + guard self.micStatus == .authorized else { + throw NSError( + domain: "ASRService", + code: -1, + userInfo: [NSLocalizedDescriptionKey: "Microphone access is not authorized."] + ) + } guard let device = self.resolvedInputDeviceForCapture() else { - DebugLogger.shared.warning("No input device is available for direct capture", source: "ASRService") - return false + throw NSError( + domain: "ASRService", + code: -1, + userInfo: [NSLocalizedDescriptionKey: "No input device is available for Direct Core Audio capture."] + ) } if let directAudioInput = self.directAudioInput, directAudioInput.deviceID == device.id { - return true + return directAudioInput } self.directAudioInput?.invalidate() self.directAudioInput = nil let pipeline = self.audioCapturePipeline - do { - let directAudioInput = try DirectCoreAudioInput(deviceID: device.id) { samples, frameCount, sampleRate, inputHostTime, inputSampleTime in - pipeline.handle( - samples: samples, - frameCount: frameCount, - sampleRate: sampleRate, - inputHostTime: inputHostTime, - inputSampleTime: inputSampleTime - ) - } - self.directAudioInput = directAudioInput - DebugLogger.shared.info( - "Prepared direct Core Audio input '\(device.name)' " + - "(\(Int(directAudioInput.sampleRate.rounded()))Hz, " + - "\(directAudioInput.hardwareBufferFrameSize) frames, reason=\(reason))", - source: "ASRService" - ) - return true - } catch { - DebugLogger.shared.warning( - "Direct Core Audio input unavailable for '\(device.name)': \(error.localizedDescription). " + - "Falling back to AVAudioEngine.", - source: "ASRService" + let directAudioInput = try DirectCoreAudioInput(deviceID: device.id) { samples, frameCount, sampleRate, inputHostTime, inputSampleTime in + pipeline.handle( + samples: samples, + frameCount: frameCount, + sampleRate: sampleRate, + inputHostTime: inputHostTime, + inputSampleTime: inputSampleTime ) - return false } + self.directAudioInput = directAudioInput + DebugLogger.shared.info( + "Prepared direct Core Audio input '\(device.name)' " + + "(\(Int(directAudioInput.sampleRate.rounded()))Hz, " + + "\(directAudioInput.hardwareBufferFrameSize) frames, reason=\(reason))", + source: "ASRService" + ) + return directAudioInput } - private func startPreferredAudioCapture() async throws { - // A non-route path may have scheduled a fire-and-forget retirement. Do - // not let capture startup overlap any final AVAudioEngine release that is - // already queued. - await self.audioEngineRetirementDrain.waitForScheduledReleases() - - let directCaptureEnabled = SettingsStore.shared.experimentalDirectAudioCaptureEnabled - if directCaptureEnabled, - self.prepareDirectAudioInputIfPossible(reason: "recording_start"), - let directAudioInput = self.directAudioInput - { + private func startConfiguredAudioCapture() async throws { + if SettingsStore.shared.experimentalDirectAudioCaptureEnabled { + // A non-route path may have scheduled a fire-and-forget retirement. + // Do not let direct capture startup overlap a queued AVAudioEngine + // release. + await self.audioEngineRetirementDrain.waitForScheduledReleases() do { + let directAudioInput = try self.prepareDirectAudioInput(reason: "recording_start") try directAudioInput.start() self.activeAudioCaptureBackend = .directCoreAudio - let callbackMs = Int( - (Double(directAudioInput.hardwareBufferFrameSize) / - directAudioInput.sampleRate * 1000).rounded() - ) + let callbackDurationMilliseconds = + Double(directAudioInput.hardwareBufferFrameSize) / + directAudioInput.sampleRate * 1000 + let callbackMs = Int(callbackDurationMilliseconds.rounded()) self.benchmarkLog( "audio_backend kind=direct_core_audio device=\(directAudioInput.deviceID) " + "frames=\(directAudioInput.hardwareBufferFrameSize) callbackMs=\(callbackMs)" ) return } catch { - DebugLogger.shared.warning( - "Direct Core Audio start failed: \(error.localizedDescription). Falling back to AVAudioEngine.", + self.directAudioInput?.invalidate() + self.directAudioInput = nil + DebugLogger.shared.error( + "Direct Core Audio capture failed: \(error.localizedDescription)", source: "ASRService" ) - directAudioInput.invalidate() - self.directAudioInput = nil + throw error } } - if directCaptureEnabled == false, let directAudioInput = self.directAudioInput { + if let directAudioInput = self.directAudioInput { directAudioInput.invalidate() self.directAudioInput = nil } - try await self.startCompatibilityAudioCapture( - reason: directCaptureEnabled ? "direct_unavailable" : "experimental_disabled" - ) + try await self.startAVAudioEngineCapture() } - private func startCompatibilityAudioCapture(reason: String) async throws { + private func startAVAudioEngineCapture() async throws { await self.audioEngineRetirementDrain.waitForScheduledReleases() - self.benchmarkLog("audio_backend kind=av_audio_engine_fallback reason=\(reason)") + self.benchmarkLog("audio_backend kind=av_audio_engine reason=faster_recording_start_disabled") try self.configureSession() try await self.startEngine() try self.setupEngineTap() @@ -989,84 +975,6 @@ final class ASRService: ObservableObject { self.activeAudioCaptureBackend = .none } - private func handleDirectCaptureDurationMismatch( - sessionID: Int, - capturedMilliseconds: Int, - elapsedMilliseconds: Int - ) async { - guard sessionID == self.benchmarkSessionID, - self.isRunning, - self.activeAudioCaptureBackend == .directCoreAudio, - self.isFallingBackFromDirectCapture == false - else { return } - - self.isFallingBackFromDirectCapture = true - defer { self.isFallingBackFromDirectCapture = false } - - let failureCount = SettingsStore.shared.directAudioCaptureConsecutiveFailures + 1 - SettingsStore.shared.directAudioCaptureConsecutiveFailures = failureCount - let shouldDisable = Self.directCaptureShouldDisable(afterFailureCount: failureCount) - if shouldDisable { - SettingsStore.shared.experimentalDirectAudioCaptureEnabled = false - } - NotificationService.showAudioCaptureFallback( - failureCount: failureCount, - experimentalSettingDisabled: shouldDisable - ) - DebugLogger.shared.error( - "DIRECT_CAPTURE_RATE_MISMATCH session=\(sessionID) " + - "capturedMs=\(capturedMilliseconds) elapsedMs=\(elapsedMilliseconds); " + - "switching to AVAudioEngine", - source: "ASRService" - ) - self.benchmarkLog( - "direct_capture_fallback reason=duration_mismatch " + - "capturedMs=\(capturedMilliseconds) elapsedMs=\(elapsedMilliseconds)" - ) - - self.audioCapturePipeline.setRecordingEnabled(false) - self.stopActiveAudioCapture() - self.directAudioInput?.invalidate() - self.directAudioInput = nil - await self.stopStreamingTimerAndAwait() - guard self.isRunning else { return } - self.audioBuffer.clear(keepingCapacity: true) - self.dictionaryTrainingAudioGeneration &+= 1 - self.lastProcessedSampleCount = 0 - self.benchmarkLastChunkSampleCount = 0 - (self.transcriptionProvider as? FluidAudioProvider)?.resetStreamingPreviewCache() - - do { - self.audioCapturePipeline.setRecordingEnabled( - true, - sessionID: sessionID, - startHostTime: mach_absolute_time() - ) - try await self.startCompatibilityAudioCapture(reason: "duration_mismatch") - let model = SettingsStore.shared.selectedSpeechModel - if model.supportsStreaming, self.isDictionaryTrainingCaptureActive == false { - self.startStreamingTranscription() - } - DebugLogger.shared.info( - "Direct capture runtime fallback to AVAudioEngine succeeded", - source: "ASRService" - ) - } catch { - self.audioCapturePipeline.setRecordingEnabled(false) - self.stopActiveAudioCapture() - DebugLogger.shared.error( - "Direct capture runtime fallback failed: \(error.localizedDescription)", - source: "ASRService" - ) - await self.stopWithoutTranscription() - NotificationCenter.default.post( - name: NSNotification.Name("ASRServiceStartFailed"), - object: nil, - userInfo: ["errorMessage": "The microphone changed unexpectedly. Please try recording again."] - ) - } - } - /// Applies the experimental capture preference immediately when idle. /// If a recording transition is already underway, start() reads the latest /// persisted value and the following session will use it. @@ -1177,15 +1085,6 @@ final class ASRService: ObservableObject { ) } }, - onDurationMismatch: { [weak self] sessionID, capturedMilliseconds, elapsedMilliseconds in - Task { @MainActor [weak self] in - await self?.handleDirectCaptureDurationMismatch( - sessionID: sessionID, - capturedMilliseconds: capturedMilliseconds, - elapsedMilliseconds: elapsedMilliseconds - ) - } - }, onLevel: { [weak self] level in // Keep Combine sends on the main queue. DispatchQueue.main.async { [weak self] in @@ -1313,7 +1212,7 @@ final class ASRService: ObservableObject { // Register the input callback and allocate its fixed ring now. This // does not start the device or show the microphone privacy indicator. - self.prewarmAudioEngineIfPossible(reason: "startup") + self.prewarmConfiguredAudioCaptureIfPossible(reason: "startup") // Check if models exist on disk and auto-load if present // This is done in a Task to support async model detection (e.g., AppleSpeechAnalyzerProvider) @@ -1329,7 +1228,7 @@ final class ASRService: ObservableObject { do { try await self.ensureAsrReady() DebugLogger.shared.info("Models auto-loaded successfully on startup", source: "ASRService") - self.prewarmAudioEngineIfPossible(reason: "startup") + self.prewarmConfiguredAudioCaptureIfPossible(reason: "startup") } catch { DebugLogger.shared.error("Failed to auto-load models on startup: \(error)", source: "ASRService") } @@ -1386,7 +1285,7 @@ final class ASRService: ObservableObject { self.micPermissionGranted = granted self.micStatus = granted ? .authorized : .denied if granted { - self.prewarmAudioEngineIfPossible(reason: "permission_granted") + self.prewarmConfiguredAudioCaptureIfPossible(reason: "permission_granted") } } } @@ -1476,7 +1375,7 @@ final class ASRService: ObservableObject { AppServices.shared.microphonePreferenceCoordinator.enforcePreferredInput(reason: "recording start") } - try await self.startPreferredAudioCapture() + try await self.startConfiguredAudioCapture() self.isDictionaryTrainingCaptureActive = forDictionaryTraining self.isRunning = true DebugLogger.shared.info("✅ Audio capture running", source: "ASRService") @@ -1659,29 +1558,13 @@ final class ASRService: ObservableObject { self.stopMonitoringDevice() DebugLogger.shared.debug("✅ Device monitoring stopped", source: "ASRService") - if self.activeAudioCaptureBackend == .directCoreAudio, - let recordingStartedAt = self.benchmarkRecordingStartedAt - { - let elapsedMilliseconds = Int( - (Date().timeIntervalSince1970 - recordingStartedAt) * 1000 - ) - let capturedMilliseconds = self.audioBuffer.count * 1000 / 16_000 - if elapsedMilliseconds >= 500, - Self.directCaptureDurationIsMismatched( - capturedMilliseconds: capturedMilliseconds, - elapsedMilliseconds: elapsedMilliseconds - ) == false - { - SettingsStore.shared.directAudioCaptureConsecutiveFailures = 0 - } - } self.stopActiveAudioCapture() self.audioCapturePipeline.finishRecording() // A prepared direct IOProc owns only fixed memory and registration; it // does not run hardware, show the mic indicator, or hold Bluetooth in // headset mode. Keep it prepared across idle periods. The heavier - // AVAudioEngine fallback retains its existing bounded timeout. + // AVAudioEngine remains available when Faster Recording Start is disabled. if self.directAudioInput != nil { self.audioEngineStandbyTask?.cancel() self.audioEngineStandbyTask = nil @@ -2632,7 +2515,7 @@ final class ASRService: ObservableObject { await self.retireAudioEngineAndWait(reason: "idle_route_change:\(request.reason)") guard request.generation == self.audioRouteRecoveryGeneration, Task.isCancelled == false else { return } - self.prewarmAudioEngineIfPossible( + self.prewarmConfiguredAudioCaptureIfPossible( reason: "idle_route_change", allowDuringRouteRecovery: true ) @@ -2658,7 +2541,7 @@ final class ASRService: ObservableObject { sessionID: self.benchmarkSessionID, startHostTime: mach_absolute_time() ) - try await self.startPreferredAudioCapture() + try await self.startConfiguredAudioCapture() if let currentDevice = self.getCurrentlyBoundInputDevice() { self.startMonitoringDevice(currentDevice.id) @@ -3938,14 +3821,13 @@ private extension ASRService { // // Audio callbacks are not main-actor isolated. Direct Core Audio arrives through -// a lock-free C ring; AVAudioEngine remains the compatibility fallback. This -// pipeline owns timestamp trimming, 16 kHz conversion, levels, and session-safe -// delivery without touching ASRService from a realtime callback. +// a lock-free C ring. AVAudioEngine is used only when Faster Recording Start is +// disabled. This pipeline owns timestamp trimming, 16 kHz conversion, levels, +// and session-safe delivery without touching ASRService from a realtime callback. private final nonisolated class AudioCapturePipeline: @unchecked Sendable { private let audioBuffer: ThreadSafeAudioBuffer private let onFirstAudio: (Int, Int, Int, Double, Int, Int) -> Void - private let onDurationMismatch: (Int, Int, Int) -> Void private let onLevel: (CGFloat) -> Void private let lock = NSLock() @@ -3954,8 +3836,6 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable { private var recordingSessionID: Int = 0 private var recordingStartHostTime: UInt64 = 0 private var recordingStopHostTime: UInt64? - private var capturedOutputFrameCount: Int = 0 - private var durationMismatchReported: Bool = false private var resampleSourceRate: Double = 0 private var resampleSourceFrameCursor: Int64 = 0 private var resampleNextSourcePosition: Double = 0 @@ -3978,12 +3858,10 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable { init( audioBuffer: ThreadSafeAudioBuffer, onFirstAudio: @escaping (Int, Int, Int, Double, Int, Int) -> Void, - onDurationMismatch: @escaping (Int, Int, Int) -> Void, onLevel: @escaping (CGFloat) -> Void ) { self.audioBuffer = audioBuffer self.onFirstAudio = onFirstAudio - self.onDurationMismatch = onDurationMismatch self.onLevel = onLevel } @@ -3998,8 +3876,6 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable { self.recordingSessionID = sessionID self.recordingStartHostTime = startHostTime == 0 ? mach_absolute_time() : startHostTime self.recordingStopHostTime = nil - self.capturedOutputFrameCount = 0 - self.durationMismatchReported = false self.resetResamplerLocked() self.lastInputSampleEnd = nil self.recordingEnabled = true @@ -4009,8 +3885,6 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable { self.recordingSessionID = 0 self.recordingStartHostTime = 0 self.recordingStopHostTime = nil - self.capturedOutputFrameCount = 0 - self.durationMismatchReported = false self.resetResamplerLocked() self.lastInputSampleEnd = nil self.levelHistory.removeAll(keepingCapacity: true) @@ -4138,28 +4012,6 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable { if shouldReportFirstAudio { self.firstAudioReported = true } - self.capturedOutputFrameCount += mono16k.count - let capturedMilliseconds = self.capturedOutputFrameCount * 1000 / 16_000 - // Compare sample duration with the hardware acquisition timeline. The - // consumer queue can be delayed under CPU pressure, but late delivery - // does not mean the captured audio clock is malformed. - let acceptedPacketEndHostTime = Self.hostTime( - inputHostTime, - advancedByFrames: acceptedRange.upperBound, - sampleRate: sampleRate - ) - let elapsedMilliseconds = Self.elapsedMilliseconds( - from: startHostTime, - to: acceptedPacketEndHostTime - ) - let shouldReportDurationMismatch = self.durationMismatchReported == false && - ASRService.directCaptureDurationIsMismatched( - capturedMilliseconds: capturedMilliseconds, - elapsedMilliseconds: elapsedMilliseconds - ) - if shouldReportDurationMismatch { - self.durationMismatchReported = true - } self.lock.unlock() self.audioBuffer.append(mono16k) @@ -4186,13 +4038,6 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable { deliveryMs ) } - if shouldReportDurationMismatch { - self.onDurationMismatch( - recordingSessionID, - capturedMilliseconds, - elapsedMilliseconds - ) - } let level = self.calculateAudioLevel(mono16k) self.onLevel(level) } diff --git a/Sources/Fluid/Services/NotificationService.swift b/Sources/Fluid/Services/NotificationService.swift index e0922581..a5bbdbb5 100644 --- a/Sources/Fluid/Services/NotificationService.swift +++ b/Sources/Fluid/Services/NotificationService.swift @@ -8,49 +8,9 @@ enum NotificationService { enum Kind { static let aiProcessingFallback = "aiProcessingFallback" - static let audioCaptureFallback = "audioCaptureFallback" static let commandModeFailure = "commandModeFailure" } - static func showAudioCaptureFallback( - failureCount: Int, - experimentalSettingDisabled: Bool - ) { - let center = UNUserNotificationCenter.current() - center.getNotificationSettings { settings in - switch settings.authorizationStatus { - case .authorized, .provisional, .ephemeral: - self.deliverAudioCaptureFallback( - failureCount: failureCount, - experimentalSettingDisabled: experimentalSettingDisabled, - using: center - ) - case .notDetermined: - center.requestAuthorization(options: [.alert]) { granted, requestError in - if let requestError { - DebugLogger.shared.warning( - "Notification permission request failed: \(requestError.localizedDescription)", - source: "NotificationService" - ) - } - guard granted else { return } - self.deliverAudioCaptureFallback( - failureCount: failureCount, - experimentalSettingDisabled: experimentalSettingDisabled, - using: center - ) - } - case .denied: - DebugLogger.shared.debug( - "Skipping audio capture fallback notification because notification permission is denied", - source: "NotificationService" - ) - @unknown default: - break - } - } - } - static func showAIProcessingFallback(error: String) { guard SettingsStore.shared.notifyAIProcessingFailures else { return } @@ -135,38 +95,6 @@ enum NotificationService { } } - private static func deliverAudioCaptureFallback( - failureCount: Int, - experimentalSettingDisabled: Bool, - using center: UNUserNotificationCenter - ) { - let content = UNMutableNotificationContent() - if experimentalSettingDisabled { - content.title = "Faster Recording Start turned off" - content.body = "FluidVoice detected malformed microphone audio three times and switched to the compatibility audio path. You can turn it back on in Settings." - } else { - content.title = "Microphone audio recovered" - content.body = "FluidVoice detected malformed audio and switched this session to the compatibility audio path. Faster Recording Start will retry next recording. (\(failureCount)/3)" - } - content.sound = nil - content.userInfo = [UserInfoKey.kind: Kind.audioCaptureFallback] - - let request = UNNotificationRequest( - identifier: "audio-capture-fallback-\(UUID().uuidString)", - content: content, - trigger: nil - ) - - center.add(request) { addError in - if let addError { - DebugLogger.shared.warning( - "Failed to show audio capture fallback notification: \(addError.localizedDescription)", - source: "NotificationService" - ) - } - } - } - private static func deliverCommandModeFailure(error: String, using center: UNUserNotificationCenter) { let content = UNMutableNotificationContent() content.title = "Command Mode needs setup" diff --git a/Sources/Fluid/UI/SettingsView.swift b/Sources/Fluid/UI/SettingsView.swift index 224cc131..5b658930 100644 --- a/Sources/Fluid/UI/SettingsView.swift +++ b/Sources/Fluid/UI/SettingsView.swift @@ -2600,9 +2600,6 @@ private extension SettingsView { get: { self.settings.experimentalDirectAudioCaptureEnabled }, set: { enabled in self.settings.experimentalDirectAudioCaptureEnabled = enabled - if enabled { - self.settings.directAudioCaptureConsecutiveFailures = 0 - } self.asr.refreshAudioCaptureBackendPreference() } ) diff --git a/Tests/FluidDictationIntegrationTests/HotkeyShortcutTests.swift b/Tests/FluidDictationIntegrationTests/HotkeyShortcutTests.swift index 26dbbd78..4cd095fb 100644 --- a/Tests/FluidDictationIntegrationTests/HotkeyShortcutTests.swift +++ b/Tests/FluidDictationIntegrationTests/HotkeyShortcutTests.swift @@ -63,37 +63,6 @@ final class HotkeyShortcutTests: XCTestCase { } } - func testDirectCaptureDurationMismatchFilter() { - XCTAssertFalse(ASRService.directCaptureDurationIsMismatched( - capturedMilliseconds: 100, - elapsedMilliseconds: 499 - )) - XCTAssertFalse(ASRService.directCaptureDurationIsMismatched( - capturedMilliseconds: 460, - elapsedMilliseconds: 500 - )) - XCTAssertFalse(ASRService.directCaptureDurationIsMismatched( - capturedMilliseconds: 700, - elapsedMilliseconds: 1000 - )) - XCTAssertFalse(ASRService.directCaptureDurationIsMismatched( - capturedMilliseconds: 1300, - elapsedMilliseconds: 1000 - )) - XCTAssertTrue(ASRService.directCaptureDurationIsMismatched( - capturedMilliseconds: 333, - elapsedMilliseconds: 1000 - )) - XCTAssertTrue(ASRService.directCaptureDurationIsMismatched( - capturedMilliseconds: 1500, - elapsedMilliseconds: 1000 - )) - XCTAssertFalse(ASRService.directCaptureShouldDisable(afterFailureCount: 1)) - XCTAssertFalse(ASRService.directCaptureShouldDisable(afterFailureCount: 2)) - XCTAssertTrue(ASRService.directCaptureShouldDisable(afterFailureCount: 3)) - XCTAssertTrue(ASRService.directCaptureShouldDisable(afterFailureCount: 4)) - } - func testShortAudioSilenceGateRejectsOnlyClearShortSilence() { let silence = [Float](repeating: 0.0005, count: 16_000) let silenceAssessment = ASRService.assessShortAudioSilence(silence) From 250f8bfbbecc5144612d4111a57a5b35c3653fb5 Mon Sep 17 00:00:00 2001 From: grohith327 Date: Tue, 28 Jul 2026 21:07:00 -0700 Subject: [PATCH 3/3] big update --- Fluid.xcodeproj/project.pbxproj | 4 + .../CoreAudioCaptureSupport.c | 135 +- .../include/CoreAudioCaptureSupport.h | 9 +- Sources/Fluid/ContentView.swift | 50 +- Sources/Fluid/Services/ASRService.swift | 732 +++++++-- .../Services/AudioCaptureReadinessGate.swift | 131 ++ .../Fluid/Services/AudioDeviceService.swift | 24 +- .../Fluid/Services/DirectCoreAudioInput.swift | 1365 ++++++++++++++++- Sources/Fluid/Services/FileLogger.swift | 4 +- .../Fluid/Services/GlobalHotkeyManager.swift | 4 +- .../DirectAudioReliabilityTests.swift | 688 +++++++++ 11 files changed, 2961 insertions(+), 185 deletions(-) create mode 100644 Sources/Fluid/Services/AudioCaptureReadinessGate.swift create mode 100644 Tests/FluidDictationIntegrationTests/DirectAudioReliabilityTests.swift diff --git a/Fluid.xcodeproj/project.pbxproj b/Fluid.xcodeproj/project.pbxproj index 2b135c8c..d475db0e 100644 --- a/Fluid.xcodeproj/project.pbxproj +++ b/Fluid.xcodeproj/project.pbxproj @@ -19,6 +19,7 @@ 272BFB5CB271489892CAE50C /* TemperatureSupportTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */; }; A62300000000000000000002 /* AudioBufferConverterTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = A62300000000000000000001 /* AudioBufferConverterTests.swift */; }; C0DE63600000000000000002 /* AudioEngineRetirementDrainTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */; }; + DA7100020000000000000002 /* DirectAudioReliabilityTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */; }; 7CDB0A2F2F3C4D5600FB7CAD /* dictation_fixture.wav in Resources */ = {isa = PBXBuildFile; fileRef = 7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */; }; 7CDB0A302F3C4D5600FB7CAD /* XCTest.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 7CDB0A2C2F3C4D5600FB7CAD /* XCTest.framework */; }; 7CE006BD2E80EBE600DDCCD6 /* AppUpdater in Frameworks */ = {isa = PBXBuildFile; productRef = 7CE006BC2E80EBE600DDCCD6 /* AppUpdater */; }; @@ -54,6 +55,7 @@ 980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TemperatureSupportTests.swift; sourceTree = ""; }; A62300000000000000000001 /* AudioBufferConverterTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioBufferConverterTests.swift; sourceTree = ""; }; C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioEngineRetirementDrainTests.swift; sourceTree = ""; }; + DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DirectAudioReliabilityTests.swift; sourceTree = ""; }; 7C078D8F2E3B339200FB7CAC /* FluidVoice Debug.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "FluidVoice Debug.app"; sourceTree = BUILT_PRODUCTS_DIR; }; 7C91B0022F42AA0100C0DEF0 /* HotkeyShortcutTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HotkeyShortcutTests.swift; sourceTree = ""; }; 7CDB0A202F3C4D5600FB7CAD /* FluidDictationIntegrationTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = FluidDictationIntegrationTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -132,6 +134,7 @@ 980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */, A62300000000000000000001 /* AudioBufferConverterTests.swift */, C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */, + DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */, ); path = FluidDictationIntegrationTests; sourceTree = ""; @@ -291,6 +294,7 @@ 272BFB5CB271489892CAE50C /* TemperatureSupportTests.swift in Sources */, A62300000000000000000002 /* AudioBufferConverterTests.swift in Sources */, C0DE63600000000000000002 /* AudioEngineRetirementDrainTests.swift in Sources */, + DA7100020000000000000002 /* DirectAudioReliabilityTests.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/Sources/CoreAudioCaptureSupport/CoreAudioCaptureSupport.c b/Sources/CoreAudioCaptureSupport/CoreAudioCaptureSupport.c index 5cebb086..d75ac226 100644 --- a/Sources/CoreAudioCaptureSupport/CoreAudioCaptureSupport.c +++ b/Sources/CoreAudioCaptureSupport/CoreAudioCaptureSupport.c @@ -22,6 +22,7 @@ typedef struct { typedef struct { AudioObjectID deviceID; + AudioStreamID streamID; AudioDeviceIOProcID ioProcID; AudioStreamBasicDescription format; uint32_t bufferFrameSize; @@ -31,11 +32,14 @@ typedef struct { _Atomic uint64_t readIndex; _Atomic uint64_t droppedPackets; _Atomic bool running; + _Atomic bool formatDirty; + _Atomic bool packetGateOpen; FVPacketSlot slots[FV_RING_CAPACITY]; } FVCapture; static OSStatus fv_get_input_stream_format( AudioObjectID deviceID, + AudioStreamID *streamID, AudioStreamBasicDescription *format ) { AudioObjectPropertyAddress streamsAddress = { @@ -57,18 +61,21 @@ static OSStatus fv_get_input_stream_format( return status != noErr ? status : kAudioHardwareUnsupportedOperationError; } - AudioStreamID streamID = kAudioObjectUnknown; + AudioStreamID resolvedStreamID = kAudioObjectUnknown; status = AudioObjectGetPropertyData( deviceID, &streamsAddress, 0, NULL, &streamsSize, - &streamID + &resolvedStreamID ); - if (status != noErr || streamID == kAudioObjectUnknown) { + if (status != noErr || resolvedStreamID == kAudioObjectUnknown) { return status != noErr ? status : kAudioHardwareBadObjectError; } + if (streamID != NULL) { + *streamID = resolvedStreamID; + } AudioObjectPropertyAddress formatAddress = { kAudioStreamPropertyVirtualFormat, @@ -77,7 +84,7 @@ static OSStatus fv_get_input_stream_format( }; UInt32 formatSize = sizeof(*format); return AudioObjectGetPropertyData( - streamID, + resolvedStreamID, &formatAddress, 0, NULL, @@ -96,6 +103,35 @@ static OSStatus fv_get_buffer_frame_size(AudioObjectID deviceID, uint32_t *frame return AudioObjectGetPropertyData(deviceID, &address, 0, NULL, &size, frameSize); } +static OSStatus fv_get_maximum_buffer_frame_size( + AudioObjectID deviceID, + uint32_t fallbackFrameSize, + uint32_t *maximumFrameSize +) { + AudioObjectPropertyAddress address = { + kAudioDevicePropertyUsesVariableBufferFrameSizes, + kAudioObjectPropertyScopeGlobal, + kAudioObjectPropertyElementMain, + }; + if (!AudioObjectHasProperty(deviceID, &address)) { + *maximumFrameSize = fallbackFrameSize; + return noErr; + } + UInt32 size = sizeof(*maximumFrameSize); + OSStatus status = AudioObjectGetPropertyData( + deviceID, + &address, + 0, + NULL, + &size, + maximumFrameSize + ); + if (status == noErr && *maximumFrameSize == 0) { + *maximumFrameSize = fallbackFrameSize; + } + return status; +} + static bool fv_format_is_supported( const AudioStreamBasicDescription *format, uint32_t *bytesPerSample @@ -239,7 +275,9 @@ static OSStatus fv_io_proc( FVCapture *capture = (FVCapture *) inClientData; if (capture == NULL || inInputData == NULL || - !atomic_load_explicit(&capture->running, memory_order_relaxed)) { + !atomic_load_explicit(&capture->running, memory_order_relaxed) || + atomic_load_explicit(&capture->formatDirty, memory_order_acquire) || + !atomic_load_explicit(&capture->packetGateOpen, memory_order_acquire)) { return noErr; } @@ -332,7 +370,11 @@ int32_t fv_core_audio_capture_create( } capture->deviceID = deviceID; - OSStatus status = fv_get_input_stream_format(deviceID, &capture->format); + OSStatus status = fv_get_input_stream_format( + deviceID, + &capture->streamID, + &capture->format + ); if (status == noErr && !fv_format_is_supported(&capture->format, &capture->bytesPerSample)) { status = kAudioHardwareUnsupportedOperationError; @@ -340,9 +382,17 @@ int32_t fv_core_audio_capture_create( if (status == noErr) { status = fv_get_buffer_frame_size(deviceID, &capture->bufferFrameSize); } + uint32_t maximumBufferFrameSize = capture->bufferFrameSize; + if (status == noErr) { + status = fv_get_maximum_buffer_frame_size( + deviceID, + capture->bufferFrameSize, + &maximumBufferFrameSize + ); + } if (status == noErr && (capture->bufferFrameSize == 0 || - capture->bufferFrameSize > FV_MAX_FRAMES_PER_PACKET)) { + maximumBufferFrameSize > FV_MAX_FRAMES_PER_PACKET)) { status = kAudioHardwareUnsupportedOperationError; } if (status != noErr) { @@ -359,6 +409,8 @@ int32_t fv_core_audio_capture_create( atomic_init(&capture->readIndex, 0); atomic_init(&capture->droppedPackets, 0); atomic_init(&capture->running, false); + atomic_init(&capture->formatDirty, false); + atomic_init(&capture->packetGateOpen, false); status = AudioDeviceCreateIOProcID( deviceID, @@ -387,6 +439,7 @@ int32_t fv_core_audio_capture_start(FVCoreAudioCaptureRef captureRef) { return noErr; } + atomic_store_explicit(&capture->packetGateOpen, false, memory_order_release); atomic_store_explicit(&capture->running, true, memory_order_release); OSStatus status = AudioDeviceStart(capture->deviceID, capture->ioProcID); if (status != noErr) { @@ -406,31 +459,41 @@ int32_t fv_core_audio_capture_stop(FVCoreAudioCaptureRef captureRef) { return noErr; } - // Keep accepting callbacks until AudioDeviceStop has synchronized with the - // IOProc. Packets acquired before the caller's stop boundary can then be - // timestamp-trimmed by the consumer instead of being dropped here. + // Close publication before synchronizing with the IOProc. The consumer + // still drains every packet already committed to the ring. + atomic_store_explicit(&capture->packetGateOpen, false, memory_order_release); OSStatus status = AudioDeviceStop(capture->deviceID, capture->ioProcID); atomic_store_explicit(&capture->running, false, memory_order_release); dispatch_semaphore_signal(capture->packetSemaphore); return status; } -void fv_core_audio_capture_destroy(FVCoreAudioCaptureRef captureRef) { +int32_t fv_core_audio_capture_destroy(FVCoreAudioCaptureRef captureRef) { FVCapture *capture = (FVCapture *) captureRef; if (capture == NULL) { - return; + return noErr; } if (atomic_load_explicit(&capture->running, memory_order_acquire)) { - (void) fv_core_audio_capture_stop(captureRef); + OSStatus stopStatus = fv_core_audio_capture_stop(captureRef); + if (stopStatus != noErr) { + return stopStatus; + } } if (capture->ioProcID != NULL) { - (void) AudioDeviceDestroyIOProcID(capture->deviceID, capture->ioProcID); + OSStatus destroyStatus = AudioDeviceDestroyIOProcID( + capture->deviceID, + capture->ioProcID + ); + if (destroyStatus != noErr) { + return destroyStatus; + } capture->ioProcID = NULL; } #if !OS_OBJECT_USE_OBJC dispatch_release(capture->packetSemaphore); #endif free(capture); + return noErr; } bool fv_core_audio_capture_wait( @@ -453,7 +516,8 @@ bool fv_core_audio_capture_peek( FVCoreAudioPacket *packet ) { FVCapture *capture = (FVCapture *) captureRef; - if (capture == NULL || packet == NULL) { + if (capture == NULL || packet == NULL || + atomic_load_explicit(&capture->formatDirty, memory_order_acquire)) { return false; } const uint64_t readIndex = @@ -515,6 +579,47 @@ bool fv_core_audio_capture_is_running(FVCoreAudioCaptureRef captureRef) { atomic_load_explicit(&capture->running, memory_order_acquire); } +void fv_core_audio_capture_mark_format_dirty(FVCoreAudioCaptureRef captureRef) { + FVCapture *capture = (FVCapture *) captureRef; + if (capture == NULL) { + return; + } + atomic_store_explicit(&capture->formatDirty, true, memory_order_release); + atomic_store_explicit(&capture->packetGateOpen, false, memory_order_release); +} + +bool fv_core_audio_capture_open_packet_gate_if_clean( + FVCoreAudioCaptureRef captureRef +) { + FVCapture *capture = (FVCapture *) captureRef; + if (capture == NULL || + !atomic_load_explicit(&capture->running, memory_order_acquire) || + atomic_load_explicit(&capture->formatDirty, memory_order_acquire)) { + return false; + } + + atomic_store_explicit(&capture->packetGateOpen, true, memory_order_release); + if (atomic_load_explicit(&capture->formatDirty, memory_order_acquire)) { + atomic_store_explicit(&capture->packetGateOpen, false, memory_order_release); + return false; + } + return true; +} + +bool fv_core_audio_capture_copy_stream_format( + FVCoreAudioCaptureRef captureRef, + AudioStreamID *streamID, + AudioStreamBasicDescription *format +) { + const FVCapture *capture = (const FVCapture *) captureRef; + if (capture == NULL || streamID == NULL || format == NULL) { + return false; + } + *streamID = capture->streamID; + *format = capture->format; + return true; +} + double fv_core_audio_capture_sample_rate(FVCoreAudioCaptureRef captureRef) { const FVCapture *capture = (const FVCapture *) captureRef; return capture == NULL ? 0.0 : capture->format.mSampleRate; diff --git a/Sources/CoreAudioCaptureSupport/include/CoreAudioCaptureSupport.h b/Sources/CoreAudioCaptureSupport/include/CoreAudioCaptureSupport.h index a8c3dd72..e9e1fc7f 100644 --- a/Sources/CoreAudioCaptureSupport/include/CoreAudioCaptureSupport.h +++ b/Sources/CoreAudioCaptureSupport/include/CoreAudioCaptureSupport.h @@ -40,7 +40,7 @@ int32_t fv_core_audio_capture_create( int32_t fv_core_audio_capture_start(FVCoreAudioCaptureRef capture); int32_t fv_core_audio_capture_stop(FVCoreAudioCaptureRef capture); -void fv_core_audio_capture_destroy(FVCoreAudioCaptureRef capture); +int32_t fv_core_audio_capture_destroy(FVCoreAudioCaptureRef capture); /// Waits until the realtime producer publishes a packet or capture stops. /// Returns true when the consumer should attempt to drain the ring. @@ -54,6 +54,13 @@ void fv_core_audio_capture_clear(FVCoreAudioCaptureRef capture); void fv_core_audio_capture_wake(FVCoreAudioCaptureRef capture); bool fv_core_audio_capture_is_running(FVCoreAudioCaptureRef capture); +void fv_core_audio_capture_mark_format_dirty(FVCoreAudioCaptureRef capture); +bool fv_core_audio_capture_open_packet_gate_if_clean(FVCoreAudioCaptureRef capture); +bool fv_core_audio_capture_copy_stream_format( + FVCoreAudioCaptureRef capture, + AudioStreamID *streamID, + AudioStreamBasicDescription *format +); double fv_core_audio_capture_sample_rate(FVCoreAudioCaptureRef capture); uint32_t fv_core_audio_capture_buffer_frame_size(FVCoreAudioCaptureRef capture); uint64_t fv_core_audio_capture_dropped_packet_count(FVCoreAudioCaptureRef capture); diff --git a/Sources/Fluid/ContentView.swift b/Sources/Fluid/ContentView.swift index 5dd4b03e..cabbaac6 100644 --- a/Sources/Fluid/ContentView.swift +++ b/Sources/Fluid/ContentView.swift @@ -3059,15 +3059,25 @@ struct ContentView: View { if shouldShowDictationOverlay { self.menuBarManager.setOverlayMode(.dictation) self.menuBarManager.showRecordingOverlayImmediately() + DebugLogger.shared.benchmark( + "APP_BENCH", + message: "overlay_phase phase=connecting", + source: "AppBenchmark" + ) } Task { - if shouldPlayStartSound, !self.asr.isRunning { - TranscriptionSoundPlayer.shared.playStartSound() - } let startOutcome = await self.asr.start(onCaptureStarted: { + if shouldPlayStartSound { + TranscriptionSoundPlayer.shared.playStartSound() + } self.captureRecordingContext() self.prewarmPrivateAIDictationIfNeeded(for: .primary) + DebugLogger.shared.benchmark( + "APP_BENCH", + message: "overlay_phase phase=recording trigger=first_pcm", + source: "AppBenchmark" + ) }) if startOutcome == .failed { self.menuBarManager.hideRecordingOverlayImmediately(reason: "asr_start_failed") @@ -3304,7 +3314,7 @@ struct ContentView: View { // Set overlay mode to command self.menuBarManager.setOverlayMode(.command) - guard !self.asr.isRunning else { return } + guard !self.asr.isRunningOrStarting else { return } self.advanceOverlayLifecycle() @@ -3313,9 +3323,16 @@ struct ContentView: View { "Starting voice recording for command", source: "ContentView" ) - TranscriptionSoundPlayer.shared.playStartSound() Task { - await self.asr.start() + let startOutcome = await self.asr.start(onCaptureStarted: { + TranscriptionSoundPlayer.shared.playStartSound() + self.appBench("overlay_phase phase=recording trigger=first_pcm mode=command") + }) + if startOutcome == .failed { + self.menuBarManager.hideRecordingOverlayImmediately( + reason: "command_asr_start_failed" + ) + } } }, rewriteModeCallback: { @@ -3344,15 +3361,22 @@ struct ContentView: View { // Set flag so stopAndProcessTranscription knows to process as rewrite self.setActiveRecordingMode(.edit) - guard !self.asr.isRunning else { return } + guard !self.asr.isRunningOrStarting else { return } self.advanceOverlayLifecycle() // Start recording immediately for the edit instruction DebugLogger.shared.info("Starting voice recording for edit mode", source: "ContentView") - TranscriptionSoundPlayer.shared.playStartSound() Task { - await self.asr.start() + let startOutcome = await self.asr.start(onCaptureStarted: { + TranscriptionSoundPlayer.shared.playStartSound() + self.appBench("overlay_phase phase=recording trigger=first_pcm mode=edit") + }) + if startOutcome == .failed { + self.menuBarManager.hideRecordingOverlayImmediately( + reason: "edit_asr_start_failed" + ) + } } }, isDictateRecordingProvider: { @@ -3686,16 +3710,18 @@ extension ContentView { self.menuBarManager.setOverlayMode(.dictation) self.menuBarManager.showRecordingOverlayImmediately() self.appBench("overlay_mode_requested mode=Dictation") + self.appBench("overlay_phase phase=connecting") } Task { let asrStartStartedAt = ProcessInfo.processInfo.systemUptime DebugLogger.shared.benchmark("APP_BENCH", message: "asr_start_call", source: "AppBenchmark") - if SettingsStore.shared.enableTranscriptionSounds, !self.asr.isRunning { - TranscriptionSoundPlayer.shared.playStartSound() - } let startOutcome = await self.asr.start(onCaptureStarted: { + if SettingsStore.shared.enableTranscriptionSounds { + TranscriptionSoundPlayer.shared.playStartSound() + } self.captureRecordingContext() self.prewarmPrivateAIDictationIfNeeded(for: slot) + self.appBench("overlay_phase phase=recording trigger=first_pcm") }) if startOutcome == .failed { self.menuBarManager.hideRecordingOverlayImmediately(reason: "asr_start_failed") diff --git a/Sources/Fluid/Services/ASRService.swift b/Sources/Fluid/Services/ASRService.swift index 55fe9a2d..c0becd00 100644 --- a/Sources/Fluid/Services/ASRService.swift +++ b/Sources/Fluid/Services/ASRService.swift @@ -189,9 +189,17 @@ final class ASRService: ObservableObject { private(set) var lastDictionaryTrainingResult: ASRTranscriptionResult? private(set) var dictionaryTrainingAudioGeneration = 0 - private(set) var isStarting: Bool = false // Guard against re-entrant start() calls + @Published private(set) var isStarting: Bool = false // Guard against re-entrant start() calls private var audioCaptureStartWaiters: [CheckedContinuation] = [] - var isRunningOrStarting: Bool { self.isRunning || self.isStarting } + var isRunningOrStarting: Bool { + self.isRunning || self.isStarting + } + + private let audioCaptureReadinessGate = AudioCaptureReadinessGate() + private let firstPCMTimeoutNanoseconds: UInt64 = 2_000_000_000 + private var audioCaptureStartGeneration: UInt64 = 0 + private var audioCaptureAttemptID: UInt64 = 0 + private var isTerminating = false private var hasCompletedFirstTranscription: Bool = false // Track if model has warmed up with first transcription private var lastBoostHitTerm: String? private var hasPendingParakeetVocabularyReload: Bool = false @@ -280,9 +288,29 @@ final class ASRService: ObservableObject { } func shutdownForTermination() async { + self.isTerminating = true + let routeRecoveryShutdownStartedAt = Date().timeIntervalSince1970 + await self.cancelAudioRouteRecoveryAndWait() + self.benchmarkLog( + "route_recovery_shutdown elapsedMs=\(self.elapsedMilliseconds(since: routeRecoveryShutdownStartedAt))" + ) + if self.isStarting, self.isRunning == false { + await self.cancelPendingAudioCaptureStart(reason: "app_termination") + } if self.isRunning { await self.stopWithoutTranscription() } + let audioEngineShutdownStartedAt = Date().timeIntervalSince1970 + await self.retireAudioEngineAndWait(reason: "app_termination") + self.benchmarkLog( + "audio_engine_shutdown elapsedMs=\(self.elapsedMilliseconds(since: audioEngineShutdownStartedAt))" + ) + let directCaptureShutdownStartedAt = Date().timeIntervalSince1970 + await self.directAudioLifecycleController.shutdown(reason: "app_termination") + self.benchmarkLog( + "direct_capture_shutdown phase=\(self.directAudioLifecycleController.snapshot.phase.rawValue) " + + "elapsedMs=\(self.elapsedMilliseconds(since: directCaptureShutdownStartedAt))" + ) let preparationTask = self.ensureReadyTask let downloadTask = self.modelDownloadTask @@ -704,11 +732,30 @@ final class ASRService: ObservableObject { let requiresIdlePrewarm: Bool } - private var directAudioInput: DirectCoreAudioInput? + private lazy var directAudioLifecycleController: DirectCoreAudioLifecycleController = { + let pipeline = self.audioCapturePipeline + return DirectCoreAudioLifecycleController( + packetHandler: { samples, frameCount, sampleRate, inputHostTime, inputSampleTime in + pipeline.handle( + samples: samples, + frameCount: frameCount, + sampleRate: sampleRate, + inputHostTime: inputHostTime, + inputSampleTime: inputSampleTime + ) + }, + onFormatInvalidated: { [weak self] invalidation in + Task { @MainActor [weak self] in + await self?.handleDirectCaptureFormatInvalidation(invalidation) + } + } + ) + }() + private var activeAudioCaptureBackend: AudioCaptureBackend = .none private var hasPreparedAudioCapture: Bool { - self.directAudioInput != nil || self.hasWarmAudioEngine + self.directAudioLifecycleController.snapshot.isPrepared || self.hasWarmAudioEngine } /// Detaches the current engine from main-actor state and returns a token that @@ -719,10 +766,6 @@ final class ASRService: ObservableObject { self.audioEngineStandbyTask?.cancel() self.audioEngineStandbyTask = nil - if let directAudioInput = self.directAudioInput { - directAudioInput.invalidate() - self.directAudioInput = nil - } self.activeAudioCaptureBackend = .none if self.isEngineTapInstalled { @@ -770,23 +813,20 @@ final class ASRService: ObservableObject { } catch { return } - self?.retireWarmAudioEngineIfIdle() + await self?.retireWarmAudioEngineIfIdle() } } - private func retireWarmAudioEngineIfIdle() { + private func retireWarmAudioEngineIfIdle() async { guard self.isRunning == false, self.isStarting == false else { return } - self.coolDownAudioEngineStandby(reason: "standby_timeout") + await self.coolDownAudioEngineStandby(reason: "standby_timeout") } - private func coolDownAudioEngineStandby(reason: String) { + private func coolDownAudioEngineStandby(reason: String) async { self.audioEngineStandbyTask?.cancel() self.audioEngineStandbyTask = nil - if let directAudioInput = self.directAudioInput { - directAudioInput.invalidate() - self.directAudioInput = nil - } + await self.directAudioLifecycleController.invalidate(reason: reason) self.activeAudioCaptureBackend = .none if self.isEngineTapInstalled { @@ -806,12 +846,18 @@ final class ASRService: ObservableObject { private func prewarmConfiguredAudioCaptureIfPossible( reason: String, allowDuringRouteRecovery: Bool = false - ) { + ) async { + guard self.isTerminating == false else { + DebugLogger.shared.debug("Audio capture prewarm skipped - app is terminating", source: "ASRService") + return + } guard self.micStatus == .authorized else { DebugLogger.shared.debug("Audio engine prewarm skipped - mic not authorized", source: "ASRService") return } - guard self.isRunning == false, self.isStarting == false else { + guard self.isRunning == false, + self.isStarting == false || allowDuringRouteRecovery + else { DebugLogger.shared.debug("Audio engine prewarm skipped - capture active", source: "ASRService") return } @@ -827,7 +873,7 @@ final class ASRService: ObservableObject { let startedAt = Date().timeIntervalSince1970 if SettingsStore.shared.experimentalDirectAudioCaptureEnabled { do { - _ = try self.prepareDirectAudioInput(reason: reason) + _ = try await self.prepareDirectAudioInput(reason: reason) self.benchmarkLog("direct_audio_prewarm reason=\(reason) elapsedMs=\(self.elapsedMilliseconds(since: startedAt))") } catch { DebugLogger.shared.warning( @@ -851,10 +897,22 @@ final class ASRService: ObservableObject { AppServices.shared.microphonePreferenceCoordinator.inputDeviceForCapture() } + private func directCoreAudioDeviceSelection() -> DirectCoreAudioDeviceSelection { + if SettingsStore.shared.microphoneSelectionMode == .manual, + let preferredUID = SettingsStore.shared.preferredInputDeviceUID, + preferredUID.isEmpty == false + { + return .preferredUID(preferredUID) + } + return .systemDefault + } + /// Prepares the direct device callback without starting hardware IO. This /// keeps the default idle state privacy-friendly while removing device and /// ring allocation from the hotkey path. - private func prepareDirectAudioInput(reason: String) throws -> DirectCoreAudioInput { + private func prepareDirectAudioInput( + reason: String + ) async throws -> DirectCoreAudioLifecycleController.Snapshot { guard self.micStatus == .authorized else { throw NSError( domain: "ASRService", @@ -862,40 +920,23 @@ final class ASRService: ObservableObject { userInfo: [NSLocalizedDescriptionKey: "Microphone access is not authorized."] ) } - guard let device = self.resolvedInputDeviceForCapture() else { - throw NSError( - domain: "ASRService", - code: -1, - userInfo: [NSLocalizedDescriptionKey: "No input device is available for Direct Core Audio capture."] - ) - } - if let directAudioInput = self.directAudioInput, - directAudioInput.deviceID == device.id - { - return directAudioInput - } - - self.directAudioInput?.invalidate() - self.directAudioInput = nil - - let pipeline = self.audioCapturePipeline - let directAudioInput = try DirectCoreAudioInput(deviceID: device.id) { samples, frameCount, sampleRate, inputHostTime, inputSampleTime in - pipeline.handle( - samples: samples, - frameCount: frameCount, - sampleRate: sampleRate, - inputHostTime: inputHostTime, - inputSampleTime: inputSampleTime - ) - } - self.directAudioInput = directAudioInput + let device = try await self.directAudioLifecycleController.resolveDevice( + selection: self.directCoreAudioDeviceSelection(), + reason: "prepare:\(reason)" + ) + let snapshot = try await self.directAudioLifecycleController.prepare( + deviceID: device.id, + deviceName: device.name, + reason: reason + ) DebugLogger.shared.info( "Prepared direct Core Audio input '\(device.name)' " + - "(\(Int(directAudioInput.sampleRate.rounded()))Hz, " + - "\(directAudioInput.hardwareBufferFrameSize) frames, reason=\(reason))", + "(\(Int((snapshot.sampleRate ?? 0).rounded()))Hz, " + + "\(snapshot.bufferFrameSize ?? 0) frames, generation=\(snapshot.generation), " + + "reason=\(reason))", source: "ASRService" ) - return directAudioInput + return snapshot } private func startConfiguredAudioCapture() async throws { @@ -905,21 +946,29 @@ final class ASRService: ObservableObject { // release. await self.audioEngineRetirementDrain.waitForScheduledReleases() do { - let directAudioInput = try self.prepareDirectAudioInput(reason: "recording_start") - try directAudioInput.start() + let device = try await self.directAudioLifecycleController.resolveDevice( + selection: self.directCoreAudioDeviceSelection(), + reason: "recording_start" + ) + let snapshot = try await self.directAudioLifecycleController.start( + deviceID: device.id, + deviceName: device.name, + reason: "recording_start" + ) + try Task.checkCancellation() self.activeAudioCaptureBackend = .directCoreAudio let callbackDurationMilliseconds = - Double(directAudioInput.hardwareBufferFrameSize) / - directAudioInput.sampleRate * 1000 + Double(snapshot.bufferFrameSize ?? 0) / + max(snapshot.sampleRate ?? 0, 1) * 1000 let callbackMs = Int(callbackDurationMilliseconds.rounded()) self.benchmarkLog( - "audio_backend kind=direct_core_audio device=\(directAudioInput.deviceID) " + - "frames=\(directAudioInput.hardwareBufferFrameSize) callbackMs=\(callbackMs)" + "audio_backend kind=direct_core_audio device=\(snapshot.deviceID ?? 0) " + + "generation=\(snapshot.generation) frames=\(snapshot.bufferFrameSize ?? 0) " + + "sampleRate=\(Int((snapshot.sampleRate ?? 0).rounded())) callbackMs=\(callbackMs)" ) return } catch { - self.directAudioInput?.invalidate() - self.directAudioInput = nil + await self.directAudioLifecycleController.invalidate(reason: "recording_start_failed") DebugLogger.shared.error( "Direct Core Audio capture failed: \(error.localizedDescription)", source: "ASRService" @@ -928,10 +977,7 @@ final class ASRService: ObservableObject { } } - if let directAudioInput = self.directAudioInput { - directAudioInput.invalidate() - self.directAudioInput = nil - } + await self.directAudioLifecycleController.invalidate(reason: "av_audio_engine_selected") try await self.startAVAudioEngineCapture() } @@ -945,24 +991,27 @@ final class ASRService: ObservableObject { self.activeAudioCaptureBackend = .audioEngine } - private func stopActiveAudioCapture() { + private func stopActiveAudioCapture( + retainDirectPreparedCapture: Bool = true, + reason: String + ) async { switch self.activeAudioCaptureBackend { case .directCoreAudio: - if let directAudioInput = self.directAudioInput { - let status = directAudioInput.stop() - if status != noErr { - DebugLogger.shared.warning( - "Direct Core Audio stop returned OSStatus \(status)", - source: "ASRService" - ) - } - let droppedPackets = directAudioInput.droppedPacketCount - if droppedPackets > 0 { - DebugLogger.shared.warning( - "Direct Core Audio dropped \(droppedPackets) packet(s)", - source: "ASRService" - ) - } + let report = await self.directAudioLifecycleController.stop( + retainPrepared: retainDirectPreparedCapture, + reason: reason + ) + if report.status != noErr { + DebugLogger.shared.warning( + "Direct Core Audio stop returned OSStatus \(report.status)", + source: "ASRService" + ) + } + if report.droppedPackets > 0 { + DebugLogger.shared.warning( + "Direct Core Audio dropped \(report.droppedPackets) packet(s)", + source: "ASRService" + ) } case .audioEngine: self.removeEngineTap() @@ -1073,25 +1122,37 @@ final class ASRService: ObservableObject { /// Handles AVAudioEngine tap processing off the @MainActor to avoid touching main-actor state /// from CoreAudio's realtime callback thread. - private lazy var audioCapturePipeline: AudioCapturePipeline = .init( - audioBuffer: self.audioBuffer, - onFirstAudio: { sessionID, sampleCount, frameLength, sampleRate, acquisitionMs, elapsedMs in - DispatchQueue.main.async { - let bufferMs = Int((Double(frameLength) / sampleRate * 1000).rounded()) - DebugLogger.shared.benchmark( - "ASR_BENCH", - message: "session=\(sessionID) first_audio sampleCount=\(sampleCount) frameLength=\(frameLength) sampleRate=\(Int(sampleRate.rounded())) bufferMs=\(bufferMs) acquisitionMs=\(acquisitionMs) elapsedMs=\(elapsedMs)", - source: "ASRBenchmark" - ) - } - }, - onLevel: { [weak self] level in - // Keep Combine sends on the main queue. - DispatchQueue.main.async { [weak self] in - self?.audioLevelSubject.send(level) + private lazy var audioCapturePipeline: AudioCapturePipeline = { + let readinessGate = self.audioCaptureReadinessGate + return AudioCapturePipeline( + audioBuffer: self.audioBuffer, + onFirstAudio: { sessionID, attemptID, sampleCount, frameLength, sampleRate, acquisitionMs, elapsedMs in + Task { + await readinessGate.signalFirstPCM( + sessionID: sessionID, + attemptID: attemptID + ) + } + DispatchQueue.main.async { + let bufferMs = Int((Double(frameLength) / sampleRate * 1000).rounded()) + DebugLogger.shared.benchmark( + "ASR_BENCH", + message: "session=\(sessionID) attempt=\(attemptID) " + + "first_audio sampleCount=\(sampleCount) frameLength=\(frameLength) " + + "sampleRate=\(Int(sampleRate.rounded())) bufferMs=\(bufferMs) " + + "acquisitionMs=\(acquisitionMs) elapsedMs=\(elapsedMs)", + source: "ASRBenchmark" + ) + } + }, + onLevel: { [weak self] level in + // Keep Combine sends on the main queue. + DispatchQueue.main.async { [weak self] in + self?.audioLevelSubject.send(level) + } } - } - ) + ) + }() init() { // CRITICAL FIX: Do NOT call any framework-triggering APIs here! @@ -1118,7 +1179,6 @@ final class ASRService: ObservableObject { } deinit { - self.directAudioInput?.invalidate() if let observer = self.vocabularyChangeObserver { NotificationCenter.default.removeObserver(observer) } @@ -1212,7 +1272,9 @@ final class ASRService: ObservableObject { // Register the input callback and allocate its fixed ring now. This // does not start the device or show the microphone privacy indicator. - self.prewarmConfiguredAudioCaptureIfPossible(reason: "startup") + Task { @MainActor [weak self] in + await self?.prewarmConfiguredAudioCaptureIfPossible(reason: "startup") + } // Check if models exist on disk and auto-load if present // This is done in a Task to support async model detection (e.g., AppleSpeechAnalyzerProvider) @@ -1228,7 +1290,7 @@ final class ASRService: ObservableObject { do { try await self.ensureAsrReady() DebugLogger.shared.info("Models auto-loaded successfully on startup", source: "ASRService") - self.prewarmConfiguredAudioCaptureIfPossible(reason: "startup") + await self.prewarmConfiguredAudioCaptureIfPossible(reason: "startup") } catch { DebugLogger.shared.error("Failed to auto-load models on startup: \(error)", source: "ASRService") } @@ -1285,7 +1347,7 @@ final class ASRService: ObservableObject { self.micPermissionGranted = granted self.micStatus = granted ? .authorized : .denied if granted { - self.prewarmConfiguredAudioCaptureIfPossible(reason: "permission_granted") + await self.prewarmConfiguredAudioCaptureIfPossible(reason: "permission_granted") } } } @@ -1332,6 +1394,12 @@ final class ASRService: ObservableObject { DebugLogger.shared.warning("⚠️ START() blocked - already running (started: \(self.isRunning), starting: \(self.isStarting))", source: "ASRService") return .alreadyActive } + guard self.isTerminating == false else { + DebugLogger.shared.warning("START() blocked - app is terminating", source: "ASRService") + return .failed + } + self.audioCaptureStartGeneration &+= 1 + let startGeneration = self.audioCaptureStartGeneration self.isStarting = true defer { self.finishAudioCaptureStart() } @@ -1339,7 +1407,16 @@ final class ASRService: ObservableObject { self.didPauseMediaForThisSession = false self.audioEngineStandbyTask?.cancel() self.audioEngineStandbyTask = nil - await self.cancelAudioRouteRecoveryAndWait() + await self.waitForPendingAudioRouteRecoveryBeforeStart() + guard startGeneration == self.audioCaptureStartGeneration, + self.isTerminating == false + else { + DebugLogger.shared.debug( + "Audio capture start cancelled during route handoff generation=\(startGeneration)", + source: "ASRService" + ) + return .failed + } DebugLogger.shared.debug("🧹 Clearing buffers and state", source: "ASRService") self.finalText.removeAll() @@ -1351,6 +1428,13 @@ final class ASRService: ObservableObject { self.isProcessingChunk = false self.skipNextChunk = false self.benchmarkSessionID += 1 + let captureSessionID = self.benchmarkSessionID + self.audioCaptureAttemptID &+= 1 + var readinessAttemptID = self.audioCaptureAttemptID + await self.audioCaptureReadinessGate.arm( + sessionID: captureSessionID, + attemptID: readinessAttemptID + ) self.benchmarkRecordingStartedAt = Date().timeIntervalSince1970 self.benchmarkStreamingChunkIndex = 0 self.benchmarkCompletedStreamingChunks = 0 @@ -1360,7 +1444,8 @@ final class ASRService: ObservableObject { (self.transcriptionProvider as? FluidAudioProvider)?.resetStreamingPreviewCache() self.audioCapturePipeline.setRecordingEnabled( true, - sessionID: self.benchmarkSessionID, + sessionID: captureSessionID, + attemptID: readinessAttemptID, startHostTime: mach_absolute_time() ) self.refreshWordBoostStatus() @@ -1371,14 +1456,108 @@ final class ASRService: ObservableObject { self.isDictionaryTrainingCaptureActive = false do { - if SettingsStore.shared.microphoneSelectionMode == .manual { + if SettingsStore.shared.microphoneSelectionMode == .manual, + SettingsStore.shared.experimentalDirectAudioCaptureEnabled == false + { AppServices.shared.microphonePreferenceCoordinator.enforcePreferredInput(reason: "recording start") } - try await self.startConfiguredAudioCapture() + let maximumStartAttempts = + SettingsStore.shared.experimentalDirectAudioCaptureEnabled ? 3 : 1 + var startAttempt = 1 + while true { + let routeGenerationAtStart = self.audioRouteRecoveryGeneration + do { + try await self.startConfiguredAudioCapture() + } catch { + guard startAttempt < maximumStartAttempts, + startGeneration == self.audioCaptureStartGeneration, + self.isTerminating == false + else { + throw error + } + readinessAttemptID = try await self.prepareAudioCaptureStartRetry( + sessionID: captureSessionID, + startGeneration: startGeneration, + completedAttempt: startAttempt, + reason: "backend_start_error:\(error.localizedDescription)" + ) + startAttempt += 1 + continue + } + self.benchmarkLog( + "first_pcm_wait_begin attempt=\(startAttempt) " + + "attemptID=\(readinessAttemptID) " + + "timeoutMs=\(self.firstPCMTimeoutNanoseconds / 1_000_000) " + + "routeGeneration=\(routeGenerationAtStart) " + + "captureGeneration=\(self.directAudioLifecycleController.snapshot.generation)" + ) + let readiness = await self.audioCaptureReadinessGate.wait( + sessionID: captureSessionID, + attemptID: readinessAttemptID, + timeoutNanoseconds: self.firstPCMTimeoutNanoseconds + ) + guard startGeneration == self.audioCaptureStartGeneration, + self.isTerminating == false + else { + throw CancellationError() + } + let routeStayedStable = + routeGenerationAtStart == self.audioRouteRecoveryGeneration && + self.pendingAudioRouteRecovery == nil && + self.isRecoveringAudioRoute == false + if readiness == .ready, routeStayedStable { + self.benchmarkLog( + "first_pcm_wait_end result=ready attempt=\(startAttempt) " + + "attemptID=\(readinessAttemptID) " + + "bufferedSamples=\(self.audioBuffer.count)" + ) + break + } + + self.benchmarkLog( + "first_pcm_wait_end result=\(readiness) attempt=\(startAttempt) " + + "attemptID=\(readinessAttemptID) " + + "routeStable=\(routeStayedStable)" + ) + if readiness == .cancelled { + throw CancellationError() + } + guard startAttempt < maximumStartAttempts else { + let message: String + switch readiness { + case .timedOut: + message = "The selected microphone started but did not deliver audio." + case .formatInvalidated: + message = "The microphone format did not stabilize after reconnecting." + case .staleSession: + message = "Audio capture was replaced before the microphone became ready." + case .cancelled: + message = "Audio capture was cancelled." + case .ready: + message = "The microphone route changed before capture became stable." + } + throw NSError( + domain: "ASRService", + code: -2, + userInfo: [NSLocalizedDescriptionKey: message] + ) + } + + readinessAttemptID = try await self.prepareAudioCaptureStartRetry( + sessionID: captureSessionID, + startGeneration: startGeneration, + completedAttempt: startAttempt, + reason: "readiness_\(readiness)_routeStable_\(routeStayedStable)" + ) + startAttempt += 1 + } self.isDictionaryTrainingCaptureActive = forDictionaryTraining self.isRunning = true - DebugLogger.shared.info("✅ Audio capture running", source: "ASRService") + DebugLogger.shared.info( + "✅ Audio capture running after first PCM (session=\(captureSessionID))", + source: "ASRService" + ) onCaptureStarted?() // Pause only after capture is live so media control cannot delay the @@ -1398,11 +1577,14 @@ final class ASRService: ObservableObject { } } - // Start monitoring the currently bound device for disconnection - if let currentDevice = getCurrentlyBoundInputDevice() { + // Direct capture already owns a required device-liveness listener + // on its off-main lifecycle queue. + if self.activeAudioCaptureBackend == .audioEngine, + let currentDevice = getCurrentlyBoundInputDevice() + { DebugLogger.shared.debug("👀 Starting device monitoring for: \(currentDevice.name)", source: "ASRService") self.startMonitoringDevice(currentDevice.id) - } else { + } else if self.activeAudioCaptureBackend == .audioEngine { DebugLogger.shared.debug("ℹ️ No device to monitor", source: "ASRService") } @@ -1420,12 +1602,30 @@ final class ASRService: ObservableObject { DebugLogger.shared.info("✅ START() completed successfully", source: "ASRService") return .started } catch { + await self.audioCaptureReadinessGate.cancel( + sessionID: captureSessionID, + attemptID: readinessAttemptID + ) self.isDictionaryTrainingCaptureActive = false self.audioCapturePipeline.setRecordingEnabled(false) self.isRunning = false - self.stopActiveAudioCapture() + await self.stopActiveAudioCapture( + retainDirectPreparedCapture: false, + reason: "start_failed" + ) await self.retireAudioEngineAndWait(reason: "start_failed") - DebugLogger.shared.error("Failed to start ASR session: \(error)", source: "ASRService") + let wasCancelled = + error is CancellationError || + startGeneration != self.audioCaptureStartGeneration || + self.isTerminating + if wasCancelled { + DebugLogger.shared.info( + "Audio capture start cancelled generation=\(startGeneration)", + source: "ASRService" + ) + } else { + DebugLogger.shared.error("Failed to start ASR session: \(error)", source: "ASRService") + } // Resume media if we paused it before the failure if self.didPauseMediaForThisSession { @@ -1434,6 +1634,8 @@ final class ASRService: ObservableObject { DebugLogger.shared.info("🎵 Resumed system media after start failure", source: "ASRService") } + guard wasCancelled == false else { return .failed } + // Provide user-friendly error feedback let errorMessage: String if let nsError = error as NSError?, nsError.domain == "ASRService" { @@ -1472,6 +1674,70 @@ final class ASRService: ObservableObject { } } + private func prepareAudioCaptureStartRetry( + sessionID: Int, + startGeneration: UInt64, + completedAttempt: Int, + reason: String + ) async throws -> UInt64 { + self.audioCapturePipeline.setRecordingEnabled(false) + await self.directAudioLifecycleController.invalidate( + reason: "start_retry_attempt_\(completedAttempt):\(reason)" + ) + // Invalidation retires the packet gate and drains accepted packets, so + // this clear cannot be followed by late PCM from the failed attempt. + self.audioBuffer.clear(keepingCapacity: true) + let routeRecoveryWasPending = self.audioRouteRecoveryTask != nil + await self.waitForPendingAudioRouteRecoveryBeforeStart() + guard startGeneration == self.audioCaptureStartGeneration, + self.isTerminating == false + else { + throw CancellationError() + } + if routeRecoveryWasPending == false { + try await Task.sleep(nanoseconds: self.audioRouteRecoveryDelayNanoseconds) + } + guard startGeneration == self.audioCaptureStartGeneration, + self.isTerminating == false + else { + throw CancellationError() + } + + self.audioCaptureAttemptID &+= 1 + let attemptID = self.audioCaptureAttemptID + await self.audioCaptureReadinessGate.arm( + sessionID: sessionID, + attemptID: attemptID + ) + self.audioCapturePipeline.setRecordingEnabled( + true, + sessionID: sessionID, + attemptID: attemptID, + startHostTime: mach_absolute_time() + ) + DebugLogger.shared.info( + "Retrying direct audio startup in the same session after \(reason) " + + "(nextAttempt=\(completedAttempt + 1))", + source: "ASRService" + ) + return attemptID + } + + func cancelPendingAudioCaptureStart(reason: String) async { + guard self.isStarting, self.isRunning == false else { return } + self.audioCaptureStartGeneration &+= 1 + let cancelledSessionID = self.benchmarkSessionID + self.benchmarkLog( + "capture_start_cancel reason=\(reason) session=\(cancelledSessionID) " + + "generation=\(self.audioCaptureStartGeneration)" + ) + await self.audioCaptureReadinessGate.cancel( + sessionID: cancelledSessionID, + attemptID: self.audioCaptureAttemptID + ) + await self.waitForPendingStart() + } + private func finishAudioCaptureStart() { self.isStarting = false let waiters = self.audioCaptureStartWaiters @@ -1523,6 +1789,9 @@ final class ASRService: ObservableObject { let stopStartedAt = Date().timeIntervalSince1970 self.benchmarkLog("stop_start ageMs=\(self.elapsedMilliseconds(since: self.benchmarkRecordingStartedAt)) bufferedSamples=\(self.audioBuffer.count)") + if self.isStarting, self.isRunning == false { + await self.cancelPendingAudioCaptureStart(reason: "recording_stop") + } guard self.isRunning else { self.isDictionaryTrainingCaptureActive = false DebugLogger.shared.warning("⚠️ STOP() - not running, returning empty string", source: "ASRService") @@ -1558,14 +1827,14 @@ final class ASRService: ObservableObject { self.stopMonitoringDevice() DebugLogger.shared.debug("✅ Device monitoring stopped", source: "ASRService") - self.stopActiveAudioCapture() + await self.stopActiveAudioCapture(reason: "recording_stop") self.audioCapturePipeline.finishRecording() // A prepared direct IOProc owns only fixed memory and registration; it // does not run hardware, show the mic indicator, or hold Bluetooth in // headset mode. Keep it prepared across idle periods. The heavier // AVAudioEngine remains available when Faster Recording Start is disabled. - if self.directAudioInput != nil { + if self.directAudioLifecycleController.snapshot.isPrepared { self.audioEngineStandbyTask?.cancel() self.audioEngineStandbyTask = nil DebugLogger.shared.debug("♻️ Direct audio capture remains prepared", source: "ASRService") @@ -1578,7 +1847,11 @@ final class ASRService: ObservableObject { // (potentially slow) final transcription pass. await MainActor.run { onCaptureStopped?() } - self.benchmarkLog("audio_capture_prepared retained=\(self.directAudioInput != nil)") + let directCaptureSnapshot = self.directAudioLifecycleController.snapshot + self.benchmarkLog( + "audio_capture_prepared retained=\(directCaptureSnapshot.isPrepared) " + + "phase=\(directCaptureSnapshot.phase.rawValue) generation=\(directCaptureSnapshot.generation)" + ) // CRITICAL FIX: Await completion of streaming task AND any pending transcriptions // This prevents use-after-free crashes (EXC_BAD_ACCESS) when clearing buffer @@ -1889,7 +2162,7 @@ final class ASRService: ObservableObject { func stopWithoutTranscription() async { if self.isStarting, self.isRunning == false { - await self.waitForPendingStart() + await self.cancelPendingAudioCaptureStart(reason: "stop_without_transcription") } guard self.isRunning else { return } defer { @@ -1912,7 +2185,10 @@ final class ASRService: ObservableObject { // Stop monitoring device self.stopMonitoringDevice() - self.stopActiveAudioCapture() + await self.stopActiveAudioCapture( + retainDirectPreparedCapture: false, + reason: "stop_without_transcription" + ) DebugLogger.shared.debug("Audio capture stopped", source: "ASRService") // Cancel/no-transcription paths stay conservative and retire the engine. @@ -2407,6 +2683,10 @@ final class ASRService: ObservableObject { reason: String, requiresIdlePrewarm: Bool = false ) { + guard self.isTerminating == false else { + self.benchmarkLog("route_recovery_ignored reason=app_terminating event=\(reason)") + return + } self.audioRouteRecoveryGeneration &+= 1 let requiresPrewarmAfterRecovery = requiresIdlePrewarm || self.pendingAudioRouteRecovery?.requiresIdlePrewarm == true @@ -2418,12 +2698,14 @@ final class ASRService: ObservableObject { self.pendingAudioRouteRecovery = request self.audioLevelSubject.send(0.0) - if self.isRunning { + if self.isRunning || self.isStarting { // Stop accepting samples immediately, but do not touch AVAudioEngine // until Core Audio has been quiet for the debounce interval. self.audioCapturePipeline.setRecordingEnabled(false) DebugLogger.shared.warning( - "Audio route changed while recording; waiting for topology quiet (\(reason), generation=\(request.generation))", + "Audio route changed during capture; waiting for topology quiet " + + "(\(reason), generation=\(request.generation), " + + "isStarting=\(self.isStarting), isRunning=\(self.isRunning))", source: "ASRService" ) } else { @@ -2469,8 +2751,37 @@ final class ASRService: ObservableObject { await self.audioEngineRetirementDrain.waitForScheduledReleases() } + /// Recording startup must let an already-scheduled route rebuild finish. + /// Cancelling it here can preserve the stale prepared generation that the + /// route event was meant to retire. + private func waitForPendingAudioRouteRecoveryBeforeStart() async { + let startedAt = Date().timeIntervalSince1970 + var waitedGenerations: [UInt64] = [] + while let task = self.audioRouteRecoveryTask { + let generation = self.audioRouteRecoveryGeneration + waitedGenerations.append(generation) + self.benchmarkLog("route_recovery_handoff_wait generation=\(generation)") + _ = await task.result + if self.pendingAudioRouteRecovery == nil, + self.isRecoveringAudioRoute == false + { + self.audioRouteRecoveryTask = nil + break + } + } + await self.audioEngineRetirementDrain.waitForScheduledReleases() + self.benchmarkLog( + "route_recovery_handoff_end generations=\(waitedGenerations) " + + "elapsedMs=\(self.elapsedMilliseconds(since: startedAt)) " + + "capturePhase=\(self.directAudioLifecycleController.snapshot.phase.rawValue)" + ) + } + private func performAudioRouteRecovery(_ request: AudioRouteRecoveryRequest) async { - guard request.generation == self.audioRouteRecoveryGeneration, Task.isCancelled == false else { return } + guard self.isTerminating == false, + request.generation == self.audioRouteRecoveryGeneration, + Task.isCancelled == false + else { return } guard self.isRecoveringAudioRoute == false else { return } self.isRecoveringAudioRoute = true @@ -2512,10 +2823,13 @@ final class ASRService: ObservableObject { reason: request.reason, requiresIdlePrewarm: true ) + await self.directAudioLifecycleController.invalidate( + reason: "idle_route_change:\(request.reason)" + ) await self.retireAudioEngineAndWait(reason: "idle_route_change:\(request.reason)") guard request.generation == self.audioRouteRecoveryGeneration, Task.isCancelled == false else { return } - self.prewarmConfiguredAudioCaptureIfPossible( + await self.prewarmConfiguredAudioCaptureIfPossible( reason: "idle_route_change", allowDuringRouteRecovery: true ) @@ -2530,20 +2844,72 @@ final class ASRService: ObservableObject { ) self.audioCapturePipeline.setRecordingEnabled(false) self.stopMonitoringDevice() - self.stopActiveAudioCapture() + await self.stopActiveAudioCapture( + retainDirectPreparedCapture: false, + reason: "active_route_recovery" + ) await self.retireAudioEngineAndWait(reason: "audio_route_recovery") guard request.generation == self.audioRouteRecoveryGeneration, Task.isCancelled == false else { return } do { + self.audioCaptureAttemptID &+= 1 + let readinessAttemptID = self.audioCaptureAttemptID + await self.audioCaptureReadinessGate.arm( + sessionID: self.benchmarkSessionID, + attemptID: readinessAttemptID + ) self.audioCapturePipeline.setRecordingEnabled( true, sessionID: self.benchmarkSessionID, + attemptID: readinessAttemptID, startHostTime: mach_absolute_time() ) try await self.startConfiguredAudioCapture() + guard request.generation == self.audioRouteRecoveryGeneration, + Task.isCancelled == false + else { + self.audioCapturePipeline.setRecordingEnabled(false) + await self.stopActiveAudioCapture( + retainDirectPreparedCapture: false, + reason: "superseded_route_recovery_start" + ) + return + } + let readiness = await self.audioCaptureReadinessGate.wait( + sessionID: self.benchmarkSessionID, + attemptID: readinessAttemptID, + timeoutNanoseconds: self.firstPCMTimeoutNanoseconds + ) + guard request.generation == self.audioRouteRecoveryGeneration, + Task.isCancelled == false + else { + self.audioCapturePipeline.setRecordingEnabled(false) + await self.stopActiveAudioCapture( + retainDirectPreparedCapture: false, + reason: "superseded_route_recovery_readiness" + ) + return + } + guard readiness == .ready else { + throw NSError( + domain: "ASRService", + code: -3, + userInfo: [ + NSLocalizedDescriptionKey: + "The replacement microphone did not deliver audio (\(readiness)).", + ] + ) + } + self.benchmarkLog( + "route_recovery_first_pcm generation=\(request.generation) " + + "attemptID=\(readinessAttemptID) " + + "bufferedSamples=\(self.audioBuffer.count)" + ) - if let currentDevice = self.getCurrentlyBoundInputDevice() { + if self.activeAudioCaptureBackend == .audioEngine, + let currentDevice = self.getCurrentlyBoundInputDevice() + { self.startMonitoringDevice(currentDevice.id) } @@ -2554,7 +2920,10 @@ final class ASRService: ObservableObject { } catch { guard request.generation == self.audioRouteRecoveryGeneration, Task.isCancelled == false else { return } self.audioCapturePipeline.setRecordingEnabled(false) - self.stopActiveAudioCapture() + await self.stopActiveAudioCapture( + retainDirectPreparedCapture: false, + reason: "active_route_recovery_failed" + ) DebugLogger.shared.error("Audio route recovery failed: \(error)", source: "ASRService") // Avoid asking stopWithoutTranscription() to await the recovery task @@ -2569,9 +2938,48 @@ final class ASRService: ObservableObject { } } + private func handleDirectCaptureFormatInvalidation( + _ invalidation: DirectCoreAudioLifecycleController.FormatInvalidation + ) async { + let captureSnapshot = self.directAudioLifecycleController.snapshot + guard invalidation.generation == captureSnapshot.generation else { + self.benchmarkLog( + "direct_format_invalidation_ignored staleGeneration=\(invalidation.generation) " + + "currentGeneration=\(captureSnapshot.generation) property=\(invalidation.reason)" + ) + return + } + let sessionID = self.benchmarkSessionID + self.audioCapturePipeline.setRecordingEnabled(false) + if self.isStarting, self.isRunning == false { + await self.audioCaptureReadinessGate.signalFormatInvalidation( + sessionID: sessionID, + attemptID: self.audioCaptureAttemptID + ) + } + self.benchmarkLog( + "direct_format_invalidation generation=\(invalidation.generation) " + + "device=\(invalidation.deviceID) property=\(invalidation.reason) " + + "wasRunning=\(invalidation.wasRunning) isStarting=\(self.isStarting)" + ) + if invalidation.reason == "audio_service_restarted" { + self.reestablishAudioHardwareListenersAfterServiceReset() + AppServices.shared.audioObserver.restartObservingAfterAudioServiceReset() + } + DebugLogger.shared.warning( + "Direct capture generation \(invalidation.generation) invalidated by " + + "\(invalidation.reason); scheduling serialized route recovery", + source: "ASRService" + ) + self.scheduleAudioRouteRecovery( + reason: "direct format changed: \(invalidation.reason)", + requiresIdlePrewarm: true + ) + } + private func handleDefaultInputChanged() { if SettingsStore.shared.microphoneSelectionMode == .manual { - if self.isRunning { + if self.isRunning || self.isStarting { AppServices.shared.microphonePreferenceCoordinator.stabilizePreferredInputAfterHardwareChange( reason: "default input changed" ) @@ -2585,7 +2993,7 @@ final class ASRService: ObservableObject { private func handleDefaultOutputChanged() { // Input-only direct capture has no output device dependency. - if self.directAudioInput != nil { + if self.directAudioLifecycleController.snapshot.isPrepared { return } @@ -2626,6 +3034,44 @@ final class ASRService: ObservableObject { private var defaultInputListenerInstalled = false private var defaultInputListenerToken: AudioObjectPropertyListenerBlock? private var defaultOutputListenerToken: AudioObjectPropertyListenerBlock? + + private func reestablishAudioHardwareListenersAfterServiceReset() { + // The reset invalidates these registrations in HAL; do not attempt to + // remove the stale tokens before installing replacements. + self.defaultInputListenerInstalled = false + self.defaultInputListenerToken = nil + self.defaultOutputListenerToken = nil + self.deviceListListenerInstalled = false + self.deviceListListenerToken = nil + self.monitoredDeviceID = nil + self.monitoredDeviceIsAliveListenerToken = nil + self.registerDefaultDeviceChangeListener() + self.registerDeviceListChangeListener() + let defaultInputReady = self.defaultInputListenerInstalled + let defaultOutputReady = self.defaultOutputListenerToken != nil + let deviceListReady = self.deviceListListenerInstalled + self.benchmarkLog( + "audio_service_restart listeners_reestablished " + + "defaultInput=\(defaultInputReady) defaultOutput=\(defaultOutputReady) " + + "deviceList=\(deviceListReady)" + ) + let logMessage = + "ASR hardware listeners after Core Audio service reset: " + + "defaultInput=\(defaultInputReady), defaultOutput=\(defaultOutputReady), " + + "deviceList=\(deviceListReady)" + if defaultInputReady, defaultOutputReady, deviceListReady { + DebugLogger.shared.warning( + "Re-registered \(logMessage)", + source: "ASRService" + ) + } else { + DebugLogger.shared.error( + "Failed to fully re-register \(logMessage)", + source: "ASRService" + ) + } + } + private func registerDefaultDeviceChangeListener() { guard self.defaultInputListenerInstalled == false || self.defaultOutputListenerToken == nil else { return } var inputAddress = AudioObjectPropertyAddress( @@ -2841,8 +3287,15 @@ final class ASRService: ObservableObject { /// Gets the currently bound input device (if determinable) private func getCurrentlyBoundInputDevice() -> AudioDevice.Device? { - if let directAudioInput = self.directAudioInput { - return AudioDevice.listInputDevices().first { $0.id == directAudioInput.deviceID } + let directSnapshot = self.directAudioLifecycleController.snapshot + if let deviceID = directSnapshot.deviceID { + return AudioDevice.Device( + id: deviceID, + uid: "", + name: directSnapshot.deviceName ?? "Direct Core Audio input", + hasInput: true, + hasOutput: false + ) } // Check if engine exists before accessing inputNode @@ -3827,13 +4280,14 @@ private extension ASRService { private final nonisolated class AudioCapturePipeline: @unchecked Sendable { private let audioBuffer: ThreadSafeAudioBuffer - private let onFirstAudio: (Int, Int, Int, Double, Int, Int) -> Void + private let onFirstAudio: (Int, UInt64, Int, Int, Double, Int, Int) -> Void private let onLevel: (CGFloat) -> Void private let lock = NSLock() private var recordingEnabled: Bool = false private var firstAudioReported: Bool = false private var recordingSessionID: Int = 0 + private var recordingAttemptID: UInt64 = 0 private var recordingStartHostTime: UInt64 = 0 private var recordingStopHostTime: UInt64? private var resampleSourceRate: Double = 0 @@ -3857,7 +4311,7 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable { init( audioBuffer: ThreadSafeAudioBuffer, - onFirstAudio: @escaping (Int, Int, Int, Double, Int, Int) -> Void, + onFirstAudio: @escaping (Int, UInt64, Int, Int, Double, Int, Int) -> Void, onLevel: @escaping (CGFloat) -> Void ) { self.audioBuffer = audioBuffer @@ -3868,12 +4322,14 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable { func setRecordingEnabled( _ enabled: Bool, sessionID: Int = 0, + attemptID: UInt64 = 0, startHostTime: UInt64 = 0 ) { self.lock.lock() if enabled { self.firstAudioReported = false self.recordingSessionID = sessionID + self.recordingAttemptID = attemptID self.recordingStartHostTime = startHostTime == 0 ? mach_absolute_time() : startHostTime self.recordingStopHostTime = nil self.resetResamplerLocked() @@ -3883,6 +4339,7 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable { if enabled == false { self.recordingEnabled = false self.recordingSessionID = 0 + self.recordingAttemptID = 0 self.recordingStartHostTime = 0 self.recordingStopHostTime = nil self.resetResamplerLocked() @@ -3966,6 +4423,7 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable { let startHostTime = self.recordingStartHostTime let stopHostTime = self.recordingStopHostTime let recordingSessionID = self.recordingSessionID + let recordingAttemptID = self.recordingAttemptID self.lock.unlock() guard let acceptedRange = Self.acceptedFrameRange( @@ -3985,7 +4443,10 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable { acceptedSamples = Array(samples[acceptedRange]) } self.lock.lock() - guard self.recordingEnabled, self.recordingSessionID == recordingSessionID else { + guard self.recordingEnabled, + self.recordingSessionID == recordingSessionID, + self.recordingAttemptID == recordingAttemptID + else { self.lock.unlock() return } @@ -4012,8 +4473,10 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable { if shouldReportFirstAudio { self.firstAudioReported = true } - self.lock.unlock() + // Keep append and first-audio attribution inside the capture lock. + // Disabling an attempt therefore returns only after every accepted + // callback has committed its PCM and queued its attempt-scoped signal. self.audioBuffer.append(mono16k) if shouldReportFirstAudio { let acceptedHostTime = Self.hostTime( @@ -4031,6 +4494,7 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable { ) self.onFirstAudio( recordingSessionID, + recordingAttemptID, Int(mono16k.count), originalFrameCount, sampleRate, @@ -4038,6 +4502,7 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable { deliveryMs ) } + self.lock.unlock() let level = self.calculateAudioLevel(mono16k) self.onLevel(level) } @@ -4109,14 +4574,13 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable { sourceSampleRate: Double ) -> [Float] { guard samples.isEmpty == false else { return [] } - if sourceSampleRate == 16_000.0 { - return samples - } - if abs(self.resampleSourceRate - sourceSampleRate) > 0.5 { self.resetResamplerLocked() self.resampleSourceRate = sourceSampleRate } + if sourceSampleRate == 16_000.0 { + return samples + } let chunkStart = Double(self.resampleSourceFrameCursor) let chunkEnd = chunkStart + Double(samples.count) diff --git a/Sources/Fluid/Services/AudioCaptureReadinessGate.swift b/Sources/Fluid/Services/AudioCaptureReadinessGate.swift new file mode 100644 index 00000000..ac024c54 --- /dev/null +++ b/Sources/Fluid/Services/AudioCaptureReadinessGate.swift @@ -0,0 +1,131 @@ +import Foundation + +nonisolated enum AudioCaptureReadinessResult: Equatable { + case ready + case cancelled + case formatInvalidated + case timedOut + case staleSession +} + +actor AudioCaptureReadinessGate { + private struct Key: Equatable { + let sessionID: Int + let attemptID: UInt64 + } + + private struct Waiter { + let id: UUID + let continuation: CheckedContinuation + } + + private var key: Key? + private var result: AudioCaptureReadinessResult? + private var waiter: Waiter? + private var timeoutTask: Task? + + func arm(sessionID: Int, attemptID: UInt64) { + self.finishCurrent(with: .cancelled) + self.key = Key(sessionID: sessionID, attemptID: attemptID) + self.result = nil + } + + func wait( + sessionID: Int, + attemptID: UInt64, + timeoutNanoseconds: UInt64 + ) async -> AudioCaptureReadinessResult { + let key = Key(sessionID: sessionID, attemptID: attemptID) + guard Task.isCancelled == false else { return .cancelled } + guard self.key == key else { return .staleSession } + if let result { + return result + } + + let waiterID = UUID() + return await withTaskCancellationHandler { + await withCheckedContinuation { continuation in + guard self.key == key else { + continuation.resume(returning: .staleSession) + return + } + if let result = self.result { + continuation.resume(returning: result) + return + } + guard self.waiter == nil else { + continuation.resume(returning: .cancelled) + return + } + + self.waiter = Waiter(id: waiterID, continuation: continuation) + self.timeoutTask?.cancel() + self.timeoutTask = Task { [weak self] in + do { + try await Task.sleep(nanoseconds: timeoutNanoseconds) + } catch { + return + } + await self?.finish( + key: key, + waiterID: waiterID, + with: .timedOut + ) + } + } + } onCancel: { + Task { + await self.finish( + key: key, + waiterID: waiterID, + with: .cancelled + ) + } + } + } + + func signalFirstPCM(sessionID: Int, attemptID: UInt64) { + self.finish( + key: Key(sessionID: sessionID, attemptID: attemptID), + with: .ready + ) + } + + func cancel(sessionID: Int, attemptID: UInt64) { + self.finish( + key: Key(sessionID: sessionID, attemptID: attemptID), + with: .cancelled + ) + } + + func signalFormatInvalidation(sessionID: Int, attemptID: UInt64) { + self.finish( + key: Key(sessionID: sessionID, attemptID: attemptID), + with: .formatInvalidated + ) + } + + private func finish( + key: Key, + waiterID: UUID? = nil, + with result: AudioCaptureReadinessResult + ) { + guard self.key == key, self.result == nil else { return } + if let waiterID, self.waiter?.id != waiterID { + return + } + self.result = result + self.timeoutTask?.cancel() + self.timeoutTask = nil + let waiter = self.waiter + self.waiter = nil + waiter?.continuation.resume(returning: result) + } + + private func finishCurrent(with result: AudioCaptureReadinessResult) { + guard let key else { return } + self.finish(key: key, with: result) + self.key = nil + self.result = nil + } +} diff --git a/Sources/Fluid/Services/AudioDeviceService.swift b/Sources/Fluid/Services/AudioDeviceService.swift index 8e899d76..9a8c8674 100644 --- a/Sources/Fluid/Services/AudioDeviceService.swift +++ b/Sources/Fluid/Services/AudioDeviceService.swift @@ -11,7 +11,7 @@ import Foundation // MARK: - Audio Device Manager -enum AudioDevice { +nonisolated enum AudioDevice { struct Device: Identifiable, Hashable { let id: AudioObjectID let uid: String @@ -230,6 +230,28 @@ final class AudioHardwareObserver: ObservableObject { self.register() } + func restartObservingAfterAudioServiceReset() { + // Core Audio discards every previously registered listener when its + // service resets, so the old tokens must not be removed or reused. + self.devicesListenerToken = nil + self.defaultInputListenerToken = nil + self.defaultOutputListenerToken = nil + self.installed = false + self.register() + self.changeTick &+= 1 + if self.installed { + DebugLogger.shared.warning( + "Re-registered audio hardware observers after Core Audio service reset", + source: "AudioHardwareObserver" + ) + } else { + DebugLogger.shared.error( + "Failed to re-register audio hardware observers after Core Audio service reset", + source: "AudioHardwareObserver" + ) + } + } + deinit { unregister() } diff --git a/Sources/Fluid/Services/DirectCoreAudioInput.swift b/Sources/Fluid/Services/DirectCoreAudioInput.swift index 3e94e4ed..7a742749 100644 --- a/Sources/Fluid/Services/DirectCoreAudioInput.swift +++ b/Sources/Fluid/Services/DirectCoreAudioInput.swift @@ -4,21 +4,388 @@ import CoreAudioCaptureSupport #endif import Foundation +nonisolated enum DirectCoreAudioDeviceSelection: Equatable { + case systemDefault + case preferredUID(String) +} + +typealias DirectCoreAudioPacketHandler = @Sendable ( + _ samples: UnsafePointer, + _ frameCount: Int, + _ sampleRate: Double, + _ inputHostTime: UInt64, + _ inputSampleTime: Int64 +) -> Void + +nonisolated struct DirectCoreAudioStreamFormatFingerprint: Equatable { + let sampleRate: Double + let formatID: AudioFormatID + let formatFlags: AudioFormatFlags + let bytesPerPacket: UInt32 + let framesPerPacket: UInt32 + let bytesPerFrame: UInt32 + let channelsPerFrame: UInt32 + let bitsPerChannel: UInt32 + + init(_ format: AudioStreamBasicDescription) { + self.sampleRate = format.mSampleRate + self.formatID = format.mFormatID + self.formatFlags = format.mFormatFlags + self.bytesPerPacket = format.mBytesPerPacket + self.framesPerPacket = format.mFramesPerPacket + self.bytesPerFrame = format.mBytesPerFrame + self.channelsPerFrame = format.mChannelsPerFrame + self.bitsPerChannel = format.mBitsPerChannel + } + + var logDescription: String { + "\(Int(self.sampleRate.rounded()))Hz/" + + "\(self.channelsPerFrame)ch/" + + "\(self.bitsPerChannel)bit/" + + "flags=0x\(String(self.formatFlags, radix: 16))/" + + "bytesPerFrame=\(self.bytesPerFrame)" + } +} + +nonisolated struct DirectCoreAudioFormatFingerprint: Equatable { + let deviceID: AudioObjectID + let streamID: AudioStreamID + let virtualFormat: DirectCoreAudioStreamFormatFingerprint + let physicalFormat: DirectCoreAudioStreamFormatFingerprint? + let inputBufferChannels: [UInt32] + let nominalSampleRate: Double + let bufferFrameSize: UInt32 + let variableBufferFrameSizeMaximum: UInt32? + let dataSourceID: UInt32? + + var logDescription: String { + let bufferLayout = self.inputBufferChannels.map(String.init).joined(separator: ",") + let physical = self.physicalFormat?.logDescription ?? "unavailable" + let dataSource = self.dataSourceID.map(String.init) ?? "unavailable" + return "device=\(self.deviceID) stream=\(self.streamID) " + + "virtual=\(self.virtualFormat.logDescription) physical=\(physical) " + + "buffers=[\(bufferLayout)] nominalHz=\(Int(self.nominalSampleRate.rounded())) " + + "bufferFrames=\(self.bufferFrameSize) " + + "variableMax=\(self.variableBufferFrameSizeMaximum.map(String.init) ?? "fixed") " + + "dataSource=\(dataSource)" + } + + static func read(deviceID: AudioObjectID) throws -> Self { + var aliveAddress = AudioObjectPropertyAddress( + mSelector: kAudioDevicePropertyDeviceIsAlive, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain + ) + let isAlive: UInt32 = try self.readScalar( + objectID: deviceID, + address: &aliveAddress, + operation: "read input device liveness" + ) + guard isAlive != 0 else { + throw Self.error( + status: kAudioHardwareBadDeviceError, + operation: "read direct Core Audio format", + detail: "device \(deviceID) is not alive" + ) + } + + var streamsAddress = AudioObjectPropertyAddress( + mSelector: kAudioDevicePropertyStreams, + mScope: kAudioObjectPropertyScopeInput, + mElement: kAudioObjectPropertyElementMain + ) + let streams: [AudioStreamID] = try self.readArray( + objectID: deviceID, + address: &streamsAddress, + operation: "read input streams" + ) + guard streams.count == 1, let streamID = streams.first else { + throw Self.error( + status: kAudioHardwareUnsupportedOperationError, + operation: "read direct Core Audio format", + detail: "expected one input stream, found \(streams.count)" + ) + } + + var virtualFormatAddress = AudioObjectPropertyAddress( + mSelector: kAudioStreamPropertyVirtualFormat, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain + ) + let virtualFormat: AudioStreamBasicDescription = try self.readScalar( + objectID: streamID, + address: &virtualFormatAddress, + operation: "read input stream virtual format" + ) + + var physicalFormatAddress = AudioObjectPropertyAddress( + mSelector: kAudioStreamPropertyPhysicalFormat, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain + ) + let physicalFormat: AudioStreamBasicDescription? = self.readOptionalScalar( + objectID: streamID, + address: &physicalFormatAddress + ) + + var nominalRateAddress = AudioObjectPropertyAddress( + mSelector: kAudioDevicePropertyNominalSampleRate, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain + ) + let nominalSampleRate: Double = try self.readScalar( + objectID: deviceID, + address: &nominalRateAddress, + operation: "read nominal input sample rate" + ) + + var bufferFrameSizeAddress = AudioObjectPropertyAddress( + mSelector: kAudioDevicePropertyBufferFrameSize, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain + ) + let bufferFrameSize: UInt32 = try self.readScalar( + objectID: deviceID, + address: &bufferFrameSizeAddress, + operation: "read input buffer frame size" + ) + + var variableBufferSizeAddress = AudioObjectPropertyAddress( + mSelector: kAudioDevicePropertyUsesVariableBufferFrameSizes, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain + ) + let variableBufferFrameSizeMaximum: UInt32? = self.readOptionalScalar( + objectID: deviceID, + address: &variableBufferSizeAddress + ) + + var streamConfigurationAddress = AudioObjectPropertyAddress( + mSelector: kAudioDevicePropertyStreamConfiguration, + mScope: kAudioObjectPropertyScopeInput, + mElement: kAudioObjectPropertyElementMain + ) + let inputBufferChannels = try self.readInputBufferChannels( + deviceID: deviceID, + address: &streamConfigurationAddress + ) + + var dataSourceAddress = AudioObjectPropertyAddress( + mSelector: kAudioDevicePropertyDataSource, + mScope: kAudioObjectPropertyScopeInput, + mElement: kAudioObjectPropertyElementMain + ) + let dataSourceID: UInt32? = self.readOptionalScalar( + objectID: deviceID, + address: &dataSourceAddress + ) + + return Self( + deviceID: deviceID, + streamID: streamID, + virtualFormat: DirectCoreAudioStreamFormatFingerprint(virtualFormat), + physicalFormat: physicalFormat.map(DirectCoreAudioStreamFormatFingerprint.init), + inputBufferChannels: inputBufferChannels, + nominalSampleRate: nominalSampleRate, + bufferFrameSize: bufferFrameSize, + variableBufferFrameSizeMaximum: variableBufferFrameSizeMaximum, + dataSourceID: dataSourceID + ) + } + + static func captured( + deviceID: AudioObjectID, + streamID: AudioStreamID, + virtualFormat: AudioStreamBasicDescription, + bufferFrameSize: UInt32 + ) throws -> Self { + let live = try self.read(deviceID: deviceID) + return Self( + deviceID: deviceID, + streamID: streamID, + virtualFormat: DirectCoreAudioStreamFormatFingerprint(virtualFormat), + physicalFormat: live.streamID == streamID ? live.physicalFormat : nil, + inputBufferChannels: live.inputBufferChannels, + nominalSampleRate: live.nominalSampleRate, + bufferFrameSize: bufferFrameSize, + variableBufferFrameSizeMaximum: live.variableBufferFrameSizeMaximum, + dataSourceID: live.dataSourceID + ) + } + + private static func readScalar( + objectID: AudioObjectID, + address: inout AudioObjectPropertyAddress, + operation: String + ) throws -> T { + let pointer = UnsafeMutablePointer.allocate(capacity: 1) + defer { pointer.deallocate() } + var size = UInt32(MemoryLayout.size) + let status = AudioObjectGetPropertyData( + objectID, + &address, + 0, + nil, + &size, + pointer + ) + guard status == noErr, size == UInt32(MemoryLayout.size) else { + throw self.error(status: status, operation: operation) + } + return pointer.move() + } + + private static func readOptionalScalar( + objectID: AudioObjectID, + address: inout AudioObjectPropertyAddress + ) -> T? { + guard AudioObjectHasProperty(objectID, &address) else { return nil } + return try? self.readScalar( + objectID: objectID, + address: &address, + operation: "read optional Core Audio property" + ) + } + + private static func readArray( + objectID: AudioObjectID, + address: inout AudioObjectPropertyAddress, + operation: String + ) throws -> [T] { + var size: UInt32 = 0 + let sizeStatus = AudioObjectGetPropertyDataSize( + objectID, + &address, + 0, + nil, + &size + ) + guard sizeStatus == noErr, size % UInt32(MemoryLayout.stride) == 0 else { + throw self.error(status: sizeStatus, operation: operation) + } + let count = Int(size) / MemoryLayout.stride + guard count > 0 else { return [] } + let storage = UnsafeMutableRawPointer.allocate( + byteCount: Int(size), + alignment: MemoryLayout.alignment + ) + defer { storage.deallocate() } + var mutableSize = size + let status = AudioObjectGetPropertyData( + objectID, + &address, + 0, + nil, + &mutableSize, + storage + ) + guard status == noErr, + mutableSize == size, + mutableSize % UInt32(MemoryLayout.stride) == 0 + else { + throw self.error( + status: status == noErr ? kAudioHardwareUnspecifiedError : status, + operation: "\(operation) without a topology size change" + ) + } + let values = storage.assumingMemoryBound(to: T.self) + return Array(UnsafeBufferPointer(start: values, count: count)) + } + + private static func readInputBufferChannels( + deviceID: AudioObjectID, + address: inout AudioObjectPropertyAddress + ) throws -> [UInt32] { + var size: UInt32 = 0 + let sizeStatus = AudioObjectGetPropertyDataSize( + deviceID, + &address, + 0, + nil, + &size + ) + guard sizeStatus == noErr, size >= MemoryLayout.size else { + throw self.error( + status: sizeStatus, + operation: "read input stream configuration size" + ) + } + + let storage = UnsafeMutableRawPointer.allocate( + byteCount: Int(size), + alignment: MemoryLayout.alignment + ) + defer { storage.deallocate() } + var mutableSize = size + let status = AudioObjectGetPropertyData( + deviceID, + &address, + 0, + nil, + &mutableSize, + storage + ) + guard status == noErr, mutableSize == size else { + throw self.error( + status: status == noErr ? kAudioHardwareUnspecifiedError : status, + operation: "read stable input stream configuration" + ) + } + let bufferCount = storage.load(as: UInt32.self) + let bufferAlignment = MemoryLayout.alignment + let firstBufferOffset = + (MemoryLayout.size + bufferAlignment - 1) & + ~(bufferAlignment - 1) + let requiredSize = + firstBufferOffset + Int(bufferCount) * MemoryLayout.stride + guard bufferCount > 0, requiredSize <= Int(mutableSize) else { + throw self.error( + status: kAudioHardwareUnspecifiedError, + operation: "validate input stream configuration" + ) + } + let buffers = UnsafeMutableAudioBufferListPointer( + storage.assumingMemoryBound(to: AudioBufferList.self) + ) + return buffers.map(\.mNumberChannels) + } + + private static func error( + status: OSStatus, + operation: String, + detail: String? = nil + ) -> NSError { + let suffix = detail.map { ": \($0)" } ?? " (OSStatus \(status))" + return NSError( + domain: NSOSStatusErrorDomain, + code: Int(status), + userInfo: [NSLocalizedDescriptionKey: "Failed to \(operation)\(suffix)."] + ) + } +} + /// Prepared, input-only Core Audio capture. /// /// `AudioDeviceIOProc` publishes native hardware-cycle PCM into a fixed SPSC /// ring in C. This Swift wrapper drains that ring away from Core Audio's /// realtime thread, so resampling, level calculation, logging, and ASR buffer /// mutation never happen in the device callback. -final class DirectCoreAudioInput { - typealias PacketHandler = @Sendable ( - _ samples: UnsafePointer, - _ frameCount: Int, - _ sampleRate: Double, - _ inputHostTime: UInt64, - _ inputSampleTime: Int64 - ) -> Void +nonisolated protocol DirectCoreAudioInputControlling: AnyObject, Sendable { + var deviceID: AudioObjectID { get } + var sampleRate: Double { get } + var hardwareBufferFrameSize: UInt32 { get } + var formatFingerprint: DirectCoreAudioFormatFingerprint { get } + var isRunning: Bool { get } + var droppedPacketCount: UInt64 { get } + + func start() throws + func stop() -> OSStatus + func markFormatDirty() + func openPacketGateIfClean() -> Bool + func invalidate() -> OSStatus +} +private final nonisolated class DirectCoreAudioInput: DirectCoreAudioInputControlling, @unchecked Sendable { private struct SendableCaptureHandle: @unchecked Sendable { let rawValue: FVCoreAudioCaptureRef } @@ -26,16 +393,18 @@ final class DirectCoreAudioInput { let deviceID: AudioObjectID let sampleRate: Double let hardwareBufferFrameSize: UInt32 + let formatFingerprint: DirectCoreAudioFormatFingerprint private var capture: FVCoreAudioCaptureRef? - private let packetHandler: PacketHandler + private var poisonedStopStatus: OSStatus? + private let packetHandler: DirectCoreAudioPacketHandler private let workerQueue = DispatchQueue( label: "com.fluidvoice.audio.direct-input-consumer", qos: .userInteractive ) private let workerGroup = DispatchGroup() - init(deviceID: AudioObjectID, packetHandler: @escaping PacketHandler) throws { + init(deviceID: AudioObjectID, packetHandler: @escaping DirectCoreAudioPacketHandler) throws { var capture: FVCoreAudioCaptureRef? let status = fv_core_audio_capture_create(deviceID, &capture) guard status == noErr, let capture else { @@ -43,14 +412,40 @@ final class DirectCoreAudioInput { } self.deviceID = deviceID + let sampleRate = fv_core_audio_capture_sample_rate(capture) + let hardwareBufferFrameSize = fv_core_audio_capture_buffer_frame_size(capture) + var streamID = AudioStreamID(kAudioObjectUnknown) + var streamFormat = AudioStreamBasicDescription() + guard fv_core_audio_capture_copy_stream_format( + capture, + &streamID, + &streamFormat + ) else { + _ = fv_core_audio_capture_destroy(capture) + throw Self.error( + status: kAudioHardwareUnspecifiedError, + operation: "read prepared direct Core Audio format" + ) + } + do { + self.formatFingerprint = try DirectCoreAudioFormatFingerprint.captured( + deviceID: deviceID, + streamID: streamID, + virtualFormat: streamFormat, + bufferFrameSize: hardwareBufferFrameSize + ) + } catch { + _ = fv_core_audio_capture_destroy(capture) + throw error + } self.capture = capture - self.sampleRate = fv_core_audio_capture_sample_rate(capture) - self.hardwareBufferFrameSize = fv_core_audio_capture_buffer_frame_size(capture) + self.sampleRate = sampleRate + self.hardwareBufferFrameSize = hardwareBufferFrameSize self.packetHandler = packetHandler } deinit { - self.invalidate() + _ = self.invalidate() } var isRunning: Bool { @@ -85,27 +480,57 @@ final class DirectCoreAudioInput { } } + func markFormatDirty() { + guard let capture else { return } + fv_core_audio_capture_mark_format_dirty(capture) + } + + func openPacketGateIfClean() -> Bool { + guard let capture else { return false } + return fv_core_audio_capture_open_packet_gate_if_clean(capture) + } + /// Stops hardware IO and synchronously drains every packet already published /// by the realtime callback before returning. @discardableResult func stop() -> OSStatus { + if let poisonedStopStatus { + return poisonedStopStatus + } guard let capture else { return noErr } let status = fv_core_audio_capture_stop(capture) fv_core_audio_capture_wake(capture) self.workerGroup.wait() + if status != noErr { + self.poisonedStopStatus = status + } return status } - func invalidate() { - guard let capture else { return } - _ = self.stop() - fv_core_audio_capture_destroy(capture) + @discardableResult + func invalidate() -> OSStatus { + guard let capture else { return self.poisonedStopStatus ?? noErr } + let stopStatus = self.stop() + guard stopStatus == noErr else { + // A failed AudioDeviceStop does not establish that HAL stopped + // invoking the IOProc. Leak the callback context intentionally + // rather than freeing memory that Core Audio may still access. + self.capture = nil + return stopStatus + } + let destroyStatus = fv_core_audio_capture_destroy(capture) + guard destroyStatus == noErr else { + self.poisonedStopStatus = destroyStatus + self.capture = nil + return destroyStatus + } self.capture = nil + return noErr } private nonisolated static func consumePackets( capture: FVCoreAudioCaptureRef, - packetHandler: PacketHandler + packetHandler: DirectCoreAudioPacketHandler ) { while true { var packet = FVCoreAudioPacket() @@ -141,3 +566,907 @@ final class DirectCoreAudioInput { ) } } + +private final nonisolated class DirectCoreAudioPacketGate: @unchecked Sendable { + private let lock = NSLock() + private var acceptingPackets = true + + func withAcceptedPacket(_ body: () -> Void) { + self.lock.lock() + guard self.acceptingPackets else { + self.lock.unlock() + return + } + body() + self.lock.unlock() + } + + func retire() { + self.lock.lock() + self.acceptingPackets = false + self.lock.unlock() + } +} + +final nonisolated class DirectCoreAudioLifecycleController: @unchecked Sendable { + enum Phase: String { + case empty + case preparing + case prepared + case starting + case running + case stopping + case invalidating + case failed + case shutDown + } + + struct Snapshot: Equatable { + let phase: Phase + let generation: UInt64 + let deviceID: AudioObjectID? + let deviceName: String? + let sampleRate: Double? + let bufferFrameSize: UInt32? + let fingerprint: DirectCoreAudioFormatFingerprint? + + var isPrepared: Bool { + switch self.phase { + case .prepared, .starting, .running, .stopping: + return true + case .empty, .preparing, .invalidating, .failed, .shutDown: + return false + } + } + } + + struct StopReport { + let status: OSStatus + let droppedPackets: UInt64 + let retainedPreparedCapture: Bool + } + + struct FormatInvalidation { + let generation: UInt64 + let deviceID: AudioObjectID + let reason: String + let wasRunning: Bool + } + + private enum LifecycleLogLevel: String { + case debug = "DEBUG" + case info = "INFO" + case warning = "WARN" + } + + typealias PacketHandler = DirectCoreAudioPacketHandler + typealias InputFactory = ( + _ deviceID: AudioObjectID, + _ packetHandler: @escaping PacketHandler + ) throws -> any DirectCoreAudioInputControlling + typealias FingerprintReader = (_ deviceID: AudioObjectID) throws + -> DirectCoreAudioFormatFingerprint + + private struct ListenerRegistration { + let objectID: AudioObjectID + var address: AudioObjectPropertyAddress + let block: AudioObjectPropertyListenerBlock + } + + private enum ListenerPolicy: Equatable { + case optional + case required + case requiredNotification + } + + private let lifecycleQueue = DispatchQueue( + label: "com.fluidvoice.audio.direct-lifecycle", + qos: .userInitiated + ) + private let listenerQueue = DispatchQueue( + label: "com.fluidvoice.audio.direct-format-listeners", + qos: .userInitiated + ) + private let snapshotLock = NSLock() + private let stoppedHardwareLock = NSLock() + private var stoppedHardwareGenerations: Set = [] + private var storedSnapshot = Snapshot( + phase: .empty, + generation: 0, + deviceID: nil, + deviceName: nil, + sampleRate: nil, + bufferFrameSize: nil, + fingerprint: nil + ) + + private let packetHandler: PacketHandler + private let inputFactory: InputFactory + private let fingerprintReader: FingerprintReader + private let onFormatInvalidated: @Sendable (FormatInvalidation) -> Void + private let installsHardwareListeners: Bool + + // lifecycleQueue-confined state + private var input: (any DirectCoreAudioInputControlling)? + private var packetGate: DirectCoreAudioPacketGate? + private var listenerRegistrations: [ListenerRegistration] = [] + private var generation: UInt64 = 0 + private var isShutDown = false + private var isPoisoned = false + private var inputDeviceName: String? + + init( + packetHandler: @escaping PacketHandler, + inputFactory: @escaping InputFactory = { deviceID, handler in + try DirectCoreAudioInput(deviceID: deviceID, packetHandler: handler) + }, + fingerprintReader: @escaping FingerprintReader = { + try DirectCoreAudioFormatFingerprint.read(deviceID: $0) + }, + installsHardwareListeners: Bool = true, + onFormatInvalidated: @escaping @Sendable (FormatInvalidation) -> Void + ) { + self.packetHandler = packetHandler + self.inputFactory = inputFactory + self.fingerprintReader = fingerprintReader + self.installsHardwareListeners = installsHardwareListeners + self.onFormatInvalidated = onFormatInvalidated + } + + deinit { + let input = self.input + let gate = self.packetGate + let registrations = self.listenerRegistrations + let listenerQueue = self.listenerQueue + let generation = self.generation + self.lifecycleQueue.async { + gate?.retire() + Self.removeListeners(registrations, queue: listenerQueue) + if registrations.isEmpty == false { + listenerQueue.sync {} + Self.log( + "Direct capture listener drain generation=\(generation) " + + "count=\(registrations.count) reason=controller_deinit", + level: .debug + ) + } + _ = input?.invalidate() + } + } + + var snapshot: Snapshot { + self.snapshotLock.lock() + defer { self.snapshotLock.unlock() } + return self.storedSnapshot + } + + func resolveDevice( + selection: DirectCoreAudioDeviceSelection, + reason: String + ) async throws -> AudioDevice.Device { + try await withCheckedThrowingContinuation { continuation in + self.lifecycleQueue.async { + let startedAt = ProcessInfo.processInfo.systemUptime + let device: AudioDevice.Device? + switch selection { + case .systemDefault: + device = AudioDevice.getDefaultInputDevice() + case let .preferredUID(uid): + device = AudioDevice.getInputDevice(byUID: uid) + } + guard let device else { + continuation.resume( + throwing: Self.error( + "No input device is available for \(selection)." + ) + ) + return + } + Self.log( + "Direct capture device resolved off-main reason=\(reason) " + + "device='\(device.name)' id=\(device.id) " + + "elapsedMs=\(Self.elapsedMilliseconds(since: startedAt))", + level: .info + ) + continuation.resume(returning: device) + } + } + } + + func prepare( + deviceID: AudioObjectID, + deviceName: String, + reason: String + ) async throws -> Snapshot { + try await withCheckedThrowingContinuation { continuation in + self.lifecycleQueue.async { + do { + try continuation.resume( + returning: self.prepareLocked( + deviceID: deviceID, + deviceName: deviceName, + reason: reason + ) + ) + } catch { + continuation.resume(throwing: error) + } + } + } + } + + func start( + deviceID: AudioObjectID, + deviceName: String, + reason: String + ) async throws -> Snapshot { + try await withCheckedThrowingContinuation { continuation in + self.lifecycleQueue.async { + do { + guard self.isShutDown == false else { + throw Self.error("Direct Core Audio lifecycle is shut down.") + } + var prepared = try self.prepareLocked( + deviceID: deviceID, + deviceName: deviceName, + reason: reason + ) + guard let input = self.input else { + throw Self.error("Direct Core Audio input was not prepared.") + } + + let validationStartedAt = ProcessInfo.processInfo.systemUptime + let currentFingerprint = try self.fingerprintReader(deviceID) + if currentFingerprint != input.formatFingerprint { + Self.log( + "Direct capture fingerprint changed before start; rebuilding " + + "generation=\(prepared.generation) old={\(input.formatFingerprint.logDescription)} " + + "new={\(currentFingerprint.logDescription)}", + level: .warning + ) + self.invalidateLocked(reason: "pre_start_fingerprint_changed") + prepared = try self.prepareLocked( + deviceID: deviceID, + deviceName: deviceName, + reason: "pre_start_fingerprint_changed" + ) + } + + guard let validatedInput = self.input else { + throw Self.error("Direct Core Audio input disappeared before start.") + } + self.publishSnapshot( + phase: .starting, + input: validatedInput, + fingerprint: validatedInput.formatFingerprint + ) + let startStartedAt = ProcessInfo.processInfo.systemUptime + Self.log( + "Direct capture start begin generation=\(self.generation) " + + "device='\(deviceName)' validationMs=" + + "\(Self.elapsedMilliseconds(since: validationStartedAt))", + level: .info + ) + try validatedInput.start() + let fingerprintAfterStart = try self.fingerprintReader(deviceID) + guard fingerprintAfterStart == validatedInput.formatFingerprint, + validatedInput.openPacketGateIfClean() + else { + throw Self.error( + "Direct Core Audio format changed while starting generation " + + "\(self.generation)." + ) + } + self.publishSnapshot( + phase: .running, + input: validatedInput, + fingerprint: validatedInput.formatFingerprint + ) + Self.log( + "Direct capture start end generation=\(self.generation) " + + "elapsedMs=\(Self.elapsedMilliseconds(since: startStartedAt)) " + + "fingerprint={\(validatedInput.formatFingerprint.logDescription)}", + level: .info + ) + continuation.resume(returning: self.snapshot) + } catch { + self.publishSnapshot( + phase: .failed, + input: self.input, + fingerprint: self.input?.formatFingerprint + ) + self.invalidateLocked(reason: "start_failed") + continuation.resume(throwing: error) + } + } + } + } + + func stop(retainPrepared: Bool, reason: String) async -> StopReport { + await withCheckedContinuation { continuation in + self.lifecycleQueue.async { + guard let input = self.input else { + continuation.resume( + returning: StopReport( + status: noErr, + droppedPackets: 0, + retainedPreparedCapture: false + ) + ) + return + } + + let startedAt = ProcessInfo.processInfo.systemUptime + self.publishSnapshot( + phase: .stopping, + input: input, + fingerprint: input.formatFingerprint + ) + Self.log( + "Direct capture stop begin generation=\(self.generation) " + + "reason=\(reason) retainPrepared=\(retainPrepared)", + level: .info + ) + let status = input.stop() + let droppedPackets = input.droppedPacketCount + if status != noErr { + self.isPoisoned = true + _ = self.invalidateLocked(reason: "stop_failed:\(reason)") + self.publishSnapshot( + phase: .failed, + input: nil, + fingerprint: nil + ) + } else if retainPrepared, self.isShutDown == false { + self.publishSnapshot( + phase: .prepared, + input: input, + fingerprint: input.formatFingerprint + ) + } else { + self.invalidateLocked(reason: reason) + } + Self.log( + "Direct capture stop end generation=\(self.generation) " + + "elapsedMs=\(Self.elapsedMilliseconds(since: startedAt)) " + + "status=\(status) droppedPackets=\(droppedPackets) " + + "retained=\(retainPrepared && self.input != nil)", + level: .info + ) + continuation.resume( + returning: StopReport( + status: status, + droppedPackets: droppedPackets, + retainedPreparedCapture: retainPrepared && self.input != nil + ) + ) + } + } + } + + func invalidate(reason: String) async { + await withCheckedContinuation { continuation in + self.lifecycleQueue.async { + _ = self.invalidateLocked(reason: reason) + continuation.resume() + } + } + } + + func shutdown(reason: String) async { + await withCheckedContinuation { continuation in + self.lifecycleQueue.async { + guard self.isShutDown == false else { + continuation.resume() + return + } + self.isShutDown = true + _ = self.invalidateLocked(reason: reason) + self.publishSnapshot(phase: .shutDown, input: nil, fingerprint: nil) + continuation.resume() + } + } + } + + private func prepareLocked( + deviceID: AudioObjectID, + deviceName: String, + reason: String + ) throws -> Snapshot { + guard self.isShutDown == false else { + throw Self.error("Direct Core Audio lifecycle is shut down.") + } + guard self.isPoisoned == false else { + throw Self.error( + "Direct Core Audio lifecycle is poisoned by a failed hardware teardown; restart the app." + ) + } + + let currentFingerprint = try self.fingerprintReader(deviceID) + if let input = self.input, + input.deviceID == deviceID, + input.formatFingerprint == currentFingerprint + { + self.inputDeviceName = deviceName + self.publishSnapshot( + phase: input.isRunning ? .running : .prepared, + input: input, + fingerprint: input.formatFingerprint + ) + Self.log( + "Direct capture prepare reuse generation=\(self.generation) " + + "reason=\(reason) fingerprint={\(currentFingerprint.logDescription)}", + level: .debug + ) + return self.snapshot + } + + if self.input != nil { + let invalidationStatus = self.invalidateLocked( + reason: "prepare_replacement:\(reason)" + ) + guard invalidationStatus == noErr, self.isPoisoned == false else { + throw Self.error( + "Direct Core Audio lifecycle is poisoned after replacement teardown failed " + + "with status \(invalidationStatus); restart the app." + ) + } + } + + self.generation &+= 1 + let generation = self.generation + self.inputDeviceName = deviceName + self.publishSnapshot( + phase: .preparing, + input: nil, + fingerprint: currentFingerprint, + deviceID: deviceID + ) + let startedAt = ProcessInfo.processInfo.systemUptime + Self.log( + "Direct capture prepare begin generation=\(generation) " + + "device='\(deviceName)' reason=\(reason) " + + "fingerprint={\(currentFingerprint.logDescription)}", + level: .info + ) + + let gate = DirectCoreAudioPacketGate() + let downstreamHandler = self.packetHandler + let input: any DirectCoreAudioInputControlling + do { + input = try self.inputFactory(deviceID) { samples, frameCount, sampleRate, inputHostTime, inputSampleTime in + gate.withAcceptedPacket { + downstreamHandler( + samples, + frameCount, + sampleRate, + inputHostTime, + inputSampleTime + ) + } + } + } catch { + gate.retire() + self.publishSnapshot(phase: .failed, input: nil, fingerprint: currentFingerprint) + throw error + } + guard input.formatFingerprint == currentFingerprint else { + gate.retire() + _ = input.invalidate() + throw Self.error( + "Direct Core Audio format changed while preparing generation \(generation)." + ) + } + + self.input = input + self.packetGate = gate + do { + if self.installsHardwareListeners { + try self.installFormatListenersLocked( + fingerprint: input.formatFingerprint, + generation: generation, + input: input + ) + } + let fingerprintAfterListeners = try self.fingerprintReader(deviceID) + guard fingerprintAfterListeners == input.formatFingerprint else { + throw Self.error( + "Direct Core Audio format changed while installing listeners." + ) + } + } catch { + _ = self.invalidateLocked(reason: "prepare_validation_failed") + throw error + } + + self.publishSnapshot( + phase: .prepared, + input: input, + fingerprint: input.formatFingerprint + ) + Self.log( + "Direct capture prepare end generation=\(generation) " + + "elapsedMs=\(Self.elapsedMilliseconds(since: startedAt)) " + + "fingerprint={\(input.formatFingerprint.logDescription)}", + level: .info + ) + return self.snapshot + } + + @discardableResult + private func invalidateLocked( + reason: String, + allowStoppedHardwareReplacement: Bool = false + ) -> OSStatus { + guard self.input != nil || self.listenerRegistrations.isEmpty == false else { + self.inputDeviceName = nil + let phase: Phase = self.isShutDown ? .shutDown : (self.isPoisoned ? .failed : .empty) + self.publishSnapshot(phase: phase, input: nil, fingerprint: nil) + return noErr + } + + let startedAt = ProcessInfo.processInfo.systemUptime + let input = self.input + self.publishSnapshot( + phase: .invalidating, + input: input, + fingerprint: input?.formatFingerprint + ) + Self.log( + "Direct capture invalidate begin generation=\(self.generation) reason=\(reason)", + level: .info + ) + self.packetGate?.retire() + self.packetGate = nil + let listenerCount = self.listenerRegistrations.count + Self.removeListeners(self.listenerRegistrations, queue: self.listenerQueue) + self.listenerRegistrations.removeAll(keepingCapacity: false) + if listenerCount > 0 { + let listenerDrainStartedAt = ProcessInfo.processInfo.systemUptime + // Removal does not guarantee that an already-dispatched listener + // block has returned. Drain it before the input can destroy the C + // capture state accessed by markFormatDirty() and isRunning. + self.listenerQueue.sync {} + Self.log( + "Direct capture listener drain generation=\(self.generation) " + + "count=\(listenerCount) " + + "elapsedMs=\(Self.elapsedMilliseconds(since: listenerDrainStartedAt))", + level: .debug + ) + } + let status = input?.invalidate() ?? noErr + let stoppedHardwareWasReported = + self.consumeStoppedHardwareNotification(generation: self.generation) + let replacementAllowed = + allowStoppedHardwareReplacement || stoppedHardwareWasReported + if status != noErr { + if replacementAllowed { + Self.log( + "Direct capture stale hardware teardown generation=\(self.generation) " + + "status=\(status); callback context quarantined, replacement allowed", + level: .warning + ) + } else { + self.isPoisoned = true + Self.log( + "Direct capture teardown failed generation=\(self.generation) " + + "status=\(status); lifecycle poisoned and callback context quarantined", + level: .warning + ) + } + } + self.input = nil + self.inputDeviceName = nil + let finalPhase: Phase = + self.isShutDown ? .shutDown : (self.isPoisoned ? .failed : .empty) + self.publishSnapshot( + phase: finalPhase, + input: nil, + fingerprint: nil + ) + Self.log( + "Direct capture invalidate end generation=\(self.generation) " + + "elapsedMs=\(Self.elapsedMilliseconds(since: startedAt)) reason=\(reason)", + level: .info + ) + return status + } + + private func installFormatListenersLocked( + fingerprint: DirectCoreAudioFormatFingerprint, + generation: UInt64, + input: any DirectCoreAudioInputControlling + ) throws { + try self.addListenerLocked( + objectID: AudioObjectID(kAudioObjectSystemObject), + selector: kAudioHardwarePropertyServiceRestarted, + scope: kAudioObjectPropertyScopeGlobal, + name: "audio_service_restarted", + generation: generation, + policy: .requiredNotification, + input: input + ) + + let deviceSelectors: [ + (AudioObjectPropertySelector, AudioObjectPropertyScope, String, ListenerPolicy) + ] = [ + (kAudioDevicePropertyDeviceIsAlive, kAudioObjectPropertyScopeGlobal, "device_is_alive", .required), + (kAudioDevicePropertyDeviceHasChanged, kAudioObjectPropertyScopeGlobal, "device_changed", .optional), + (kAudioDevicePropertyStreams, kAudioObjectPropertyScopeInput, "streams", .required), + (kAudioDevicePropertyStreamConfiguration, kAudioObjectPropertyScopeInput, "stream_configuration", .required), + (kAudioDevicePropertyNominalSampleRate, kAudioObjectPropertyScopeGlobal, "nominal_sample_rate", .required), + (kAudioDevicePropertyBufferFrameSize, kAudioObjectPropertyScopeGlobal, "buffer_frame_size", .required), + (kAudioDevicePropertyUsesVariableBufferFrameSizes, kAudioObjectPropertyScopeGlobal, "variable_buffer_frame_size", .optional), + (kAudioDevicePropertyDataSource, kAudioObjectPropertyScopeInput, "data_source", .optional), + (kAudioDevicePropertyIOStoppedAbnormally, kAudioObjectPropertyScopeGlobal, "io_stopped_abnormally", .requiredNotification), + ] + for (selector, scope, name, policy) in deviceSelectors { + try self.addListenerLocked( + objectID: fingerprint.deviceID, + selector: selector, + scope: scope, + name: name, + generation: generation, + policy: policy, + input: input + ) + } + + let streamSelectors: [(AudioObjectPropertySelector, String, ListenerPolicy)] = [ + (kAudioStreamPropertyVirtualFormat, "virtual_format", .required), + (kAudioStreamPropertyPhysicalFormat, "physical_format", .optional), + (kAudioStreamPropertyIsActive, "stream_is_active", .required), + ] + for (selector, name, policy) in streamSelectors { + try self.addListenerLocked( + objectID: fingerprint.streamID, + selector: selector, + scope: kAudioObjectPropertyScopeGlobal, + name: name, + generation: generation, + policy: policy, + input: input + ) + } + } + + private func addListenerLocked( + objectID: AudioObjectID, + selector: AudioObjectPropertySelector, + scope: AudioObjectPropertyScope, + name: String, + generation: UInt64, + policy: ListenerPolicy, + input: any DirectCoreAudioInputControlling + ) throws { + var address = AudioObjectPropertyAddress( + mSelector: selector, + mScope: scope, + mElement: kAudioObjectPropertyElementMain + ) + if policy != .requiredNotification, + AudioObjectHasProperty(objectID, &address) == false + { + if policy == .required { + throw Self.error( + "Required Core Audio listener property is unavailable: \(name)." + ) + } + Self.log( + "Direct capture listener unavailable generation=\(generation) property=\(name)", + level: .debug + ) + return + } + + let invalidationHandler = self.onFormatInvalidated + let block: AudioObjectPropertyListenerBlock = { [weak self, weak input] listenerObjectID, _ in + let deviceIsAlive = + name == "device_is_alive" + ? Self.readDeviceLiveness(objectID: listenerObjectID) + : nil + let hardwareIsKnownStopped = + name == "audio_service_restarted" || + name == "io_stopped_abnormally" || + (name == "device_is_alive" && deviceIsAlive != true) + if hardwareIsKnownStopped { + self?.recordStoppedHardwareNotification(generation: generation) + } + if name == "device_is_alive" { + Self.log( + "Direct capture liveness notification generation=\(generation) " + + "device=\(listenerObjectID) " + + "alive=\(deviceIsAlive.map(String.init) ?? "unreadable") " + + "knownStopped=\(hardwareIsKnownStopped)", + level: hardwareIsKnownStopped ? .warning : .info + ) + } + input?.markFormatDirty() + invalidationHandler( + FormatInvalidation( + generation: generation, + deviceID: input?.deviceID ?? kAudioObjectUnknown, + reason: name, + wasRunning: input?.isRunning ?? false + ) + ) + self?.lifecycleQueue.async { [weak self] in + self?.handleFormatInvalidationLocked( + generation: generation, + reason: name + ) + } + } + let status = AudioObjectAddPropertyListenerBlock( + objectID, + &address, + self.listenerQueue, + block + ) + guard status == noErr else { + if policy != .optional { + throw Self.error( + "Failed to register required Core Audio listener \(name) " + + "(OSStatus \(status))." + ) + } + Self.log( + "Direct capture listener registration failed generation=\(generation) " + + "property=\(name) status=\(status)", + level: .warning + ) + return + } + self.listenerRegistrations.append( + ListenerRegistration(objectID: objectID, address: address, block: block) + ) + } + + private func handleFormatInvalidationLocked(generation: UInt64, reason: String) { + guard generation == self.generation else { + _ = self.consumeStoppedHardwareNotification(generation: generation) + return + } + + let hardwareIsKnownStopped = + self.consumeStoppedHardwareNotification(generation: generation) + guard let input = self.input else { + if hardwareIsKnownStopped, self.isPoisoned { + self.isPoisoned = false + self.publishSnapshot(phase: .empty, input: nil, fingerprint: nil) + Self.log( + "Direct capture cleared teardown poison generation=\(generation) " + + "after delayed \(reason) notification", + level: .warning + ) + } + return + } + + let wasRunning = input.isRunning + Self.log( + "Direct capture format invalidated generation=\(generation) " + + "reason=\(reason) wasRunning=\(wasRunning) " + + "fingerprint={\(input.formatFingerprint.logDescription)}", + level: .warning + ) + self.invalidateLocked( + reason: "format_listener:\(reason)", + allowStoppedHardwareReplacement: hardwareIsKnownStopped + ) + } + + private func recordStoppedHardwareNotification(generation: UInt64) { + self.stoppedHardwareLock.withLock { + self.stoppedHardwareGenerations.insert(generation) + } + } + + private func consumeStoppedHardwareNotification(generation: UInt64) -> Bool { + self.stoppedHardwareLock.withLock { + self.stoppedHardwareGenerations.remove(generation) != nil + } + } + + private static func readDeviceLiveness(objectID: AudioObjectID) -> Bool? { + var address = AudioObjectPropertyAddress( + mSelector: kAudioDevicePropertyDeviceIsAlive, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain + ) + var isAlive: UInt32 = 0 + var size = UInt32(MemoryLayout.size) + let status = AudioObjectGetPropertyData( + objectID, + &address, + 0, + nil, + &size, + &isAlive + ) + guard status == noErr, size == UInt32(MemoryLayout.size) else { + return nil + } + return isAlive != 0 + } + + #if DEBUG + func simulateStoppedHardwareNotificationForTesting( + generation: UInt64, + reason: String + ) async { + self.recordStoppedHardwareNotification(generation: generation) + await withCheckedContinuation { continuation in + self.lifecycleQueue.async { + self.handleFormatInvalidationLocked( + generation: generation, + reason: reason + ) + continuation.resume() + } + } + } + #endif + + private func publishSnapshot( + phase: Phase, + input: (any DirectCoreAudioInputControlling)?, + fingerprint: DirectCoreAudioFormatFingerprint?, + deviceID: AudioObjectID? = nil + ) { + let snapshot = Snapshot( + phase: phase, + generation: self.generation, + deviceID: input?.deviceID ?? deviceID, + deviceName: self.inputDeviceName, + sampleRate: input?.sampleRate ?? fingerprint?.virtualFormat.sampleRate, + bufferFrameSize: input?.hardwareBufferFrameSize ?? fingerprint?.bufferFrameSize, + fingerprint: fingerprint + ) + self.snapshotLock.lock() + self.storedSnapshot = snapshot + self.snapshotLock.unlock() + } + + private static func removeListeners( + _ registrations: [ListenerRegistration], + queue: DispatchQueue + ) { + for registration in registrations { + var address = registration.address + let status = AudioObjectRemovePropertyListenerBlock( + registration.objectID, + &address, + queue, + registration.block + ) + if status != noErr, status != kAudioHardwareBadObjectError { + Self.log( + "Direct capture listener removal failed object=\(registration.objectID) " + + "selector=\(address.mSelector) status=\(status)", + level: .warning + ) + } + } + } + + private static func log(_ message: String, level: LifecycleLogLevel) { + let uptime = ProcessInfo.processInfo.systemUptime + let line = "[\(level.rawValue)] [DirectCoreAudioLifecycle] " + + "t=\(String(format: "%.6f", uptime)) \(message)" + FileLogger.shared.append(line: line) + print(line) + } + + private static func elapsedMilliseconds(since startedAt: TimeInterval) -> Int { + Int(((ProcessInfo.processInfo.systemUptime - startedAt) * 1000).rounded()) + } + + private static func error(_ message: String) -> NSError { + NSError( + domain: "DirectCoreAudioLifecycle", + code: -1, + userInfo: [NSLocalizedDescriptionKey: message] + ) + } +} diff --git a/Sources/Fluid/Services/FileLogger.swift b/Sources/Fluid/Services/FileLogger.swift index ec2bd002..1430fffe 100644 --- a/Sources/Fluid/Services/FileLogger.swift +++ b/Sources/Fluid/Services/FileLogger.swift @@ -2,7 +2,7 @@ import Darwin import Foundation /// A lightweight file-backed logger that mirrors in-app debug logs to disk for diagnostics. -final class FileLogger { +final nonisolated class FileLogger: @unchecked Sendable { static let shared = FileLogger() private let queue = DispatchQueue(label: "file.logger.queue", qos: .utility) @@ -196,7 +196,7 @@ final class FileLogger { } } -private func fileLoggerSignalHandler(_ signalNumber: Int32) { +private nonisolated func fileLoggerSignalHandler(_ signalNumber: Int32) { FileLogger.writeCrashSignalLine(signalNumber) signal(signalNumber, SIG_DFL) raise(signalNumber) diff --git a/Sources/Fluid/Services/GlobalHotkeyManager.swift b/Sources/Fluid/Services/GlobalHotkeyManager.swift index 3e33cf56..0a634cfd 100644 --- a/Sources/Fluid/Services/GlobalHotkeyManager.swift +++ b/Sources/Fluid/Services/GlobalHotkeyManager.swift @@ -1822,8 +1822,8 @@ final class GlobalHotkeyManager: NSObject { @MainActor private func stopRecordingInternal() async { if self.asrService.isStarting, self.asrService.isRunning == false { - DebugLogger.shared.debug("Waiting for audio capture start before stopping", source: "GlobalHotkeyManager") - await self.asrService.waitForPendingStart() + DebugLogger.shared.debug("Cancelling pending audio capture start", source: "GlobalHotkeyManager") + await self.asrService.cancelPendingAudioCaptureStart(reason: "hotkey_released") } guard self.asrService.isRunning else { return } guard !self.asrService.isDictionaryTrainingCaptureActive else { diff --git a/Tests/FluidDictationIntegrationTests/DirectAudioReliabilityTests.swift b/Tests/FluidDictationIntegrationTests/DirectAudioReliabilityTests.swift new file mode 100644 index 00000000..21d1efb7 --- /dev/null +++ b/Tests/FluidDictationIntegrationTests/DirectAudioReliabilityTests.swift @@ -0,0 +1,688 @@ +import CoreAudio +@testable import FluidVoice_Debug +import Foundation +import XCTest + +final class DirectAudioReliabilityTests: XCTestCase { + func testReadinessGatePreservesFirstPCMThatArrivesBeforeWait() async { + let gate = AudioCaptureReadinessGate() + await gate.arm(sessionID: 41, attemptID: 1) + await gate.signalFirstPCM(sessionID: 41, attemptID: 1) + + let result = await gate.wait( + sessionID: 41, + attemptID: 1, + timeoutNanoseconds: 1_000_000 + ) + + XCTAssertEqual(result, .ready) + } + + func testReadinessGateCancelsWaitAndIgnoresStalePCM() async { + let gate = AudioCaptureReadinessGate() + await gate.arm(sessionID: 41, attemptID: 1) + let firstWait = Task { + await gate.wait( + sessionID: 41, + attemptID: 1, + timeoutNanoseconds: 1_000_000_000 + ) + } + await Task.yield() + await gate.cancel(sessionID: 41, attemptID: 1) + let firstResult = await firstWait.value + XCTAssertEqual(firstResult, .cancelled) + + await gate.arm(sessionID: 41, attemptID: 2) + await gate.signalFirstPCM(sessionID: 41, attemptID: 1) + let staleResult = await gate.wait( + sessionID: 41, + attemptID: 1, + timeoutNanoseconds: 1_000_000 + ) + XCTAssertEqual(staleResult, .staleSession) + + await gate.signalFirstPCM(sessionID: 41, attemptID: 2) + let replacementResult = await gate.wait( + sessionID: 41, + attemptID: 2, + timeoutNanoseconds: 1_000_000 + ) + XCTAssertEqual(replacementResult, .ready) + } + + func testReadinessGateTimesOutWithoutPCM() async { + let gate = AudioCaptureReadinessGate() + await gate.arm(sessionID: 41, attemptID: 1) + + let result = await gate.wait( + sessionID: 41, + attemptID: 1, + timeoutNanoseconds: 1_000_000 + ) + + XCTAssertEqual(result, .timedOut) + } + + func testReadinessWaitRespondsPromptlyToTaskCancellation() async { + let gate = AudioCaptureReadinessGate() + await gate.arm(sessionID: 41, attemptID: 1) + let waitTask = Task { + await gate.wait( + sessionID: 41, + attemptID: 1, + timeoutNanoseconds: 10_000_000_000 + ) + } + await Task.yield() + + let cancelledAt = ProcessInfo.processInfo.systemUptime + waitTask.cancel() + let result = await waitTask.value + let cancellationMilliseconds = + (ProcessInfo.processInfo.systemUptime - cancelledAt) * 1000 + + XCTAssertEqual(result, .cancelled) + XCTAssertLessThan(cancellationMilliseconds, 100) + } + + func testFingerprintMismatchInvalidatesOldCaptureBeforeStartingReplacement() async throws { + let oldFingerprint = makeFingerprint(sampleRate: 48_000, bufferFrameSize: 512) + let newFingerprint = makeFingerprint(sampleRate: 24_000, bufferFrameSize: 256) + let recorder = DirectAudioEventRecorder() + let factory = FakeDirectAudioInputFactory( + fingerprints: [oldFingerprint, newFingerprint], + recorder: recorder + ) + let fingerprintReader = ScriptedFingerprintReader( + values: [ + oldFingerprint, + oldFingerprint, + oldFingerprint, + newFingerprint, + newFingerprint, + newFingerprint, + newFingerprint, + ] + ) + let controller = DirectCoreAudioLifecycleController( + packetHandler: { _, _, _, _, _ in }, + inputFactory: { deviceID, _ in + try factory.make(deviceID: deviceID) + }, + fingerprintReader: { _ in + try fingerprintReader.read() + }, + installsHardwareListeners: false, + onFormatInvalidated: { _ in } + ) + + _ = try await controller.prepare( + deviceID: oldFingerprint.deviceID, + deviceName: "Test microphone", + reason: "test_prewarm" + ) + let running = try await controller.start( + deviceID: oldFingerprint.deviceID, + deviceName: "Test microphone", + reason: "test_start" + ) + await controller.shutdown(reason: "test_complete") + + XCTAssertEqual(running.fingerprint, newFingerprint) + XCTAssertEqual( + recorder.events, + [ + "make:48000", + "invalidate:48000", + "make:24000", + "start:24000", + "invalidate:24000", + ] + ) + XCTAssertFalse(recorder.events.contains("start:48000")) + XCTAssertTrue(recorder.executedOnlyOffMain) + XCTAssertEqual(recorder.maximumConcurrentOperations, 1) + } + + func testPrepareRollsBackWhenPostInstallFingerprintReadFails() async throws { + let fingerprint = makeFingerprint(sampleRate: 48_000, bufferFrameSize: 512) + let recorder = DirectAudioEventRecorder() + let factory = FakeDirectAudioInputFactory( + fingerprints: [fingerprint], + recorder: recorder + ) + let fingerprintReader = ScriptedFingerprintReader(values: [fingerprint]) + let controller = DirectCoreAudioLifecycleController( + packetHandler: { _, _, _, _, _ in }, + inputFactory: { deviceID, _ in + try factory.make(deviceID: deviceID) + }, + fingerprintReader: { _ in + try fingerprintReader.read() + }, + installsHardwareListeners: false, + onFormatInvalidated: { _ in } + ) + + do { + _ = try await controller.prepare( + deviceID: fingerprint.deviceID, + deviceName: "Test microphone", + reason: "test_prepare_failure" + ) + XCTFail("Expected the final fingerprint read to fail.") + } catch { + XCTAssertEqual(controller.snapshot.phase, .empty) + XCTAssertEqual( + recorder.events, + ["make:48000", "invalidate:48000"] + ) + } + } + + func testConcurrentLifecycleCommandsNeverOverlap() async throws { + let fingerprint = makeFingerprint(sampleRate: 48_000, bufferFrameSize: 512) + let recorder = DirectAudioEventRecorder(operationDelayMicroseconds: 2000) + let factory = FakeDirectAudioInputFactory( + fingerprints: [fingerprint], + recorder: recorder + ) + let controller = DirectCoreAudioLifecycleController( + packetHandler: { _, _, _, _, _ in }, + inputFactory: { deviceID, _ in + try factory.make(deviceID: deviceID) + }, + fingerprintReader: { _ in fingerprint }, + installsHardwareListeners: false, + onFormatInvalidated: { _ in } + ) + _ = try await controller.prepare( + deviceID: fingerprint.deviceID, + deviceName: "Test microphone", + reason: "test_prepare" + ) + + let commands = (0..<12).map { index in + Task { + if index.isMultiple(of: 2) { + _ = try? await controller.start( + deviceID: fingerprint.deviceID, + deviceName: "Test microphone", + reason: "concurrent_start" + ) + } else { + _ = await controller.stop( + retainPrepared: true, + reason: "concurrent_stop" + ) + } + } + } + for command in commands { + await command.value + } + await controller.shutdown(reason: "test_complete") + + XCTAssertTrue(recorder.executedOnlyOffMain) + XCTAssertEqual(recorder.maximumConcurrentOperations, 1) + } + + func testFailedStopPoisonsLifecycleAndPreventsReplacement() async throws { + let fingerprint = makeFingerprint(sampleRate: 48_000, bufferFrameSize: 512) + let recorder = DirectAudioEventRecorder() + let input = FailingStopDirectAudioInput( + fingerprint: fingerprint, + recorder: recorder + ) + let controller = DirectCoreAudioLifecycleController( + packetHandler: { _, _, _, _, _ in }, + inputFactory: { _, _ in input }, + fingerprintReader: { _ in fingerprint }, + installsHardwareListeners: false, + onFormatInvalidated: { _ in } + ) + _ = try await controller.start( + deviceID: fingerprint.deviceID, + deviceName: "Test microphone", + reason: "test_start" + ) + + let report = await controller.stop( + retainPrepared: true, + reason: "test_stop_failure" + ) + + XCTAssertNotEqual(report.status, noErr) + XCTAssertFalse(report.retainedPreparedCapture) + XCTAssertEqual(controller.snapshot.phase, .failed) + do { + _ = try await controller.prepare( + deviceID: fingerprint.deviceID, + deviceName: "Replacement microphone", + reason: "must_not_replace" + ) + XCTFail("A poisoned lifecycle must not create a replacement.") + } catch { + XCTAssertTrue(error.localizedDescription.contains("poisoned")) + } + XCTAssertEqual(recorder.events, ["start:48000", "stop_failed", "quarantine"]) + } + + func testFailedReplacementTeardownDoesNotCreateConcurrentInput() async throws { + let oldFingerprint = makeFingerprint(sampleRate: 48_000, bufferFrameSize: 512) + let newFingerprint = makeFingerprint(sampleRate: 24_000, bufferFrameSize: 256) + let recorder = DirectAudioEventRecorder() + let factory = FailingReplacementFactory( + fingerprint: oldFingerprint, + recorder: recorder + ) + let fingerprintReader = ScriptedFingerprintReader( + values: [oldFingerprint, oldFingerprint, newFingerprint] + ) + let controller = DirectCoreAudioLifecycleController( + packetHandler: { _, _, _, _, _ in }, + inputFactory: { deviceID, _ in + factory.make(deviceID: deviceID) + }, + fingerprintReader: { _ in + try fingerprintReader.read() + }, + installsHardwareListeners: false, + onFormatInvalidated: { _ in } + ) + + _ = try await controller.prepare( + deviceID: oldFingerprint.deviceID, + deviceName: "Old microphone", + reason: "test_initial_prepare" + ) + + do { + _ = try await controller.prepare( + deviceID: newFingerprint.deviceID, + deviceName: "Replacement microphone", + reason: "test_failed_replacement" + ) + XCTFail("A failed teardown must prevent replacement creation.") + } catch { + XCTAssertTrue(error.localizedDescription.contains("poisoned")) + } + + XCTAssertEqual(controller.snapshot.phase, .failed) + XCTAssertEqual(factory.creationCount, 1) + XCTAssertEqual(recorder.events, ["quarantine"]) + } + + func testDeviceRemovalFailureQuarantinesOldInputAndAllowsReplacement() async throws { + let oldFingerprint = makeFingerprint(sampleRate: 48_000, bufferFrameSize: 512) + let newFingerprint = makeFingerprint(sampleRate: 24_000, bufferFrameSize: 256) + let recorder = DirectAudioEventRecorder() + let factory = RecoveringAfterStoppedHardwareFactory( + oldFingerprint: oldFingerprint, + newFingerprint: newFingerprint, + recorder: recorder + ) + let fingerprintReader = ScriptedFingerprintReader( + values: [oldFingerprint, oldFingerprint, newFingerprint, newFingerprint] + ) + let controller = DirectCoreAudioLifecycleController( + packetHandler: { _, _, _, _, _ in }, + inputFactory: { deviceID, _ in + factory.make(deviceID: deviceID) + }, + fingerprintReader: { _ in + try fingerprintReader.read() + }, + installsHardwareListeners: false, + onFormatInvalidated: { _ in } + ) + + _ = try await controller.prepare( + deviceID: oldFingerprint.deviceID, + deviceName: "Old microphone", + reason: "test_initial_prepare" + ) + await controller.invalidate(reason: "teardown_before_notification") + XCTAssertEqual(controller.snapshot.phase, .failed) + await controller.simulateStoppedHardwareNotificationForTesting( + generation: controller.snapshot.generation, + reason: "device_is_alive" + ) + let replacement = try await controller.prepare( + deviceID: newFingerprint.deviceID, + deviceName: "Replacement microphone", + reason: "test_recovery" + ) + + XCTAssertEqual(replacement.phase, .prepared) + XCTAssertEqual(replacement.fingerprint, newFingerprint) + XCTAssertEqual(factory.creationCount, 2) + XCTAssertEqual(recorder.events, ["quarantine", "make:24000"]) + await controller.shutdown(reason: "test_complete") + } +} + +private nonisolated func makeFingerprint( + sampleRate: Double, + bufferFrameSize: UInt32 +) -> DirectCoreAudioFormatFingerprint { + DirectCoreAudioFormatFingerprint( + deviceID: 7001, + streamID: 7002, + virtualFormat: DirectCoreAudioStreamFormatFingerprint( + AudioStreamBasicDescription( + mSampleRate: sampleRate, + mFormatID: kAudioFormatLinearPCM, + mFormatFlags: kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked, + mBytesPerPacket: 4, + mFramesPerPacket: 1, + mBytesPerFrame: 4, + mChannelsPerFrame: 1, + mBitsPerChannel: 32, + mReserved: 0 + ) + ), + physicalFormat: nil, + inputBufferChannels: [1], + nominalSampleRate: sampleRate, + bufferFrameSize: bufferFrameSize, + variableBufferFrameSizeMaximum: nil, + dataSourceID: nil + ) +} + +private final nonisolated class FakeDirectAudioInputFactory: @unchecked Sendable { + private let lock = NSLock() + private var fingerprints: [DirectCoreAudioFormatFingerprint] + private let recorder: DirectAudioEventRecorder + + init( + fingerprints: [DirectCoreAudioFormatFingerprint], + recorder: DirectAudioEventRecorder + ) { + self.fingerprints = fingerprints + self.recorder = recorder + } + + func make(deviceID: AudioObjectID) throws -> any DirectCoreAudioInputControlling { + try self.lock.withLock { + guard self.fingerprints.isEmpty == false else { + throw NSError( + domain: "DirectAudioReliabilityTests", + code: -1, + userInfo: [NSLocalizedDescriptionKey: "No fake input remains."] + ) + } + let fingerprint = self.fingerprints.removeFirst() + self.recorder.record("make:\(Int(fingerprint.virtualFormat.sampleRate))") + return FakeDirectAudioInput( + deviceID: deviceID, + fingerprint: fingerprint, + recorder: self.recorder + ) + } + } +} + +private final nonisolated class ScriptedFingerprintReader: @unchecked Sendable { + private let lock = NSLock() + private var values: [DirectCoreAudioFormatFingerprint] + + init(values: [DirectCoreAudioFormatFingerprint]) { + self.values = values + } + + func read() throws -> DirectCoreAudioFormatFingerprint { + try self.lock.withLock { + guard self.values.isEmpty == false else { + throw NSError( + domain: "DirectAudioReliabilityTests", + code: -2, + userInfo: [NSLocalizedDescriptionKey: "No fingerprint remains."] + ) + } + return self.values.removeFirst() + } + } +} + +private final nonisolated class FakeDirectAudioInput: + DirectCoreAudioInputControlling, + @unchecked Sendable +{ + let deviceID: AudioObjectID + let sampleRate: Double + let hardwareBufferFrameSize: UInt32 + let formatFingerprint: DirectCoreAudioFormatFingerprint + + private let recorder: DirectAudioEventRecorder + private var running = false + + init( + deviceID: AudioObjectID, + fingerprint: DirectCoreAudioFormatFingerprint, + recorder: DirectAudioEventRecorder + ) { + self.deviceID = deviceID + self.sampleRate = fingerprint.virtualFormat.sampleRate + self.hardwareBufferFrameSize = fingerprint.bufferFrameSize + self.formatFingerprint = fingerprint + self.recorder = recorder + } + + var isRunning: Bool { + self.running + } + + var droppedPacketCount: UInt64 { + 0 + } + + func start() throws { + self.recorder.perform( + "start:\(Int(self.sampleRate))" + ) { + self.running = true + } + } + + func stop() -> OSStatus { + self.recorder.perform( + "stop:\(Int(self.sampleRate))" + ) { + self.running = false + } + return noErr + } + + func markFormatDirty() {} + + func openPacketGateIfClean() -> Bool { + true + } + + func invalidate() -> OSStatus { + self.recorder.perform( + "invalidate:\(Int(self.sampleRate))" + ) { + self.running = false + } + return noErr + } +} + +private final nonisolated class FailingStopDirectAudioInput: + DirectCoreAudioInputControlling, + @unchecked Sendable +{ + let deviceID: AudioObjectID + let sampleRate: Double + let hardwareBufferFrameSize: UInt32 + let formatFingerprint: DirectCoreAudioFormatFingerprint + + private let recorder: DirectAudioEventRecorder + private var running = false + private let failureStatus = OSStatus(-50) + + init( + fingerprint: DirectCoreAudioFormatFingerprint, + recorder: DirectAudioEventRecorder + ) { + self.deviceID = fingerprint.deviceID + self.sampleRate = fingerprint.virtualFormat.sampleRate + self.hardwareBufferFrameSize = fingerprint.bufferFrameSize + self.formatFingerprint = fingerprint + self.recorder = recorder + } + + var isRunning: Bool { + self.running + } + + var droppedPacketCount: UInt64 { + 0 + } + + func start() throws { + self.running = true + self.recorder.record("start:\(Int(self.sampleRate))") + } + + func stop() -> OSStatus { + self.running = false + self.recorder.record("stop_failed") + return self.failureStatus + } + + func markFormatDirty() {} + + func openPacketGateIfClean() -> Bool { + true + } + + func invalidate() -> OSStatus { + self.recorder.record("quarantine") + return self.failureStatus + } +} + +private final nonisolated class FailingReplacementFactory: @unchecked Sendable { + private let lock = NSLock() + private let fingerprint: DirectCoreAudioFormatFingerprint + private let recorder: DirectAudioEventRecorder + private var creations = 0 + + init( + fingerprint: DirectCoreAudioFormatFingerprint, + recorder: DirectAudioEventRecorder + ) { + self.fingerprint = fingerprint + self.recorder = recorder + } + + var creationCount: Int { + self.lock.withLock { self.creations } + } + + func make(deviceID _: AudioObjectID) -> any DirectCoreAudioInputControlling { + self.lock.withLock { + self.creations += 1 + } + return FailingStopDirectAudioInput( + fingerprint: self.fingerprint, + recorder: self.recorder + ) + } +} + +private final nonisolated class RecoveringAfterStoppedHardwareFactory: @unchecked Sendable { + private let lock = NSLock() + private let oldFingerprint: DirectCoreAudioFormatFingerprint + private let newFingerprint: DirectCoreAudioFormatFingerprint + private let recorder: DirectAudioEventRecorder + private var creations = 0 + + init( + oldFingerprint: DirectCoreAudioFormatFingerprint, + newFingerprint: DirectCoreAudioFormatFingerprint, + recorder: DirectAudioEventRecorder + ) { + self.oldFingerprint = oldFingerprint + self.newFingerprint = newFingerprint + self.recorder = recorder + } + + var creationCount: Int { + self.lock.withLock { self.creations } + } + + func make(deviceID: AudioObjectID) -> any DirectCoreAudioInputControlling { + let creation = self.lock.withLock { + self.creations += 1 + return self.creations + } + if creation == 1 { + return FailingStopDirectAudioInput( + fingerprint: self.oldFingerprint, + recorder: self.recorder + ) + } + self.recorder.record("make:\(Int(self.newFingerprint.virtualFormat.sampleRate))") + return FakeDirectAudioInput( + deviceID: deviceID, + fingerprint: self.newFingerprint, + recorder: self.recorder + ) + } +} + +private final nonisolated class DirectAudioEventRecorder: @unchecked Sendable { + private let lock = NSLock() + private let operationDelayMicroseconds: useconds_t + private var recordedEvents: [String] = [] + private var operationCount = 0 + private var maximumOperationCount = 0 + private var allOperationsOffMain = true + + init(operationDelayMicroseconds: useconds_t = 0) { + self.operationDelayMicroseconds = operationDelayMicroseconds + } + + var events: [String] { + self.lock.withLock { self.recordedEvents } + } + + var maximumConcurrentOperations: Int { + self.lock.withLock { self.maximumOperationCount } + } + + var executedOnlyOffMain: Bool { + self.lock.withLock { self.allOperationsOffMain } + } + + func record(_ event: String) { + self.lock.withLock { + self.recordedEvents.append(event) + } + } + + func perform(_ event: String, body: () -> Void) { + self.lock.withLock { + self.operationCount += 1 + self.maximumOperationCount = max( + self.maximumOperationCount, + self.operationCount + ) + self.allOperationsOffMain = self.allOperationsOffMain && Thread.isMainThread == false + self.recordedEvents.append(event) + } + if self.operationDelayMicroseconds > 0 { + usleep(self.operationDelayMicroseconds) + } + body() + self.lock.withLock { + self.operationCount -= 1 + } + } +}