Skip to content

Improve core audio resiliency #732

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
grohith327 merged 3 commits into from
Jul 31, 2026
Merged

Improve core audio resiliency #732

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9d5e87f
Enable core audio flow by default
grohith327 Jul 29, 2026
9d597a5
remove the duration mistmach check
grohith327 Jul 29, 2026
250f8bf
big update
grohith327 Jul 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions Fluid.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
272BFB5CB271489892CAE50C /* TemperatureSupportTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */; };
A62300000000000000000002 /* AudioBufferConverterTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = A62300000000000000000001 /* AudioBufferConverterTests.swift */; };
C0DE63600000000000000002 /* AudioEngineRetirementDrainTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */; };
DA7100020000000000000002 /* DirectAudioReliabilityTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */; };
7CDB0A2F2F3C4D5600FB7CAD /* dictation_fixture.wav in Resources */ = {isa = PBXBuildFile; fileRef = 7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */; };
7CDB0A302F3C4D5600FB7CAD /* XCTest.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 7CDB0A2C2F3C4D5600FB7CAD /* XCTest.framework */; };
7CE006BD2E80EBE600DDCCD6 /* AppUpdater in Frameworks */ = {isa = PBXBuildFile; productRef = 7CE006BC2E80EBE600DDCCD6 /* AppUpdater */; };
Expand Down Expand Up @@ -54,6 +55,7 @@
980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TemperatureSupportTests.swift; sourceTree = "<group>"; };
A62300000000000000000001 /* AudioBufferConverterTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioBufferConverterTests.swift; sourceTree = "<group>"; };
C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioEngineRetirementDrainTests.swift; sourceTree = "<group>"; };
DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DirectAudioReliabilityTests.swift; sourceTree = "<group>"; };
7C078D8F2E3B339200FB7CAC /* FluidVoice Debug.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "FluidVoice Debug.app"; sourceTree = BUILT_PRODUCTS_DIR; };
7C91B0022F42AA0100C0DEF0 /* HotkeyShortcutTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HotkeyShortcutTests.swift; sourceTree = "<group>"; };
7CDB0A202F3C4D5600FB7CAD /* FluidDictationIntegrationTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = FluidDictationIntegrationTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
Expand Down Expand Up @@ -132,6 +134,7 @@
980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */,
A62300000000000000000001 /* AudioBufferConverterTests.swift */,
C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */,
DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */,
);
path = FluidDictationIntegrationTests;
sourceTree = "<group>";
Expand Down Expand Up @@ -291,6 +294,7 @@
272BFB5CB271489892CAE50C /* TemperatureSupportTests.swift in Sources */,
A62300000000000000000002 /* AudioBufferConverterTests.swift in Sources */,
C0DE63600000000000000002 /* AudioEngineRetirementDrainTests.swift in Sources */,
DA7100020000000000000002 /* DirectAudioReliabilityTests.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down
135 changes: 120 additions & 15 deletions Sources/CoreAudioCaptureSupport/CoreAudioCaptureSupport.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ typedef struct {

typedef struct {
AudioObjectID deviceID;
AudioStreamID streamID;
AudioDeviceIOProcID ioProcID;
AudioStreamBasicDescription format;
uint32_t bufferFrameSize;
Expand All @@ -31,11 +32,14 @@ typedef struct {
_Atomic uint64_t readIndex;
_Atomic uint64_t droppedPackets;
_Atomic bool running;
_Atomic bool formatDirty;
_Atomic bool packetGateOpen;
FVPacketSlot slots[FV_RING_CAPACITY];
} FVCapture;

static OSStatus fv_get_input_stream_format(
AudioObjectID deviceID,
AudioStreamID *streamID,
AudioStreamBasicDescription *format
) {
AudioObjectPropertyAddress streamsAddress = {
Expand All @@ -57,18 +61,21 @@ static OSStatus fv_get_input_stream_format(
return status != noErr ? status : kAudioHardwareUnsupportedOperationError;
}

AudioStreamID streamID = kAudioObjectUnknown;
AudioStreamID resolvedStreamID = kAudioObjectUnknown;
status = AudioObjectGetPropertyData(
deviceID,
&streamsAddress,
0,
NULL,
&streamsSize,
&streamID
&resolvedStreamID
);
if (status != noErr || streamID == kAudioObjectUnknown) {
if (status != noErr || resolvedStreamID == kAudioObjectUnknown) {
return status != noErr ? status : kAudioHardwareBadObjectError;
}
if (streamID != NULL) {
*streamID = resolvedStreamID;
}

AudioObjectPropertyAddress formatAddress = {
kAudioStreamPropertyVirtualFormat,
Expand All @@ -77,7 +84,7 @@ static OSStatus fv_get_input_stream_format(
};
UInt32 formatSize = sizeof(*format);
return AudioObjectGetPropertyData(
streamID,
resolvedStreamID,
&formatAddress,
0,
NULL,
Expand All @@ -96,6 +103,35 @@ static OSStatus fv_get_buffer_frame_size(AudioObjectID deviceID, uint32_t *frame
return AudioObjectGetPropertyData(deviceID, &address, 0, NULL, &size, frameSize);
}

static OSStatus fv_get_maximum_buffer_frame_size(
AudioObjectID deviceID,
uint32_t fallbackFrameSize,
uint32_t *maximumFrameSize
) {
AudioObjectPropertyAddress address = {
kAudioDevicePropertyUsesVariableBufferFrameSizes,
kAudioObjectPropertyScopeGlobal,
kAudioObjectPropertyElementMain,
};
if (!AudioObjectHasProperty(deviceID, &address)) {
*maximumFrameSize = fallbackFrameSize;
return noErr;
}
UInt32 size = sizeof(*maximumFrameSize);
OSStatus status = AudioObjectGetPropertyData(
deviceID,
&address,
0,
NULL,
&size,
maximumFrameSize
);
if (status == noErr && *maximumFrameSize == 0) {
*maximumFrameSize = fallbackFrameSize;
}
return status;
}

static bool fv_format_is_supported(
const AudioStreamBasicDescription *format,
uint32_t *bytesPerSample
Expand Down Expand Up @@ -239,7 +275,9 @@ static OSStatus fv_io_proc(

FVCapture *capture = (FVCapture *) inClientData;
if (capture == NULL || inInputData == NULL ||
!atomic_load_explicit(&capture->running, memory_order_relaxed)) {
!atomic_load_explicit(&capture->running, memory_order_relaxed) ||
atomic_load_explicit(&capture->formatDirty, memory_order_acquire) ||
!atomic_load_explicit(&capture->packetGateOpen, memory_order_acquire)) {
return noErr;
}

Expand Down Expand Up @@ -332,17 +370,29 @@ int32_t fv_core_audio_capture_create(
}
capture->deviceID = deviceID;

OSStatus status = fv_get_input_stream_format(deviceID, &capture->format);
OSStatus status = fv_get_input_stream_format(
deviceID,
&capture->streamID,
&capture->format
);
if (status == noErr &&
!fv_format_is_supported(&capture->format, &capture->bytesPerSample)) {
status = kAudioHardwareUnsupportedOperationError;
}
if (status == noErr) {
status = fv_get_buffer_frame_size(deviceID, &capture->bufferFrameSize);
}
uint32_t maximumBufferFrameSize = capture->bufferFrameSize;
if (status == noErr) {
status = fv_get_maximum_buffer_frame_size(
deviceID,
capture->bufferFrameSize,
&maximumBufferFrameSize
);
}
if (status == noErr &&
(capture->bufferFrameSize == 0 ||
capture->bufferFrameSize > FV_MAX_FRAMES_PER_PACKET)) {
maximumBufferFrameSize > FV_MAX_FRAMES_PER_PACKET)) {
status = kAudioHardwareUnsupportedOperationError;
}
if (status != noErr) {
Expand All @@ -359,6 +409,8 @@ int32_t fv_core_audio_capture_create(
atomic_init(&capture->readIndex, 0);
atomic_init(&capture->droppedPackets, 0);
atomic_init(&capture->running, false);
atomic_init(&capture->formatDirty, false);
atomic_init(&capture->packetGateOpen, false);

status = AudioDeviceCreateIOProcID(
deviceID,
Expand Down Expand Up @@ -387,6 +439,7 @@ int32_t fv_core_audio_capture_start(FVCoreAudioCaptureRef captureRef) {
return noErr;
}

atomic_store_explicit(&capture->packetGateOpen, false, memory_order_release);
atomic_store_explicit(&capture->running, true, memory_order_release);
OSStatus status = AudioDeviceStart(capture->deviceID, capture->ioProcID);
if (status != noErr) {
Expand All @@ -406,31 +459,41 @@ int32_t fv_core_audio_capture_stop(FVCoreAudioCaptureRef captureRef) {
return noErr;
}

// Keep accepting callbacks until AudioDeviceStop has synchronized with the
// IOProc. Packets acquired before the caller's stop boundary can then be
// timestamp-trimmed by the consumer instead of being dropped here.
// Close publication before synchronizing with the IOProc. The consumer
// still drains every packet already committed to the ring.
atomic_store_explicit(&capture->packetGateOpen, false, memory_order_release);
OSStatus status = AudioDeviceStop(capture->deviceID, capture->ioProcID);
Comment on lines +462 to 465

@chatgpt-codex-connector chatgpt-codex-connector Bot Jul 29, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Keep accepting timestamped packets until the IOProc stops

When stop races the next hardware callback, closing packetGateOpen before AudioDeviceStop causes that callback to return without publishing even if its timestamped buffer contains frames acquired before the stop boundary. ASRService.stop() explicitly marks a host-time boundary and expects the synchronously drained final packet to be trimmed to it, so this can cut the last hardware buffer—and potentially the final phoneme—from recordings. Keep publication open until AudioDeviceStop has synchronized with the IOProc, then let the pipeline discard frames after the recorded boundary.

Useful? React with 👍 / 👎.

atomic_store_explicit(&capture->running, false, memory_order_release);
dispatch_semaphore_signal(capture->packetSemaphore);
return status;
}

void fv_core_audio_capture_destroy(FVCoreAudioCaptureRef captureRef) {
int32_t fv_core_audio_capture_destroy(FVCoreAudioCaptureRef captureRef) {
FVCapture *capture = (FVCapture *) captureRef;
if (capture == NULL) {
return;
return noErr;
}
if (atomic_load_explicit(&capture->running, memory_order_acquire)) {
(void) fv_core_audio_capture_stop(captureRef);
OSStatus stopStatus = fv_core_audio_capture_stop(captureRef);
if (stopStatus != noErr) {
return stopStatus;
}
}
if (capture->ioProcID != NULL) {
(void) AudioDeviceDestroyIOProcID(capture->deviceID, capture->ioProcID);
OSStatus destroyStatus = AudioDeviceDestroyIOProcID(
capture->deviceID,
capture->ioProcID
);
if (destroyStatus != noErr) {
return destroyStatus;
}
capture->ioProcID = NULL;
}
#if !OS_OBJECT_USE_OBJC
dispatch_release(capture->packetSemaphore);
#endif
free(capture);
return noErr;
}

bool fv_core_audio_capture_wait(
Expand All @@ -453,7 +516,8 @@ bool fv_core_audio_capture_peek(
FVCoreAudioPacket *packet
) {
FVCapture *capture = (FVCapture *) captureRef;
if (capture == NULL || packet == NULL) {
if (capture == NULL || packet == NULL ||
atomic_load_explicit(&capture->formatDirty, memory_order_acquire)) {
return false;
}
const uint64_t readIndex =
Expand Down Expand Up @@ -515,6 +579,47 @@ bool fv_core_audio_capture_is_running(FVCoreAudioCaptureRef captureRef) {
atomic_load_explicit(&capture->running, memory_order_acquire);
}

void fv_core_audio_capture_mark_format_dirty(FVCoreAudioCaptureRef captureRef) {
FVCapture *capture = (FVCapture *) captureRef;
if (capture == NULL) {
return;
}
atomic_store_explicit(&capture->formatDirty, true, memory_order_release);
atomic_store_explicit(&capture->packetGateOpen, false, memory_order_release);
}

bool fv_core_audio_capture_open_packet_gate_if_clean(
FVCoreAudioCaptureRef captureRef
) {
FVCapture *capture = (FVCapture *) captureRef;
if (capture == NULL ||
!atomic_load_explicit(&capture->running, memory_order_acquire) ||
atomic_load_explicit(&capture->formatDirty, memory_order_acquire)) {
return false;
}

atomic_store_explicit(&capture->packetGateOpen, true, memory_order_release);
if (atomic_load_explicit(&capture->formatDirty, memory_order_acquire)) {
atomic_store_explicit(&capture->packetGateOpen, false, memory_order_release);
return false;
}
return true;
}

bool fv_core_audio_capture_copy_stream_format(
FVCoreAudioCaptureRef captureRef,
AudioStreamID *streamID,
AudioStreamBasicDescription *format
) {
const FVCapture *capture = (const FVCapture *) captureRef;
if (capture == NULL || streamID == NULL || format == NULL) {
return false;
}
*streamID = capture->streamID;
*format = capture->format;
return true;
}

double fv_core_audio_capture_sample_rate(FVCoreAudioCaptureRef captureRef) {
const FVCapture *capture = (const FVCapture *) captureRef;
return capture == NULL ? 0.0 : capture->format.mSampleRate;
Expand Down
9 changes: 8 additions & 1 deletion Sources/CoreAudioCaptureSupport/include/CoreAudioCaptureSupport.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ int32_t fv_core_audio_capture_create(

int32_t fv_core_audio_capture_start(FVCoreAudioCaptureRef capture);
int32_t fv_core_audio_capture_stop(FVCoreAudioCaptureRef capture);
void fv_core_audio_capture_destroy(FVCoreAudioCaptureRef capture);
int32_t fv_core_audio_capture_destroy(FVCoreAudioCaptureRef capture);

/// Waits until the realtime producer publishes a packet or capture stops.
/// Returns true when the consumer should attempt to drain the ring.
Expand All @@ -54,6 +54,13 @@ void fv_core_audio_capture_clear(FVCoreAudioCaptureRef capture);
void fv_core_audio_capture_wake(FVCoreAudioCaptureRef capture);

bool fv_core_audio_capture_is_running(FVCoreAudioCaptureRef capture);
void fv_core_audio_capture_mark_format_dirty(FVCoreAudioCaptureRef capture);
bool fv_core_audio_capture_open_packet_gate_if_clean(FVCoreAudioCaptureRef capture);
bool fv_core_audio_capture_copy_stream_format(
FVCoreAudioCaptureRef capture,
AudioStreamID *streamID,
AudioStreamBasicDescription *format
);
double fv_core_audio_capture_sample_rate(FVCoreAudioCaptureRef capture);
uint32_t fv_core_audio_capture_buffer_frame_size(FVCoreAudioCaptureRef capture);
uint64_t fv_core_audio_capture_dropped_packet_count(FVCoreAudioCaptureRef capture);
Expand Down
Loading
Loading