diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index ea2fc1c04..6e35ac8bb 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -14,6 +14,7 @@ dependencies = [ "berd-voice", "bytes", "bzip2 0.6.1", + "cc", "chrono", "dirs", "doctor", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 62bc64a2b..ccc523bca 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -19,6 +19,7 @@ exclude = ["plugins/app-test-driver"] [build-dependencies] tauri-build = { version = "2", features = [] } +cc = "1" [dependencies] anyhow = "1" diff --git a/src-tauri/build.rs b/src-tauri/build.rs index 34874ccd2..69ecf7ee2 100644 --- a/src-tauri/build.rs +++ b/src-tauri/build.rs @@ -7,5 +7,18 @@ fn main() { std::env::var("BERD_APP_VERSION").unwrap_or_else(|_| env!("CARGO_PKG_VERSION").to_owned()); println!("cargo:rustc-env=BERD_BUILD_VERSION={app_version}"); + #[cfg(target_os = "macos")] + { + println!("cargo:rerun-if-changed=native/siri_tts_bridge.h"); + println!("cargo:rerun-if-changed=native/siri_tts_bridge.m"); + cc::Build::new() + .file("native/siri_tts_bridge.m") + .flag("-fobjc-arc") + .compile("berd_siri_tts_bridge"); + for framework in ["Foundation", "AVFoundation", "AudioToolbox", "CoreAudio"] { + println!("cargo:rustc-link-lib=framework={framework}"); + } + } + tauri_build::build() } diff --git a/src-tauri/crates/berd-voice/src/lib.rs b/src-tauri/crates/berd-voice/src/lib.rs index 9f25705b9..8b0abc784 100644 --- a/src-tauri/crates/berd-voice/src/lib.rs +++ b/src-tauri/crates/berd-voice/src/lib.rs @@ -3,5 +3,6 @@ mod pocket; pub use pocket::{ - load_text_to_speech, load_voice_style, PocketTts, StreamingTextChunks, VoiceStyle, SAMPLE_RATE, + load_text_to_speech, load_voice_style, take_streaming_text_chunks, PocketTts, + StreamingTextChunks, VoiceStyle, SAMPLE_RATE, }; diff --git a/src-tauri/crates/berd-voice/src/pocket.rs b/src-tauri/crates/berd-voice/src/pocket.rs index f0e357904..0d120a093 100644 --- a/src-tauri/crates/berd-voice/src/pocket.rs +++ b/src-tauri/crates/berd-voice/src/pocket.rs @@ -29,6 +29,31 @@ pub const SAMPLE_RATE: u32 = 24_000; const TTS_NUM_THREADS: usize = 1; +/// Drain stable, sentence-aware chunks from text that may still be growing. +/// +/// This backend-neutral form uses a word-count budget. It lets system speech +/// engines share Berd's first-sentence latency behavior without loading a +/// Pocket model solely to segment text. +pub fn take_streaming_text_chunks( + text: &str, + first_chunk_pending: bool, + flush: bool, +) -> Result { + let (ready, pending, first_chunk_pending) = + pocket_april::take_streaming_chunks_at_natural_boundaries( + text, + 50, + first_chunk_pending, + flush, + |candidate| Ok(candidate.split_whitespace().count()), + )?; + Ok(StreamingTextChunks { + ready, + pending, + first_chunk_pending, + }) +} + thread_local! { static ACTIVE_SYNTHESIS_ENGINES: RefCell> = const { RefCell::new(Vec::new()) }; } diff --git a/src-tauri/crates/berd-voice/src/pocket_april.rs b/src-tauri/crates/berd-voice/src/pocket_april.rs index fc47cbdfb..534bd4834 100644 --- a/src-tauri/crates/berd-voice/src/pocket_april.rs +++ b/src-tauri/crates/berd-voice/src/pocket_april.rs @@ -898,7 +898,7 @@ where Ok(chunks) } -fn take_streaming_chunks_at_natural_boundaries( +pub(crate) fn take_streaming_chunks_at_natural_boundaries( text: &str, max_tokens: usize, mut first_chunk_pending: bool, diff --git a/src-tauri/native/siri_tts_bridge.h b/src-tauri/native/siri_tts_bridge.h new file mode 100644 index 000000000..92b98e1c6 --- /dev/null +++ b/src-tauri/native/siri_tts_bridge.h @@ -0,0 +1,81 @@ +#ifndef BERD_SIRI_TTS_BRIDGE_H +#define BERD_SIRI_TTS_BRIDGE_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/// Returns a malloc-owned JSON array of Siri voices for the requested language +/// prefix. Each item contains name, language, sizeBytes, and installed. Returns +/// NULL and sets error_out on failure. +char *berd_siri_tts_catalog_json(const char *language_prefix, char **error_out); + +/// Returns the locale tags represented in the complete Siri voice catalog. +/// This does not perform per-voice daemon validation. +char *berd_siri_tts_languages_json(char **error_out); + +/// Downloads and validates one exact Siri voice. This call blocks until the +/// voice is usable or the timeout elapses. +bool berd_siri_tts_download_voice( + const char *language, + const char *voice_name, + double timeout_seconds, + char **error_out +); + +typedef bool (*BerdSiriTTSShouldStop)(void *context); +typedef void (*BerdSiriTTSPlaybackStarted)(void *context); + +/// Plays the small per-voice sample bundled with macOS. This works before the +/// full Siri voice has been downloaded. +bool berd_siri_tts_play_sample( + const char *voice_name, + const char *language, + float rate, + BerdSiriTTSShouldStop should_stop, + void *context, + char **error_out +); + +/// Opaque streaming player. Text chunks are synthesized in order while +/// previously queued audio continues playing. +void *berd_siri_tts_stream_create( + const char *language, + const char *voice_name, + float rate, + BerdSiriTTSPlaybackStarted playback_started, + void *context, + char **error_out +); +bool berd_siri_tts_stream_enqueue(void *stream, const char *text, char **error_out); +void berd_siri_tts_stream_finish(void *stream); +bool berd_siri_tts_stream_is_finished(void *stream); +uint64_t berd_siri_tts_stream_progress(void *stream); +char *berd_siri_tts_stream_copy_error(void *stream); +void berd_siri_tts_stream_cancel(void *stream); +void berd_siri_tts_stream_release(void *stream); + +/// Synthesizes one utterance through sirittsd and streams its audio packets to +/// the default macOS output. This call blocks until playback completes. +bool berd_siri_tts_speak( + const char *text, + const char *language, + const char *voice_name, + float rate, + BerdSiriTTSShouldStop should_stop, + BerdSiriTTSPlaybackStarted playback_started, + void *context, + char **error_out +); + +/// Frees strings returned by this bridge. +void berd_siri_tts_free_string(char *value); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src-tauri/native/siri_tts_bridge.m b/src-tauri/native/siri_tts_bridge.m new file mode 100644 index 000000000..e0558a71f --- /dev/null +++ b/src-tauri/native/siri_tts_bridge.m @@ -0,0 +1,1070 @@ +#import "siri_tts_bridge.h" + +#import +#import +#import +#import +#import +#import + +static NSString *const BerdSiriTTSErrorDomain = @"com.block.berd.sirittsd"; +static void *BerdSiriSpeechQueueKey = &BerdSiriSpeechQueueKey; + +@protocol BerdSiriTTSAvailabilityProtocol +- (void)downloadedVoicesMatching:(id)voice + reply:(void (^)(NSArray *voices))reply; +@end + +@protocol BerdSiriTTSSubscribeProtocol +- (void)subscribeWithVoices:(NSArray *)voices + clientId:(NSString *)clientId + accessoryId:(NSString *)accessoryId + reply:(void (^)(NSError *_Nullable error))reply; +@end + +@protocol BerdSiriTTSDaemonProtocol +- (void)synthesizeWithRequest:(id)request + reply:(void (^)(NSError *_Nullable error))reply; +- (void)cancelWithRequest:(id)request; +@end + +@protocol BerdSiriTTSSessionDelegate +- (void)didGenerateAudioWithRequestId:(uint64_t)requestId audio:(id)audio; +- (void)didGenerateWordTimingsWithRequestId:(uint64_t)requestId wordTimingInfo:(id)info; +- (void)didReportInstrumentWithRequestId:(uint64_t)requestId instrumentationMetrics:(id)metrics; +- (void)didStartSpeakingWithRequestId:(uint64_t)requestId; +- (void)pingWithReply:(void (^)(void))reply; +- (void)event:(uint64_t)event eventData:(id)data; +- (void)internalEvent:(uint64_t)event internalEventData:(id)data; +@end + +static NSError *BerdError(NSInteger code, NSString *message) { + return [NSError errorWithDomain:BerdSiriTTSErrorDomain + code:code + userInfo:@{NSLocalizedDescriptionKey : message}]; +} + +static void BerdSetError(char **errorOut, NSError *error) { + if (!errorOut || !error) return; + *errorOut = strdup(error.localizedDescription.UTF8String ?: "Siri TTS failed"); +} + +static BOOL BerdLoadFramework(NSString *path, NSError **error) { + if (dlopen(path.UTF8String, RTLD_NOW) != NULL) return YES; + if (error) { + const char *detail = dlerror(); + NSString *message = detail + ? [NSString stringWithUTF8String:detail] + : [NSString stringWithFormat:@"Could not load %@", path.lastPathComponent]; + *error = BerdError(1, message); + } + return NO; +} + +static BOOL BerdLoadSiriTTS(NSError **error) { + return BerdLoadFramework( + @"/System/Library/PrivateFrameworks/SiriTTSService.framework/SiriTTSService", + error + ); +} + +static NSSet *BerdRequestClasses(void) { + NSMutableSet *classes = [NSMutableSet setWithObjects: + NSString.class, NSNumber.class, NSData.class, NSURL.class, + NSUUID.class, NSValue.class, NSArray.class, NSDictionary.class, + NSError.class, nil]; + for (NSString *name in @[ + @"SiriTTSSynthesisRequest", @"SiriTTSSpeechRequest", + @"SiriTTSSynthesisVoice", @"SiriTTSBaseRequest", + @"SiriTTSAudibleContext", @"SiriTTSSynthesisContext", + @"SiriTTSProsodyProperties" + ]) { + Class cls = objc_getClass(name.UTF8String); + if (cls) [classes addObject:cls]; + } + return classes; +} + +static NSSet *BerdCallbackClasses(void) { + NSMutableSet *classes = [BerdRequestClasses() mutableCopy]; + for (NSString *name in @[ + @"SiriTTSAudioData", @"SiriTTSWordTimingInfo", + @"SiriTTSInstrumentationMetrics" + ]) { + Class cls = objc_getClass(name.UTF8String); + if (cls) [classes addObject:cls]; + } + return classes; +} + +static id BerdCreateVoice(NSString *language, NSString *name, NSError **error) { + if (!BerdLoadSiriTTS(error)) return nil; + Class cls = objc_getClass("SiriTTSSynthesisVoice"); + SEL selector = @selector(initWithLanguage:name:); + if (!cls || ![cls instancesRespondToSelector:selector]) { + if (error) *error = BerdError(2, @"Siri voices are unavailable on this macOS version."); + return nil; + } + typedef id (*InitializeVoice)(id, SEL, id, id); + InitializeVoice initialize = (InitializeVoice)objc_msgSend; + return initialize([cls alloc], selector, language, name); +} + +static NSDictionary *BerdVoiceDictionary(id voice) { + return @{ + @"name" : [voice valueForKey:@"name"] ?: @"", + @"language" : [voice valueForKey:@"language"] ?: @"", + @"version" : [voice valueForKey:@"version"] ?: @0, + }; +} + +typedef void (^BerdAudioHandler)( + NSData *data, + AudioStreamBasicDescription format, + UInt32 packetCount, + NSData *_Nullable packetDescriptions +); + +@interface BerdSiriSessionDelegateImpl : NSObject +@property(nonatomic, copy) BerdAudioHandler audioHandler; +@end + +@implementation BerdSiriSessionDelegateImpl +- (void)didGenerateAudioWithRequestId:(uint64_t)requestId audio:(id)audio { + (void)requestId; + NSData *data = [audio valueForKey:@"audioData"]; + if (!data.length) return; + NSNumber *packetCount = [audio valueForKey:@"packetCount"] ?: @1; + NSData *packetDescriptions = [audio valueForKey:@"packetDescriptions"]; + SEL selector = NSSelectorFromString(@"asbd"); + if (![audio respondsToSelector:selector]) return; + typedef AudioStreamBasicDescription (*GetASBD)(id, SEL); + GetASBD getASBD = (GetASBD)[audio methodForSelector:selector]; + self.audioHandler(data, getASBD(audio, selector), + packetCount.unsignedIntValue, packetDescriptions); +} +- (void)didGenerateWordTimingsWithRequestId:(uint64_t)requestId wordTimingInfo:(id)info { + (void)requestId; (void)info; +} +- (void)didReportInstrumentWithRequestId:(uint64_t)requestId instrumentationMetrics:(id)metrics { + (void)requestId; (void)metrics; +} +- (void)didStartSpeakingWithRequestId:(uint64_t)requestId { (void)requestId; } +- (void)pingWithReply:(void (^)(void))reply { if (reply) reply(); } +- (void)event:(uint64_t)event eventData:(id)data { (void)event; (void)data; } +- (void)internalEvent:(uint64_t)event internalEventData:(id)data { (void)event; (void)data; } +@end + +@interface BerdSiriSynthesisSession : NSObject +@property(nonatomic, strong) NSXPCConnection *connection; +@property(nonatomic, strong) BerdSiriSessionDelegateImpl *delegate; +@property(nonatomic, strong) id request; +@property(nonatomic, copy) void (^completion)(NSError *_Nullable error); +@property(nonatomic, assign) BOOL finished; +- (instancetype)initWithAudioHandler:(BerdAudioHandler)audioHandler; +- (void)synthesizeText:(NSString *)text language:(NSString *)language + voiceName:(NSString *)voiceName rate:(float)rate + completion:(void (^)(NSError *_Nullable error))completion; +- (void)cancel; +@end + +@implementation BerdSiriSynthesisSession +- (instancetype)initWithAudioHandler:(BerdAudioHandler)audioHandler { + self = [super init]; + if (self) { + _delegate = [BerdSiriSessionDelegateImpl new]; + _delegate.audioHandler = audioHandler; + } + return self; +} +- (void)finish:(NSError *)error { + void (^completion)(NSError *) = nil; + @synchronized (self) { + if (self.finished) return; + self.finished = YES; + completion = self.completion; + self.completion = nil; + } + if (completion) completion(error); +} +- (void)synthesizeText:(NSString *)text language:(NSString *)language + voiceName:(NSString *)voiceName rate:(float)rate + completion:(void (^)(NSError *))completion { + NSError *error = nil; + id voice = BerdCreateVoice(language, voiceName, &error); + if (!voice) { + completion(error ?: BerdError(12, @"Could not create Siri voice.")); + return; + } + Class requestClass = objc_getClass("SiriTTSSynthesisRequest"); + SEL selector = @selector(initWithText:voice:); + if (!requestClass || ![requestClass instancesRespondToSelector:selector]) { + completion(BerdError(13, @"Siri synthesis requests are unavailable.")); + return; + } + typedef id (*InitializeRequest)(id, SEL, id, id); + id request = ((InitializeRequest)objc_msgSend)([requestClass alloc], selector, text, voice); + if (rate != 1.0f && [request respondsToSelector:@selector(setRate:)]) { + ((void (*)(id, SEL, float))objc_msgSend)(request, @selector(setRate:), rate); + } + + NSXPCConnection *connection = [[NSXPCConnection alloc] + initWithMachServiceName:@"com.apple.sirittsd" options:0]; + NSXPCInterface *remote = [NSXPCInterface + interfaceWithProtocol:@protocol(BerdSiriTTSDaemonProtocol)]; + [remote setClasses:BerdRequestClasses() + forSelector:@selector(synthesizeWithRequest:reply:) + argumentIndex:0 + ofReply:NO]; + [remote setClasses:BerdRequestClasses() + forSelector:@selector(cancelWithRequest:) + argumentIndex:0 + ofReply:NO]; + connection.remoteObjectInterface = remote; + NSXPCInterface *exported = [NSXPCInterface + interfaceWithProtocol:@protocol(BerdSiriTTSSessionDelegate)]; + NSSet *callbackClasses = BerdCallbackClasses(); + [exported setClasses:callbackClasses + forSelector:@selector(didGenerateAudioWithRequestId:audio:) + argumentIndex:1 + ofReply:NO]; + [exported setClasses:callbackClasses + forSelector:@selector(didGenerateWordTimingsWithRequestId:wordTimingInfo:) + argumentIndex:1 + ofReply:NO]; + [exported setClasses:callbackClasses + forSelector:@selector(didReportInstrumentWithRequestId:instrumentationMetrics:) + argumentIndex:1 + ofReply:NO]; + connection.exportedInterface = exported; + connection.exportedObject = self.delegate; + + self.request = request; + self.connection = connection; + self.completion = completion; + self.finished = NO; + __weak typeof(self) weakSelf = self; + connection.interruptionHandler = ^{ + [weakSelf finish:BerdError(14, @"Siri synthesis connection was interrupted.")]; + }; + connection.invalidationHandler = ^{ + [weakSelf finish:BerdError(15, @"Siri synthesis connection was invalidated.")]; + }; + [connection resume]; + id proxy = + [connection remoteObjectProxyWithErrorHandler:^(NSError *proxyError) { + [weakSelf finish:proxyError]; + }]; + [proxy synthesizeWithRequest:request reply:^(NSError *replyError) { + [weakSelf finish:replyError]; + }]; +} +- (void)cancel { + if (self.connection && self.request) { + id proxy = + [self.connection remoteObjectProxyWithErrorHandler:^(__unused NSError *error) {}]; + [proxy cancelWithRequest:self.request]; + } + [self finish:BerdError(NSUserCancelledError, @"Siri synthesis cancelled.")]; + [self.connection invalidate]; + self.connection = nil; + self.request = nil; +} +- (void)dealloc { [self.connection invalidate]; } +@end + +@interface BerdSiriSpeechPlayer : NSObject +@property(nonatomic, strong) dispatch_queue_t queue; +@property(nonatomic, strong) AVAudioEngine *engine; +@property(nonatomic, strong) AVAudioPlayerNode *player; +@property(nonatomic, strong) AVAudioConverter *converter; +@property(nonatomic, strong) BerdSiriSynthesisSession *session; +@property(nonatomic, strong) NSMutableArray *pendingTexts; +@property(nonatomic, strong) dispatch_semaphore_t completionSemaphore; +@property(nonatomic, strong) NSError *error; +@property(nonatomic, assign) NSInteger pendingBuffers; +@property(nonatomic, assign) BOOL inputFinished; +@property(nonatomic, assign) BOOL playbackStarted; +@property(nonatomic, assign) BOOL finished; +@property(nonatomic, assign) uint64_t progressGeneration; +@property(nonatomic, assign) BerdSiriTTSPlaybackStarted startedCallback; +@property(nonatomic, assign) void *callbackContext; +@property(nonatomic, copy) NSString *language; +@property(nonatomic, copy) NSString *voiceName; +@property(nonatomic, assign) float rate; +- (void)enqueueText:(NSString *)text; +- (void)finishInput; +- (void)cancel; +@end + +@implementation BerdSiriSpeechPlayer +- (instancetype)init { + self = [super init]; + if (self) { + _queue = dispatch_queue_create("com.block.berd.sirittsd", DISPATCH_QUEUE_SERIAL); + dispatch_queue_set_specific(_queue, BerdSiriSpeechQueueKey, + BerdSiriSpeechQueueKey, NULL); + _completionSemaphore = dispatch_semaphore_create(0); + _pendingTexts = [NSMutableArray array]; + } + return self; +} +- (void)finish:(NSError *)error { + if (self.finished) return; + self.finished = YES; + self.error = error; + self.progressGeneration += 1; + dispatch_semaphore_signal(self.completionSemaphore); +} +- (void)finishIfReady { + if (self.inputFinished && !self.session && self.pendingTexts.count == 0 && + self.pendingBuffers == 0) { + [self finish:self.error]; + } +} +- (AVAudioPCMBuffer *)decodeData:(NSData *)data + format:(AudioStreamBasicDescription)description + packetCount:(UInt32)packetCount + packetDescriptions:(NSData *)packetDescriptions + error:(NSError **)error { + if (description.mFormatID == kAudioFormatLinearPCM) { + if (description.mBytesPerFrame == 0) { + if (error) *error = BerdError(16, @"Siri returned an invalid PCM format."); + return nil; + } + AVAudioFormat *format = [[AVAudioFormat alloc] initWithStreamDescription:&description]; + AVAudioFrameCount count = (AVAudioFrameCount)(data.length / description.mBytesPerFrame); + AVAudioPCMBuffer *buffer = [[AVAudioPCMBuffer alloc] + initWithPCMFormat:format frameCapacity:count]; + buffer.frameLength = count; + AudioBuffer *destination = &buffer.mutableAudioBufferList->mBuffers[0]; + memcpy(destination->mData, data.bytes, MIN(data.length, destination->mDataByteSize)); + return buffer; + } + if (description.mFormatID != kAudioFormatOpus) { + if (error) *error = BerdError(17, @"Siri returned an unsupported audio format."); + return nil; + } + AVAudioFormat *source = [[AVAudioFormat alloc] initWithStreamDescription:&description]; + AVAudioFormat *destination = [[AVAudioFormat alloc] + initWithCommonFormat:AVAudioPCMFormatFloat32 + sampleRate:description.mSampleRate + channels:description.mChannelsPerFrame + interleaved:NO]; + if (!self.converter) self.converter = [[AVAudioConverter alloc] initFromFormat:source + toFormat:destination]; + UInt32 count = MAX(packetCount, 1); + AVAudioCompressedBuffer *compressed = [[AVAudioCompressedBuffer alloc] + initWithFormat:source packetCapacity:count maximumPacketSize:MAX((UInt32)data.length, 1)]; + compressed.byteLength = (UInt32)data.length; + compressed.packetCount = count; + memcpy(compressed.data, data.bytes, data.length); + if (packetDescriptions.length >= count * sizeof(AudioStreamPacketDescription)) { + memcpy(compressed.packetDescriptions, packetDescriptions.bytes, + count * sizeof(AudioStreamPacketDescription)); + } else if (count == 1) { + compressed.packetDescriptions[0] = (AudioStreamPacketDescription){ + .mStartOffset = 0, .mVariableFramesInPacket = 0, + .mDataByteSize = (UInt32)data.length, + }; + } else { + if (error) *error = BerdError(18, @"Siri omitted Opus packet descriptions."); + return nil; + } + AVAudioPCMBuffer *pcm = [[AVAudioPCMBuffer alloc] + initWithPCMFormat:destination frameCapacity:count * 5760]; + __block BOOL supplied = NO; + NSError *conversionError = nil; + AVAudioConverterOutputStatus status = [self.converter + convertToBuffer:pcm error:&conversionError + withInputFromBlock:^AVAudioBuffer *(AVAudioPacketCount requested, + AVAudioConverterInputStatus *inputStatus) { + (void)requested; + if (supplied) { + *inputStatus = AVAudioConverterInputStatus_NoDataNow; + return nil; + } + supplied = YES; + *inputStatus = AVAudioConverterInputStatus_HaveData; + return compressed; + }]; + if (conversionError || status == AVAudioConverterOutputStatus_Error) { + if (error) *error = conversionError ?: BerdError(19, @"Could not decode Siri audio."); + return nil; + } + return pcm.frameLength ? pcm : nil; +} +- (BOOL)ensurePlayer:(AVAudioFormat *)format error:(NSError **)error { + if (self.player) return YES; + self.engine = [AVAudioEngine new]; + self.player = [AVAudioPlayerNode new]; + [self.engine attachNode:self.player]; + [self.engine connect:self.player to:self.engine.mainMixerNode format:format]; + [self.engine prepare]; + if (![self.engine startAndReturnError:error]) return NO; + return YES; +} +- (void)enqueueData:(NSData *)data format:(AudioStreamBasicDescription)format + packetCount:(UInt32)packetCount packetDescriptions:(NSData *)packetDescriptions { + if (self.finished || !data.length) return; + self.progressGeneration += 1; + NSError *error = nil; + AVAudioPCMBuffer *buffer = [self decodeData:data format:format packetCount:packetCount + packetDescriptions:packetDescriptions error:&error]; + if (error || !buffer) { + if (error) [self finish:error]; + return; + } + if (![self ensurePlayer:buffer.format error:&error]) { + [self finish:error]; + return; + } + self.pendingBuffers += 1; + [self.player scheduleBuffer:buffer completionCallbackType:AVAudioPlayerNodeCompletionDataPlayedBack + completionHandler:^(__unused AVAudioPlayerNodeCompletionCallbackType type) { + dispatch_async(self.queue, ^{ + self.pendingBuffers = MAX(0, self.pendingBuffers - 1); + self.progressGeneration += 1; + [self finishIfReady]; + }); + }]; + if (!self.playbackStarted) { + self.playbackStarted = YES; + [self.player play]; + if (self.startedCallback) self.startedCallback(self.callbackContext); + } +} +- (void)startNextSynthesis { + if (self.finished || self.session || self.pendingTexts.count == 0) { + [self finishIfReady]; + return; + } + NSString *text = self.pendingTexts.firstObject; + [self.pendingTexts removeObjectAtIndex:0]; + self.progressGeneration += 1; + __weak typeof(self) weakSelf = self; + self.session = [[BerdSiriSynthesisSession alloc] + initWithAudioHandler:^(NSData *data, AudioStreamBasicDescription format, + UInt32 packetCount, NSData *descriptions) { + dispatch_async(weakSelf.queue, ^{ + [weakSelf enqueueData:data format:format packetCount:packetCount + packetDescriptions:descriptions]; + }); + }]; + [self.session synthesizeText:text language:self.language voiceName:self.voiceName rate:self.rate + completion:^(NSError *error) { + dispatch_async(weakSelf.queue, ^{ + weakSelf.progressGeneration += 1; + weakSelf.session = nil; + if (error && error.code != NSUserCancelledError) { + [weakSelf finish:error]; + return; + } + [weakSelf startNextSynthesis]; + }); + }]; +} +- (void)enqueueText:(NSString *)text { + dispatch_async(self.queue, ^{ + if (self.finished || self.inputFinished || !text.length) return; + [self.pendingTexts addObject:text]; + self.progressGeneration += 1; + [self startNextSynthesis]; + }); +} +- (void)finishInput { + dispatch_async(self.queue, ^{ + self.inputFinished = YES; + self.progressGeneration += 1; + [self startNextSynthesis]; + [self finishIfReady]; + }); +} +- (void)cancel { + void (^cancelWork)(void) = ^{ + if (self.finished) return; + self.startedCallback = NULL; + self.callbackContext = NULL; + [self.session cancel]; + [self.player stop]; + [self.engine stop]; + [self finish:BerdError(NSUserCancelledError, @"Siri playback cancelled.")]; + }; + if (dispatch_get_specific(BerdSiriSpeechQueueKey)) cancelWork(); + else dispatch_sync(self.queue, cancelWork); +} +@end + +static void BerdDownloadedVoices( + NSString *language, + NSString *voiceName, + void (^completion)(NSArray *> *, NSError *) +) { + NSError *error = nil; + id voice = BerdCreateVoice(language, voiceName, &error); + if (!voice) { + completion(nil, error ?: BerdError(3, @"Could not create Siri voice query.")); + return; + } + + NSXPCConnection *connection = [[NSXPCConnection alloc] + initWithMachServiceName:@"com.apple.sirittsd" options:0]; + NSXPCInterface *interface = [NSXPCInterface + interfaceWithProtocol:@protocol(BerdSiriTTSAvailabilityProtocol)]; + NSSet *classes = BerdRequestClasses(); + [interface setClasses:classes + forSelector:@selector(downloadedVoicesMatching:reply:) + argumentIndex:0 + ofReply:NO]; + [interface setClasses:classes + forSelector:@selector(downloadedVoicesMatching:reply:) + argumentIndex:0 + ofReply:YES]; + connection.remoteObjectInterface = interface; + + __block BOOL replied = NO; + void (^finish)(NSArray *, NSError *) = ^(NSArray *voices, NSError *finishError) { + @synchronized (connection) { + if (replied) return; + replied = YES; + } + NSMutableArray *result = [NSMutableArray arrayWithCapacity:voices.count]; + for (id resolved in voices ?: @[]) [result addObject:BerdVoiceDictionary(resolved)]; + completion(result, finishError); + [connection invalidate]; + }; + connection.invalidationHandler = ^{ + finish(nil, BerdError(4, @"Siri voice query connection was invalidated.")); + }; + [connection resume]; + id proxy = + [connection remoteObjectProxyWithErrorHandler:^(NSError *proxyError) { + finish(nil, proxyError); + }]; + [proxy downloadedVoicesMatching:voice reply:^(NSArray *voices) { + finish(voices, nil); + }]; + dispatch_after( + dispatch_time(DISPATCH_TIME_NOW, (int64_t)(3 * NSEC_PER_SEC)), + dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0), + ^{ finish(nil, BerdError(5, @"Timed out validating Siri voice.")); } + ); +} + +static NSDictionary *BerdDownloadedVoiceSync( + NSString *language, + NSString *voiceName, + NSError **error +) { + dispatch_semaphore_t semaphore = dispatch_semaphore_create(0); + __block NSArray *> *voices = nil; + __block NSError *replyError = nil; + BerdDownloadedVoices(language, voiceName, ^(NSArray *reply, NSError *failure) { + voices = reply; + replyError = failure; + dispatch_semaphore_signal(semaphore); + }); + dispatch_semaphore_wait( + semaphore, + dispatch_time(DISPATCH_TIME_NOW, (int64_t)(4 * NSEC_PER_SEC)) + ); + if (replyError) { + if (error) *error = replyError; + return nil; + } + NSString *normalizedLanguage = + [[language stringByReplacingOccurrencesOfString:@"_" withString:@"-"] lowercaseString]; + for (NSDictionary *candidate in voices ?: @[]) { + NSString *candidateName = candidate[@"name"]; + NSString *candidateLanguage = candidate[@"language"]; + NSString *normalizedCandidate = + [[candidateLanguage stringByReplacingOccurrencesOfString:@"_" withString:@"-"] lowercaseString]; + if ([candidateName caseInsensitiveCompare:voiceName] == NSOrderedSame && + [normalizedCandidate isEqualToString:normalizedLanguage]) { + return candidate; + } + } + return nil; +} + +static NSArray *> *BerdDiscoverVoices( + NSString *languagePrefix, + NSError **error +) { + if (!BerdLoadFramework( + @"/System/Library/PrivateFrameworks/TextToSpeech.framework/TextToSpeech", + error + )) return nil; + Class managerClass = objc_getClass("TTSAXResourceManager"); + if (!managerClass) { + if (error) *error = BerdError(6, @"Siri voice catalog is unavailable."); + return nil; + } + SEL sharedSelector = @selector(sharedInstance); + SEL voicesSelector = @selector(allVoices:); + if (![managerClass respondsToSelector:sharedSelector]) { + if (error) *error = BerdError(6, @"Siri voice catalog manager is unavailable."); + return nil; + } + typedef id (*SendNoArguments)(id, SEL); + typedef id (*SendObject)(id, SEL, id); + id manager = ((SendNoArguments)objc_msgSend)(managerClass, sharedSelector); + if (![manager respondsToSelector:voicesSelector]) { + if (error) *error = BerdError(6, @"Siri voice catalog lookup is unavailable."); + return nil; + } + NSArray *resources = ((SendObject)objc_msgSend)(manager, voicesSelector, nil); + NSString *normalizedPrefix = + [[languagePrefix stringByReplacingOccurrencesOfString:@"_" withString:@"-"] lowercaseString]; + NSMutableDictionary *> *byKey = + [NSMutableDictionary dictionary]; + for (id resource in resources ?: @[]) { + NSString *identifier = [resource valueForKey:@"identifier"]; + if (![identifier hasPrefix:@"com.apple.siri.natural."]) continue; + NSString *language = [resource valueForKey:@"language"] ?: @""; + NSString *normalizedLanguage = + [[language stringByReplacingOccurrencesOfString:@"_" withString:@"-"] lowercaseString]; + if (normalizedPrefix.length && ![normalizedLanguage isEqualToString:normalizedPrefix]) continue; + NSString *name = [resource valueForKey:@"name"] ?: @""; + if (!name.length || !language.length) continue; + NSString *key = [NSString stringWithFormat:@"%@|%@", name.lowercaseString, + normalizedLanguage]; + byKey[key] = @{ + @"name" : name, + @"language" : language, + @"sizeBytes" : [resource valueForKey:@"assetSize"] ?: @0, + }; + } + return [[byKey allValues] sortedArrayUsingComparator: + ^NSComparisonResult(NSDictionary *left, NSDictionary *right) { + NSComparisonResult language = [left[@"language"] + localizedCaseInsensitiveCompare:right[@"language"]]; + return language != NSOrderedSame + ? language + : [left[@"name"] localizedCaseInsensitiveCompare:right[@"name"]]; + }]; +} + +static BOOL BerdSubscribeVoiceSync(NSString *language, NSString *voiceName, NSError **error) { + NSError *voiceError = nil; + id voice = BerdCreateVoice(language, voiceName, &voiceError); + if (!voice) { + if (error) *error = voiceError; + return NO; + } + NSXPCConnection *connection = [[NSXPCConnection alloc] + initWithMachServiceName:@"com.apple.sirittsd" options:0]; + NSXPCInterface *interface = [NSXPCInterface + interfaceWithProtocol:@protocol(BerdSiriTTSSubscribeProtocol)]; + [interface setClasses:BerdRequestClasses() + forSelector:@selector(subscribeWithVoices:clientId:accessoryId:reply:) + argumentIndex:0 + ofReply:NO]; + connection.remoteObjectInterface = interface; + dispatch_semaphore_t semaphore = dispatch_semaphore_create(0); + __block NSError *replyError = nil; + __block BOOL replied = NO; + void (^finish)(NSError *) = ^(NSError *failure) { + @synchronized (connection) { + if (replied) return; + replyError = failure; + replied = YES; + } + dispatch_semaphore_signal(semaphore); + [connection invalidate]; + }; + [connection resume]; + id proxy = + [connection remoteObjectProxyWithErrorHandler:^(NSError *proxyError) { + finish(proxyError); + }]; + [proxy subscribeWithVoices:@[ voice ] + clientId:@"com.apple.speech" + accessoryId:@"" + reply:^(NSError *failure) { finish(failure); }]; + long wait = dispatch_semaphore_wait( + semaphore, + dispatch_time(DISPATCH_TIME_NOW, (int64_t)(10 * NSEC_PER_SEC)) + ); + if (wait != 0) finish(BerdError(7, @"Timed out subscribing to Siri voice.")); + if (replyError) { + if (error) *error = replyError; + return NO; + } + return YES; +} + +static BOOL BerdTriggerDownload(NSString *language, NSError **error) { + if (!BerdLoadFramework( + @"/System/Library/PrivateFrameworks/UnifiedAssetFramework.framework/UnifiedAssetFramework", + error + )) return NO; + Class serviceClass = objc_getClass("UAFAssetUtilitiesService"); + if (!serviceClass) { + if (error) *error = BerdError(8, @"Siri voice download service is unavailable."); + return NO; + } + id service = [serviceClass new]; + SEL switchSelector = NSSelectorFromString(@"switchLanguage:"); + SEL downloadSelector = NSSelectorFromString(@"downloadSiriAssets"); + if (![service respondsToSelector:switchSelector] || + ![service respondsToSelector:downloadSelector]) { + if (error) *error = BerdError(9, @"Siri voice download API is unavailable."); + return NO; + } + NSString *normalized = [language stringByReplacingOccurrencesOfString:@"-" withString:@"_"]; + ((void (*)(id, SEL, id))objc_msgSend)(service, switchSelector, normalized); + ((void (*)(id, SEL))objc_msgSend)(service, downloadSelector); + return YES; +} + +char *berd_siri_tts_catalog_json(const char *languagePrefix, char **errorOut) { + @autoreleasepool { + if (errorOut) *errorOut = NULL; + NSString *prefix = languagePrefix + ? [NSString stringWithUTF8String:languagePrefix] + : @""; + NSError *error = nil; + NSArray *> *candidates = + BerdDiscoverVoices(prefix, &error); + if (!candidates) { + BerdSetError(errorOut, error); + return NULL; + } + + dispatch_group_t group = dispatch_group_create(); + NSMutableDictionary *installed = [NSMutableDictionary dictionary]; + for (NSDictionary *candidate in candidates) { + NSString *name = candidate[@"name"]; + NSString *language = candidate[@"language"]; + NSString *key = [NSString stringWithFormat:@"%@|%@", name.lowercaseString, + language.lowercaseString]; + dispatch_group_enter(group); + BerdDownloadedVoices(language, name, ^(NSArray *voices, NSError *failure) { + BOOL exact = NO; + if (!failure) { + for (NSDictionary *voice in voices) { + if ([voice[@"name"] caseInsensitiveCompare:name] == NSOrderedSame && + [voice[@"language"] caseInsensitiveCompare:language] == NSOrderedSame) { + exact = YES; + break; + } + } + } + @synchronized (installed) { installed[key] = @(exact); } + dispatch_group_leave(group); + }); + } + dispatch_group_wait( + group, + dispatch_time(DISPATCH_TIME_NOW, (int64_t)(4 * NSEC_PER_SEC)) + ); + + NSMutableArray *result = [NSMutableArray arrayWithCapacity:candidates.count]; + for (NSDictionary *candidate in candidates) { + NSString *key = [NSString stringWithFormat:@"%@|%@", + [candidate[@"name"] lowercaseString], + [candidate[@"language"] lowercaseString]]; + NSMutableDictionary *voice = [candidate mutableCopy]; + voice[@"installed"] = installed[key] ?: @NO; + [result addObject:voice]; + } + NSData *json = [NSJSONSerialization dataWithJSONObject:result options:0 error:&error]; + if (!json) { + BerdSetError(errorOut, error); + return NULL; + } + NSString *encoded = [[NSString alloc] initWithData:json encoding:NSUTF8StringEncoding]; + return strdup(encoded.UTF8String); + } +} + +char *berd_siri_tts_languages_json(char **errorOut) { + @autoreleasepool { + if (errorOut) *errorOut = NULL; + NSError *error = nil; + NSArray *> *candidates = + BerdDiscoverVoices(@"", &error); + if (!candidates) { + BerdSetError(errorOut, error); + return NULL; + } + NSMutableSet *languages = [NSMutableSet set]; + for (NSDictionary *candidate in candidates) { + NSString *language = candidate[@"language"]; + if (language.length) [languages addObject:language]; + } + NSArray *sorted = [[languages allObjects] + sortedArrayUsingSelector:@selector(localizedCaseInsensitiveCompare:)]; + NSData *json = [NSJSONSerialization dataWithJSONObject:sorted options:0 error:&error]; + if (!json) { + BerdSetError(errorOut, error); + return NULL; + } + NSString *encoded = [[NSString alloc] initWithData:json encoding:NSUTF8StringEncoding]; + return strdup(encoded.UTF8String); + } +} + +bool berd_siri_tts_download_voice( + const char *languageValue, + const char *voiceNameValue, + double timeoutSeconds, + char **errorOut +) { + @autoreleasepool { + if (errorOut) *errorOut = NULL; + if (!languageValue || !voiceNameValue) { + BerdSetError(errorOut, BerdError(10, @"A Siri voice name and language are required.")); + return false; + } + NSString *language = [NSString stringWithUTF8String:languageValue]; + NSString *voiceName = [NSString stringWithUTF8String:voiceNameValue]; + NSError *error = nil; + if (BerdDownloadedVoiceSync(language, voiceName, &error)) return true; + error = nil; + if (!BerdSubscribeVoiceSync(language, voiceName, &error) || + !BerdTriggerDownload(language, &error)) { + BerdSetError(errorOut, error); + return false; + } + + NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:MAX(1, timeoutSeconds)]; + while ([deadline timeIntervalSinceNow] > 0) { + error = nil; + if (BerdDownloadedVoiceSync(language, voiceName, &error)) return true; + [NSThread sleepForTimeInterval:2]; + } + BerdSetError(errorOut, BerdError(11, + [NSString stringWithFormat:@"Timed out downloading %@ (%@).", voiceName, language])); + return false; + } +} + +static NSURL *BerdSiriVoiceSampleURL(NSString *voiceName, NSString *language) { + NSURL *root = [NSURL fileURLWithPath: + @"/System/Library/AssetsV2/com_apple_MobileAsset_TTSAXResourceModelAssets" + isDirectory:YES]; + NSArray *assets = [[NSFileManager defaultManager] + contentsOfDirectoryAtURL:root + includingPropertiesForKeys:nil + options:NSDirectoryEnumerationSkipsHiddenFiles + error:nil]; + NSString *normalizedName = voiceName.lowercaseString; + NSString *normalizedLanguage = + [[language stringByReplacingOccurrencesOfString:@"_" withString:@"-"] lowercaseString]; + NSString *suffix = [NSString stringWithFormat:@"_%@_%@_premium.caf", + normalizedName, + normalizedLanguage]; + NSMutableArray *matches = [NSMutableArray array]; + for (NSURL *asset in assets ?: @[]) { + NSURL *contents = [[asset URLByAppendingPathComponent:@"AssetData" isDirectory:YES] + URLByAppendingPathComponent:@"Contents" isDirectory:YES]; + NSArray *samples = [[NSFileManager defaultManager] + contentsOfDirectoryAtURL:contents + includingPropertiesForKeys:nil + options:NSDirectoryEnumerationSkipsHiddenFiles + error:nil]; + for (NSURL *sample in samples ?: @[]) { + if ([sample.lastPathComponent.lowercaseString hasSuffix:suffix]) { + [matches addObject:sample]; + } + } + } + [matches sortUsingComparator:^NSComparisonResult(NSURL *left, NSURL *right) { + NSInteger (^rank)(NSURL *) = ^NSInteger(NSURL *url) { + NSString *name = url.lastPathComponent.lowercaseString; + if ([name containsString:@"gryphon-neuralax_"]) return 0; + if ([name containsString:@"gryphon-neural_"]) return 1; + return 2; + }; + NSInteger leftRank = rank(left); + NSInteger rightRank = rank(right); + if (leftRank != rightRank) { + return leftRank < rightRank ? NSOrderedAscending : NSOrderedDescending; + } + return [left.lastPathComponent localizedCaseInsensitiveCompare:right.lastPathComponent]; + }]; + return matches.firstObject; +} + +bool berd_siri_tts_play_sample( + const char *voiceNameValue, + const char *languageValue, + float rate, + BerdSiriTTSShouldStop shouldStop, + void *context, + char **errorOut +) { + @autoreleasepool { + if (errorOut) *errorOut = NULL; + if (!voiceNameValue || !languageValue) { + BerdSetError(errorOut, + BerdError(23, @"A Siri voice name and language are required.")); + return false; + } + NSString *voiceName = [NSString stringWithUTF8String:voiceNameValue]; + NSString *language = [NSString stringWithUTF8String:languageValue]; + NSURL *sampleURL = BerdSiriVoiceSampleURL(voiceName, language); + if (!sampleURL) { + BerdSetError(errorOut, BerdError(24, + [NSString stringWithFormat:@"No system preview is available for %@ (%@).", + voiceName, language])); + return false; + } + NSError *error = nil; + AVAudioPlayer *player = [[AVAudioPlayer alloc] + initWithContentsOfURL:sampleURL error:&error]; + if (!player) { + BerdSetError(errorOut, + error ?: BerdError(25, @"Could not open the Siri voice preview.")); + return false; + } + player.enableRate = YES; + player.rate = MAX(0.5f, MIN(2.0f, rate)); + if (![player prepareToPlay] || ![player play]) { + BerdSetError(errorOut, BerdError(26, @"Could not play the Siri voice preview.")); + return false; + } + while (player.isPlaying) { + if (shouldStop && shouldStop(context)) { + [player stop]; + return true; + } + [NSThread sleepForTimeInterval:0.01]; + } + return true; + } +} + +void *berd_siri_tts_stream_create( + const char *languageValue, + const char *voiceNameValue, + float rate, + BerdSiriTTSPlaybackStarted playbackStarted, + void *context, + char **errorOut +) { + @autoreleasepool { + if (errorOut) *errorOut = NULL; + if (!languageValue || !voiceNameValue) { + BerdSetError(errorOut, BerdError(20, @"A Siri voice name and language are required.")); + return NULL; + } + NSString *language = [NSString stringWithUTF8String:languageValue]; + NSString *voiceName = [NSString stringWithUTF8String:voiceNameValue]; + NSError *validationError = nil; + if (!BerdDownloadedVoiceSync(language, voiceName, &validationError)) { + BerdSetError(errorOut, validationError ?: BerdError(21, + [NSString stringWithFormat:@"Siri voice %@ (%@) is not installed.", + voiceName, language])); + return NULL; + } + BerdSiriSpeechPlayer *player = [BerdSiriSpeechPlayer new]; + player.language = language; + player.voiceName = voiceName; + player.rate = MAX(0.25f, MIN(4.0f, rate)); + player.startedCallback = playbackStarted; + player.callbackContext = context; + return (__bridge_retained void *)player; + } +} + +bool berd_siri_tts_stream_enqueue(void *stream, const char *textValue, char **errorOut) { + @autoreleasepool { + if (errorOut) *errorOut = NULL; + if (!stream || !textValue) { + BerdSetError(errorOut, BerdError(22, @"An active Siri stream and text are required.")); + return false; + } + BerdSiriSpeechPlayer *player = (__bridge BerdSiriSpeechPlayer *)stream; + NSString *text = [NSString stringWithUTF8String:textValue]; + if (!text.length) return true; + [player enqueueText:text]; + return true; + } +} + +void berd_siri_tts_stream_finish(void *stream) { + if (!stream) return; + [(__bridge BerdSiriSpeechPlayer *)stream finishInput]; +} + +bool berd_siri_tts_stream_is_finished(void *stream) { + if (!stream) return true; + BerdSiriSpeechPlayer *player = (__bridge BerdSiriSpeechPlayer *)stream; + __block BOOL finished = NO; + dispatch_sync(player.queue, ^{ finished = player.finished; }); + return finished; +} + +uint64_t berd_siri_tts_stream_progress(void *stream) { + if (!stream) return 0; + BerdSiriSpeechPlayer *player = (__bridge BerdSiriSpeechPlayer *)stream; + __block uint64_t progress = 0; + dispatch_sync(player.queue, ^{ progress = player.progressGeneration; }); + return progress; +} + +char *berd_siri_tts_stream_copy_error(void *stream) { + if (!stream) return strdup("Siri stream is unavailable"); + BerdSiriSpeechPlayer *player = (__bridge BerdSiriSpeechPlayer *)stream; + __block NSString *message = nil; + dispatch_sync(player.queue, ^{ message = player.error.localizedDescription; }); + return message.length ? strdup(message.UTF8String) : NULL; +} + +void berd_siri_tts_stream_cancel(void *stream) { + if (!stream) return; + [(__bridge BerdSiriSpeechPlayer *)stream cancel]; +} + +void berd_siri_tts_stream_release(void *stream) { + if (!stream) return; + CFBridgingRelease(stream); +} + +bool berd_siri_tts_speak( + const char *textValue, + const char *languageValue, + const char *voiceNameValue, + float rate, + BerdSiriTTSShouldStop shouldStop, + BerdSiriTTSPlaybackStarted playbackStarted, + void *context, + char **errorOut +) { + @autoreleasepool { + if (errorOut) *errorOut = NULL; + if (!textValue || !languageValue || !voiceNameValue) { + BerdSetError(errorOut, BerdError(20, @"Text, voice name, and language are required.")); + return false; + } + void *stream = berd_siri_tts_stream_create( + languageValue, voiceNameValue, rate, playbackStarted, context, errorOut); + if (!stream) return false; + if (!berd_siri_tts_stream_enqueue(stream, textValue, errorOut)) { + berd_siri_tts_stream_release(stream); + return false; + } + berd_siri_tts_stream_finish(stream); + BerdSiriSpeechPlayer *player = (__bridge BerdSiriSpeechPlayer *)stream; + while (dispatch_semaphore_wait( + player.completionSemaphore, + dispatch_time(DISPATCH_TIME_NOW, (int64_t)(10 * NSEC_PER_MSEC))) != 0) { + if (shouldStop && shouldStop(context)) berd_siri_tts_stream_cancel(stream); + } + char *streamError = berd_siri_tts_stream_copy_error(stream); + berd_siri_tts_stream_release(stream); + if (streamError) { + if (errorOut) *errorOut = streamError; + else free(streamError); + return false; + } + return true; + } +} + +void berd_siri_tts_free_string(char *value) { + free(value); +} diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index 1545fab10..465a05e20 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -44,6 +44,7 @@ pub mod pull_requests; pub mod renderer; pub mod runtime_config; pub mod security_threshold; +pub mod siri_voice; pub mod skill_marketplace; pub mod system; pub mod telemetry; diff --git a/src-tauri/src/commands/pocket_voice.rs b/src-tauri/src/commands/pocket_voice.rs index ebcd90331..32dd45fa5 100644 --- a/src-tauri/src/commands/pocket_voice.rs +++ b/src-tauri/src/commands/pocket_voice.rs @@ -342,7 +342,7 @@ fn selected_output_device() -> Option { } #[cfg(target_os = "macos")] -fn effective_output_device_name(configured: Option<&str>) -> Option { +pub(crate) fn effective_output_device_name(configured: Option<&str>) -> Option { use rodio::cpal::traits::HostTrait; if let Some(name) = configured { @@ -360,7 +360,7 @@ fn effective_output_device_name(configured: Option<&str>) -> Option { configured.map(ToOwned::to_owned) } -fn output_device_uses_speakers(output_device: Option<&str>) -> bool { +pub(crate) fn output_device_uses_speakers(output_device: Option<&str>) -> bool { output_device.is_some_and(|name| name.to_lowercase().contains("speaker")) } diff --git a/src-tauri/src/commands/siri_voice.rs b/src-tauri/src/commands/siri_voice.rs new file mode 100644 index 000000000..8595c0735 --- /dev/null +++ b/src-tauri/src/commands/siri_voice.rs @@ -0,0 +1,1106 @@ +//! macOS SiriTTSD voice discovery, download, and selection. + +#[cfg(target_os = "macos")] +use std::ffi::{CStr, CString}; +use std::fs; +#[cfg(target_os = "macos")] +use std::os::raw::c_char; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +#[cfg(target_os = "macos")] +use std::sync::mpsc; +use std::sync::{Arc, Mutex}; +#[cfg(target_os = "macos")] +use std::time::{Duration, Instant}; + +use serde::{Deserialize, Serialize}; +#[cfg(target_os = "macos")] +use tauri::Emitter; +use tauri::{AppHandle, Manager}; + +use super::native_voice::NativeVoiceState; +#[cfg(target_os = "macos")] +use super::pocket_voice::{effective_output_device_name, output_device_uses_speakers}; + +#[derive(Clone, Debug, Default)] +pub struct SiriVoiceState { + runtime: Arc>, +} + +#[derive(Debug, Default)] +struct SiriVoiceRuntime { + active: Option>, + owner_window: Option, + #[cfg(target_os = "macos")] + stream: Option, +} + +#[cfg(target_os = "macos")] +#[derive(Debug)] +struct ActiveSiriStream { + id: String, + sender: mpsc::Sender, +} + +#[cfg_attr(not(target_os = "macos"), allow(dead_code))] +#[derive(Debug)] +enum SiriStreamCommand { + Append(String), + Flush, + Finish, + Stop, +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +enum SiriStreamEventState { + Started, + Completed, + Interrupted, + Failed, +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct SiriStreamEvent { + stream_id: String, + state: SiriStreamEventState, + error: Option, +} + +#[cfg(target_os = "macos")] +const SIRI_STREAM_EVENT: &str = "siri-voice:stream-event"; +#[cfg(target_os = "macos")] +const SIRI_STREAM_STALL_TIMEOUT: Duration = Duration::from_secs(60); +const MIN_PLAYBACK_SPEED: f32 = 0.5; +const MAX_PLAYBACK_SPEED: f32 = 2.0; +static SIRI_SETTINGS_LOCK: Mutex<()> = Mutex::new(()); +static SIRI_SETTINGS_TEMP_COUNTER: AtomicU64 = AtomicU64::new(0); + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct SiriVoice { + name: String, + language: String, + size_bytes: u64, + installed: bool, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct SiriVoiceSelection { + name: String, + language: String, +} + +#[derive(Clone, Debug, Serialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct SiriVoiceStatus { + supported: bool, + available_languages: Vec, + selected_voice: Option, + selected_voice_installed: bool, + playback_speed: f32, + voices: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +struct SiriVoiceSettings { + selected_voice: Option, + #[serde(default = "default_playback_speed")] + playback_speed: f32, +} + +fn default_playback_speed() -> f32 { + 1.0 +} + +impl Default for SiriVoiceSettings { + fn default() -> Self { + Self { + selected_voice: None, + playback_speed: default_playback_speed(), + } + } +} + +fn settings_path(app: &AppHandle) -> Result { + app.path() + .app_data_dir() + .map(|path| path.join("siri-tts").join("settings.json")) + .map_err(|error| format!("resolve Siri TTS settings directory: {error}")) +} + +fn read_settings(path: &Path) -> SiriVoiceSettings { + fs::read(path) + .ok() + .and_then(|data| serde_json::from_slice(&data).ok()) + .unwrap_or_default() +} + +fn write_settings(path: &Path, settings: &SiriVoiceSettings) -> Result<(), String> { + let parent = path + .parent() + .ok_or_else(|| "Siri TTS settings path has no parent".to_string())?; + fs::create_dir_all(parent).map_err(|error| format!("create Siri TTS settings: {error}"))?; + let data = serde_json::to_vec_pretty(settings) + .map_err(|error| format!("encode Siri TTS settings: {error}"))?; + let temporary = path.with_extension(format!( + "json.{}.{}.tmp", + std::process::id(), + SIRI_SETTINGS_TEMP_COUNTER.fetch_add(1, Ordering::Relaxed), + )); + fs::write(&temporary, data).map_err(|error| format!("write Siri TTS settings: {error}"))?; + fs::rename(&temporary, path).map_err(|error| { + let _ = fs::remove_file(&temporary); + format!("publish Siri TTS settings: {error}") + }) +} + +fn update_settings( + path: &Path, + update: impl FnOnce(&mut SiriVoiceSettings) -> bool, +) -> Result { + let _guard = SIRI_SETTINGS_LOCK + .lock() + .map_err(|_| "Siri TTS settings lock was poisoned".to_string())?; + let mut settings = read_settings(path); + if update(&mut settings) { + write_settings(path, &settings)?; + } + Ok(settings) +} + +#[cfg(target_os = "macos")] +unsafe extern "C" { + fn berd_siri_tts_catalog_json( + language_prefix: *const c_char, + error_out: *mut *mut c_char, + ) -> *mut c_char; + fn berd_siri_tts_languages_json(error_out: *mut *mut c_char) -> *mut c_char; + fn berd_siri_tts_download_voice( + language: *const c_char, + voice_name: *const c_char, + timeout_seconds: f64, + error_out: *mut *mut c_char, + ) -> bool; + fn berd_siri_tts_play_sample( + voice_name: *const c_char, + language: *const c_char, + rate: f32, + should_stop: Option bool>, + context: *mut std::ffi::c_void, + error_out: *mut *mut c_char, + ) -> bool; + fn berd_siri_tts_speak( + text: *const c_char, + language: *const c_char, + voice_name: *const c_char, + rate: f32, + should_stop: Option bool>, + playback_started: Option, + context: *mut std::ffi::c_void, + error_out: *mut *mut c_char, + ) -> bool; + fn berd_siri_tts_stream_create( + language: *const c_char, + voice_name: *const c_char, + rate: f32, + playback_started: Option, + context: *mut std::ffi::c_void, + error_out: *mut *mut c_char, + ) -> *mut std::ffi::c_void; + fn berd_siri_tts_stream_enqueue( + stream: *mut std::ffi::c_void, + text: *const c_char, + error_out: *mut *mut c_char, + ) -> bool; + fn berd_siri_tts_stream_finish(stream: *mut std::ffi::c_void); + fn berd_siri_tts_stream_is_finished(stream: *mut std::ffi::c_void) -> bool; + fn berd_siri_tts_stream_progress(stream: *mut std::ffi::c_void) -> u64; + fn berd_siri_tts_stream_copy_error(stream: *mut std::ffi::c_void) -> *mut c_char; + fn berd_siri_tts_stream_cancel(stream: *mut std::ffi::c_void); + fn berd_siri_tts_stream_release(stream: *mut std::ffi::c_void); + fn berd_siri_tts_free_string(value: *mut c_char); +} + +#[cfg(target_os = "macos")] +unsafe extern "C" fn should_stop_siri_playback(context: *mut std::ffi::c_void) -> bool { + if context.is_null() { + return false; + } + // SAFETY: The pointer comes from an Arc kept alive for the + // entire synchronous bridge call. + let active = unsafe { &*(context.cast::()) }; + !active.load(Ordering::SeqCst) +} + +#[cfg(any(test, target_os = "macos"))] +fn begin_playback(state: &SiriVoiceState, owner_window: &str) -> Result, String> { + let mut runtime = state + .runtime + .lock() + .map_err(|_| "Siri playback state lock was poisoned".to_string())?; + if runtime.active.is_some() { + return Err("Siri voice playback is already active".to_string()); + } + let token = Arc::new(AtomicBool::new(true)); + runtime.active = Some(token.clone()); + runtime.owner_window = Some(owner_window.to_string()); + Ok(token) +} + +#[cfg(any(test, target_os = "macos"))] +fn finish_playback(state: &SiriVoiceState, completed: &Arc) { + if let Ok(mut runtime) = state.runtime.lock() { + if runtime + .active + .as_ref() + .is_some_and(|current| Arc::ptr_eq(current, completed)) + { + runtime.active = None; + runtime.owner_window = None; + #[cfg(target_os = "macos")] + { + runtime.stream = None; + } + } + } +} + +#[cfg(target_os = "macos")] +#[derive(Debug)] +struct SiriStreamWatchdog { + progress: u64, + last_progress_at: Instant, +} + +#[cfg(target_os = "macos")] +impl SiriStreamWatchdog { + fn new(progress: u64, now: Instant) -> Self { + Self { + progress, + last_progress_at: now, + } + } + + fn observe(&mut self, progress: u64, now: Instant) -> bool { + if progress != self.progress { + self.progress = progress; + self.last_progress_at = now; + return false; + } + now.duration_since(self.last_progress_at) >= SIRI_STREAM_STALL_TIMEOUT + } +} + +#[cfg(target_os = "macos")] +fn take_bridge_string(value: *mut c_char) -> Option { + if value.is_null() { + return None; + } + // SAFETY: The Objective-C bridge returns a NUL-terminated malloc-owned + // string. Copy it before releasing it through the paired bridge function. + let result = unsafe { CStr::from_ptr(value) } + .to_string_lossy() + .into_owned(); + unsafe { berd_siri_tts_free_string(value) }; + Some(result) +} + +#[cfg(target_os = "macos")] +fn bridge_error(error: *mut c_char, fallback: &str) -> String { + take_bridge_string(error).unwrap_or_else(|| fallback.to_string()) +} + +#[cfg(target_os = "macos")] +struct SiriStreamCallbackContext { + app: AppHandle, + stream_id: String, +} + +#[cfg(target_os = "macos")] +unsafe extern "C" fn siri_playback_started(context: *mut std::ffi::c_void) { + if context.is_null() { + return; + } + // SAFETY: The stream worker owns this boxed context until after the native + // player has completed and been released. + let context = unsafe { &*(context.cast::()) }; + let _ = context.app.emit( + SIRI_STREAM_EVENT, + SiriStreamEvent { + stream_id: context.stream_id.clone(), + state: SiriStreamEventState::Started, + error: None, + }, + ); +} + +#[cfg(target_os = "macos")] +fn emit_stream_event( + app: &AppHandle, + stream_id: &str, + state: SiriStreamEventState, + error: Option, +) { + let _ = app.emit( + SIRI_STREAM_EVENT, + SiriStreamEvent { + stream_id: stream_id.to_string(), + state, + error, + }, + ); +} + +#[cfg(target_os = "macos")] +fn discover_voices(language_prefix: &str) -> Result, String> { + let prefix = CString::new(language_prefix) + .map_err(|_| "Siri voice language cannot contain NUL bytes".to_string())?; + let mut error = std::ptr::null_mut(); + // SAFETY: The bridge copies the input string synchronously and returns + // owned strings through its documented allocation contract. + let json = unsafe { berd_siri_tts_catalog_json(prefix.as_ptr(), &mut error) }; + let json = take_bridge_string(json) + .ok_or_else(|| bridge_error(error, "Could not load the Siri voice catalog"))?; + serde_json::from_str(&json).map_err(|error| format!("decode Siri voice catalog: {error}")) +} + +#[cfg(target_os = "macos")] +fn discover_languages() -> Result, String> { + let mut error = std::ptr::null_mut(); + // SAFETY: Returned strings follow the bridge allocation contract. + let json = unsafe { berd_siri_tts_languages_json(&mut error) }; + let json = take_bridge_string(json) + .ok_or_else(|| bridge_error(error, "Could not load Siri voice languages"))?; + serde_json::from_str(&json).map_err(|error| format!("decode Siri voice languages: {error}")) +} + +#[cfg(not(target_os = "macos"))] +fn discover_voices(_language_prefix: &str) -> Result, String> { + Ok(Vec::new()) +} + +#[cfg(not(target_os = "macos"))] +fn discover_languages() -> Result, String> { + Ok(Vec::new()) +} + +fn normalize_language(value: &str) -> String { + value.replace('_', "-").to_lowercase() +} + +fn find_voice<'a>( + voices: &'a [SiriVoice], + selection: &SiriVoiceSelection, +) -> Option<&'a SiriVoice> { + let language = normalize_language(&selection.language); + voices.iter().find(|voice| { + voice.name.eq_ignore_ascii_case(&selection.name) + && normalize_language(&voice.language) == language + }) +} + +fn first_installed_voice(voices: &[SiriVoice]) -> Option { + voices + .iter() + .find(|voice| voice.installed) + .map(|voice| SiriVoiceSelection { + name: voice.name.clone(), + language: voice.language.clone(), + }) +} + +fn choose_installed_voice( + preferred_voices: &[SiriVoice], + load_all_voices: impl FnOnce() -> Result, String>, +) -> Result, String> { + if let Some(selection) = first_installed_voice(preferred_voices) { + return Ok(Some(selection)); + } + + Ok(first_installed_voice(&load_all_voices()?)) +} + +fn status(app: &AppHandle, language_prefix: &str) -> Result { + let voices = discover_voices(language_prefix)?; + let available_languages = discover_languages()?; + let path = settings_path(app)?; + let automatic_selection = if read_settings(&path).selected_voice.is_none() { + choose_installed_voice(&voices, || discover_voices(""))? + } else { + None + }; + let settings = update_settings(&path, |settings| { + if settings.selected_voice.is_none() && automatic_selection.is_some() { + settings.selected_voice = automatic_selection; + true + } else { + false + } + })?; + let selected_voice_installed = settings.selected_voice.as_ref().is_some_and(|selection| { + find_voice(&voices, selection).is_some_and(|voice| voice.installed) + || discover_voices(&selection.language) + .ok() + .and_then(|selected| find_voice(&selected, selection).cloned()) + .is_some_and(|voice| voice.installed) + }); + Ok(SiriVoiceStatus { + supported: cfg!(target_os = "macos"), + available_languages, + selected_voice: settings.selected_voice, + selected_voice_installed, + playback_speed: settings + .playback_speed + .clamp(MIN_PLAYBACK_SPEED, MAX_PLAYBACK_SPEED), + voices, + }) +} + +#[tauri::command] +pub async fn get_siri_voice_status( + app: AppHandle, + language_prefix: Option, +) -> Result { + let prefix = language_prefix.unwrap_or_default(); + tauri::async_runtime::spawn_blocking(move || status(&app, prefix.trim())) + .await + .map_err(|error| format!("Siri voice catalog task failed: {error}"))? +} + +#[tauri::command] +pub async fn select_siri_voice(app: AppHandle, voice: SiriVoiceSelection) -> Result<(), String> { + let prefix = voice.language.clone(); + let candidate = voice.clone(); + let installed = tauri::async_runtime::spawn_blocking(move || { + let voices = discover_voices(&prefix)?; + Ok::<_, String>(find_voice(&voices, &candidate).is_some_and(|voice| voice.installed)) + }) + .await + .map_err(|error| format!("Siri voice validation task failed: {error}"))??; + if !installed { + return Err(format!( + "Siri voice {} ({}) must be downloaded before selection", + voice.name, voice.language + )); + } + update_settings(&settings_path(&app)?, |settings| { + settings.selected_voice = Some(voice); + true + }) + .map(|_| ()) +} + +#[tauri::command] +pub fn set_siri_playback_speed(app: AppHandle, speed: f32) -> Result<(), String> { + if !speed.is_finite() || !(MIN_PLAYBACK_SPEED..=MAX_PLAYBACK_SPEED).contains(&speed) { + return Err("Siri playback speed must be between 0.5 and 2.0".to_string()); + } + let path = settings_path(&app)?; + update_settings(&path, |settings| { + settings.playback_speed = speed; + true + }) + .map(|_| ()) +} + +#[tauri::command] +pub async fn download_siri_voice(app: AppHandle, voice: SiriVoiceSelection) -> Result<(), String> { + #[cfg(not(target_os = "macos"))] + { + let _ = (app, voice); + Err("Siri TTS is only available on macOS".to_string()) + } + + #[cfg(target_os = "macos")] + { + let language = CString::new(voice.language.clone()) + .map_err(|_| "Siri voice language cannot contain NUL bytes".to_string())?; + let name = CString::new(voice.name.clone()) + .map_err(|_| "Siri voice name cannot contain NUL bytes".to_string())?; + tauri::async_runtime::spawn_blocking(move || { + let mut error = std::ptr::null_mut(); + // SAFETY: Inputs stay alive for the blocking call and returned + // errors follow the bridge string ownership contract. + let downloaded = unsafe { + berd_siri_tts_download_voice(language.as_ptr(), name.as_ptr(), 300.0, &mut error) + }; + downloaded + .then_some(()) + .ok_or_else(|| bridge_error(error, "Siri voice download failed")) + }) + .await + .map_err(|error| format!("Siri voice download task failed: {error}"))??; + let _ = app; + Ok(()) + } +} + +#[cfg(target_os = "macos")] +fn enqueue_native_stream(stream: *mut std::ffi::c_void, text: &str) -> Result<(), String> { + let text = + CString::new(text).map_err(|_| "Siri speech text cannot contain NUL bytes".to_string())?; + let mut error = std::ptr::null_mut(); + // SAFETY: The native stream remains owned by the worker for this call and + // the bridge copies the text before returning. + let accepted = unsafe { berd_siri_tts_stream_enqueue(stream, text.as_ptr(), &mut error) }; + accepted + .then_some(()) + .ok_or_else(|| bridge_error(error, "Siri stream rejected text")) +} + +#[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] +fn run_siri_stream( + app: AppHandle, + stream_id: String, + selection: SiriVoiceSelection, + speed: f32, + active: Arc, + receiver: mpsc::Receiver, +) -> Result { + let language = CString::new(selection.language) + .map_err(|_| "Siri voice language cannot contain NUL bytes".to_string())?; + let name = CString::new(selection.name) + .map_err(|_| "Siri voice name cannot contain NUL bytes".to_string())?; + let callback_context = Box::new(SiriStreamCallbackContext { + app: app.clone(), + stream_id: stream_id.clone(), + }); + let callback_context = Box::into_raw(callback_context); + let mut error = std::ptr::null_mut(); + // SAFETY: Strings remain alive through creation. The callback context is + // released only after the native stream has finished and is released. + let stream = unsafe { + berd_siri_tts_stream_create( + language.as_ptr(), + name.as_ptr(), + speed, + Some(siri_playback_started), + callback_context.cast(), + &mut error, + ) + }; + if stream.is_null() { + // SAFETY: Native creation failed, so no callback retained the box. + unsafe { drop(Box::from_raw(callback_context)) }; + return Err(bridge_error(error, "Could not start Siri voice stream")); + } + + let result = (|| { + let mut pending = String::new(); + let mut first_chunk_pending = true; + let mut finishing = false; + let mut watchdog: Option = None; + loop { + if !active.load(Ordering::SeqCst) { + unsafe { berd_siri_tts_stream_cancel(stream) }; + return Ok(SiriStreamEventState::Interrupted); + } + if finishing && unsafe { berd_siri_tts_stream_is_finished(stream) } { + let native_error = + take_bridge_string(unsafe { berd_siri_tts_stream_copy_error(stream) }); + return native_error.map_or(Ok(SiriStreamEventState::Completed), Err); + } + if let Some(watchdog) = watchdog.as_mut() { + let progress = unsafe { berd_siri_tts_stream_progress(stream) }; + if watchdog.observe(progress, Instant::now()) { + unsafe { berd_siri_tts_stream_cancel(stream) }; + return Err("Siri synthesis stopped making progress".to_string()); + } + } + + let command = match receiver.recv_timeout(Duration::from_millis(10)) { + Ok(command) => command, + Err(mpsc::RecvTimeoutError::Timeout) => continue, + Err(mpsc::RecvTimeoutError::Disconnected) => SiriStreamCommand::Stop, + }; + match command { + SiriStreamCommand::Append(text) if !finishing => { + pending.push_str(&text); + let split = berd_voice::take_streaming_text_chunks( + &pending, + first_chunk_pending, + false, + )?; + pending = split.pending; + first_chunk_pending = split.first_chunk_pending; + for ready in split.ready { + enqueue_native_stream(stream, ready.trim())?; + } + } + SiriStreamCommand::Flush if !finishing => { + let split = berd_voice::take_streaming_text_chunks( + &pending, + first_chunk_pending, + true, + )?; + pending = split.pending; + first_chunk_pending = split.first_chunk_pending; + for ready in split.ready { + enqueue_native_stream(stream, ready.trim())?; + } + } + SiriStreamCommand::Finish if !finishing => { + let split = berd_voice::take_streaming_text_chunks( + &pending, + first_chunk_pending, + true, + )?; + for ready in split.ready { + enqueue_native_stream(stream, ready.trim())?; + } + pending.clear(); + finishing = true; + unsafe { berd_siri_tts_stream_finish(stream) }; + watchdog = Some(SiriStreamWatchdog::new( + unsafe { berd_siri_tts_stream_progress(stream) }, + Instant::now(), + )); + } + SiriStreamCommand::Stop => { + active.store(false, Ordering::SeqCst); + unsafe { berd_siri_tts_stream_cancel(stream) }; + return Ok(SiriStreamEventState::Interrupted); + } + _ => {} + } + } + })(); + + unsafe { + berd_siri_tts_stream_release(stream); + drop(Box::from_raw(callback_context)); + } + result +} + +#[tauri::command] +pub fn start_siri_voice_stream( + app: AppHandle, + webview_window: tauri::WebviewWindow, + state: tauri::State<'_, SiriVoiceState>, + native_voice: tauri::State<'_, NativeVoiceState>, + stream_id: String, +) -> Result<(), String> { + #[cfg(not(target_os = "macos"))] + { + let _ = (app, webview_window, state, native_voice, stream_id); + Err("Siri TTS is only available on macOS".to_string()) + } + + #[cfg(target_os = "macos")] + { + if stream_id.trim().is_empty() { + return Err("Siri voice stream id cannot be empty".to_string()); + } + let settings = read_settings(&settings_path(&app)?); + let selection = settings.selected_voice.ok_or_else(|| { + "Select an installed Siri voice in Voice settings before using Siri TTS".to_string() + })?; + let active = begin_playback(&state, webview_window.label())?; + let capture_suppression = + output_device_uses_speakers(effective_output_device_name(None).as_deref()).then(|| { + log::info!("[voice-echo-guard] speaker output detected"); + native_voice.suppress_capture() + }); + let (sender, receiver) = mpsc::channel(); + { + let mut runtime = state + .runtime + .lock() + .map_err(|_| "Siri playback state lock was poisoned".to_string())?; + runtime.stream = Some(ActiveSiriStream { + id: stream_id.clone(), + sender, + }); + } + let playback_state = state.inner().clone(); + let playback_active = active.clone(); + tauri::async_runtime::spawn_blocking(move || { + let _capture_suppression = capture_suppression; + let result = run_siri_stream( + app.clone(), + stream_id.clone(), + selection, + settings + .playback_speed + .clamp(MIN_PLAYBACK_SPEED, MAX_PLAYBACK_SPEED), + active.clone(), + receiver, + ); + let (event_state, error) = match result { + Ok(state) => (state, None), + Err(_error) if !active.load(Ordering::SeqCst) => { + (SiriStreamEventState::Interrupted, None) + } + Err(error) => (SiriStreamEventState::Failed, Some(error)), + }; + emit_stream_event(&app, &stream_id, event_state, error); + finish_playback(&playback_state, &playback_active); + }); + Ok(()) + } +} + +#[tauri::command] +pub fn append_siri_voice_stream( + state: tauri::State<'_, SiriVoiceState>, + stream_id: String, + text: String, +) -> Result<(), String> { + if text.is_empty() { + return Ok(()); + } + send_stream_command(&state, &stream_id, SiriStreamCommand::Append(text)) +} + +#[tauri::command] +pub fn flush_siri_voice_stream( + state: tauri::State<'_, SiriVoiceState>, + stream_id: String, +) -> Result<(), String> { + send_stream_command(&state, &stream_id, SiriStreamCommand::Flush) +} + +#[tauri::command] +pub fn finish_siri_voice_stream( + state: tauri::State<'_, SiriVoiceState>, + stream_id: String, +) -> Result<(), String> { + send_stream_command(&state, &stream_id, SiriStreamCommand::Finish) +} + +#[cfg(target_os = "macos")] +fn send_stream_command( + state: &SiriVoiceState, + stream_id: &str, + command: SiriStreamCommand, +) -> Result<(), String> { + let runtime = state + .runtime + .lock() + .map_err(|_| "Siri playback state lock was poisoned".to_string())?; + let stream = runtime + .stream + .as_ref() + .filter(|stream| stream.id == stream_id) + .ok_or_else(|| format!("Siri voice stream is not active: {stream_id}"))?; + stream + .sender + .send(command) + .map_err(|_| format!("Siri voice stream worker stopped: {stream_id}")) +} + +#[cfg(not(target_os = "macos"))] +fn send_stream_command( + _state: &SiriVoiceState, + _stream_id: &str, + _command: SiriStreamCommand, +) -> Result<(), String> { + Err("Siri TTS is only available on macOS".to_string()) +} + +#[tauri::command] +pub async fn preview_siri_voice( + app: AppHandle, + webview_window: tauri::WebviewWindow, + state: tauri::State<'_, SiriVoiceState>, + native_voice: tauri::State<'_, NativeVoiceState>, + voice: SiriVoiceSelection, +) -> Result<(), String> { + #[cfg(not(target_os = "macos"))] + { + let _ = (app, webview_window, state, native_voice, voice); + Err("Siri TTS is only available on macOS".to_string()) + } + + #[cfg(target_os = "macos")] + { + let speed = read_settings(&settings_path(&app)?) + .playback_speed + .clamp(MIN_PLAYBACK_SPEED, MAX_PLAYBACK_SPEED); + let text = CString::new("Hello. This is a preview of my voice.").expect("static preview"); + let language = CString::new(voice.language.clone()) + .map_err(|_| "Siri voice language cannot contain NUL bytes".to_string())?; + let name = CString::new(voice.name.clone()) + .map_err(|_| "Siri voice name cannot contain NUL bytes".to_string())?; + let active = begin_playback(&state, webview_window.label())?; + let capture_suppression = + output_device_uses_speakers(effective_output_device_name(None).as_deref()).then(|| { + log::info!("[voice-echo-guard] speaker output detected"); + native_voice.suppress_capture() + }); + let playback_state = state.inner().clone(); + let playback_active = active.clone(); + tauri::async_runtime::spawn_blocking(move || { + let _capture_suppression = capture_suppression; + let result = (|| { + let mut error = std::ptr::null_mut(); + // SAFETY: The bridge copies all strings synchronously. The Arc + // keeps the callback context alive until the call returns. + let context = Arc::as_ptr(&playback_active).cast_mut().cast(); + let sample_played = unsafe { + berd_siri_tts_play_sample( + name.as_ptr(), + language.as_ptr(), + speed, + Some(should_stop_siri_playback), + context, + &mut error, + ) + }; + if sample_played { + return Ok(()); + } + + let sample_error = bridge_error(error, "No system preview is available"); + let voices = discover_voices(&voice.language)?; + if !find_voice(&voices, &voice).is_some_and(|candidate| candidate.installed) { + return Err(sample_error); + } + + error = std::ptr::null_mut(); + // SAFETY: The bridge copies all strings synchronously and the + // callback context remains alive for the duration of the call. + let spoken = unsafe { + berd_siri_tts_speak( + text.as_ptr(), + language.as_ptr(), + name.as_ptr(), + speed, + Some(should_stop_siri_playback), + None, + context, + &mut error, + ) + }; + spoken + .then_some(()) + .ok_or_else(|| bridge_error(error, "Siri voice preview failed")) + })(); + finish_playback(&playback_state, &playback_active); + result + }) + .await + .map_err(|error| format!("Siri voice preview task failed: {error}"))? + } +} + +#[tauri::command] +pub fn stop_siri_voice(state: tauri::State<'_, SiriVoiceState>) -> Result { + stop_siri_playback(&state) +} + +fn stop_siri_playback(state: &SiriVoiceState) -> Result { + stop_siri_playback_for_owner(state, None) +} + +fn stop_siri_playback_for_owner( + state: &SiriVoiceState, + owner_window: Option<&str>, +) -> Result { + let runtime = state + .runtime + .lock() + .map_err(|_| "Siri playback state lock was poisoned".to_string())?; + if owner_window.is_some_and(|owner| runtime.owner_window.as_deref() != Some(owner)) { + return Ok(false); + } + let Some(active) = runtime.active.as_ref() else { + return Ok(false); + }; + active.store(false, Ordering::SeqCst); + #[cfg(target_os = "macos")] + if let Some(stream) = runtime.stream.as_ref() { + let _ = stream.sender.send(SiriStreamCommand::Stop); + } + Ok(true) +} + +impl SiriVoiceState { + pub(crate) fn stop_for_window_destroyed(&self, window_label: &str) -> bool { + stop_siri_playback_for_owner(self, Some(window_label)).unwrap_or_else(|error| { + log::warn!("Failed to stop Siri playback for a destroyed window: {error}"); + false + }) + } + + pub(crate) fn stop_for_app_exit(&self) { + if let Err(error) = stop_siri_playback(self) { + log::warn!("Failed to stop Siri playback during app exit: {error}"); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn voice_lookup_normalizes_language_and_name_case() { + let voices = vec![SiriVoice { + name: "Aaron".to_string(), + language: "en_US".to_string(), + size_bytes: 10, + installed: true, + }]; + let selected = SiriVoiceSelection { + name: "aaron".to_string(), + language: "EN-us".to_string(), + }; + assert_eq!(find_voice(&voices, &selected), voices.first()); + } + + #[test] + fn settings_default_without_a_selected_voice() { + let directory = tempfile::tempdir().expect("tempdir"); + assert_eq!( + read_settings(&directory.path().join("missing.json")).selected_voice, + None + ); + } + + #[test] + fn concurrent_settings_updates_preserve_both_fields() { + let directory = tempfile::tempdir().expect("tempdir"); + let path = Arc::new(directory.path().join("settings.json")); + let (selection_entered_tx, selection_entered_rx) = std::sync::mpsc::channel(); + let (release_selection_tx, release_selection_rx) = std::sync::mpsc::channel(); + + let selection_path = path.clone(); + let selection_writer = std::thread::spawn(move || { + update_settings(&selection_path, |settings| { + selection_entered_tx.send(()).expect("signal settings read"); + release_selection_rx + .recv() + .expect("release selection write"); + settings.selected_voice = Some(SiriVoiceSelection { + name: "Aaron".to_string(), + language: "en-US".to_string(), + }); + true + }) + .expect("write selected voice"); + }); + + selection_entered_rx + .recv() + .expect("selection acquired lock"); + assert!(matches!( + SIRI_SETTINGS_LOCK.try_lock(), + Err(std::sync::TryLockError::WouldBlock) + )); + let speed_path = path.clone(); + let (speed_started_tx, speed_started_rx) = std::sync::mpsc::channel(); + let speed_writer = std::thread::spawn(move || { + speed_started_tx.send(()).expect("signal speed update"); + update_settings(&speed_path, |settings| { + settings.playback_speed = 1.5; + true + }) + .expect("write playback speed"); + }); + speed_started_rx.recv().expect("speed update started"); + release_selection_tx.send(()).expect("release selection"); + selection_writer.join().expect("selection writer"); + speed_writer.join().expect("speed writer"); + + let settings = read_settings(&path); + assert_eq!( + settings.selected_voice, + Some(SiriVoiceSelection { + name: "Aaron".to_string(), + language: "en-US".to_string(), + }) + ); + assert_eq!(settings.playback_speed, 1.5); + serde_json::from_slice::(&fs::read(&*path).expect("settings JSON")) + .expect("valid settings JSON"); + } + + #[test] + fn auto_selection_uses_an_installed_siri_voice() { + let voices = vec![ + SiriVoice { + name: "Quinn".to_string(), + language: "en-US".to_string(), + size_bytes: 10, + installed: false, + }, + SiriVoice { + name: "Aaron".to_string(), + language: "en-US".to_string(), + size_bytes: 10, + installed: true, + }, + ]; + + assert_eq!( + first_installed_voice(&voices), + Some(SiriVoiceSelection { + name: "Aaron".to_string(), + language: "en-US".to_string(), + }) + ); + } + + #[test] + fn auto_selection_falls_back_to_an_installed_voice_outside_the_filter() { + let filtered_voices = vec![SiriVoice { + name: "Aaron".to_string(), + language: "en-US".to_string(), + size_bytes: 10, + installed: false, + }]; + + assert_eq!( + choose_installed_voice(&filtered_voices, || { + Ok(vec![SiriVoice { + name: "Catherine".to_string(), + language: "en-AU".to_string(), + size_bytes: 10, + installed: true, + }]) + }), + Ok(Some(SiriVoiceSelection { + name: "Catherine".to_string(), + language: "en-AU".to_string(), + })) + ); + } + + #[test] + fn window_destroy_stops_only_its_owned_siri_playback() { + let state = SiriVoiceState::default(); + let active = begin_playback(&state, "session-window").expect("start playback"); + + assert!(!state.stop_for_window_destroyed("other-window")); + assert!(active.load(Ordering::SeqCst)); + + assert!(state.stop_for_window_destroyed("session-window")); + assert!(!active.load(Ordering::SeqCst)); + + finish_playback(&state, &active); + assert!(begin_playback(&state, "next-window").is_ok()); + } + + #[cfg(target_os = "macos")] + #[test] + fn stream_watchdog_times_out_only_after_progress_stalls() { + let started = Instant::now(); + let mut watchdog = SiriStreamWatchdog::new(1, started); + + assert!(!watchdog.observe(2, started + SIRI_STREAM_STALL_TIMEOUT)); + assert!(!watchdog.observe( + 2, + started + SIRI_STREAM_STALL_TIMEOUT + Duration::from_millis(1), + )); + assert!(watchdog.observe(2, started + SIRI_STREAM_STALL_TIMEOUT * 2,)); + } +} diff --git a/src-tauri/src/commands/window_session.rs b/src-tauri/src/commands/window_session.rs index 8f8874edf..6af73a640 100644 --- a/src-tauri/src/commands/window_session.rs +++ b/src-tauri/src/commands/window_session.rs @@ -688,6 +688,9 @@ pub fn open_session_window( .state::() .stop_for_window_destroyed(); } + app_for_close + .state::() + .stop_for_window_destroyed(&label_for_close); reg_for_close.release_label(&label_for_close); let _ = emit_snapshot(&app_for_close, ®_for_close); } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index c7541d2a1..f90b7b950 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -213,6 +213,7 @@ pub fn run() { app.manage(commands::agent_setup::AgentSetupRegistry::default()); app.manage(commands::model_setup::ModelSetupRegistry::default()); app.manage(commands::pocket_voice::PocketVoiceState::default()); + app.manage(commands::siri_voice::SiriVoiceState::default()); app.manage(commands::native_voice::NativeVoiceState::default()); app.manage(commands::voice_capture::VoiceCaptureState::default()); app.manage(commands::telemetry::TelemetryAuthState::new( @@ -628,6 +629,16 @@ pub fn run() { commands::pocket_voice::finish_pocket_voice_stream, commands::pocket_voice::stop_pocket_voice, commands::pocket_voice::remove_voice_model, + commands::siri_voice::get_siri_voice_status, + commands::siri_voice::select_siri_voice, + commands::siri_voice::download_siri_voice, + commands::siri_voice::set_siri_playback_speed, + commands::siri_voice::preview_siri_voice, + commands::siri_voice::start_siri_voice_stream, + commands::siri_voice::append_siri_voice_stream, + commands::siri_voice::flush_siri_voice_stream, + commands::siri_voice::finish_siri_voice_stream, + commands::siri_voice::stop_siri_voice, commands::native_voice::get_native_voice_conversation_status, commands::native_voice::drain_native_voice_conversation_transcripts, commands::native_voice::acknowledge_native_voice_conversation_transcript, @@ -666,6 +677,8 @@ pub fn run() { app.state::().stop_all(); app.state::() .stop_for_app_exit(); + app.state::() + .stop_for_app_exit(); services::acp::goose_serve::GooseServeProcess::kill_singleton(); } #[cfg(target_os = "macos")] diff --git a/src/app/AppShell.tsx b/src/app/AppShell.tsx index 4606476c0..c08504812 100644 --- a/src/app/AppShell.tsx +++ b/src/app/AppShell.tsx @@ -225,7 +225,10 @@ import { OnboardingFlow } from "@/features/onboarding/ui/OnboardingFlow"; import { useOnboardingState } from "@/features/onboarding/model"; import { useVoiceConversationStore } from "@/features/voice-conversation/stores/voiceConversationStore"; import { usePocketVoiceSetup } from "@/features/voice-conversation/hooks/usePocketVoiceSetup"; +import { useSiriVoiceSetup } from "@/features/voice-conversation/hooks/useSiriVoiceSetup"; import { PocketVoiceSetupDialog } from "@/features/voice-conversation/ui/PocketVoiceSetupDialog"; +import { useVoiceOutputPreference } from "@/features/voice-conversation/lib/voiceOutputPreference"; +import { isVoiceSetupReady } from "@/features/voice-conversation/lib/voiceSetupReadiness"; import { cancelPendingVoiceStart, continuePendingVoiceStart, @@ -714,6 +717,15 @@ export function AppShell({ const globalPocketVoiceSetup = usePocketVoiceSetup( capabilities.voiceConversation, ); + const globalVoiceOutput = useVoiceOutputPreference(); + const globalSiriVoiceSetup = useSiriVoiceSetup( + capabilities.voiceConversation && globalVoiceOutput.backend === "siri", + ); + const globalVoiceReady = isVoiceSetupReady( + globalPocketVoiceSetup.status, + globalSiriVoiceSetup.status, + globalVoiceOutput.backend, + ); const [globalPocketVoiceSetupOpen, setGlobalPocketVoiceSetupOpen] = useState(false); const pendingGlobalVoiceStartRef = @@ -3206,7 +3218,7 @@ export function AppShell({ setupComplete = false, ): Promise => { if (!capabilities.voiceConversation) return Promise.resolve(false); - if (!setupComplete && globalPocketVoiceSetup.status?.installed !== true) { + if (!setupComplete && !globalVoiceReady) { const pending = deferPendingVoiceStart( pendingGlobalVoiceStartRef, payload, @@ -3296,7 +3308,7 @@ export function AppShell({ [ capabilities.voiceConversation, createNewTab, - globalPocketVoiceSetup.status?.installed, + globalVoiceReady, guardAppNavigation, handleNavigateToSession, patchSession, @@ -5298,8 +5310,7 @@ export function AppShell({ capabilities.voiceConversation ? { enabled: true, - ready: - globalPocketVoiceSetup.status?.installed === true, + ready: globalVoiceReady, onStart: handleGlobalVoiceConversationStart, } : undefined @@ -5319,6 +5330,9 @@ export function AppShell({ onOpenChange={handleGlobalPocketVoiceSetupOpenChange} onUseSelected={handleGlobalPocketVoiceUseSelected} setup={globalPocketVoiceSetup} + siriSetup={globalSiriVoiceSetup} + backend={globalVoiceOutput.backend} + onBackendChange={globalVoiceOutput.setBackend} /> state.requestStart, ); @@ -245,7 +257,7 @@ export function ChatView({ onSend: controller.handleSend, enabled: capabilities.voiceConversation, isGooseSession: controller.selectedProvider === "goose", - pocketReady: pocketVoiceSetup.status?.installed === true, + pocketReady: voiceReady, onPocketSetupRequired: () => { pendingPocketVoiceStartRef.current = sessionId; setPocketVoiceSetupOpen(true); @@ -992,6 +1004,9 @@ export function ChatView({ onOpenChange={handlePocketVoiceSetupOpenChange} onUseSelected={handlePocketVoiceUseSelected} setup={pocketVoiceSetup} + siriSetup={siriVoiceSetup} + backend={voiceOutput.backend} + onBackendChange={voiceOutput.setBackend} /> ({ + invoke: vi.fn(), +})); + +vi.mock("@tauri-apps/api/core", () => ({ invoke: mocks.invoke })); + +import { getSiriVoiceStatus } from "./siriVoice"; + +describe("Siri voice API", () => { + beforeEach(() => { + mocks.invoke.mockReset(); + }); + + it("passes the selected language to voice discovery", async () => { + const status = { supported: true, voices: [] }; + mocks.invoke.mockResolvedValue(status); + + await expect(getSiriVoiceStatus("fr-CA")).resolves.toBe(status); + + expect(mocks.invoke).toHaveBeenCalledWith("get_siri_voice_status", { + languagePrefix: "fr-CA", + }); + }); + + it("coalesces only matching regional locale requests", async () => { + let resolveRequest: ((value: unknown) => void) | undefined; + mocks.invoke.mockReturnValue( + new Promise((resolve) => { + resolveRequest = resolve; + }), + ); + + const first = getSiriVoiceStatus("en-US", { coalesce: true }); + const joined = getSiriVoiceStatus("en_US", { coalesce: true }); + const separate = getSiriVoiceStatus("en-AU", { coalesce: true }); + + expect(first).toBe(joined); + expect(separate).not.toBe(first); + expect(mocks.invoke).toHaveBeenCalledTimes(2); + resolveRequest?.({ supported: true, voices: [] }); + await Promise.all([first, joined, separate]); + }); +}); diff --git a/src/features/voice-conversation/api/siriVoice.ts b/src/features/voice-conversation/api/siriVoice.ts new file mode 100644 index 000000000..2b72bbc56 --- /dev/null +++ b/src/features/voice-conversation/api/siriVoice.ts @@ -0,0 +1,94 @@ +import { invoke } from "@tauri-apps/api/core"; +import { listen, type UnlistenFn } from "@tauri-apps/api/event"; + +export interface SiriVoice { + name: string; + language: string; + sizeBytes: number; + installed: boolean; +} + +export interface SiriVoiceSelection { + name: string; + language: string; +} + +export interface SiriVoiceStatus { + supported: boolean; + availableLanguages: string[]; + selectedVoice: SiriVoiceSelection | null; + selectedVoiceInstalled: boolean; + playbackSpeed: number; + voices: SiriVoice[]; +} + +const statusRequests = new Map>(); + +export function getSiriVoiceStatus( + languagePrefix: string, + options?: { coalesce?: boolean }, +): Promise { + const key = languagePrefix.replaceAll("_", "-").toLowerCase(); + const current = statusRequests.get(key); + if (options?.coalesce && current) return current; + const request = invoke("get_siri_voice_status", { + languagePrefix, + }).finally(() => { + if (statusRequests.get(key) === request) statusRequests.delete(key); + }); + statusRequests.set(key, request); + return request; +} + +export function downloadSiriVoice(voice: SiriVoiceSelection): Promise { + return invoke("download_siri_voice", { voice }); +} + +export function selectSiriVoice(voice: SiriVoiceSelection): Promise { + return invoke("select_siri_voice", { voice }); +} + +export function previewSiriVoice(voice: SiriVoiceSelection): Promise { + return invoke("preview_siri_voice", { voice }); +} + +export function setSiriPlaybackSpeed(speed: number): Promise { + return invoke("set_siri_playback_speed", { speed }); +} + +export interface SiriVoiceStreamEvent { + streamId: string; + state: "started" | "completed" | "interrupted" | "failed"; + error: string | null; +} + +export function startSiriVoiceStream(streamId: string): Promise { + return invoke("start_siri_voice_stream", { streamId }); +} + +export function appendSiriVoiceStream( + streamId: string, + text: string, +): Promise { + return invoke("append_siri_voice_stream", { streamId, text }); +} + +export function flushSiriVoiceStream(streamId: string): Promise { + return invoke("flush_siri_voice_stream", { streamId }); +} + +export function finishSiriVoiceStream(streamId: string): Promise { + return invoke("finish_siri_voice_stream", { streamId }); +} + +export function stopSiriVoice(): Promise { + return invoke("stop_siri_voice"); +} + +export function listenToSiriVoiceStream( + onEvent: (event: SiriVoiceStreamEvent) => void, +): Promise { + return listen("siri-voice:stream-event", (event) => + onEvent(event.payload), + ); +} diff --git a/src/features/voice-conversation/hooks/useSiriVoiceSetup.test.ts b/src/features/voice-conversation/hooks/useSiriVoiceSetup.test.ts new file mode 100644 index 000000000..0aea84a47 --- /dev/null +++ b/src/features/voice-conversation/hooks/useSiriVoiceSetup.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { + availableLocales, + canonicalLocale, + chooseAvailableLocale, + initialSelectedVoiceLocale, +} from "./useSiriVoiceSetup"; + +describe("Siri voice locales", () => { + it("preserves exact regional variants", () => { + expect(availableLocales(["en_US", "en-AU", "en-IN", "en-US"])).toEqual([ + "en-AU", + "en-IN", + "en-US", + ]); + expect(canonicalLocale("en_US")).toBe("en-US"); + }); + + it("uses the exact system locale when it is available", () => { + expect(chooseAvailableLocale("en-US", ["en-AU", "en-IN", "en-US"])).toBe( + "en-US", + ); + }); + + it("falls back to a regional variant without adding an all-language option", () => { + expect(chooseAvailableLocale("en-CA", ["en-AU", "en-IN"])).toBe("en-AU"); + }); + + it("opens on the selected voice's regional locale", () => { + expect( + initialSelectedVoiceLocale("en-US", ["en-AU", "en-US"], { + name: "Catherine", + language: "en_AU", + }), + ).toBe("en-AU"); + }); +}); diff --git a/src/features/voice-conversation/hooks/useSiriVoiceSetup.ts b/src/features/voice-conversation/hooks/useSiriVoiceSetup.ts new file mode 100644 index 000000000..4d4d8d67b --- /dev/null +++ b/src/features/voice-conversation/hooks/useSiriVoiceSetup.ts @@ -0,0 +1,272 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { + downloadSiriVoice, + getSiriVoiceStatus, + previewSiriVoice, + selectSiriVoice, + setSiriPlaybackSpeed, + type SiriVoice, + type SiriVoiceSelection, + type SiriVoiceStatus, +} from "../api/siriVoice"; + +function canonicalLocale(locale: string): string { + try { + return new Intl.Locale(locale.replaceAll("_", "-")).toString(); + } catch { + return locale.replaceAll("_", "-") || "en-US"; + } +} + +function primaryLanguage(locale: string): string { + try { + return new Intl.Locale(canonicalLocale(locale)).language; + } catch { + return canonicalLocale(locale).split("-", 1)[0]?.toLowerCase() || "en"; + } +} + +function availableLocales(locales: string[]): string[] { + return Array.from(new Set(locales.map(canonicalLocale))).sort(); +} + +function chooseAvailableLocale(preferred: string, available: string[]): string { + const exact = canonicalLocale(preferred); + if (available.includes(exact)) return exact; + const preferredLanguage = primaryLanguage(exact); + return ( + available.find( + (candidate) => primaryLanguage(candidate) === preferredLanguage, + ) ?? + available[0] ?? + exact + ); +} + +function initialSelectedVoiceLocale( + current: string, + available: string[], + selectedVoice: SiriVoiceSelection, +): string { + const selectedLocale = canonicalLocale(selectedVoice.language); + return available.includes(selectedLocale) ? selectedLocale : current; +} + +function voiceKey(voice: SiriVoiceSelection): string { + return `${voice.name.toLowerCase()}|${voice.language.toLowerCase()}`; +} + +const SIRI_VOICE_SETTINGS_CHANGED = "berd:siri-voice-settings-changed"; + +export interface SiriVoiceSetup { + status: SiriVoiceStatus | null; + language: string; + languages: string[]; + loading: boolean; + error: string | null; + downloadingVoiceKey: string | null; + previewingVoiceKey: string | null; + setLanguage: (language: string) => void; + setPlaybackSpeed: (speed: number) => Promise; + downloadVoice: (voice: SiriVoice) => Promise; + previewVoice: (voice: SiriVoice) => Promise; + selectVoice: (voice: SiriVoice) => Promise; +} + +export function useSiriVoiceSetup(enabled = true): SiriVoiceSetup { + const [status, setStatus] = useState(null); + const [language, setLanguage] = useState(() => + canonicalLocale( + typeof navigator === "undefined" ? "en-US" : navigator.language, + ), + ); + const [loading, setLoading] = useState(enabled); + const [error, setError] = useState(null); + const [downloadingVoiceKey, setDownloadingVoiceKey] = useState( + null, + ); + const [previewingVoiceKey, setPreviewingVoiceKey] = useState( + null, + ); + const languageRef = useRef(language); + const initialSelectedLocaleAppliedRef = useRef(false); + const languageSelectedByUserRef = useRef(false); + languageRef.current = language; + + const selectLanguage = useCallback((nextLanguage: string) => { + languageSelectedByUserRef.current = true; + setLanguage(canonicalLocale(nextLanguage)); + }, []); + + const refresh = useCallback(async (prefix: string) => { + const next = await getSiriVoiceStatus(prefix, { coalesce: true }); + if (canonicalLocale(languageRef.current) === canonicalLocale(prefix)) { + setStatus(next); + setError(null); + } + return next; + }, []); + + useEffect(() => { + if (!enabled || !window.__TAURI_INTERNALS__) { + setStatus(null); + setLoading(false); + return; + } + let active = true; + setLoading(true); + setError(null); + void getSiriVoiceStatus(language, { coalesce: true }) + .then((next) => { + if (active) setStatus(next); + }) + .catch((nextError) => { + if (active) setError(String(nextError)); + }) + .finally(() => { + if (active) setLoading(false); + }); + return () => { + active = false; + }; + }, [enabled, language]); + + useEffect(() => { + if (!enabled || !window.__TAURI_INTERNALS__) return; + const handleSettingsChanged = () => { + void refresh(language).catch((nextError) => { + setError(String(nextError)); + }); + }; + window.addEventListener(SIRI_VOICE_SETTINGS_CHANGED, handleSettingsChanged); + return () => { + window.removeEventListener( + SIRI_VOICE_SETTINGS_CHANGED, + handleSettingsChanged, + ); + }; + }, [enabled, language, refresh]); + + const languages = useMemo( + () => availableLocales(status?.availableLanguages ?? []), + [status?.availableLanguages], + ); + + useEffect(() => { + if (languages.length === 0 || languages.includes(language)) return; + setLanguage(chooseAvailableLocale(language, languages)); + }, [language, languages]); + + useEffect(() => { + if ( + initialSelectedLocaleAppliedRef.current || + languageSelectedByUserRef.current || + !status?.selectedVoice + ) { + return; + } + initialSelectedLocaleAppliedRef.current = true; + const selectedLocale = initialSelectedVoiceLocale( + language, + languages, + status.selectedVoice, + ); + if (selectedLocale !== language) { + setLanguage(selectedLocale); + } + }, [language, languages, status?.selectedVoice]); + + useEffect(() => { + if (!enabled || !window.__TAURI_INTERNALS__) return; + const handleFocus = () => { + void refresh(language).catch((nextError) => { + setError(String(nextError)); + }); + }; + window.addEventListener("focus", handleFocus); + return () => window.removeEventListener("focus", handleFocus); + }, [enabled, language, refresh]); + + const downloadVoice = useCallback( + async (voice: SiriVoice) => { + const selection = { name: voice.name, language: voice.language }; + setError(null); + setDownloadingVoiceKey(voiceKey(selection)); + try { + await downloadSiriVoice(selection); + window.dispatchEvent(new Event(SIRI_VOICE_SETTINGS_CHANGED)); + await refresh(language); + } catch (nextError) { + setError(String(nextError)); + } finally { + setDownloadingVoiceKey(null); + } + }, + [language, refresh], + ); + + const previewVoice = useCallback(async (voice: SiriVoice) => { + const selection = { name: voice.name, language: voice.language }; + setError(null); + setPreviewingVoiceKey(voiceKey(selection)); + try { + await previewSiriVoice(selection); + } catch (nextError) { + setError(String(nextError)); + } finally { + setPreviewingVoiceKey(null); + } + }, []); + + const selectVoice = useCallback( + async (voice: SiriVoice) => { + setError(null); + try { + await selectSiriVoice({ name: voice.name, language: voice.language }); + window.dispatchEvent(new Event(SIRI_VOICE_SETTINGS_CHANGED)); + await refresh(language); + } catch (nextError) { + setError(String(nextError)); + } + }, + [language, refresh], + ); + + const setPlaybackSpeed = useCallback( + async (speed: number) => { + setError(null); + try { + await setSiriPlaybackSpeed(speed); + window.dispatchEvent(new Event(SIRI_VOICE_SETTINGS_CHANGED)); + await refresh(language); + } catch (nextError) { + setError(String(nextError)); + } + }, + [language, refresh], + ); + + return { + status, + language, + languages, + loading, + error, + downloadingVoiceKey, + previewingVoiceKey, + setLanguage: selectLanguage, + setPlaybackSpeed, + downloadVoice, + previewVoice, + selectVoice, + }; +} + +export { + availableLocales, + canonicalLocale, + chooseAvailableLocale, + initialSelectedVoiceLocale, + primaryLanguage, + voiceKey, +}; diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts index 3d0c56120..87d9416f5 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts @@ -5,12 +5,19 @@ import { useVoiceConversationStore } from "../stores/voiceConversationStore"; import type { PocketVoiceStreamEvent } from "../api/pocketVoice"; const mocks = vi.hoisted(() => ({ + backend: "pocket" as "pocket" | "siri", start: vi.fn<(streamId: string) => Promise>(), append: vi.fn<(streamId: string, text: string) => Promise>(), flush: vi.fn<(streamId: string) => Promise>(), finish: vi.fn<(streamId: string) => Promise>(), stop: vi.fn<() => Promise>(), streamHandler: null as ((event: PocketVoiceStreamEvent) => void) | null, + siriStart: vi.fn<(streamId: string) => Promise>(), + siriAppend: vi.fn<(streamId: string, text: string) => Promise>(), + siriFlush: vi.fn<(streamId: string) => Promise>(), + siriFinish: vi.fn<(streamId: string) => Promise>(), + siriStop: vi.fn<() => Promise>(), + siriStreamHandler: null as ((event: PocketVoiceStreamEvent) => void) | null, })); vi.mock("../api/pocketVoice", () => ({ @@ -28,6 +35,25 @@ vi.mock("../api/pocketVoice", () => ({ }, })); +vi.mock("../api/siriVoice", () => ({ + startSiriVoiceStream: (streamId: string) => mocks.siriStart(streamId), + appendSiriVoiceStream: (streamId: string, text: string) => + mocks.siriAppend(streamId, text), + flushSiriVoiceStream: (streamId: string) => mocks.siriFlush(streamId), + finishSiriVoiceStream: (streamId: string) => mocks.siriFinish(streamId), + stopSiriVoice: () => mocks.siriStop(), + listenToSiriVoiceStream: async ( + handler: (event: PocketVoiceStreamEvent) => void, + ) => { + mocks.siriStreamHandler = handler; + return vi.fn(); + }, +})); + +vi.mock("./voiceOutputPreference", () => ({ + getVoiceOutputBackend: () => mocks.backend, +})); + import { startNativeAssistantSpeech, stopNativeAssistantSpeech, @@ -60,12 +86,19 @@ function emit( describe("native assistant speech stream", () => { beforeEach(() => { + mocks.backend = "pocket"; mocks.start.mockReset().mockResolvedValue(); mocks.append.mockReset().mockResolvedValue(); mocks.flush.mockReset().mockResolvedValue(); mocks.finish.mockReset().mockResolvedValue(); mocks.stop.mockReset().mockResolvedValue(true); mocks.streamHandler = null; + mocks.siriStart.mockReset().mockResolvedValue(); + mocks.siriAppend.mockReset().mockResolvedValue(); + mocks.siriFlush.mockReset().mockResolvedValue(); + mocks.siriFinish.mockReset().mockResolvedValue(); + mocks.siriStop.mockReset().mockResolvedValue(true); + mocks.siriStreamHandler = null; useChatStore.setState({ messagesBySession: {}, sessionStateById: {}, @@ -120,6 +153,27 @@ describe("native assistant speech stream", () => { }); }); + it("routes the complete utterance stream through Siri when selected", async () => { + mocks.backend = "siri"; + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: "Hello from Siri." }], "completed"), + ]); + + await vi.waitFor(() => { + expect(mocks.siriStart).toHaveBeenCalledTimes(1); + expect(mocks.siriAppend).toHaveBeenCalledWith( + mocks.siriStart.mock.calls[0]?.[0], + "Hello from Siri.", + ); + expect(mocks.siriFinish).toHaveBeenCalledTimes(1); + }); + expect(mocks.start).not.toHaveBeenCalled(); + expect(mocks.append).not.toHaveBeenCalled(); + }); + it("preserves the first live reply while speech is arming", async () => { const history = assistant( [{ type: "text", text: "Historical response." }], diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts index 0b3d1c6dc..dffcef13e 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts @@ -8,6 +8,16 @@ import { stopPocketVoice, type PocketVoiceStreamEvent, } from "../api/pocketVoice"; +import { + appendSiriVoiceStream, + finishSiriVoiceStream, + flushSiriVoiceStream, + listenToSiriVoiceStream, + startSiriVoiceStream, + stopSiriVoice, + type SiriVoiceStreamEvent, +} from "../api/siriVoice"; +import { getVoiceOutputBackend } from "./voiceOutputPreference"; import { useVoiceConversationStore } from "../stores/voiceConversationStore"; type SpeechFailureHandler = (text: string, error: unknown) => void; @@ -38,6 +48,7 @@ let generation = 0; let commandEpoch = 0; let activeSpeechSessionId: string | null = null; let activeUtterance: ActiveUtterance | null = null; +let stopActiveVoice: () => Promise = stopPocketVoice; const pendingNotices = new Map(); const recordedNoticeKeys = new Set(); @@ -140,7 +151,9 @@ function queueStreamCommand( }); } -function handleStreamEvent(event: PocketVoiceStreamEvent) { +function handleStreamEvent( + event: PocketVoiceStreamEvent | SiriVoiceStreamEvent, +) { const utterance = activeUtterance; if (!utterance || utterance.id !== event.streamId) return; const voice = useVoiceConversationStore.getState(); @@ -201,9 +214,9 @@ function interruptActiveUtterance() { ); utterance.onTerminal(); } - void stopPocketVoice().catch(() => undefined); + void stopActiveVoice().catch(() => undefined); commandQueue = commandQueue.then(async () => { - await stopPocketVoice().catch(() => undefined); + await stopActiveVoice().catch(() => undefined); }); } @@ -227,15 +240,34 @@ export function startNativeAssistantSpeech( stopNativeAssistantSpeech(); activeSpeechSessionId = sessionId; const activeGeneration = generation; - streamListenerReady = listenToPocketVoiceStream(handleStreamEvent).then( - (unlisten) => { + const streamBackend = + getVoiceOutputBackend() === "siri" + ? { + start: startSiriVoiceStream, + append: appendSiriVoiceStream, + flush: flushSiriVoiceStream, + finish: finishSiriVoiceStream, + stop: stopSiriVoice, + listen: listenToSiriVoiceStream, + } + : { + start: startPocketVoiceStream, + append: appendPocketVoiceStream, + flush: flushPocketVoiceStream, + finish: finishPocketVoiceStream, + stop: stopPocketVoice, + listen: listenToPocketVoiceStream, + }; + stopActiveVoice = streamBackend.stop; + streamListenerReady = streamBackend + .listen(handleStreamEvent) + .then((unlisten) => { if (activeGeneration !== generation) { unlisten(); return; } stopStreamSubscription = unlisten; - }, - ); + }); const initialMessages = useChatStore.getState().messagesBySession[sessionId] ?? []; @@ -291,7 +323,7 @@ export function startNativeAssistantSpeech( utterance, async () => { await streamListenerReady; - await startPocketVoiceStream(utterance.id); + await streamBackend.start(utterance.id); }, onFailure, ); @@ -356,7 +388,7 @@ export function startNativeAssistantSpeech( utterance.text += delta; queueStreamCommand( utterance, - () => appendPocketVoiceStream(utterance.id, delta), + () => streamBackend.append(utterance.id, delta), onFailure, ); } @@ -365,7 +397,7 @@ export function startNativeAssistantSpeech( if (crossedToolBoundary && utterance && !utterance.finishing) { queueStreamCommand( utterance, - () => flushPocketVoiceStream(utterance.id), + () => streamBackend.flush(utterance.id), onFailure, ); } @@ -373,7 +405,7 @@ export function startNativeAssistantSpeech( utterance.finishing = true; queueStreamCommand( utterance, - () => finishPocketVoiceStream(utterance.id), + () => streamBackend.finish(utterance.id), onFailure, ); } diff --git a/src/features/voice-conversation/lib/voiceOutputPreference.test.ts b/src/features/voice-conversation/lib/voiceOutputPreference.test.ts new file mode 100644 index 000000000..2048afa2a --- /dev/null +++ b/src/features/voice-conversation/lib/voiceOutputPreference.test.ts @@ -0,0 +1,51 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + getDefaultVoiceOutputBackend, + getVoiceOutputBackend, +} from "./voiceOutputPreference"; + +const originalNavigator = globalThis.navigator; + +function setPlatform(userAgent: string) { + Object.defineProperty(globalThis, "navigator", { + configurable: true, + value: { userAgent }, + }); +} + +describe("voice output preference", () => { + beforeEach(() => window.localStorage.clear()); + + afterEach(() => { + window.localStorage.clear(); + Object.defineProperty(globalThis, "navigator", { + configurable: true, + value: originalNavigator, + }); + }); + + it("defaults to Pocket on macOS", () => { + setPlatform("Macintosh"); + expect(getDefaultVoiceOutputBackend()).toBe("pocket"); + expect(getVoiceOutputBackend()).toBe("pocket"); + }); + + it("defaults to Pocket and rejects persisted Siri off macOS", () => { + setPlatform("Windows"); + window.localStorage.setItem("goose:voice-output-backend", "siri"); + expect(getDefaultVoiceOutputBackend()).toBe("pocket"); + expect(getVoiceOutputBackend()).toBe("pocket"); + }); + + it("preserves an explicit Pocket choice on macOS", () => { + setPlatform("Macintosh"); + window.localStorage.setItem("goose:voice-output-backend", "pocket"); + expect(getVoiceOutputBackend()).toBe("pocket"); + }); + + it("preserves an explicit Siri choice on macOS", () => { + setPlatform("Macintosh"); + window.localStorage.setItem("goose:voice-output-backend", "siri"); + expect(getVoiceOutputBackend()).toBe("siri"); + }); +}); diff --git a/src/features/voice-conversation/lib/voiceOutputPreference.ts b/src/features/voice-conversation/lib/voiceOutputPreference.ts new file mode 100644 index 000000000..d1735c747 --- /dev/null +++ b/src/features/voice-conversation/lib/voiceOutputPreference.ts @@ -0,0 +1,79 @@ +import { useCallback, useSyncExternalStore } from "react"; +import { getPlatform } from "@/shared/lib/platform"; + +export type VoiceOutputBackend = "pocket" | "siri"; + +const STORAGE_KEY = "goose:voice-output-backend"; +const CHANGED_EVENT = "goose:voice-output-backend-changed"; +export function getDefaultVoiceOutputBackend(): VoiceOutputBackend { + return "pocket"; +} + +function normalize(value: unknown): VoiceOutputBackend { + if (value === "siri") { + return getPlatform() === "mac" ? "siri" : "pocket"; + } + return value === "pocket" ? value : getDefaultVoiceOutputBackend(); +} + +export function getVoiceOutputBackend(): VoiceOutputBackend { + if (typeof window === "undefined") return getDefaultVoiceOutputBackend(); + try { + return normalize(window.localStorage.getItem(STORAGE_KEY)); + } catch { + return getDefaultVoiceOutputBackend(); + } +} + +const listeners = new Set<() => void>(); +let removeWindowListeners: (() => void) | undefined; + +function notify() { + for (const listener of listeners) listener(); +} + +function subscribe(listener: () => void) { + if (typeof window === "undefined") return () => {}; + listeners.add(listener); + if (!removeWindowListeners) { + const handleStorage = (event: StorageEvent) => { + if (event.key === STORAGE_KEY || event.key === null) notify(); + }; + window.addEventListener(CHANGED_EVENT, notify); + window.addEventListener("storage", handleStorage); + removeWindowListeners = () => { + window.removeEventListener(CHANGED_EVENT, notify); + window.removeEventListener("storage", handleStorage); + }; + } + return () => { + listeners.delete(listener); + if (listeners.size === 0) { + removeWindowListeners?.(); + removeWindowListeners = undefined; + } + }; +} + +export function setVoiceOutputBackend(backend: VoiceOutputBackend): void { + if (typeof window === "undefined") return; + const value = normalize(backend); + try { + window.localStorage.setItem(STORAGE_KEY, value); + } catch { + // Keep the current in-memory renderer usable when storage is unavailable. + } + window.dispatchEvent(new CustomEvent(CHANGED_EVENT, { detail: { value } })); +} + +export function useVoiceOutputPreference() { + const backend = useSyncExternalStore( + subscribe, + getVoiceOutputBackend, + getDefaultVoiceOutputBackend, + ); + const setBackend = useCallback((value: VoiceOutputBackend) => { + setVoiceOutputBackend(value); + }, []); + return { backend, setBackend }; +} diff --git a/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts b/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts new file mode 100644 index 000000000..d90b72457 --- /dev/null +++ b/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest"; +import type { PocketVoiceStatus } from "../api/pocketVoice"; +import type { SiriVoiceStatus } from "../api/siriVoice"; +import { isVoiceSetupReady } from "./voiceSetupReadiness"; + +const pocket = { + installed: true, + pocketInstalled: true, + parakeetInstalled: true, +} as PocketVoiceStatus; + +const siri = { + supported: true, + selectedVoice: { name: "Aaron", language: "en-US" }, + selectedVoiceInstalled: true, +} as SiriVoiceStatus; + +describe("voice setup readiness", () => { + it("requires Parakeet and Pocket for the Pocket backend", () => { + expect(isVoiceSetupReady(pocket, null, "pocket")).toBe(true); + expect( + isVoiceSetupReady( + { ...pocket, parakeetInstalled: false }, + null, + "pocket", + ), + ).toBe(false); + }); + + it("requires Parakeet and an installed selected Siri voice", () => { + expect(isVoiceSetupReady(pocket, siri, "siri")).toBe(true); + expect( + isVoiceSetupReady( + pocket, + { ...siri, selectedVoiceInstalled: false }, + "siri", + ), + ).toBe(false); + }); +}); diff --git a/src/features/voice-conversation/lib/voiceSetupReadiness.ts b/src/features/voice-conversation/lib/voiceSetupReadiness.ts new file mode 100644 index 000000000..e8b868c1c --- /dev/null +++ b/src/features/voice-conversation/lib/voiceSetupReadiness.ts @@ -0,0 +1,15 @@ +import type { PocketVoiceStatus } from "../api/pocketVoice"; +import type { SiriVoiceStatus } from "../api/siriVoice"; +import type { VoiceOutputBackend } from "./voiceOutputPreference"; + +export function isVoiceSetupReady( + pocket: PocketVoiceStatus | null, + siri: SiriVoiceStatus | null, + backend: VoiceOutputBackend, +): boolean { + if (!pocket?.parakeetInstalled) return false; + if (backend === "pocket") return pocket.pocketInstalled; + return Boolean( + siri?.supported && siri.selectedVoice && siri.selectedVoiceInstalled, + ); +} diff --git a/src/features/voice-conversation/ui/PocketVoiceSetupDialog.test.tsx b/src/features/voice-conversation/ui/PocketVoiceSetupDialog.test.tsx index fde6ccb94..cf857881a 100644 --- a/src/features/voice-conversation/ui/PocketVoiceSetupDialog.test.tsx +++ b/src/features/voice-conversation/ui/PocketVoiceSetupDialog.test.tsx @@ -57,7 +57,7 @@ describe("PocketVoiceSetupDialog", () => { ...overrides, }); - it("names Voice Conversation and explains that both models are required", () => { + it("puts speech input before speech output", () => { renderWithProviders( { ).toBeInTheDocument(); expect( screen.getByText( - "Install both Pocket TTS and Parakeet STT to use Voice Conversation.", + "Install speech recognition and choose how Berd speaks during Voice Conversation.", ), ).toBeInTheDocument(); + const input = screen.getByRole("heading", { name: "Speech input" }); + const output = screen.getByRole("heading", { name: "Speech output" }); + expect( + input.compareDocumentPosition(output) & Node.DOCUMENT_POSITION_FOLLOWING, + ).toBeTruthy(); }); it("hands an installed setup back to the initiating voice action exactly once", async () => { @@ -102,6 +107,50 @@ describe("PocketVoiceSetupDialog", () => { expect(onOpenChange).not.toHaveBeenCalled(); }); + it("accepts an installed Siri voice without requiring Pocket TTS", async () => { + const onUseSelected = vi.fn(); + renderWithProviders( + , + ); + + await userEvent.click( + screen.getByRole("button", { name: "Use selected voice" }), + ); + expect(onUseSelected).toHaveBeenCalledTimes(1); + }); + it("keeps both missing model actions independently clickable", async () => { const installModel = vi.fn().mockResolvedValue(undefined); renderWithProviders( @@ -117,9 +166,8 @@ describe("PocketVoiceSetupDialog", () => { expect( screen.getAllByRole("button", { name: "Download model" }), ).toHaveLength(2); - const buttons = screen.getAllByRole("button", { name: "Download model" }); - await userEvent.click(buttons[0]); - await userEvent.click(buttons[1]); + await userEvent.click(screen.getByTestId("voice-model-pocket-download")); + await userEvent.click(screen.getByTestId("voice-model-parakeet-download")); expect(installModel).toHaveBeenNthCalledWith(1, "pocket"); expect(installModel).toHaveBeenNthCalledWith(2, "parakeet"); }); @@ -340,9 +388,7 @@ describe("PocketVoiceSetupDialog", () => { />, ); - await userEvent.click( - screen.getAllByRole("button", { name: "Remove model" })[1], - ); + await userEvent.click(screen.getByTestId("voice-model-parakeet-remove")); expect( screen.getByRole("heading", { name: "Remove Parakeet STT?" }), ).toBeInTheDocument(); diff --git a/src/features/voice-conversation/ui/PocketVoiceSetupDialog.tsx b/src/features/voice-conversation/ui/PocketVoiceSetupDialog.tsx index 261c23ca8..54e558b9d 100644 --- a/src/features/voice-conversation/ui/PocketVoiceSetupDialog.tsx +++ b/src/features/voice-conversation/ui/PocketVoiceSetupDialog.tsx @@ -16,8 +16,20 @@ import { } from "@/shared/ui/dialog"; import { Progress } from "@/shared/ui/progress"; import { RadioGroup, RadioGroupItem } from "@/shared/ui/radio-group"; -import type { PocketVoiceSetup } from "../hooks/usePocketVoiceSetup"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/shared/ui/select"; +import { getPlatform } from "@/shared/lib/platform"; import type { VoiceModelKind } from "../api/pocketVoice"; +import type { PocketVoiceSetup } from "../hooks/usePocketVoiceSetup"; +import type { SiriVoiceSetup } from "../hooks/useSiriVoiceSetup"; +import type { VoiceOutputBackend } from "../lib/voiceOutputPreference"; +import { isVoiceSetupReady } from "../lib/voiceSetupReadiness"; +import { SiriVoiceSettings } from "./SiriVoiceSettings"; function formatBytes(bytes: number): string { return `${(bytes / 1_000_000).toFixed(1)} MB`; @@ -28,14 +40,22 @@ export function PocketVoiceSetupDialog({ onOpenChange, onUseSelected, setup, + siriSetup, + backend = "pocket", + onBackendChange, }: { open: boolean; onOpenChange: (open: boolean) => void; onUseSelected?: () => void; setup: PocketVoiceSetup; + siriSetup?: SiriVoiceSetup; + backend?: VoiceOutputBackend; + onBackendChange?: (backend: VoiceOutputBackend) => void; }) { const { t } = useTranslation("settings"); const { status } = setup; + const siriSupported = getPlatform() === "mac"; + const ready = isVoiceSetupReady(status, siriSetup?.status ?? null, backend); return ( @@ -45,10 +65,50 @@ export function PocketVoiceSetupDialog({ {t("voice.description")} - +
+

{t("voice.speechInput")}

+ +
+
+

{t("voice.speechOutput")}

+ {siriSupported && siriSetup && onBackendChange ? ( +
+ + +
+ ) : null} + {backend === "siri" && siriSetup ? ( + + ) : ( + + )} +
- {status?.installed ? ( + {ready ? ( + ))} + + + ) : null} + + {setup.error ? ( +

+ {setup.error} +

+ ) : null} + + {setup.loading ? ( +

+ {t("voice.siriLoading")} +

+ ) : groups.length === 0 ? ( +

+ {t("voice.siriNoVoices")} +

+ ) : ( +
+ {groups.map((group) => ( +
+

+ {localeLabel(group.locale)} +

+
+ {group.voices.map((voice) => { + const key = voiceKey(voice); + const selected = key === selectedKey; + const downloading = setup.downloadingVoiceKey === key; + const previewing = setup.previewingVoiceKey === key; + return ( +
+
+
+ {voice.name} + {selected ? ( + + ) : null} +
+

+ {voice.installed + ? t("voice.siriInstalled") + : t("voice.siriDownloadSize", { + size: formatBytes(voice.sizeBytes), + })} +

+
+ + {voice.installed ? ( + + ) : ( + + )} +
+ ); + })} +
+
+ ))} +
+ )} + + ); +} + +export { groupVoicesByLocale, languageLabel, localeLabel }; diff --git a/src/features/voice-conversation/ui/VoiceSettings.tsx b/src/features/voice-conversation/ui/VoiceSettings.tsx index 5b65afcb4..8b87025d8 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.tsx @@ -1,11 +1,27 @@ import { useTranslation } from "react-i18next"; +import { getPlatform } from "@/shared/lib/platform"; +import { SettingsPage } from "@/shared/ui/SettingsPage"; +import { SettingsRow } from "@/shared/ui/settings-row"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/shared/ui/select"; import { usePocketVoiceSetup } from "../hooks/usePocketVoiceSetup"; +import { useSiriVoiceSetup } from "../hooks/useSiriVoiceSetup"; +import type { VoiceOutputBackend } from "../lib/voiceOutputPreference"; +import { useVoiceOutputPreference } from "../lib/voiceOutputPreference"; import { PocketVoiceSetupContent } from "./PocketVoiceSetupDialog"; -import { SettingsPage } from "@/shared/ui/SettingsPage"; +import { SiriVoiceSettings } from "./SiriVoiceSettings"; export function VoiceSettings() { const { t } = useTranslation("settings"); const setup = usePocketVoiceSetup(); + const output = useVoiceOutputPreference(); + const siriSetup = useSiriVoiceSetup(output.backend === "siri"); + const siriSupported = getPlatform() === "mac"; return ( -
- +
+

{t("voice.speechInput")}

+ +
+
+

{t("voice.speechOutput")}

+ + output.setBackend(value as VoiceOutputBackend) + } + > + + + + + + {t("voice.backendPocket")} + + {siriSupported ? ( + {t("voice.backendSiri")} + ) : null} + + + } + /> + {output.backend === "siri" ? ( + + ) : ( + + )}
); diff --git a/src/shared/i18n/locales/en/settings.json b/src/shared/i18n/locales/en/settings.json index 5420287a2..7505c73e8 100644 --- a/src/shared/i18n/locales/en/settings.json +++ b/src/shared/i18n/locales/en/settings.json @@ -865,8 +865,12 @@ } }, "voice": { - "description": "Install both Pocket TTS and Parakeet STT to use Voice Conversation.", + "backendPocket": "Pocket TTS", + "backendSiri": "Siri voices (macOS)", + "description": "Install speech recognition and choose how Berd speaks during Voice Conversation.", "download": "Download model", + "downloadVoice": "Download {{voice}}", + "downloadingVoice": "Downloading {{voice}}", "downloadPhase": { "complete": "Installed", "downloading": "Downloading", @@ -881,8 +885,11 @@ "modelMissingSize": "Not installed · {{size}} download", "modelNotInstalled": "Not installed", "notNow": "Not now", + "outputBackend": "Speech engine", + "outputBackendDescription": "Choose how Berd speaks assistant responses.", "playbackSpeed": "Playback speed", "playing": "Playing", + "playingVoice": "Playing preview for {{voice}}", "preview": "Preview", "previewVoice": "Preview {{voice}}", "removeModel": "Remove model", @@ -891,9 +898,23 @@ "removeModelTitle": "Remove {{model}}?", "removingModel": "Removing model…", "retryDownload": "Retry model download", - "settingsDescription": "Install Pocket TTS, choose from 12 voices, and preview them through your selected output device.", + "settingsDescription": "Choose how Berd speaks, install speech recognition, and preview available voices.", + "siriDownloadSize": "Available · {{size}} download", + "siriDownloading": "Downloading…", + "siriInstalled": "Installed", + "siriLanguage": "Language", + "siriLanguageDescription": "Choose the exact language and regional voice you want to use.", + "siriLoading": "Loading Siri voices…", + "siriNoVoices": "No Siri voices are available for this language.", + "siriSelected": "Selected", + "siriUnsupported": "Siri voices are available on macOS only.", + "siriUseVoice": "Use voice", + "speechInput": "Speech input", + "speechOutput": "Speech output", "title": "Voice conversation", + "selectedVoice": "Selected voice: {{voice}}", "useSelected": "Use selected voice", + "useVoice": "Use {{voice}}", "voiceLabel": "Pocket TTS voice" } } diff --git a/src/shared/i18n/locales/es/settings.json b/src/shared/i18n/locales/es/settings.json index bd709b42e..e56e4fd53 100644 --- a/src/shared/i18n/locales/es/settings.json +++ b/src/shared/i18n/locales/es/settings.json @@ -868,8 +868,12 @@ } }, "voice": { - "description": "Instala Pocket TTS y Parakeet STT para usar Conversación por voz.", + "backendPocket": "Pocket TTS", + "backendSiri": "Voces de Siri (macOS)", + "description": "Instala el reconocimiento de voz y elige cómo habla Berd durante la conversación por voz.", "download": "Descargar modelo", + "downloadVoice": "Descargar {{voice}}", + "downloadingVoice": "Descargando {{voice}}", "downloadPhase": { "complete": "Instalado", "downloading": "Descargando", @@ -884,8 +888,11 @@ "modelMissingSize": "No instalado · descarga de {{size}}", "modelNotInstalled": "No instalado", "notNow": "Ahora no", + "outputBackend": "Motor de voz", + "outputBackendDescription": "Elige cómo Berd reproduce las respuestas del asistente.", "playbackSpeed": "Velocidad de reproducción", "playing": "Reproduciendo", + "playingVoice": "Reproduciendo la muestra de {{voice}}", "preview": "Escuchar", "previewVoice": "Escuchar {{voice}}", "removeModel": "Eliminar modelo", @@ -894,9 +901,23 @@ "removeModelTitle": "¿Eliminar {{model}}?", "removingModel": "Eliminando modelo…", "retryDownload": "Reintentar descarga del modelo", - "settingsDescription": "Instala Pocket TTS, elige entre 12 voces y escúchalas en el dispositivo de salida seleccionado.", + "settingsDescription": "Elige cómo habla Berd, instala el reconocimiento de voz y escucha las voces disponibles.", + "siriDownloadSize": "Disponible · descarga de {{size}}", + "siriDownloading": "Descargando…", + "siriInstalled": "Instalada", + "siriLanguage": "Idioma", + "siriLanguageDescription": "Elige el idioma exacto y la voz regional que quieres usar.", + "siriLoading": "Cargando voces de Siri…", + "siriNoVoices": "No hay voces de Siri disponibles para este idioma.", + "siriSelected": "Seleccionada", + "siriUnsupported": "Las voces de Siri solo están disponibles en macOS.", + "siriUseVoice": "Usar voz", + "speechInput": "Entrada de voz", + "speechOutput": "Salida de voz", "title": "Conversación por voz", + "selectedVoice": "Voz seleccionada: {{voice}}", "useSelected": "Usar la voz seleccionada", + "useVoice": "Usar {{voice}}", "voiceLabel": "Voz de Pocket TTS" } }