From 10be773401cf6e81c2a236f5a19eeb1d9b05ef99 Mon Sep 17 00:00:00 2001 From: Senad Date: Sat, 17 Oct 2026 20:58:41 +0200 Subject: [PATCH 01/19] Keep the llama KV cache across keystrokes on sliding-window models Gemma's sliding-window attention routed every request through the eight-token checkpoint path, so most keystrokes re-decoded the whole prompt (about 200 ms at 490 tokens, 2 s at 2,500 on Gemma 4 E4B). The native context now uses swa_full, which keeps every SWA position so ordinary suffix trims stay valid; checkpoints remain for recurrent and hybrid models only. Measured: a 3-token edit costs 22-41 ms instead of a full rebuild, with no change to per-token decode time. Reuse also no longer depends on things the cache does not: - the output-token budget is out of both sampling fingerprints, so performance tuning that moves it keeps the cache; - the core keeps any shared token prefix without requiring the hint tracker's byte hint, which was missing after every cancelled request and whenever a growing field changed its frame; - a sampler change swaps the native chain in place (updateSampling) instead of destroying the sequence. Real-model tests check that hintless and sampler-changed reuse keep the prompt and produce exactly the fresh result. --- .../Request/SuggestionConfiguration.swift | 10 +- .../Runtime/Llama/LlamaRuntimeCore.swift | 47 ++++++-- .../Runtime/Llama/LlamaSuggestionEngine.swift | 12 +- .../LlamaPromptCacheHintTrackerTests.swift | 12 +- .../LlamaRuntimeCoreIntegrationTests.swift | 48 ++++++++ .../cotabbyinference-upstream-pending.patch | 111 +++++++++++++----- 6 files changed, 188 insertions(+), 52 deletions(-) diff --git a/Cotabby/Models/Suggestion/Request/SuggestionConfiguration.swift b/Cotabby/Models/Suggestion/Request/SuggestionConfiguration.swift index 4654b8975..72300fefd 100644 --- a/Cotabby/Models/Suggestion/Request/SuggestionConfiguration.swift +++ b/Cotabby/Models/Suggestion/Request/SuggestionConfiguration.swift @@ -146,11 +146,11 @@ struct SuggestionConfiguration: Equatable, Sendable { // can hold after the preface), not by a word cap: a 150-word cap left most of a long // document out of the prompt, and the model's sense of the topic with it. The caps here // only stop pathological fields (a megabyte of log text) from being windowed each poll. - // Latency honesty: where KV prefix reuse works (dense models), the window is prefilled - // once per field and every keystroke after that decodes only the delta (measured: one - // token); the hybrid/SWA catalog models reject partial trims and re-prefill per request, - // so there the wider window costs prefill only in long-document sessions, which is - // exactly where it buys quality. + // Latency honesty: dense and sliding-window models (Gemma, via the native `swa_full` + // cache) prefill the window once per field, and every keystroke after that decodes only + // the delta. Hybrid/recurrent catalog models restore from a near-caret checkpoint and + // re-prefill when an edit lands before it, so there the wider window costs prefill only + // in long-document sessions, which is exactly where it buys quality. maxPrefixWords: 2400, maxPrefixCharacters: 14000, // Apple's on-device model has a 4096-token shared context. Even with instructions plus diff --git a/Cotabby/Services/Runtime/Llama/LlamaRuntimeCore.swift b/Cotabby/Services/Runtime/Llama/LlamaRuntimeCore.swift index 1a18da0f8..1d21d5598 100644 --- a/Cotabby/Services/Runtime/Llama/LlamaRuntimeCore.swift +++ b/Cotabby/Services/Runtime/Llama/LlamaRuntimeCore.swift @@ -35,6 +35,10 @@ nonisolated final class LlamaRuntimeCore: @unchecked Sendable { private var autocompletePromptBytes: [UInt8] = [] private var autocompletePromptTokens: [Int32] = [] private var autocompleteSamplingFingerprint: SamplingFingerprint? + /// Prompt tokens the most recent generate/prefill kept from native KV instead of decoding + /// (0 after a fresh build). Diagnostics and tests read it to prove reuse actually happened; + /// equal outputs alone cannot distinguish a reused cache from a silent rebuild. + private(set) var lastReusedPromptTokens = 0 /// The sequence the in-flight autocomplete operation is decoding into, published for /// `abortInFlightGeneration` to target from the canceller's thread. Guarded by its own lock @@ -317,13 +321,13 @@ nonisolated final class LlamaRuntimeCore: @unchecked Sendable { let healingPrefix = healing.replayBytes // A byte-fallback vocabulary can replay at most one token per prefix byte. Reserve a // separate bounded allowance so healing cannot consume the user's output-token budget. + // A front-truncated prompt shifts every token position, so its token prefix no longer + // matches the cached one and reuse simply finds nothing in common; no flag is needed. let maxPromptTokens = max(1, preparedRuntime.contextWindowTokens - options.maxPredictionTokens - healingPrefix.count) - let truncated = tokens.count > maxPromptTokens return PreparedPrompt( promptBytes: promptBytes, promptTokens: Array(tokens.suffix(maxPromptTokens)), - cachedPrefixBytes: truncated ? nil : cachedPrefixBytes, healingPrefix: healingPrefix, fingerprint: SamplingFingerprint(options: options) ) @@ -332,7 +336,6 @@ nonisolated final class LlamaRuntimeCore: @unchecked Sendable { private struct PreparedPrompt { let promptBytes: [UInt8] let promptTokens: [Int32] - let cachedPrefixBytes: Int? let healingPrefix: [UInt8] let fingerprint: SamplingFingerprint } @@ -561,6 +564,12 @@ nonisolated final class LlamaRuntimeCore: @unchecked Sendable { /// retain generated tokens or a cancellation flag; restoration is mandatory before reuse, /// even when both prompt descriptions are identical. A missed checkpoint rebuilds cold. /// Must be called while holding `autocompleteLock`. + /// + /// Reuse is decided here from the token prefix alone. KV contents are a pure function of the + /// token sequence and its positions, so a shared token prefix is always safe to keep no matter + /// which field, frame, or request produced it; the caller's byte hint is diagnostic only. + /// Requiring the hint used to discard the cache whenever the hint tracker had no record + /// (every cancelled request) or saw the field's frame grow while the user typed. private func obtainAutocompleteSequence( preparation: PreparedPrompt, options: LlamaGenerationOptions @@ -568,16 +577,11 @@ nonisolated final class LlamaRuntimeCore: @unchecked Sendable { let promptTokens = preparation.promptTokens let promptBytes = preparation.promptBytes let fingerprint = preparation.fingerprint - let cachedPrefixBytes = preparation.cachedPrefixBytes let healingPrefix = preparation.healingPrefix if autocompleteSequenceID >= 0, - let cachedPrefixBytes, cachedPrefixBytes > 0, - autocompleteSamplingFingerprint == fingerprint { + adoptSamplingIfNeeded(fingerprint, options: options) { - let confirmedCommonBytes = min( - cachedPrefixBytes, - Self.commonPrefixCount(autocompletePromptBytes, promptBytes) - ) + let confirmedCommonBytes = Self.commonPrefixCount(autocompletePromptBytes, promptBytes) if confirmedCommonBytes > 0 { let commonTokenPrefix = Self.commonPrefixCount(autocompletePromptTokens, promptTokens) @@ -630,6 +634,7 @@ nonisolated final class LlamaRuntimeCore: @unchecked Sendable { return try buildFreshSequence(promptTokens: promptTokens, healingPrefix: healingPrefix, options: options) } } + lastReusedPromptTokens = reusableTokenCount CotabbyLogger.runtime.debug( "KV prefix reused", metadata: [ @@ -652,11 +657,26 @@ nonisolated final class LlamaRuntimeCore: @unchecked Sendable { return try buildFreshSequence(promptTokens: promptTokens, healingPrefix: healingPrefix, options: options) } + /// Brings the live sequence's sampler in line with this request without touching its KV. + /// A sampling change (temperature, top-k, single-line mask) only affects how the next tokens + /// are drawn, never the cached prompt, so the native sampler chain is rebuilt in place instead + /// of destroying the reusable prefix. Returns false when the swap fails; the caller then + /// rebuilds the sequence from scratch. Must be called while holding `autocompleteLock`. + private func adoptSamplingIfNeeded(_ fingerprint: SamplingFingerprint, options: LlamaGenerationOptions) -> Bool { + guard autocompleteSamplingFingerprint != fingerprint else { return true } + guard engine.updateSampling(autocompleteSequenceID, Self.samplingConfig(from: options)) else { + return false + } + autocompleteSamplingFingerprint = fingerprint + return true + } + private func buildFreshSequence( promptTokens: [Int32], healingPrefix: [UInt8], options: LlamaGenerationOptions ) throws -> Int32 { + lastReusedPromptTokens = 0 let config = Self.samplingConfig(from: options) let seqID = engine.createSequence(config) guard seqID >= 0 else { @@ -783,9 +803,11 @@ nonisolated final class LlamaRuntimeCore: @unchecked Sendable { return index } - /// Generation knobs that intentionally break KV reuse when changed. + /// The settings baked into the native sampler chain. A change swaps the chain in place + /// (`adoptSamplingIfNeeded`); it never invalidates the KV prefix. The output-token budget is + /// deliberately absent: the Swift decode loop enforces it, so performance tuning that moves the + /// budget between requests must not look like a different sampler. private struct SamplingFingerprint: Equatable { - let maxPredictionTokens: Int let temperature: Double let topK: Int let topP: Double @@ -795,7 +817,6 @@ nonisolated final class LlamaRuntimeCore: @unchecked Sendable { let singleLine: Bool init(options: LlamaGenerationOptions) { - maxPredictionTokens = options.maxPredictionTokens temperature = options.temperature topK = options.topK topP = options.topP diff --git a/Cotabby/Services/Runtime/Llama/LlamaSuggestionEngine.swift b/Cotabby/Services/Runtime/Llama/LlamaSuggestionEngine.swift index a3baa0dd1..958ae82f3 100644 --- a/Cotabby/Services/Runtime/Llama/LlamaSuggestionEngine.swift +++ b/Cotabby/Services/Runtime/Llama/LlamaSuggestionEngine.swift @@ -289,9 +289,10 @@ final class LlamaSuggestionEngine { extension LlamaSuggestionEngine: SuggestionGenerating {} -/// Tracks the last successful llama prompt so the engine can pass a conservative byte-prefix hint -/// into `LlamaRuntimeManager.generate`. This type deliberately does not own correctness: native KV -/// state is still validated by `LlamaRuntimeCore` after tokenization. +/// Tracks the last successful llama prompt so the engine can pass a byte-prefix hint into +/// `LlamaRuntimeManager.generate` for diagnostics (`cache_hint_bytes` in the logs). This type +/// deliberately does not own correctness or gate reuse: `LlamaRuntimeCore` keeps whatever token +/// prefix the new prompt shares with native KV, whether or not a hint was offered. struct LlamaPromptCacheHintTracker: Equatable { private var lastRequest: CachedRequest? @@ -387,8 +388,10 @@ private extension LlamaPromptCacheHintTracker { } } + /// Mirrors the native sampler settings. The output-token budget is excluded for the same reason + /// as in `LlamaRuntimeCore`: performance tuning moves it between requests without changing + /// anything a cached prompt depends on. struct SamplingFingerprint: Equatable { - let maxPredictionTokens: Int let temperature: Double let topK: Int let topP: Double @@ -397,7 +400,6 @@ private extension LlamaPromptCacheHintTracker { let randomSeed: UInt32? init(request: SuggestionRequest) { - maxPredictionTokens = request.maxPredictionTokens temperature = request.temperature topK = request.topK topP = request.topP diff --git a/CotabbyTests/Services/Runtime/Llama/LlamaPromptCacheHintTrackerTests.swift b/CotabbyTests/Services/Runtime/Llama/LlamaPromptCacheHintTrackerTests.swift index 504f46b45..27ef0928c 100644 --- a/CotabbyTests/Services/Runtime/Llama/LlamaPromptCacheHintTrackerTests.swift +++ b/CotabbyTests/Services/Runtime/Llama/LlamaPromptCacheHintTrackerTests.swift @@ -56,6 +56,15 @@ final class LlamaPromptCacheHintTrackerTests: XCTestCase { XCTAssertNil(tracker.cachedPrefixBytes(for: makeRequest(prompt: "hello!", topK: 40))) } + /// Performance tuning moves the output budget between requests; the cached prompt does not + /// depend on it, so the hint must survive the change. + func test_cacheHint_survivesOutputBudgetChange() { + var tracker = LlamaPromptCacheHintTracker() + tracker.recordSuccessfulRequest(makeRequest(prompt: "hello", maxPredictionTokens: 26)) + + XCTAssertEqual(tracker.cachedPrefixBytes(for: makeRequest(prompt: "hello!", maxPredictionTokens: 6)), 5) + } + func test_cacheHint_invalidatesWhenProcessChanges() { var tracker = LlamaPromptCacheHintTracker() tracker.recordSuccessfulRequest(makeRequest(prompt: "hello", processIdentifier: 100)) @@ -123,6 +132,7 @@ final class LlamaPromptCacheHintTrackerTests: XCTestCase { elementIdentifier: String = "field", processIdentifier: Int32 = 123, topK: Int = 20, + maxPredictionTokens: Int = 8, inputFrameRect: CGRect? = nil ) -> SuggestionRequest { let context = CotabbyTestFixtures.focusedInputContext( @@ -137,7 +147,7 @@ final class LlamaPromptCacheHintTrackerTests: XCTestCase { prefixText: prompt, prompt: prompt, generation: context.generation, - maxPredictionTokens: 8, + maxPredictionTokens: maxPredictionTokens, temperature: 0.1, topK: topK, topP: 0.7, diff --git a/CotabbyTests/Services/Runtime/Llama/LlamaRuntimeCoreIntegrationTests.swift b/CotabbyTests/Services/Runtime/Llama/LlamaRuntimeCoreIntegrationTests.swift index fb0c528b0..bb1f6c675 100644 --- a/CotabbyTests/Services/Runtime/Llama/LlamaRuntimeCoreIntegrationTests.swift +++ b/CotabbyTests/Services/Runtime/Llama/LlamaRuntimeCoreIntegrationTests.swift @@ -111,6 +111,54 @@ final class LlamaRuntimeCoreIntegrationTests: XCTestCase { } } + /// Reuse is decided from the token prefix, not from the caller's byte hint. A request without a + /// hint (the tracker had no record, e.g. after a cancelled request) must still keep the shared + /// prompt and produce exactly the fresh result. With a sliding-window model this is the case + /// `swa_full` enables: the trim lands inside the window instead of rebuilding. + func testHintlessEditReusesPromptAndMatchesFreshRequest() throws { + let core = try makeCore() + defer { core.shutdown() } + let original = "Hi Alex, thanks for sending the project update. I will review the schedule and send you my" + for edited in [original + " feedback", String(original.dropLast(2)) + "our feedback"] { + core.resetPromptCache() + _ = try core.generate(prompt: original, options: options) + let reused = try core.generate(prompt: edited, options: options) + XCTAssertGreaterThan(core.lastReusedPromptTokens, 0, "A shared token prefix must be kept without a hint: \(edited)") + core.resetPromptCache() + let fresh = try core.generate(prompt: edited, options: options) + XCTAssertEqual(core.lastReusedPromptTokens, 0) + XCTAssertEqual(reused.text, fresh.text, "Hintless reuse changed the completion for: \(edited)") + } + } + + /// A different output budget or sampler setting must not discard the cached prompt: the budget + /// lives in the Swift loop and the sampler chain is swapped in place. The reused request must + /// still sample exactly like a fresh sequence built with the new settings. + func testSamplingChangeKeepsPromptAndMatchesFreshRequest() throws { + let core = try makeCore() + defer { core.shutdown() } + let prompt = "Hi Alex, thanks for sending the project update. I will review the schedule and send you my" + for configuration in sampledConfigurations { + core.resetPromptCache() + _ = try core.generate(prompt: prompt, options: options) + let changed = LlamaGenerationOptions( + maxPredictionTokens: configuration.options.maxPredictionTokens - 4, + temperature: configuration.options.temperature, + topK: configuration.options.topK, + topP: configuration.options.topP, + minP: configuration.options.minP, + repetitionPenalty: configuration.options.repetitionPenalty, + seed: configuration.options.seed, + stopAtArgmaxEOG: false + ) + let reused = try core.generate(prompt: prompt, options: changed) + XCTAssertGreaterThan(core.lastReusedPromptTokens, 0, "\(configuration.label): sampling change must keep the prompt") + core.resetPromptCache() + let fresh = try core.generate(prompt: prompt, options: changed) + XCTAssertEqual(reused.text, fresh.text, "\(configuration.label): swapped sampler diverged from a fresh one") + } + } + func testCancelledRetainedTailCanBeRestoredByNextRequest() throws { let core = try makeCore() defer { core.shutdown() } diff --git a/patches/cotabbyinference-upstream-pending.patch b/patches/cotabbyinference-upstream-pending.patch index d2de3d353..7f3446b1a 100644 --- a/patches/cotabbyinference-upstream-pending.patch +++ b/patches/cotabbyinference-upstream-pending.patch @@ -1,5 +1,5 @@ diff --git a/README.md b/README.md -index 1b2aca3..673e8de 100644 +index 1b2aca3..eac988c 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Cotabby LlamaRuntimeCore @@ -47,7 +47,7 @@ index 1b2aca3..673e8de 100644 ## Generation Semantics -@@ -119,16 +122,55 @@ Cotabby controls the maximum token count in Swift. The engine controls token sel +@@ -119,16 +122,60 @@ Cotabby controls the maximum token count in Swift. The engine controls token sel sampling selected visible text; - `logprob`: the selected token's raw-model log-probability when enabled. @@ -72,7 +72,10 @@ index 1b2aca3..673e8de 100644 -pending feedback token. Cotabby independently validates request continuity, UTF-8 prefix, token -prefix, and sampling compatibility before calling it. +`trimKV` restores a prefix in slot zero and invalidates its pending seed/feedback token. Ordinary -+attention uses direct suffix removal. Recurrent, hybrid, and sliding-window models use one ++and sliding-window attention use direct suffix removal: the context is created with `swa_full`, ++so the SWA cache has one cell per context position and never overwrites a row a later trim ++would need (a default-sized SWA cache keeps only `n_swa + n_ubatch` cells). Recurrent and hybrid ++models, whose recurrent state cannot be rewound by removing KV, use one +`PARTIAL_ONLY | ON_DEVICE` checkpoint near the prompt tail, retaining full attention KV in place. +The checkpoint's tensor memory is capped at 128 MiB and restoration replays at most eight prompt +tokens without sampling. Device storage belongs to the llama context's slot-zero checkpoint and @@ -81,9 +84,11 @@ index 1b2aca3..673e8de 100644 +A request that edits before the saved checkpoint, a very short prompt without a nonempty checkpoint, +or a model exceeding the cap returns a cache miss; callers rebuild that request. A miss must not +permanently disable reuse for the model. Saving a newer near-caret checkpoint intentionally gives -+up deeper backspace history. Gemma's sliding-window cache supports suffix removal, but restoring -+its saved window also protects rows evicted during prediction. No model-family-name heuristics -+are used: llama model metadata selects the partial-state path. ++up deeper backspace history. No model-family-name heuristics are used: llama model metadata ++selects the partial-state path. ++ ++`updateSampling(sequence, config)` swaps the sampler chain of the live sequence without touching ++its KV, so a request whose sampling settings differ from the cached one still reuses the prompt. + +`decodePrompt` resets the sampler and accepts only committed prompt tokens. Discarded predictions +therefore never pollute penalties or RNG state; `llama_sampler_sample` already accepts its sample, @@ -108,7 +113,7 @@ index 1b2aca3..673e8de 100644 ## Testing -@@ -144,10 +186,31 @@ Run the full native path with a local GGUF: +@@ -144,10 +191,31 @@ Run the full native path with a local GGUF: COTABBY_TEST_MODEL_PATH=/absolute/path/model.gguf swift test ~~~ @@ -142,7 +147,7 @@ index 1b2aca3..673e8de 100644 ## Requirements diff --git a/Sources/CotabbyInferenceEngine/CotabbyInferenceEngine.cpp b/Sources/CotabbyInferenceEngine/CotabbyInferenceEngine.cpp -index 5fc867f..b9b6aa6 100644 +index 5fc867f..7ff725a 100644 --- a/Sources/CotabbyInferenceEngine/CotabbyInferenceEngine.cpp +++ b/Sources/CotabbyInferenceEngine/CotabbyInferenceEngine.cpp @@ -1,4 +1,5 @@ @@ -337,16 +342,34 @@ index 5fc867f..b9b6aa6 100644 } // Masks every "starts a new word" token (decoded text begins with whitespace) in the logits -@@ -424,6 +555,8 @@ EngineStatus CotabbyInferenceEngine::loadModel(const char* path, int gpu_layers, +@@ -424,6 +555,13 @@ EngineStatus CotabbyInferenceEngine::loadModel(const char* path, int gpu_layers, impl_->context_window_tokens = context_window_tokens; impl_->batch_size = batch_size; impl_->gpu_layer_count = gpu_layers; ++ // Only recurrent state cannot be rewound by removing a KV suffix. Sliding-window attention ++ // can, provided its cache keeps every position (`swa_full` below); a default-sized SWA cache ++ // holds only n_swa + n_ubatch cells and overwrites older ones, which is what used to force ++ // SWA models such as Gemma onto the eight-token checkpoint path and a full prompt re-decode ++ // after nearly every keystroke. + impl_->needs_prompt_checkpoint = llama_model_is_recurrent(impl_->model) || -+ llama_model_is_hybrid(impl_->model) || llama_model_n_swa(impl_->model) > 0; ++ llama_model_is_hybrid(impl_->model); // Performance cores only — see resolveDecodeThreadCount. hardware_concurrency() counted // every logical core including efficiency cores, which both slows barriered matmuls and // burns extra package power for nothing when layers are Metal-offloaded anyway. -@@ -471,6 +604,7 @@ void CotabbyInferenceEngine::unloadModel() { +@@ -439,6 +577,12 @@ EngineStatus CotabbyInferenceEngine::loadModel(const char* path, int gpu_layers, + ctx_params.n_threads = static_cast(impl_->thread_count); + ctx_params.n_threads_batch = static_cast(impl_->thread_count); + ctx_params.offload_kqv = true; ++ // Size the sliding-window cache like the full-attention cache so no position inside the ++ // context window is ever overwritten. That keeps `llama_memory_seq_rm` suffix trims valid for ++ // SWA models, which is what lets each keystroke decode only its changed tail. The cost is KV ++ // memory proportional to n_ctx for the SWA layers; Cotabby's single 4K sequence keeps that ++ // bounded. Models without SWA layers ignore the flag. ++ ctx_params.swa_full = true; + + impl_->shared_ctx = llama_init_from_model(impl_->model, ctx_params); + if (!impl_->shared_ctx) { +@@ -471,6 +615,7 @@ void CotabbyInferenceEngine::unloadModel() { } impl_->vocab = nullptr; impl_->model_path.clear(); @@ -354,7 +377,7 @@ index 5fc867f..b9b6aa6 100644 if (impl_->backend_initialized) { llama_backend_free(); -@@ -495,6 +629,7 @@ int32_t CotabbyInferenceEngine::createSequence(SamplingConfig config) { +@@ -495,10 +640,31 @@ int32_t CotabbyInferenceEngine::createSequence(SamplingConfig config) { auto state = std::make_unique(); state->external_id = id; state->sampler = sampler; @@ -362,7 +385,31 @@ index 5fc867f..b9b6aa6 100644 impl_->sequence = std::move(state); return id; } -@@ -574,6 +709,9 @@ EngineStatus CotabbyInferenceEngine::decodePrompt(int32_t sequence_id, + ++bool CotabbyInferenceEngine::updateSampling(int32_t sequence_id, SamplingConfig config) { ++ if (!impl_ || !impl_->model || !impl_->shared_ctx) return false; ++ llama_sampler* sampler = impl_->buildSampler(config); ++ if (!sampler) return false; ++ ++ // Hold the context lock as well: sampleNext reads the sampler under decode_mutex, so swapping ++ // it here can never race a token being sampled by an in-flight generation. ++ std::lock_guard sequence_lock(impl_->sequence_mutex); ++ std::lock_guard decode_lock(impl_->decode_mutex); ++ if (!impl_->sequence || impl_->sequence->external_id != sequence_id) { ++ llama_sampler_free(sampler); ++ return false; ++ } ++ SequenceState& seq = *impl_->sequence; ++ if (seq.sampler) llama_sampler_free(seq.sampler); ++ seq.sampler = sampler; ++ seq.single_line = config.single_line; ++ return true; ++} ++ + void CotabbyInferenceEngine::destroySequence(int32_t sequence_id) { + if (!impl_) return; + +@@ -574,6 +740,9 @@ EngineStatus CotabbyInferenceEngine::decodePrompt(int32_t sequence_id, SequenceState* seq = impl_->findSequence(sequence_id); if (!seq) return EngineStatus::error; @@ -372,7 +419,7 @@ index 5fc867f..b9b6aa6 100644 if (seq->cancelled.load(std::memory_order_acquire)) { return EngineStatus::cancelled; -@@ -587,6 +725,12 @@ EngineStatus CotabbyInferenceEngine::decodePrompt(int32_t sequence_id, +@@ -587,6 +756,12 @@ EngineStatus CotabbyInferenceEngine::decodePrompt(int32_t sequence_id, int cursor = 0; int end = token_count; int total_end_position = start_position + token_count; @@ -385,7 +432,7 @@ index 5fc867f..b9b6aa6 100644 while (cursor < end) { if (seq->cancelled.load(std::memory_order_acquire)) { -@@ -595,6 +739,10 @@ EngineStatus CotabbyInferenceEngine::decodePrompt(int32_t sequence_id, +@@ -595,6 +770,10 @@ EngineStatus CotabbyInferenceEngine::decodePrompt(int32_t sequence_id, } int chunk_end = std::min(cursor + batch_cap, end); @@ -396,7 +443,7 @@ index 5fc867f..b9b6aa6 100644 int chunk_size = chunk_end - cursor; batch.n_tokens = static_cast(chunk_size); -@@ -612,20 +760,37 @@ EngineStatus CotabbyInferenceEngine::decodePrompt(int32_t sequence_id, +@@ -612,20 +791,37 @@ EngineStatus CotabbyInferenceEngine::decodePrompt(int32_t sequence_id, } if (llama_decode(impl_->shared_ctx, batch) != 0) { @@ -435,7 +482,7 @@ index 5fc867f..b9b6aa6 100644 impl_->maskNewWordStarts(-1); seq->force_word_continuation = false; } -@@ -633,10 +798,10 @@ EngineStatus CotabbyInferenceEngine::decodePrompt(int32_t sequence_id, +@@ -633,10 +829,10 @@ EngineStatus CotabbyInferenceEngine::decodePrompt(int32_t sequence_id, // Seed sample: take one token from the prompt's final logits row. The seed will be returned by // the next sampleNext call as-is and feedback-decoded by the call after that. llama_token seed = llama_sampler_sample(seq->sampler, impl_->shared_ctx, -1); @@ -448,7 +495,7 @@ index 5fc867f..b9b6aa6 100644 seq->has_seed_token = true; seq->has_pending_input = false; -@@ -668,6 +833,10 @@ SampleResult CotabbyInferenceEngine::sampleNext(int32_t sequence_id) { +@@ -668,6 +864,10 @@ SampleResult CotabbyInferenceEngine::sampleNext(int32_t sequence_id) { result.is_eos = true; return result; } @@ -459,7 +506,7 @@ index 5fc867f..b9b6aa6 100644 if (seq->cancelled.load(std::memory_order_acquire)) { result.was_cancelled = true; -@@ -732,20 +901,37 @@ SampleResult CotabbyInferenceEngine::sampleNext(int32_t sequence_id) { +@@ -732,20 +932,37 @@ SampleResult CotabbyInferenceEngine::sampleNext(int32_t sequence_id) { batch.logits[0] = 1; const int status = llama_decode(impl_->shared_ctx, batch); @@ -499,7 +546,7 @@ index 5fc867f..b9b6aa6 100644 seq->last_piece.resize(64); while (true) { const int written = llama_token_to_piece( -@@ -775,7 +961,6 @@ SampleResult CotabbyInferenceEngine::sampleNext(int32_t sequence_id) { +@@ -775,7 +992,6 @@ SampleResult CotabbyInferenceEngine::sampleNext(int32_t sequence_id) { // Feedback decode advanced KV by one position; record the just-sampled // token as input for the next call. @@ -507,7 +554,7 @@ index 5fc867f..b9b6aa6 100644 seq->pending_input_token = result.token; seq->has_pending_input = true; return result; -@@ -789,6 +974,10 @@ bool CotabbyInferenceEngine::trimKV(int32_t sequence_id, int keep_positions) { +@@ -789,6 +1005,10 @@ bool CotabbyInferenceEngine::trimKV(int32_t sequence_id, int keep_positions) { if (!impl_->shared_ctx) return false; SequenceState* seq = impl_->findSequence(sequence_id); if (!seq) return false; @@ -518,7 +565,7 @@ index 5fc867f..b9b6aa6 100644 llama_memory_t memory = llama_get_memory(impl_->shared_ctx); if (!memory) return false; -@@ -796,20 +985,34 @@ bool CotabbyInferenceEngine::trimKV(int32_t sequence_id, int keep_positions) { +@@ -796,20 +1016,34 @@ bool CotabbyInferenceEngine::trimKV(int32_t sequence_id, int keep_positions) { // Serialize with prompt and feedback decode; never remove KV while llama is mutating it. std::lock_guard lock(impl_->decode_mutex); @@ -559,7 +606,7 @@ index 5fc867f..b9b6aa6 100644 } return ok; } -@@ -822,6 +1025,23 @@ void CotabbyInferenceEngine::setForceWordContinuation(int32_t sequence_id, bool +@@ -822,6 +1056,23 @@ void CotabbyInferenceEngine::setForceWordContinuation(int32_t sequence_id, bool } } @@ -583,7 +630,7 @@ index 5fc867f..b9b6aa6 100644 void CotabbyInferenceEngine::setComputeLogprob(int32_t sequence_id, bool enabled) { if (!impl_) return; SequenceState* seq = impl_->findSequence(sequence_id); -@@ -835,9 +1055,12 @@ void CotabbyInferenceEngine::setComputeLogprob(int32_t sequence_id, bool enabled +@@ -835,9 +1086,12 @@ void CotabbyInferenceEngine::setComputeLogprob(int32_t sequence_id, bool enabled // --------------------------------------------------------------------------- void CotabbyInferenceEngine::cancelSequence(int32_t sequence_id) { @@ -599,7 +646,7 @@ index 5fc867f..b9b6aa6 100644 } } -@@ -860,3 +1083,16 @@ int CotabbyInferenceEngine::getThreadCount() const { +@@ -860,3 +1114,16 @@ int CotabbyInferenceEngine::getThreadCount() const { int CotabbyInferenceEngine::getGPULayerCount() const { return impl_->gpu_layer_count; } @@ -772,7 +819,7 @@ index 0000000..15712fb + +} // namespace cotabby diff --git a/Sources/CotabbyInferenceEngine/include/CotabbyInferenceEngine.h b/Sources/CotabbyInferenceEngine/include/CotabbyInferenceEngine.h -index 383bcaf..d15151e 100644 +index 383bcaf..c35f0d9 100644 --- a/Sources/CotabbyInferenceEngine/include/CotabbyInferenceEngine.h +++ b/Sources/CotabbyInferenceEngine/include/CotabbyInferenceEngine.h @@ -40,6 +40,15 @@ enum class EngineStatus : int { @@ -791,7 +838,15 @@ index 383bcaf..d15151e 100644 class CotabbyInferenceEngine { public: CotabbyInferenceEngine(); -@@ -64,6 +73,11 @@ public: +@@ -61,9 +70,19 @@ public: + // alive; destroy the current sequence before creating its replacement. + int32_t createSequence(SamplingConfig config); + void destroySequence(int32_t sequence_id); ++ // Replaces the live sequence's sampler chain while keeping its KV cache. Sampler state never ++ // depends on KV contents: decodePrompt resets it and re-accepts the prompt tokens, so a ++ // changed temperature/top-k/single-line request can reuse the cached prompt instead of ++ // destroying it. Returns false when the sequence is unknown or the chain cannot be built. ++ bool updateSampling(int32_t sequence_id, SamplingConfig config); // Tokenization (thread-safe, read-only on vocab) std::vector tokenize(const char* text, int text_length) const; @@ -803,7 +858,7 @@ index 383bcaf..d15151e 100644 // Prompt decoding EngineStatus decodePrompt(int32_t sequence_id, const int32_t* tokens, int token_count, -@@ -94,6 +108,7 @@ public: +@@ -94,6 +113,7 @@ public: int getBatchSize() const; int getThreadCount() const; int getGPULayerCount() const; From bd94e2c90b116d9548e39112fbc554daa5b7df2c Mon Sep 17 00:00:00 2001 From: Senad Date: Sat, 17 Oct 2026 20:58:41 +0200 Subject: [PATCH 02/19] Keep the screen excerpt unless the screen materially changed The visual-context refresh re-captures every 3 s, and the capture usually includes the field being typed into. Any textual difference, including OCR misreads and the user's own typing, replaced the screen section of the prompt and fired onInjectedContextReady, which discarded the KV cache from that section on (up to ~1,200 tokens) and retired the visible suggestion while the user typed. VisualContextChangePolicy compares refreshes line by line, ignoring lines that echo the focused field and tolerating OCR jitter. Only a material change (a new message, navigation, scrolling) replaces the excerpt; otherwise the prompt keeps its text and the capture time is renewed so expiry semantics hold. --- Cotabby.xcodeproj/project.pbxproj | 10 +++ .../Visual/VisualContextCoordinator.swift | 20 ++++- .../Context/VisualContextChangePolicy.swift | 87 +++++++++++++++++++ .../VisualContextCoordinatorTests.swift | 21 +++++ .../VisualContextChangePolicyTests.swift | 66 ++++++++++++++ 5 files changed, 200 insertions(+), 4 deletions(-) create mode 100644 Cotabby/Support/Context/VisualContextChangePolicy.swift create mode 100644 CotabbyTests/Support/Context/VisualContextChangePolicyTests.swift diff --git a/Cotabby.xcodeproj/project.pbxproj b/Cotabby.xcodeproj/project.pbxproj index efe7ecb17..a7e88fd82 100644 --- a/Cotabby.xcodeproj/project.pbxproj +++ b/Cotabby.xcodeproj/project.pbxproj @@ -36,6 +36,7 @@ 05CFA72F4FF44A0CDBEFED3C /* PostExhaustionAcceptanceState.swift in Sources */ = {isa = PBXBuildFile; fileRef = 22FA75F9AD318B9084B23ECF /* PostExhaustionAcceptanceState.swift */; }; 067371AD898788392F158EDE /* PhrasePredictionScreenContext.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6E6A9735EBD163A2E8AD67A5 /* PhrasePredictionScreenContext.swift */; }; 07639673B5A52D2075A9ED57 /* VisualContextModels.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4A51B1559F4CE6728D39C959 /* VisualContextModels.swift */; }; + 07C1D9A7919366C977AC1623 /* VisualContextChangePolicy.swift in Sources */ = {isa = PBXBuildFile; fileRef = 357BDB23F653C34D18233A49 /* VisualContextChangePolicy.swift */; }; 07E54011F501E14AC15DE2B7 /* OnboardingTemplateFeatureList.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7174EF88BA41ACC0A74CA582 /* OnboardingTemplateFeatureList.swift */; }; 081EB4DF65AD5736DE7FBFDE /* MirrorOverlayLayoutTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2D962AC9D24D00957562CA7B /* MirrorOverlayLayoutTests.swift */; }; 085BB87581DFFA260A630E24 /* CotabbyBrand.swift in Sources */ = {isa = PBXBuildFile; fileRef = D3C1C9A9BF1F74C16E4401EC /* CotabbyBrand.swift */; }; @@ -419,6 +420,7 @@ 67545937A4CAA6B1A1B4B86A /* AXTextGeometryResolver.swift in Sources */ = {isa = PBXBuildFile; fileRef = 98423150126CC5969C2886BA /* AXTextGeometryResolver.swift */; }; 676533C6555BAC0C4A263576 /* HuggingFaceSearchServiceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5A9DE35F3B6F0DFEB9538692 /* HuggingFaceSearchServiceTests.swift */; }; 67A5529348C03A5B97BEFFC5 /* BlockBreakAlignmentTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D681DDF8E81F868C1BC92CB4 /* BlockBreakAlignmentTests.swift */; }; + 67A8673F38C847E921CC8BEE /* VisualContextChangePolicyTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4C8F4DA76268B231B9F2C7D /* VisualContextChangePolicyTests.swift */; }; 67C36958132793C76843F875 /* LlamaTypingSessionEvalTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = A936CA937FA629F828938516 /* LlamaTypingSessionEvalTests.swift */; }; 68DA5F93B7185B4F5E6DB4C3 /* it.txt in Resources */ = {isa = PBXBuildFile; fileRef = 0397F1DACB094A0F6A66BC0E /* it.txt */; }; 6952E5B030740443C684F263 /* SettingsSearchRanker.swift in Sources */ = {isa = PBXBuildFile; fileRef = 57B29DAC600626783649475A /* SettingsSearchRanker.swift */; }; @@ -531,6 +533,7 @@ 83B25F54B74EE35787D5DBD4 /* de-100k.txt in Resources */ = {isa = PBXBuildFile; fileRef = 4B8665A5495891F9E3DDA48B /* de-100k.txt */; }; 83CFB4EA330BE0201DA9F03A /* TextLayoutCaretEstimator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1BF6C6A11D5B5A7B987AFB9C /* TextLayoutCaretEstimator.swift */; }; 840B11AFD82E64D5452CC7A3 /* WordPrefixIndex.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6428AF104998876A60AC602C /* WordPrefixIndex.swift */; }; + 845687C938075DC9C7074F9C /* VisualContextChangePolicy.swift in Sources */ = {isa = PBXBuildFile; fileRef = 357BDB23F653C34D18233A49 /* VisualContextChangePolicy.swift */; }; 8456BDA5B716AB3B0AFF19E4 /* THIRD_PARTY_LICENSES.md in Resources */ = {isa = PBXBuildFile; fileRef = 5FF4FF5C43E650FF46F4BE76 /* THIRD_PARTY_LICENSES.md */; }; 84C7C981F9243D1DB1BC90A4 /* PermissionOverlayWindowController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4213A267AE55FE56A071646F /* PermissionOverlayWindowController.swift */; }; 84E9E1F4D55CEB8A9692734E /* MenuBarPresentationObserver.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9DE86A49339BB507D197629D /* MenuBarPresentationObserver.swift */; }; @@ -1164,6 +1167,7 @@ 341A1BBA636AEBA2491F6A26 /* FocusTrackingModelTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FocusTrackingModelTests.swift; sourceTree = ""; }; 34EF2406E7A384F3325AAF9A /* SettingsIndex.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsIndex.swift; sourceTree = ""; }; 353191D1D8A1C655E1B5F562 /* CapturedInputEventTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CapturedInputEventTests.swift; sourceTree = ""; }; + 357BDB23F653C34D18233A49 /* VisualContextChangePolicy.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VisualContextChangePolicy.swift; sourceTree = ""; }; 35AA2C8F42B510F013D86C3C /* InsertedTextAdvanceTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InsertedTextAdvanceTests.swift; sourceTree = ""; }; 35C0B587D81D87ACB952C95E /* SuggestionSettingsStoreTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SuggestionSettingsStoreTests.swift; sourceTree = ""; }; 365AE69E4E1A3DD3486D203A /* CredentialFieldDetectorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CredentialFieldDetectorTests.swift; sourceTree = ""; }; @@ -1662,6 +1666,7 @@ F3D991D3536ED989C89AF44D /* EmojiUsageStoreTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EmojiUsageStoreTests.swift; sourceTree = ""; }; F47CD9DF50E7192A70D1870E /* CotabbyDebugOptions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CotabbyDebugOptions.swift; sourceTree = ""; }; F4B98B1ABB36E600D7ECA185 /* SpeculativeAcceptanceContextTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SpeculativeAcceptanceContextTests.swift; sourceTree = ""; }; + F4C8F4DA76268B231B9F2C7D /* VisualContextChangePolicyTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VisualContextChangePolicyTests.swift; sourceTree = ""; }; F6809474A7C1B57F1D13AA6F /* SuggestionDismissalMemoryTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SuggestionDismissalMemoryTests.swift; sourceTree = ""; }; F7014E6E969FD4579028BAC4 /* TypingCadence.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TypingCadence.swift; sourceTree = ""; }; F7F3F983901FC3648076B23D /* MacroReferenceSheet.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MacroReferenceSheet.swift; sourceTree = ""; }; @@ -2668,6 +2673,7 @@ 7736E6B390C822F91CC1DA99 /* OCRTextHygiene.swift */, D182046E27CB170D943E8F86 /* PromptContextSanitizer.swift */, 27B03589D82D0ADCF41B9CD0 /* SurfaceContextComposer.swift */, + 357BDB23F653C34D18233A49 /* VisualContextChangePolicy.swift */, 47457FED99065966328047EB /* VisualContextExcerptSelector.swift */, DEF8F1A40EAA82773C07F8B9 /* VisualContextStartCoalescer.swift */, ); @@ -3462,6 +3468,7 @@ 64B0584FA16465453F04CA8C /* OCRTextHygieneTests.swift */, 0DA66559D50874865032EE8C /* PromptContextSanitizerTests.swift */, 652973AF18C20D51F81F76C5 /* SurfaceContextComposerTests.swift */, + F4C8F4DA76268B231B9F2C7D /* VisualContextChangePolicyTests.swift */, 17B5B1AE537B78F54A32BC63 /* VisualContextExcerptSelectorTests.swift */, 70F68E9A9255F607C4439ED5 /* VisualContextStartCoalescerTests.swift */, ); @@ -4289,6 +4296,7 @@ C0468063222C5FEA7C1CFAB3 /* TypoCaseTransfer.swift in Sources */, A88F3C7039E8DDB71C5D6246 /* TypoGate.swift in Sources */, B2A6C1474B9F3A6E26F488C9 /* UnitConversionEvaluator.swift in Sources */, + 07C1D9A7919366C977AC1623 /* VisualContextChangePolicy.swift in Sources */, 0A15FAD03A93D57372D267E0 /* VisualContextCoordinator.swift in Sources */, 0DBDD71AB17F87D6558DC07D /* VisualContextExcerptSelector.swift in Sources */, 948AD3BD25C113539DF12FD7 /* VisualContextModels.swift in Sources */, @@ -4630,6 +4638,7 @@ CF391BC4A3C80C41048D64D1 /* TypoCaseTransfer.swift in Sources */, 48A2F371756723299597F5A1 /* TypoGate.swift in Sources */, 2DD6D14058C7FD05DBE1AD60 /* UnitConversionEvaluator.swift in Sources */, + 845687C938075DC9C7074F9C /* VisualContextChangePolicy.swift in Sources */, E9E4CC657771DF9F4C56183C /* VisualContextCoordinator.swift in Sources */, 540C1F9A70A5C7DB988FBDD4 /* VisualContextExcerptSelector.swift in Sources */, 07639673B5A52D2075A9ED57 /* VisualContextModels.swift in Sources */, @@ -4919,6 +4928,7 @@ F25F1800B79BF85AF07048C3 /* TypoCaseTransferTests.swift in Sources */, C3F63C89E82ED82DF24BD603 /* TypoGateTests.swift in Sources */, 9CD0DEFE83D73972985D2F1C /* UnitConversionEvaluatorTests.swift in Sources */, + 67A8673F38C847E921CC8BEE /* VisualContextChangePolicyTests.swift in Sources */, E5B8A88A1BC2F84444EABDE0 /* VisualContextCoordinatorTests.swift in Sources */, 9259C21BDBE17A814F0C6431 /* VisualContextExcerptSelectorTests.swift in Sources */, DC6FEE1602B6891CA4D0CDCA /* VisualContextStartCoalescerTests.swift in Sources */, diff --git a/Cotabby/Services/Visual/VisualContextCoordinator.swift b/Cotabby/Services/Visual/VisualContextCoordinator.swift index a5d09c5ae..334e2deb5 100644 --- a/Cotabby/Services/Visual/VisualContextCoordinator.swift +++ b/Cotabby/Services/Visual/VisualContextCoordinator.swift @@ -217,6 +217,7 @@ final class VisualContextCoordinator { excerpt, for: session.sessionID, identity: snapshotContext.identity, + fieldText: snapshotContext.precedingText + snapshotContext.trailingText, capturedAt: capturedAt ) } catch is CancellationError { @@ -332,10 +333,17 @@ final class VisualContextCoordinator { } /// Commits the generated screenshot excerpt and reports readiness for the still-focused field. + /// + /// A refresh that only differs by OCR noise or by the user's own typing keeps the excerpt text + /// already in the prompt (`VisualContextChangePolicy`) and only renews its capture time. The + /// new capture still proves the screen looks the same, so expiry semantics hold, while the + /// prompt stays byte-identical: the KV cache keeps the screen section and the visible + /// suggestion is not retired. Only a material change replaces the text and notifies. private func applyExcerpt( _ excerpt: VisualContextExcerpt, for sessionID: UUID, identity: FocusedInputIdentity, + fieldText: String, capturedAt: TimeInterval ) { guard activeAugmentationSession?.sessionID == sessionID, @@ -350,14 +358,18 @@ final class VisualContextCoordinator { return } - let changed = activeAugmentationSession?.excerpt?.text != excerpt.text + let previousText = activeAugmentationSession?.excerpt?.text + let changed = VisualContextChangePolicy.isMaterialChange( + previous: previousText, next: excerpt.text, fieldText: fieldText + ) + let committed = changed ? excerpt : VisualContextExcerpt(text: previousText ?? excerpt.text) activeAugmentationSession?.status = .ready - activeAugmentationSession?.excerpt = excerpt + activeAugmentationSession?.excerpt = committed status = .ready - latestExcerpt = excerpt.text + latestExcerpt = committed.text excerptCapturedAt = capturedAt scheduleExpiry(sessionID: sessionID, capturedAt: capturedAt) - CotabbyLogger.app.debug("Visual context ready: \(excerpt.text.count) chars") + CotabbyLogger.app.debug("Visual context ready: \(committed.text.count) chars\(changed ? "" : " (unchanged; kept prompt text)")") publishState() if changed { onInjectedContextReady?(identity) } } diff --git a/Cotabby/Support/Context/VisualContextChangePolicy.swift b/Cotabby/Support/Context/VisualContextChangePolicy.swift new file mode 100644 index 000000000..514e7cd36 --- /dev/null +++ b/Cotabby/Support/Context/VisualContextChangePolicy.swift @@ -0,0 +1,87 @@ +import Foundation + +/// File overview: +/// Decides whether a refreshed screen excerpt differs from the one already in the prompt in a way +/// the model should see, or only by OCR noise and the user's own typing. +/// +/// Why this exists: `VisualContextCoordinator` re-captures the screen every few seconds, and the +/// capture usually contains the very field the user is typing into. Treating every textual +/// difference as new context made each refresh replace the screen section of the prompt, which +/// discards the KV cache from that section onward (up to ~1,200 tokens to re-decode) and retires +/// the visible suggestion through `onInjectedContextReady`, so ghost text vanished every refresh +/// while the user typed. Keeping the previous excerpt when nothing material changed holds the +/// prompt byte-stable across refreshes; real changes (a new chat message, navigation, scrolling) +/// still replace it immediately. +/// +/// The comparison is line based because OCR jitter is local: a misread letter changes a word or +/// two inside a line, while a genuine update adds or removes whole lines. +nonisolated enum VisualContextChangePolicy { + /// A refreshed line counts as the same line when it shares at least this fraction of its words + /// with some previous line. OCR misreads ("includina" for "including") stay well above it. + static let lineMatchThreshold = 0.5 + /// A line made mostly of words the user has in the focused field is their own text rendered + /// on screen, which the prompt already carries as the caret prefix. + static let fieldEchoThreshold = 0.6 + /// Lines shorter than this many words (timestamps, button labels, "Edited") are too noisy to + /// count as new content on their own. + static let minimumLineWords = 3 + /// Content that disappears without anything new appearing (a scrolled-off line) only matters + /// once enough of it is gone to make the old excerpt misleading. + static let removedLinesThreshold = 3 + + /// - Parameters: + /// - previous: The excerpt currently in the prompt, or nil when none is. + /// - next: The freshly captured excerpt. + /// - fieldText: The focused field's text around the caret, used to ignore the user's own + /// typing where it shows up in the capture. + /// - Returns: true when `next` should replace `previous` in the prompt. + static func isMaterialChange(previous: String?, next: String, fieldText: String) -> Bool { + guard let previous else { return true } + if previous == next { return false } + + let fieldWords = Set(words(in: fieldText)) + let previousLines = contentLines(previous, fieldWords: fieldWords) + let nextLines = contentLines(next, fieldWords: fieldWords) + + let added = nextLines.filter { line in + !previousLines.contains { similarity(line, $0) >= lineMatchThreshold } + } + if !added.isEmpty { return true } + + let removed = previousLines.filter { line in + !nextLines.contains { similarity(line, $0) >= lineMatchThreshold } + } + return removed.count >= removedLinesThreshold + } + + /// The word sets of the lines that carry screen content: long enough to mean something and + /// not an echo of the field the user is typing in. + private static func contentLines(_ text: String, fieldWords: Set) -> [Set] { + text.split(whereSeparator: \.isNewline).compactMap { rawLine in + let lineWords = words(in: String(rawLine)) + guard lineWords.count >= minimumLineWords else { return nil } + let lineSet = Set(lineWords) + if !fieldWords.isEmpty { + let echoed = Double(lineSet.intersection(fieldWords).count) / Double(lineSet.count) + if echoed >= fieldEchoThreshold { return nil } + } + return lineSet + } + } + + /// Jaccard similarity of two word sets. + private static func similarity(_ lhs: Set, _ rhs: Set) -> Double { + let union = lhs.union(rhs).count + guard union > 0 else { return 1 } + return Double(lhs.intersection(rhs).count) / Double(union) + } + + /// Lowercased letter/digit runs of two or more characters. Single characters are mostly OCR + /// debris (bullets, stray glyphs) and would make short lines look different for no reason. + private static func words(in text: String) -> [String] { + text.lowercased() + .split(whereSeparator: { !$0.isLetter && !$0.isNumber }) + .filter { $0.count >= 2 } + .map(String.init) + } +} diff --git a/CotabbyTests/Services/Visual/VisualContextCoordinatorTests.swift b/CotabbyTests/Services/Visual/VisualContextCoordinatorTests.swift index a5846f0e9..be0255a32 100644 --- a/CotabbyTests/Services/Visual/VisualContextCoordinatorTests.swift +++ b/CotabbyTests/Services/Visual/VisualContextCoordinatorTests.swift @@ -23,6 +23,27 @@ final class VisualContextCoordinatorTests: XCTestCase { XCTAssertEqual(coordinator.latestExcerpt, generator.text) } + /// A refresh that only misreads a letter must keep the excerpt already in the prompt, so the + /// prompt stays byte-identical (KV reuse) and no prediction is retired. + func test_ocrJitterKeepsPromptTextAndDoesNotNotify() async throws { + let generator = StubVisualContextGenerator() + let coordinator = makeCoordinator(generator) + let snapshot = CotabbyTestFixtures.focusedInputSnapshot(precedingText: "Draft") + coordinator.refreshContextProvider = { snapshot } + var notifications = 0 + coordinator.onInjectedContextReady = { _ in notifications += 1 } + coordinator.startSessionIfNeeded(for: snapshot, configuration: .local) + defer { coordinator.cancel(resetState: true) } + try await waitUntil { generator.contexts.count >= 1 && coordinator.status == .ready } + let original = coordinator.latestExcerpt + generator.text = "Project agenda and deadlinc" + try await waitUntil { generator.contexts.count >= 2 } + try await waitUntil { generator.completedCount >= 2 } + XCTAssertEqual(notifications, 1) + XCTAssertEqual(coordinator.latestExcerpt, original, "OCR noise must not replace the prompt's screen text") + XCTAssertEqual(coordinator.status, .ready) + } + func test_permissionRevocationStopsRefreshAndClearsExcerpt() async throws { let generator = StubVisualContextGenerator() var allowed = true diff --git a/CotabbyTests/Support/Context/VisualContextChangePolicyTests.swift b/CotabbyTests/Support/Context/VisualContextChangePolicyTests.swift new file mode 100644 index 000000000..ee20aee03 --- /dev/null +++ b/CotabbyTests/Support/Context/VisualContextChangePolicyTests.swift @@ -0,0 +1,66 @@ +import XCTest +@testable import Cotabby + +/// Pure-function tests for deciding when a refreshed screen excerpt should replace the one in the +/// prompt. The cases mirror what a refresh actually sees while someone types: OCR misreads, the +/// user's own text growing inside the captured field, and genuine screen updates. +final class VisualContextChangePolicyTests: XCTestCase { + private let chat = """ + Alex Morgan 10:42 + Can you send the updated proposal before Friday? + The pricing section still lists last quarter's numbers. + Sam Lee 10:44 + I will check with finance and get back to you today. + """ + + func test_firstExcerptIsAlwaysMaterial() { + XCTAssertTrue(VisualContextChangePolicy.isMaterialChange(previous: nil, next: chat, fieldText: "")) + } + + func test_identicalExcerptIsNotMaterial() { + XCTAssertFalse(VisualContextChangePolicy.isMaterialChange(previous: chat, next: chat, fieldText: "")) + } + + func test_ocrMisreadsAreNotMaterial() { + let jittered = chat + .replacingOccurrences(of: "updated proposal", with: "updatcd proposal") + .replacingOccurrences(of: "quarter's", with: "quarters") + .replacingOccurrences(of: "10:44", with: "10.44") + XCTAssertFalse(VisualContextChangePolicy.isMaterialChange(previous: chat, next: jittered, fieldText: "")) + } + + /// The capture usually includes the field being typed into. Its growing text must not count + /// as new screen content: the prompt already carries it as the caret prefix. + func test_userTypingInsideCapturedFieldIsNotMaterial() { + let typed = "Thanks Alex, I will update the pricing section and send the proposal" + let previous = chat + "\nThanks Alex, I will update" + let next = chat + "\n" + typed + XCTAssertFalse(VisualContextChangePolicy.isMaterialChange(previous: previous, next: next, fieldText: typed)) + } + + func test_newIncomingMessageIsMaterial() { + let next = chat + "\nAlex Morgan 10:47\nAlso please include the revised delivery timeline." + XCTAssertTrue(VisualContextChangePolicy.isMaterialChange(previous: chat, next: next, fieldText: "")) + } + + func test_navigationToAnotherConversationIsMaterial() { + let other = """ + Priya Shah 09:15 + The staging cluster is down again after the last deploy. + Could someone from platform take a look this morning? + """ + XCTAssertTrue(VisualContextChangePolicy.isMaterialChange(previous: chat, next: other, fieldText: "")) + } + + func test_shortChromeChangesAreNotMaterial() { + let next = chat.replacingOccurrences(of: "Alex Morgan 10:42", with: "Alex Morgan 10:42 Edited") + XCTAssertFalse(VisualContextChangePolicy.isMaterialChange(previous: chat, next: next, fieldText: "")) + } + + func test_contentDisappearingIsMaterialOnlyPastThreshold() { + let lines = chat.split(separator: "\n").map(String.init) + let oneGone = lines.dropFirst(2).joined(separator: "\n") + XCTAssertFalse(VisualContextChangePolicy.isMaterialChange(previous: chat, next: oneGone, fieldText: "")) + XCTAssertTrue(VisualContextChangePolicy.isMaterialChange(previous: chat, next: "", fieldText: "")) + } +} From 04cb4643d71697592089b6201aac73caba27ac08 Mon Sep 17 00:00:00 2001 From: Senad Date: Sat, 17 Oct 2026 20:58:41 +0200 Subject: [PATCH 03/19] Order prompt context by volatility and prewarm with the same context History examples change every few words, more often than the pinned clipboard or the now-stable screen excerpt. Placed before them, each history step re-decoded the whole screen section too; they now follow it. The focus-time prewarm also renders the pinned clipboard and any ready screen excerpt, so the first real request shares its bytes instead of diverging right after the surface line. The quality and recall eval suites score identically before and after (0.853 and 0.826, same per-case outcomes on Gemma 4 E4B). --- .../SuggestionCoordinator+Input.swift | 9 +++++++ .../BaseCompletionPromptRenderer.swift | 19 ++++++++++----- .../SuggestionCoordinatorInputTests.swift | 21 ++++++++++++++++ .../BaseCompletionPromptRendererTests.swift | 24 +++++++++++++++++++ 4 files changed, 67 insertions(+), 6 deletions(-) diff --git a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Input.swift b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Input.swift index 4ec63409a..5511250b9 100644 --- a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Input.swift +++ b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Input.swift @@ -220,10 +220,19 @@ extension SuggestionCoordinator { // null the freshly primed session out from under us. await self.awaitCachedGenerationContextResetIfNeeded() let prewarmContext = FocusedInputContext(snapshot: rawContext, generation: 0) + // Build the prompt from the same context sources the first real request will read + // (`SuggestionCoordinator+Prediction`): the pinned clipboard and any ready screen + // excerpt. The prefill is only worth its decode if the next request shares its bytes; + // leaving these out made the first request diverge right after the surface line and + // re-decode everything the prewarm had already paid for. let request = SuggestionRequestFactory.buildRequest( context: prewarmContext, settings: settings, configuration: configuration, + clipboardContext: self.pinnedClipboardContext(rawContext: rawContext), + visualContextSummary: self.permissionManager.screenRecordingGranted + ? self.visualContextCoordinator.excerpt(for: prewarmContext) + : nil, historyExamples: self.historyExamples(for: prewarmContext) ).request await suggestionEngine.prewarm(for: request) diff --git a/Cotabby/Support/Prompting/BaseCompletionPromptRenderer.swift b/Cotabby/Support/Prompting/BaseCompletionPromptRenderer.swift index 1bf822de0..02ef03f65 100644 --- a/Cotabby/Support/Prompting/BaseCompletionPromptRenderer.swift +++ b/Cotabby/Support/Prompting/BaseCompletionPromptRenderer.swift @@ -63,9 +63,12 @@ enum BaseCompletionPromptRenderer { // the whole blob lands. sections.append(Self.contextSection("notes", "Notes the writer keeps in mind: \(notes)", priority: 40, maxChars: 1300)) } - if let history = Self.historySection(historyExamples) { - sections.append(history) - } + // Sections are ordered from least to most volatile so a change invalidates as little of the + // cached prompt as possible: everything after the first changed byte is re-decoded. The + // clipboard is pinned per field and the screen excerpt only changes on a material update + // (`VisualContextChangePolicy`), while history examples move every few words. History + // therefore follows the screen text: placed before it, each history step re-decoded the + // whole screen section (up to ~1,200 tokens) as well. if let clip = Self.nonEmpty(clipboardContext) { sections.append(Self.contextSection("clipboard", "On the clipboard: \(clip)", priority: 35, maxChars: 400)) } @@ -74,6 +77,9 @@ enum BaseCompletionPromptRenderer { "screen", "Nearby on screen: \(screen)", priority: screenPriority, maxChars: maxScreenCharacters )) } + if let history = Self.historySection(historyExamples) { + sections.append(history) + } let followingText = String(trailingText.prefix(max(0, maxSuffixCharacters))) if !followingText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { // Base models do not share a universal fill-in-the-middle token format. Describe the @@ -144,9 +150,10 @@ enum BaseCompletionPromptRenderer { /// The user's own earlier sentences, quoted. A base model conditions strongly on nearby text in /// the same voice, so a couple of real examples pull its word choice toward how this person - /// writes. Sits after the stable preface and before the per-keystroke clipboard and screen - /// sections: the examples change only every few words (`TypingHistoryQuery.stableText`), so - /// placing them earlier keeps more of the prompt's head reusable from the KV cache. + /// writes. Sits after the clipboard and screen sections and before the caret text: the examples + /// change every few words (`TypingHistoryQuery.stableText`), more often than the pinned + /// clipboard or the screen excerpt, so placing them last keeps the larger sections reusable + /// from the KV cache. /// /// All or nothing, like the "following" section: a budget-trimmed history section could end /// on an unclosed quote, and the model would then read the live caret text as part of that diff --git a/CotabbyTests/App/Coordinators/Suggestion/SuggestionCoordinatorInputTests.swift b/CotabbyTests/App/Coordinators/Suggestion/SuggestionCoordinatorInputTests.swift index 51087d36a..43527390a 100644 --- a/CotabbyTests/App/Coordinators/Suggestion/SuggestionCoordinatorInputTests.swift +++ b/CotabbyTests/App/Coordinators/Suggestion/SuggestionCoordinatorInputTests.swift @@ -304,6 +304,27 @@ final class SuggestionCoordinatorInputTests: SuggestionCoordinatorRigTestCase { XCTAssertEqual(rig.engine.prewarmedRequests.first?.generation, 0) } + /// The focus prefill is only reusable if the first real request renders the same prompt bytes, + /// so the prewarm must carry the same pinned clipboard context a prediction would. + func test_fieldSwitchPrewarmIncludesPinnedClipboardContext() async { + let rig = retained(makeCoordinatorRig()) + _ = rig.interactionState.materializeContext(from: rig.focusProvider.snapshot.context!) + rig.clipboardFilter.filtered = "Quarterly numbers attached" + + let otherApp = CotabbyTestFixtures.focusedInputSnapshot(processIdentifier: 456, precedingText: "Hi") + rig.coordinator.handleSupportedSnapshot(FocusSnapshot( + applicationName: otherApp.applicationName, + bundleIdentifier: otherApp.bundleIdentifier, + capability: .supported, + context: otherApp + )) + + await waitUntil("Engine was never prewarmed for the new field") { + !rig.engine.prewarmedRequests.isEmpty + } + XCTAssertEqual(rig.engine.prewarmedRequests.first?.clipboardContext, "Quarterly numbers attached") + } + func test_handleSupportedSnapshot_withoutContextDisablesOutright() { let rig = retained(makeCoordinatorRig()) let bareSnapshot = FocusSnapshot( diff --git a/CotabbyTests/Support/Prompting/BaseCompletionPromptRendererTests.swift b/CotabbyTests/Support/Prompting/BaseCompletionPromptRendererTests.swift index 3db10bbb9..fb462d566 100644 --- a/CotabbyTests/Support/Prompting/BaseCompletionPromptRendererTests.swift +++ b/CotabbyTests/Support/Prompting/BaseCompletionPromptRendererTests.swift @@ -90,6 +90,30 @@ final class BaseCompletionPromptRendererTests: XCTestCase { ) } + /// Context runs from least to most volatile so a change re-decodes as little cached prompt as + /// possible: pinned clipboard, then the screen excerpt, then history examples (which move + /// every few words), then the following text and the caret prefix. + func test_historyExamplesFollowClipboardAndScreenText() { + let prompt = BaseCompletionPromptRenderer.prompt( + prefixText: "the meeting is at ", + applicationName: "Mail", + userName: nil, + trailingText: " on Friday.", + clipboardContext: "zoom link", + visualContextSummary: "Calendar: Q3 planning 3pm", + historyExamples: ["See you at the standup."] + ) + XCTAssertEqual( + prompt, + [ + "On the clipboard: zoom link", + "Nearby on screen: Calendar: Q3 planning 3pm", + "Earlier writing by the same author:\n“See you at the standup.”", + "Later in the same passage:\n“ on Friday.”" + ].joined(separator: "\n") + "\n\nthe meeting is at " + ) + } + /// The following text is cut to `maxSuffixCharacters` and quoted, keeping its whitespace. func test_followingTextIsBoundedQuotedContextBeforeExactPrefix() { let prefix = "We are meeting " From d38b89e85dd3481478a1f00d65bc80a7ceda969f Mon Sep 17 00:00:00 2001 From: Senad Date: Sat, 17 Oct 2026 20:58:42 +0200 Subject: [PATCH 04/19] Default to short suggestions At 15-20 ms per generated token on a 4B-class model, the 12-20 word default (26 tokens) spent most of each request on words that were rarely kept while the next keystroke cancelled it. The default is now 4-7 words; onboarding's Quick tier is 2-4, Everyday and Custom 4-7, and Powerful keeps 12-20. Saved choices are untouched, and the existing continuation prefetch extends a short suggestion as the user types through it. In the typing-session replays, useful streamed suggestions rose from 13/40 to 20/40 and cancelled requests fell from 82 to 58 together with the cache changes. --- Cotabby/Models/Onboarding/OnboardingTemplate.swift | 9 ++++++--- .../Suggestion/Request/SuggestionConfiguration.swift | 11 ++++++++--- .../Suggestion/Request/SuggestionRequestFactory.swift | 2 +- .../OnboardingTemplateRecommenderTests.swift | 3 ++- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/Cotabby/Models/Onboarding/OnboardingTemplate.swift b/Cotabby/Models/Onboarding/OnboardingTemplate.swift index ed11e0b9d..60c3d97f7 100644 --- a/Cotabby/Models/Onboarding/OnboardingTemplate.swift +++ b/Cotabby/Models/Onboarding/OnboardingTemplate.swift @@ -86,16 +86,19 @@ enum OnboardingTemplate: String, CaseIterable, Identifiable, Equatable, Sendable } } + /// Short suggestions are both faster (fewer generated tokens) and more often exactly right, and + /// accepting one extends it with a continuation, so every tier but Powerful starts short. + /// Powerful keeps the long preset for users who explicitly choose longer suggestions. var wordCountPreset: SuggestionWordCountPreset { switch self { case .quick: - return .fourToSeven + return .twoToFour case .everyday: - return .sevenToTwelve + return .fourToSeven case .powerful: return .twelveToTwenty case .custom: - return .sevenToTwelve + return .fourToSeven } } diff --git a/Cotabby/Models/Suggestion/Request/SuggestionConfiguration.swift b/Cotabby/Models/Suggestion/Request/SuggestionConfiguration.swift index 72300fefd..7f0d447ef 100644 --- a/Cotabby/Models/Suggestion/Request/SuggestionConfiguration.swift +++ b/Cotabby/Models/Suggestion/Request/SuggestionConfiguration.swift @@ -110,8 +110,8 @@ struct SuggestionConfiguration: Equatable, Sendable { let focusPollIntervalMilliseconds: Int - /// Output reserve built into the configured llama prompt budget. It covers the default - /// single-line request (12-20 words, 26 tokens) with room to spare; a request that asks for + /// Output reserve built into the configured llama prompt budget. It covers the longest + /// single-line preset (12-20 words, 26 tokens) with room to spare; a request that asks for /// more has the excess taken out of its own prompt (`SuggestionRequestFactory.promptTokenBudget`). static let llamaPromptOutputCeilingTokens = 50 /// Margin for BOS plus token-estimator error; the estimator skews conservative, so real @@ -167,7 +167,12 @@ struct SuggestionConfiguration: Equatable, Sendable { // endpoint backend can carry that prompt off the Mac, so a real name is an explicit opt-in // and a placeholder would sign every user's mail with someone else's name. defaultUserName: nil, - defaultWordCountPreset: .twelveToTwenty, + // Short by default: at ~15-20 ms per generated token on a 4B-class model, a 12-20 word + // budget (26 tokens) spent most of each request decoding words the user rarely kept, while + // the next keystroke was already cancelling it. A 4-7 word suggestion (10 tokens) lands in + // a fraction of that, is right more often, and accepting it schedules the continuation + // that extends it. Longer presets stay one selection away in Settings. + defaultWordCountPreset: .fourToSeven, focusPollIntervalMilliseconds: 50 ) } diff --git a/Cotabby/Support/Suggestion/Request/SuggestionRequestFactory.swift b/Cotabby/Support/Suggestion/Request/SuggestionRequestFactory.swift index e8711a205..087814600 100644 --- a/Cotabby/Support/Suggestion/Request/SuggestionRequestFactory.swift +++ b/Cotabby/Support/Suggestion/Request/SuggestionRequestFactory.swift @@ -297,7 +297,7 @@ enum SuggestionRequestFactory { /// The prompt's token budget for one request. The configured budget holds back /// `llamaPromptOutputCeilingTokens` of the context window for output, but a request can ask for /// more: a 50-word range reaches 100 tokens in a 2-tokens-per-word language, and multi-line - /// doubles budgets up to 120 (the default 12-20 range becomes 52). `LlamaRuntimeCore` reserves + /// doubles budgets up to 120 (the 12-20 range becomes 52). `LlamaRuntimeCore` reserves /// the whole decode before admitting the prompt, so an overflowing prompt loses its oldest raw /// tokens there: the preface is cut mid-section and the request forfeits KV prefix reuse. /// Taking the excess out of the budget here lets the renderer trim by section priority instead. diff --git a/CotabbyTests/Support/Onboarding/OnboardingTemplateRecommenderTests.swift b/CotabbyTests/Support/Onboarding/OnboardingTemplateRecommenderTests.swift index c1cc902c9..5564976a6 100644 --- a/CotabbyTests/Support/Onboarding/OnboardingTemplateRecommenderTests.swift +++ b/CotabbyTests/Support/Onboarding/OnboardingTemplateRecommenderTests.swift @@ -40,12 +40,13 @@ final class OnboardingTemplateRecommenderTests: XCTestCase { func testAppleIntelligenceStillCarriesTierBehaviorFlags() { let quick = OnboardingTemplateRecommender.resolvePlan(for: .quick, engine: .appleIntelligence) - XCTAssertEqual(quick.wordCountPreset, .fourToSeven) + XCTAssertEqual(quick.wordCountPreset, .twoToFour) XCTAssertFalse(quick.enablesFastMode) XCTAssertFalse(quick.enablesMultiLine) XCTAssertFalse(quick.enablesClipboardContext) let everyday = OnboardingTemplateRecommender.resolvePlan(for: .everyday, engine: .appleIntelligence) + XCTAssertEqual(everyday.wordCountPreset, .fourToSeven) XCTAssertFalse(everyday.enablesFastMode) XCTAssertFalse(everyday.enablesMultiLine) XCTAssertTrue(everyday.enablesClipboardContext) From 3d84ef61c4de1401c0b319ee0c5d7eb83c37c192 Mon Sep 17 00:00:00 2001 From: Senad Date: Sat, 17 Oct 2026 21:20:10 +0200 Subject: [PATCH 05/19] Cut the waits between a keystroke and visible ghost text Live traces of Cotabby Dev in TextEdit showed keystroke-to-ghost at about 230 ms p50 even though a warm request had text ready in 40-60 ms: - The debounce was keyed to the full generation latency, so every complete 4-7 word suggestion (100-200 ms) bought the maximum 55 ms wait in front of the next keystroke. In-process engines now key it to the latency to the request's first streamed text, which is what a superseded request actually costs with KV reuse: 0 ms when that is under 60 ms, 10 ms under 120 ms, the old tiers otherwise. The HTTP endpoint keeps its trailing-edge pause. - The mid-word quiet window held a suggestion back for 1.2x the typing interval (80-220 ms, ~156 ms at 130 ms per letter). It now spans 0.4x the interval, bounded to 20-50 ms; it runs from the last letter and overlaps the generation, so it suppresses only sub-keystroke flashes. --- .../SuggestionCoordinator+Prediction.swift | 8 +++-- ...ggestionCoordinator+TypingPrediction.swift | 4 +++ .../Suggestion/SuggestionCoordinator.swift | 6 ++++ .../Suggestion/Request/DebouncePolicy.swift | 16 +++++++++ .../Suggestion/Streaming/TypingCadence.swift | 10 ++++-- .../Request/DebouncePolicyTests.swift | 36 +++++++++++++++++++ .../Streaming/TypingCadenceTests.swift | 14 ++++---- 7 files changed, 81 insertions(+), 13 deletions(-) diff --git a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Prediction.swift b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Prediction.swift index 3853fd634..d06a8dc98 100644 --- a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Prediction.swift +++ b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Prediction.swift @@ -26,12 +26,14 @@ extension SuggestionCoordinator { return } - // The debounce window adapts to the last generation latency: snappier when the model is - // fast, calmer when it is slow (fewer doomed generations to cancel). The configured value - // is the fallback until a first latency exists. + // The debounce window adapts to how quickly the engine produces text: none when a warm + // request shows its first text fast, calmer when the model is slow (fewer doomed + // generations to cancel). The configured value is the fallback until a latency exists. let debounceMilliseconds = DebouncePolicy.milliseconds( lastGenerationLatencyMilliseconds: lastLatencyByEngine[settingsSnapshot.selectedEngine], + lastFirstTextLatencyMilliseconds: + lastFirstTextLatencyByEngine[settingsSnapshot.selectedEngine], fallback: settingsSnapshot.debounceMilliseconds, engine: settingsSnapshot.selectedEngine ) diff --git a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+TypingPrediction.swift b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+TypingPrediction.swift index b56e49b25..ddfeaed36 100644 --- a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+TypingPrediction.swift +++ b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+TypingPrediction.swift @@ -67,6 +67,10 @@ extension SuggestionCoordinator { /// presentation; the hidden buffer still lets a complete answer survive intervening typing. func receiveTypingPartial(_ result: SuggestionResult, workID: UInt64, showPartials: Bool) { guard workController.isCurrent(workID) else { return } + if firstTextRecordedWorkID != workID { + firstTextRecordedWorkID = workID + lastFirstTextLatencyByEngine[settingsSnapshot.selectedEngine] = Int((result.latency * 1_000).rounded()) + } if var candidate = typingPrediction { guard !candidate.isFinal, result.generation == candidate.context.generation else { return } guard candidate.receive(result, final: false) else { diff --git a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator.swift b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator.swift index 298ec3811..824653388 100644 --- a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator.swift +++ b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator.swift @@ -77,6 +77,12 @@ final class SuggestionCoordinator: ObservableObject { /// a slow HTTP backend retain an appropriate debounce without leaking that latency into the /// in-process engines. var lastLatencyByEngine: [SuggestionEngineKind: Int] = [:] + /// Latency to the first streamed text of the most recent request per backend: what starting a + /// request costs once the KV prefix is warm. `DebouncePolicy` prefers it, because a superseded + /// request stops at its next token while a debounce delays every keystroke. Recorded once per + /// work item (`firstTextRecordedWorkID`), only for the request that is still current. + var lastFirstTextLatencyByEngine: [SuggestionEngineKind: Int] = [:] + var firstTextRecordedWorkID: UInt64? // Synchronous input/focus callbacks cannot directly `await`, so resets are represented as a // barrier task that the next generation must cross before it can ask the runtime for output. var cacheResetSequence: UInt64 = 0 diff --git a/Cotabby/Support/Suggestion/Request/DebouncePolicy.swift b/Cotabby/Support/Suggestion/Request/DebouncePolicy.swift index 2f43e24ee..ddace1c8e 100644 --- a/Cotabby/Support/Suggestion/Request/DebouncePolicy.swift +++ b/Cotabby/Support/Suggestion/Request/DebouncePolicy.swift @@ -7,9 +7,17 @@ import Foundation /// that cannot keep up (each cancel still costs a decode setup and teardown). Keying the debounce /// to the most recent generation latency makes fast machines snappier and slow machines calmer, /// with no configuration. The configured value remains the fallback until a first latency exists. +/// +/// For the in-process engines the cost a debounce saves is the cost of *starting* a request, not +/// of finishing it: a superseded generation is cancelled between tokens, and with KV prefix reuse a +/// warm request decodes only the typed delta before its first text appears. So when the latency to +/// that first text is known and small, the debounce shrinks to nothing; keying it to the full +/// latency instead made every short-but-complete suggestion (100-200 ms) buy the maximum 55 ms +/// wait in front of the next keystroke, the largest single delay before visible text. nonisolated enum DebouncePolicy { static func milliseconds( lastGenerationLatencyMilliseconds: Int?, + lastFirstTextLatencyMilliseconds: Int? = nil, fallback: Int, engine: SuggestionEngineKind = .llamaOpenSource ) -> Int { @@ -28,6 +36,14 @@ nonisolated enum DebouncePolicy { } } + if let firstText = lastFirstTextLatencyMilliseconds, firstText > 0 { + switch firstText { + case ...60: return 0 + case ...120: return 10 + default: break + } + } + guard let last = lastGenerationLatencyMilliseconds, last > 0 else { return fallback } diff --git a/Cotabby/Support/Suggestion/Streaming/TypingCadence.swift b/Cotabby/Support/Suggestion/Streaming/TypingCadence.swift index 17124b829..94d022724 100644 --- a/Cotabby/Support/Suggestion/Streaming/TypingCadence.swift +++ b/Cotabby/Support/Suggestion/Streaming/TypingCadence.swift @@ -34,9 +34,13 @@ nonisolated struct TypingCadence { guard self.identityKey == identityKey, let last = lastLetterAt, CaretWordContext.unfinishedWord(in: precedingText) != nil else { return 0 } let ordered = intervals.sorted() - // These are bounded presentation limits, not a new global debounce. Before enough rhythm - // exists, 80ms merely absorbs an ordinary brief hesitation; a completed word pays no delay. - let quietWindow = ordered.count >= 2 ? min(0.22, max(0.08, ordered[ordered.count / 2] * 1.2)) : 0.08 + // These are bounded presentation limits, not a new global debounce, and they only suppress + // a mid-word ghost that would flash for a fraction of a keystroke. The window runs from the + // last letter, so it overlaps the generation itself: a warm request needs ~40-60 ms to + // produce text, and by then a window of at most 50 ms has usually elapsed and costs + // nothing. The previous 80-220 ms window (1.2x the typing interval) held every mid-word + // suggestion back for a whole keystroke on top of generation. A completed word pays none. + let quietWindow = ordered.count >= 2 ? min(0.05, max(0.02, ordered[ordered.count / 2] * 0.4)) : 0.04 return max(0, quietWindow - max(0, time - last)) } } diff --git a/CotabbyTests/Support/Suggestion/Request/DebouncePolicyTests.swift b/CotabbyTests/Support/Suggestion/Request/DebouncePolicyTests.swift index f184073dd..609e2c1d3 100644 --- a/CotabbyTests/Support/Suggestion/Request/DebouncePolicyTests.swift +++ b/CotabbyTests/Support/Suggestion/Request/DebouncePolicyTests.swift @@ -40,6 +40,42 @@ final class DebouncePolicyTests: XCTestCase { assertDebounce(cases, fallback: 20, engine: .appleIntelligence) } + /// A fast time to first text removes the debounce for in-process engines even when the full + /// suggestion took long; a slow first text falls back to the full-latency tiers. + func test_localEngines_firstTextLatencyTiers() { + let cases: [(firstText: Int?, full: Int?, expected: Int)] = [ + (1, 900, 0), (60, 900, 0), + (61, 900, 10), (120, 900, 10), + (121, 900, 55), (121, 100, 25), + (0, 45, 15), (nil, 45, 15) + ] + for engine in [SuggestionEngineKind.llamaOpenSource, .appleIntelligence] { + for testCase in cases { + XCTAssertEqual( + DebouncePolicy.milliseconds( + lastGenerationLatencyMilliseconds: testCase.full, + lastFirstTextLatencyMilliseconds: testCase.firstText, + fallback: 20, + engine: engine + ), + testCase.expected, + "\(engine) first \(String(describing: testCase.firstText)) full \(String(describing: testCase.full))" + ) + } + } + } + + /// The endpoint keeps its trailing-edge pause regardless of first-text latency. + func test_endpoint_ignoresFirstTextLatency() { + XCTAssertEqual( + DebouncePolicy.milliseconds( + lastGenerationLatencyMilliseconds: 200, lastFirstTextLatencyMilliseconds: 20, + fallback: 20, engine: .openAICompatible + ), + 100 + ) + } + func test_defaultEngineIsLocal() { XCTAssertEqual(DebouncePolicy.milliseconds(lastGenerationLatencyMilliseconds: 45, fallback: 20), 15) } diff --git a/CotabbyTests/Support/Suggestion/Streaming/TypingCadenceTests.swift b/CotabbyTests/Support/Suggestion/Streaming/TypingCadenceTests.swift index 7286048f1..d5aeca54c 100644 --- a/CotabbyTests/Support/Suggestion/Streaming/TypingCadenceTests.swift +++ b/CotabbyTests/Support/Suggestion/Streaming/TypingCadenceTests.swift @@ -7,7 +7,7 @@ final class TypingCadenceTests: XCTestCase { for (index, letter) in ["s", "c", "h", "e"].enumerated() { cadence.record(identityKey: 1, characters: letter, at: Double(index) * 0.1) } - XCTAssertEqual(cadence.remainingDelay(identityKey: 1, precedingText: "sche", at: 0.32), 0.1, accuracy: 0.001) + XCTAssertEqual(cadence.remainingDelay(identityKey: 1, precedingText: "sche", at: 0.32), 0.02, accuracy: 0.001) XCTAssertEqual(cadence.remainingDelay(identityKey: 1, precedingText: "sche", at: 0.5), 0) XCTAssertEqual(cadence.remainingDelay(identityKey: 2, precedingText: "sche", at: 0.32), 0) cadence.record(identityKey: 1, characters: " ", at: 0.33) @@ -15,19 +15,19 @@ final class TypingCadenceTests: XCTestCase { } func testQuietWindowTracksMedianRhythmWithinItsBounds() { - // Slow but steady typing (500ms gaps) would ask for 600ms; the window is capped at 220ms. + // Slow but steady typing (500ms gaps) would ask for 200ms; the window is capped at 50ms. var slow = TypingCadence() for (index, letter) in ["a", "b", "c"].enumerated() { slow.record(identityKey: 1, characters: letter, at: Double(index) * 0.5) } - XCTAssertEqual(slow.remainingDelay(identityKey: 1, precedingText: "abc", at: 1.0), 0.22, accuracy: 0.001) + XCTAssertEqual(slow.remainingDelay(identityKey: 1, precedingText: "abc", at: 1.0), 0.05, accuracy: 0.001) - // Very fast typing (30ms gaps) would ask for 36ms; the window never drops below 80ms. + // Very fast typing (30ms gaps) would ask for 12ms; the window never drops below 20ms. var fast = TypingCadence() for (index, letter) in ["a", "b", "c"].enumerated() { fast.record(identityKey: 1, characters: letter, at: Double(index) * 0.03) } - XCTAssertEqual(fast.remainingDelay(identityKey: 1, precedingText: "abc", at: 0.06), 0.08, accuracy: 0.001) + XCTAssertEqual(fast.remainingDelay(identityKey: 1, precedingText: "abc", at: 0.06), 0.02, accuracy: 0.001) } func testSwitchingFieldsDiscardsThePreviousFieldsRhythm() { @@ -38,7 +38,7 @@ final class TypingCadenceTests: XCTestCase { cadence.record(identityKey: 2, characters: "x", at: 1.1) XCTAssertEqual(cadence.remainingDelay(identityKey: 1, precedingText: "abc", at: 1.1), 0) // The new field starts without intervals, so only the default brief-hesitation window applies. - XCTAssertEqual(cadence.remainingDelay(identityKey: 2, precedingText: "x", at: 1.1), 0.08, accuracy: 0.001) + XCTAssertEqual(cadence.remainingDelay(identityKey: 2, precedingText: "x", at: 1.1), 0.04, accuracy: 0.001) } func testNonFiniteTimestampsAreIgnoredAndNonWordPrefixesPayNoDelay() { @@ -56,7 +56,7 @@ final class TypingCadenceTests: XCTestCase { var cadence = TypingCadence() cadence.record(identityKey: 1, characters: "s", at: 0) cadence.record(identityKey: 1, characters: "c", at: 5) - XCTAssertEqual(cadence.remainingDelay(identityKey: 1, precedingText: "sc", at: 5), 0.08, accuracy: 0.001) + XCTAssertEqual(cadence.remainingDelay(identityKey: 1, precedingText: "sc", at: 5), 0.04, accuracy: 0.001) cadence.record(identityKey: 1, characters: "pasted", at: 5.01) XCTAssertEqual(cadence.remainingDelay(identityKey: 1, precedingText: "pasted", at: 5.02), 0) } From f5d6b35d45d1578c329f79c4b35bef9eee071331 Mon Sep 17 00:00:00 2001 From: Senad Date: Sat, 17 Oct 2026 21:43:13 +0200 Subject: [PATCH 06/19] Poll densely for the host's AX publish in its first 60 ms The host-publish wait polled at +10 ms and then every 30 ms. A native host that published just after the first retry left the request waiting a further ~30 ms (live TextEdit trace: keystroke to request 50 ms instead of ~20 ms). The first retry is now at 6 ms and polls run every 8 ms until 60 ms, where native hosts publish, before backing off to 30 ms for slow Chromium hosts. --- .../SuggestionCoordinator+Input.swift | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Input.swift b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Input.swift index 5511250b9..8ec01ab26 100644 --- a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Input.swift +++ b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Input.swift @@ -363,7 +363,16 @@ extension SuggestionCoordinator { /// own tap callback before the host has even processed the key and therefore always misses. A /// short first retry catches fast-publishing native apps in ~10ms instead of making them wait a /// full steady interval; slow Chromium hosts fall through to `hostPublishPollIntervalMs` below. - private static let hostPublishFirstPollIntervalMs = 10 + private static let hostPublishFirstPollIntervalMs = 6 + + /// Native hosts publish a keystroke to AX within roughly 10-40 ms, so the wait polls densely + /// during that window and only then backs off to `hostPublishPollIntervalMs` for slow Chromium + /// hosts. With a 30 ms step from the start, a publish that landed just after the first retry + /// waited a further ~30 ms (live TextEdit trace: key-to-request 50 ms instead of ~20 ms), the + /// largest remaining wait before generation. The dense phase costs a handful of ~2-3 ms AX + /// reads per keystroke and ends after `hostPublishFastPhaseMs`. + private static let hostPublishFastPollIntervalMs = 8 + private static let hostPublishFastPhaseMs = 60 /// Schedules a fresh prediction once the host app has actually published the new /// contenteditable text to AX. The previous fix waited a fixed 150ms — see PR #376 — but the @@ -491,8 +500,11 @@ extension SuggestionCoordinator { // produces no AX change (rare but possible — e.g. dead-key composition) would never get // its next prediction. // We already waited the short first interval before entering this method; remaining polls - // use the steady cadence until the ceiling. - let interval = Self.hostPublishPollIntervalMs + // stay dense while native hosts usually publish, then use the steady cadence until the + // ceiling. + let interval = elapsedMs < Self.hostPublishFastPhaseMs + ? Self.hostPublishFastPollIntervalMs + : Self.hostPublishPollIntervalMs let nextElapsed = elapsedMs + interval guard nextElapsed < Self.hostPublishWaitCeilingMs else { if baseline.requiresTextChange { From 09435224cf0f925bb517298f132dbe754acb61b8 Mon Sep 17 00:00:00 2001 From: Senad Date: Sat, 17 Oct 2026 21:43:13 +0200 Subject: [PATCH 07/19] Refresh typing-history examples in larger steps after 24 words Each change of the history query re-decodes the examples section and everything after it (~100 ms on Gemma 4 E4B), and with a query that moved every 8 words this was the most frequent cache miss left while typing a longer text. Short texts keep 8-word blocks so a brief message still gets examples early; from 24 words the query moves in 24-word steps. --- Cotabby/Support/History/TypingHistoryIndex.swift | 15 +++++++++++++-- .../Support/History/TypingHistoryIndexTests.swift | 14 ++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/Cotabby/Support/History/TypingHistoryIndex.swift b/Cotabby/Support/History/TypingHistoryIndex.swift index f52a0ac05..3ee7276b1 100644 --- a/Cotabby/Support/History/TypingHistoryIndex.swift +++ b/Cotabby/Support/History/TypingHistoryIndex.swift @@ -16,9 +16,20 @@ nonisolated struct TypingHistoryQuery: Equatable, Sendable { /// in that head. Querying on every keystroke would change the examples, and with them the cached /// prefix, on nearly every request. Rounding to blocks of `wordsPerBlock` words keeps the /// examples fixed while a few words are typed and refreshes them as the topic moves on. - static func stableText(from precedingText: String, wordsPerBlock: Int = 8, maxWords: Int = 48) -> String { + /// + /// Short blocks apply until `laterWordsPerBlock` words so a brief message still gets examples + /// early; after that the query moves in `laterWordsPerBlock` steps (8, 16, 24, 48, ...). Each + /// step re-decodes the examples section and everything + /// after it (~100 ms on a 4B-class model), so refreshing every 8 words throughout a long text + /// was the most frequent cache miss left while typing. + static func stableText( + from precedingText: String, wordsPerBlock: Int = 8, laterWordsPerBlock: Int = 24, maxWords: Int = 48 + ) -> String { let words = precedingText.split(whereSeparator: \.isWhitespace) - let completeBlocks = (words.count / wordsPerBlock) * wordsPerBlock + let step = max(wordsPerBlock, laterWordsPerBlock) + let completeBlocks = words.count < step + ? (words.count / wordsPerBlock) * wordsPerBlock + : (words.count / step) * step guard completeBlocks > 0 else { return "" } let window = words[.. Date: Sat, 17 Oct 2026 21:55:06 +0200 Subject: [PATCH 08/19] Start generating at the keystroke instead of after the host's AX publish A typed character used to wait for the host to publish it over Accessibility (10-50 ms in TextEdit, longer in Chromium) before the request could start. A plain single-character keystroke now dispatches the generation at once against the text the host is about to publish, reusing the post-acceptance speculation path. The host-publish poll that follows validates it: an exact publish keeps the in-flight request, anything else (smart quotes, dead keys, IME, auto-pairing, an earlier key not yet in the snapshot) schedules the ordinary request and retires this one. Streamed partials of a speculative request are accepted by content, as apply already did for its final result: once the live field reads what was speculated they show, and before that they stay hidden, so a wrong bet costs one discarded generation and never a ghost at a wrong caret. The endpoint engine keeps its debounced path, and the existing speculative-prefetch kill switch disables this too. --- .../SuggestionCoordinator+Input.swift | 15 +++++- .../SuggestionCoordinator+Prediction.swift | 54 +++++++++++++++++-- .../SuggestionCoordinatorInputTests.swift | 34 ++++++++++++ 3 files changed, 97 insertions(+), 6 deletions(-) diff --git a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Input.swift b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Input.swift index 8ec01ab26..af8967f29 100644 --- a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Input.swift +++ b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Input.swift @@ -343,11 +343,23 @@ extension SuggestionCoordinator { // *before* the host app processes the keystroke, so a synchronous `refreshNow()` here // reads pre-keystroke text and feeds it into generation. The result is a suggestion // that looks like the typed character was ignored — see - // `schedulePredictionAfterHostPublishDelay` for the full rationale. + // `schedulePredictionAfterHostPublishDelay` for the full rationale. A plain typed + // character can still start generating now, against the text the host is about to + // publish; the poll then validates that bet instead of gating the request. + speculateOnTypedCharacter(event) schedulePredictionAfterHostPublishDelay() } } + /// Starts the keystroke-time speculative generation for a plain typed character. The base is + /// the last captured snapshot; when it does not yet include an earlier unpublished key, the + /// speculated text simply misses and the publish poll falls back to the ordinary request. + private func speculateOnTypedCharacter(_ event: CapturedInputEvent) { + guard event.kind == .textMutation, !event.characters.isEmpty, + let raw = focusModel.snapshot.context else { return } + dispatchSpeculativeTypingGeneration(rawContext: raw, typed: event.characters) + } + /// Maximum wall time we'll wait for the host app to publish post-keystroke AX before giving /// up and generating against whatever's there. Chosen empirically: long enough to cover /// Chrome's slower contenteditable publish on a busy page, short enough that the user can @@ -567,6 +579,7 @@ extension SuggestionCoordinator { clearDiagnostics: false ) if event.shouldSchedulePrediction { + speculateOnTypedCharacter(event) schedulePredictionAfterHostPublishDelay() } return false diff --git a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Prediction.swift b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Prediction.swift index d06a8dc98..bf34e3c30 100644 --- a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Prediction.swift +++ b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Prediction.swift @@ -254,11 +254,37 @@ extension SuggestionCoordinator { guard !userDefaults.bool(forKey: Self.speculativePrefetchDisabledDefaultsKey) else { return } guard !insertionChunk.isEmpty else { return } - let optimistic = SpeculativeAcceptanceContext.optimisticSnapshot( - after: rawContext, - inserting: insertionChunk + dispatchSpeculativeGeneration( + optimistic: SpeculativeAcceptanceContext.optimisticSnapshot(after: rawContext, inserting: insertionChunk), + message: "Started the post-acceptance generation against the expected post-insert text." ) + } + /// Starts generating for a plain typed character at keystroke time, against the text the host + /// is about to publish, instead of after the host-publish poll sees it (10-50 ms in native + /// hosts, longer in Chromium). The poll that follows validates exactly as it does for + /// post-acceptance speculation: a matching publish keeps this request, anything else (smart + /// quotes, dead keys, IME, auto-pairing, a key the host had not yet applied to the snapshot) + /// schedules the ordinary request and retires this one. Results and streamed partials are only + /// shown once the live text matches, so a wrong bet costs one discarded generation, never a + /// ghost at the wrong caret. + /// + /// Endpoint requests keep their debounced path: speculative traffic must not multiply requests + /// to a configured server. + func dispatchSpeculativeTypingGeneration(rawContext: FocusedInputSnapshot, typed characters: String) { + guard settingsSnapshot.selectedEngine != .openAICompatible, + !userDefaults.bool(forKey: Self.speculativePrefetchDisabledDefaultsKey), + characters.count == 1, let typed = characters.first, !typed.isNewline, + rawContext.selection.length == 0, !rawContext.isSecure else { return } + dispatchSpeculativeGeneration( + optimistic: SpeculativeAcceptanceContext.optimisticSnapshot(after: rawContext, inserting: characters), + message: "Started generating against the typed character before the host published it." + ) + } + + /// Shared body of the speculative paths: gate, build the request for `optimistic`, and run it + /// with no debounce while `pendingSpeculativeContext` lets the publish poll validate it. + private func dispatchSpeculativeGeneration(optimistic: FocusedInputSnapshot, message: String) { // Same pre-generation gates the ordinary cycle applies, minus their UI side effects: a // speculative request must not spend a decode on text the normal path would refuse (too // little text) or suppress (typo gate). The post-publish regeneration still runs the full @@ -308,7 +334,7 @@ extension SuggestionCoordinator { "speculative-generating", workID: workID, generation: context.generation, - message: "Started the post-acceptance generation against the expected post-insert text.", + message: message, prompt: requestBuildResult.promptPreview ) } @@ -578,7 +604,25 @@ extension SuggestionCoordinator { } return rebased } - return liveContext.generation == partial.generation ? partial : nil + if liveContext.generation == partial.generation { return partial } + // A speculative request was built against text the host had not published yet, so any + // read of the pre-publish snapshot moves the live generation past the partial's. Like + // `apply`, accept it by content instead: once the live field reads exactly what was + // speculated, the partial is current; before that it stays hidden. + if let expected = pendingSpeculativeContext, + expected.sessionIdentity == liveContext.sessionIdentity, + expected.contentSignature == liveContext.contentSignature, + expected.generation == partial.generation { + return SuggestionResult( + generation: liveContext.generation, + rawText: partial.rawText, + text: partial.text, + latency: partial.latency, + suppressionReason: partial.suppressionReason, + spacingIsExact: partial.spacingIsExact + ) + } + return nil } /// Runs the typo gate for the current word. Returns `true` when it handled the cycle by suppressing, diff --git a/CotabbyTests/App/Coordinators/Suggestion/SuggestionCoordinatorInputTests.swift b/CotabbyTests/App/Coordinators/Suggestion/SuggestionCoordinatorInputTests.swift index 43527390a..e911f5cff 100644 --- a/CotabbyTests/App/Coordinators/Suggestion/SuggestionCoordinatorInputTests.swift +++ b/CotabbyTests/App/Coordinators/Suggestion/SuggestionCoordinatorInputTests.swift @@ -325,6 +325,40 @@ final class SuggestionCoordinatorInputTests: SuggestionCoordinatorRigTestCase { XCTAssertEqual(rig.engine.prewarmedRequests.first?.clipboardContext, "Quarterly numbers attached") } + // MARK: - Keystroke-time speculation + + /// A plain typed character starts generating at once against the text the host is about to + /// publish; the host-publish poll only validates it. + func test_typedCharacterStartsSpeculativeGenerationBeforeHostPublish() async { + let rig = retained(makeCoordinatorRig( + snapshot: CotabbyTestFixtures.focusedInputSnapshot(precedingText: "Thanks for the quick rep") + )) + _ = rig.coordinator.handleInputEvent( + CapturedInputEvent(kind: .textMutation, keyCode: 37, characters: "l", flags: []) + ) + + XCTAssertEqual(rig.coordinator.pendingSpeculativeContext?.precedingText, "Thanks for the quick repl") + XCTAssertEqual(rig.coordinator.state, .generating) + await waitUntil("The speculative request never reached the engine") { + !rig.engine.requests.isEmpty + } + XCTAssertEqual(rig.engine.requests.first?.context.precedingText, "Thanks for the quick repl") + } + + /// Deletions, multi-character input, and the speculation kill switch keep the ordinary + /// publish-gated request. + func test_typedSpeculationSkipsDeletionsAndHonorsKillSwitch() { + let rig = retained(makeCoordinatorRig( + snapshot: CotabbyTestFixtures.focusedInputSnapshot(precedingText: "Thanks for the quick rep") + )) + _ = rig.coordinator.handleInputEvent(CapturedInputEvent(kind: .textMutation, keyCode: 51, characters: "", flags: [])) + XCTAssertNil(rig.coordinator.pendingSpeculativeContext, "Backspace has no character to speculate with") + + rig.coordinator.userDefaults.set(true, forKey: SuggestionCoordinator.speculativePrefetchDisabledDefaultsKey) + _ = rig.coordinator.handleInputEvent(CapturedInputEvent(kind: .textMutation, keyCode: 37, characters: "l", flags: [])) + XCTAssertNil(rig.coordinator.pendingSpeculativeContext, "The kill switch must skip typing speculation") + } + func test_handleSupportedSnapshot_withoutContextDisablesOutright() { let rig = retained(makeCoordinatorRig()) let bareSnapshot = FocusSnapshot( From 2978ef0f119d45dd1a945a41aa19e139050c1f89 Mon Sep 17 00:00:00 2001 From: Senad Date: Sat, 17 Oct 2026 21:55:06 +0200 Subject: [PATCH 09/19] Smooth the first-text debounce over recent requests The debounce read only the last request's time to first text, so one cache miss (a context refresh re-decoding a whole section) put the next keystroke behind the 55 ms full-latency tier. It now reads the median of the last three, and a known-but-slow first text pauses 20 ms instead of falling back to the full-latency tiers. --- .../SuggestionCoordinator+Prediction.swift | 2 +- ...uggestionCoordinator+TypingPrediction.swift | 2 +- .../Suggestion/SuggestionCoordinator.swift | 18 +++++++++++++++--- .../Suggestion/Request/DebouncePolicy.swift | 2 +- .../Request/DebouncePolicyTests.swift | 5 +++-- 5 files changed, 21 insertions(+), 8 deletions(-) diff --git a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Prediction.swift b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Prediction.swift index bf34e3c30..51a7bcc9a 100644 --- a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Prediction.swift +++ b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Prediction.swift @@ -33,7 +33,7 @@ extension SuggestionCoordinator { lastGenerationLatencyMilliseconds: lastLatencyByEngine[settingsSnapshot.selectedEngine], lastFirstTextLatencyMilliseconds: - lastFirstTextLatencyByEngine[settingsSnapshot.selectedEngine], + typicalFirstTextLatency(engine: settingsSnapshot.selectedEngine), fallback: settingsSnapshot.debounceMilliseconds, engine: settingsSnapshot.selectedEngine ) diff --git a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+TypingPrediction.swift b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+TypingPrediction.swift index ddfeaed36..8acdcfd41 100644 --- a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+TypingPrediction.swift +++ b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+TypingPrediction.swift @@ -69,7 +69,7 @@ extension SuggestionCoordinator { guard workController.isCurrent(workID) else { return } if firstTextRecordedWorkID != workID { firstTextRecordedWorkID = workID - lastFirstTextLatencyByEngine[settingsSnapshot.selectedEngine] = Int((result.latency * 1_000).rounded()) + recordFirstTextLatency(Int((result.latency * 1_000).rounded()), engine: settingsSnapshot.selectedEngine) } if var candidate = typingPrediction { guard !candidate.isFinal, result.generation == candidate.context.generation else { return } diff --git a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator.swift b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator.swift index 824653388..bebf08ef9 100644 --- a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator.swift +++ b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator.swift @@ -77,12 +77,24 @@ final class SuggestionCoordinator: ObservableObject { /// a slow HTTP backend retain an appropriate debounce without leaking that latency into the /// in-process engines. var lastLatencyByEngine: [SuggestionEngineKind: Int] = [:] - /// Latency to the first streamed text of the most recent request per backend: what starting a + /// Latency to the first streamed text of the most recent requests per backend: what starting a /// request costs once the KV prefix is warm. `DebouncePolicy` prefers it, because a superseded /// request stops at its next token while a debounce delays every keystroke. Recorded once per - /// work item (`firstTextRecordedWorkID`), only for the request that is still current. - var lastFirstTextLatencyByEngine: [SuggestionEngineKind: Int] = [:] + /// work item (`firstTextRecordedWorkID`), only for the request that is still current, and read + /// as the median of the last few so one cache miss (a context refresh re-decoding a whole + /// section) cannot put the next keystroke behind the slow tier. + var recentFirstTextLatenciesByEngine: [SuggestionEngineKind: [Int]] = [:] var firstTextRecordedWorkID: UInt64? + + func recordFirstTextLatency(_ milliseconds: Int, engine: SuggestionEngineKind) { + recentFirstTextLatenciesByEngine[engine, default: []].append(milliseconds) + recentFirstTextLatenciesByEngine[engine] = Array(recentFirstTextLatenciesByEngine[engine, default: []].suffix(3)) + } + + func typicalFirstTextLatency(engine: SuggestionEngineKind) -> Int? { + let recent = recentFirstTextLatenciesByEngine[engine, default: []].sorted() + return recent.isEmpty ? nil : recent[recent.count / 2] + } // Synchronous input/focus callbacks cannot directly `await`, so resets are represented as a // barrier task that the next generation must cross before it can ask the runtime for output. var cacheResetSequence: UInt64 = 0 diff --git a/Cotabby/Support/Suggestion/Request/DebouncePolicy.swift b/Cotabby/Support/Suggestion/Request/DebouncePolicy.swift index ddace1c8e..3087eeb6d 100644 --- a/Cotabby/Support/Suggestion/Request/DebouncePolicy.swift +++ b/Cotabby/Support/Suggestion/Request/DebouncePolicy.swift @@ -40,7 +40,7 @@ nonisolated enum DebouncePolicy { switch firstText { case ...60: return 0 case ...120: return 10 - default: break + default: return 20 } } diff --git a/CotabbyTests/Support/Suggestion/Request/DebouncePolicyTests.swift b/CotabbyTests/Support/Suggestion/Request/DebouncePolicyTests.swift index 609e2c1d3..9a04705aa 100644 --- a/CotabbyTests/Support/Suggestion/Request/DebouncePolicyTests.swift +++ b/CotabbyTests/Support/Suggestion/Request/DebouncePolicyTests.swift @@ -41,12 +41,13 @@ final class DebouncePolicyTests: XCTestCase { } /// A fast time to first text removes the debounce for in-process engines even when the full - /// suggestion took long; a slow first text falls back to the full-latency tiers. + /// suggestion took long; a slow first text keeps a short 20 ms pause instead of the 55 ms + /// full-latency tier, since cancelling a superseded local request costs at most one token. func test_localEngines_firstTextLatencyTiers() { let cases: [(firstText: Int?, full: Int?, expected: Int)] = [ (1, 900, 0), (60, 900, 0), (61, 900, 10), (120, 900, 10), - (121, 900, 55), (121, 100, 25), + (121, 900, 20), (121, 100, 20), (0, 45, 15), (nil, 45, 15) ] for engine in [SuggestionEngineKind.llamaOpenSource, .appleIntelligence] { From 9e8468c8c2adae5b8170bdbc20100a45faf6ff53 Mon Sep 17 00:00:00 2001 From: Senad Date: Sat, 17 Oct 2026 22:09:41 +0200 Subject: [PATCH 10/19] Speculate from the typed-through text, not a lagging snapshot Typing through a visible suggestion advances the session without a new AX capture, so the snapshot a mismatching key speculated from could still miss the letters just typed through; the speculation then missed validation and fell back to the publish-gated request (4 of 8 in a live TextEdit run). The mismatch path now speculates from the session's base text plus its consumed characters whenever that extends the snapshot. --- .../Suggestion/SuggestionCoordinator+Input.swift | 15 ++++++++++----- .../SuggestionCoordinator+Prediction.swift | 13 +++++++++++-- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Input.swift b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Input.swift index af8967f29..1b94d11e3 100644 --- a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Input.swift +++ b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Input.swift @@ -352,12 +352,17 @@ extension SuggestionCoordinator { } /// Starts the keystroke-time speculative generation for a plain typed character. The base is - /// the last captured snapshot; when it does not yet include an earlier unpublished key, the - /// speculated text simply misses and the publish poll falls back to the ordinary request. - private func speculateOnTypedCharacter(_ event: CapturedInputEvent) { + /// the last captured snapshot, extended by the letters a visible suggestion was typed through + /// (`session`), which the snapshot may not include yet. When it still misses an unpublished key, + /// the speculated text simply misses and the publish poll falls back to the ordinary request. + private func speculateOnTypedCharacter(_ event: CapturedInputEvent, after session: ActiveSuggestionSession? = nil) { guard event.kind == .textMutation, !event.characters.isEmpty, let raw = focusModel.snapshot.context else { return } - dispatchSpeculativeTypingGeneration(rawContext: raw, typed: event.characters) + let typedThrough = session.flatMap { session -> String? in + guard session.baseContext.elementIdentifier == raw.elementIdentifier else { return nil } + return session.baseContext.precedingText + String(session.fullText.prefix(session.consumedCharacterCount)) + } + dispatchSpeculativeTypingGeneration(rawContext: raw, typed: event.characters, knownPrecedingText: typedThrough) } /// Maximum wall time we'll wait for the host app to publish post-keystroke AX before giving @@ -579,7 +584,7 @@ extension SuggestionCoordinator { clearDiagnostics: false ) if event.shouldSchedulePrediction { - speculateOnTypedCharacter(event) + speculateOnTypedCharacter(event, after: session) schedulePredictionAfterHostPublishDelay() } return false diff --git a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Prediction.swift b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Prediction.swift index 51a7bcc9a..32c6a25dc 100644 --- a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Prediction.swift +++ b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Prediction.swift @@ -271,13 +271,22 @@ extension SuggestionCoordinator { /// /// Endpoint requests keep their debounced path: speculative traffic must not multiply requests /// to a configured server. - func dispatchSpeculativeTypingGeneration(rawContext: FocusedInputSnapshot, typed characters: String) { + /// + /// `knownPrecedingText` is the text before the caret as Cotabby already knows it when the + /// captured snapshot lags: typing through a visible suggestion advances the session without a + /// new AX capture, so the snapshot can still miss those letters. It is used only when it + /// extends the snapshot's own text. + func dispatchSpeculativeTypingGeneration( + rawContext: FocusedInputSnapshot, typed characters: String, knownPrecedingText: String? = nil + ) { guard settingsSnapshot.selectedEngine != .openAICompatible, !userDefaults.bool(forKey: Self.speculativePrefetchDisabledDefaultsKey), characters.count == 1, let typed = characters.first, !typed.isNewline, rawContext.selection.length == 0, !rawContext.isSecure else { return } + let base = knownPrecedingText.flatMap { $0.hasPrefix(rawContext.precedingText) ? $0 : nil } + ?? rawContext.precedingText dispatchSpeculativeGeneration( - optimistic: SpeculativeAcceptanceContext.optimisticSnapshot(after: rawContext, inserting: characters), + optimistic: SpeculativeAcceptanceContext.optimisticSnapshot(after: rawContext, precedingText: base + characters), message: "Started generating against the typed character before the host published it." ) } From b4c358a9d7bd0a8f46e37668d6994fdede2de6a8 Mon Sep 17 00:00:00 2001 From: Senad Date: Sat, 17 Oct 2026 22:38:55 +0200 Subject: [PATCH 11/19] Speculate the character Shift produces, and log speculation misses In a live run most typing-speculation misses fell on capital letters and on "?": the key event's Unicode string did not carry Shift, so the speculation inserted "s" where the host inserted "S". Letters now follow Shift XOR Caps Lock; a shifted symbol depends on the keyboard layout, so it keeps the publish-gated request instead of guessing. A miss now logs (debug) how the publish differed from the speculated text: lengths, selections, and the first differing offset, never the text itself. --- .../SuggestionCoordinator+Input.swift | 41 ++++++++++++++++++- .../SuggestionCoordinatorInputTests.swift | 17 ++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Input.swift b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Input.swift index 1b94d11e3..21a2a4f84 100644 --- a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Input.swift +++ b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Input.swift @@ -1,3 +1,4 @@ +import CoreGraphics import Foundation import Logging @@ -356,13 +357,25 @@ extension SuggestionCoordinator { /// (`session`), which the snapshot may not include yet. When it still misses an unpublished key, /// the speculated text simply misses and the publish poll falls back to the ordinary request. private func speculateOnTypedCharacter(_ event: CapturedInputEvent, after session: ActiveSuggestionSession? = nil) { - guard event.kind == .textMutation, !event.characters.isEmpty, + guard event.kind == .textMutation, let typed = Self.speculatedCharacters(for: event), let raw = focusModel.snapshot.context else { return } let typedThrough = session.flatMap { session -> String? in guard session.baseContext.elementIdentifier == raw.elementIdentifier else { return nil } return session.baseContext.precedingText + String(session.fullText.prefix(session.consumedCharacterCount)) } - dispatchSpeculativeTypingGeneration(rawContext: raw, typed: event.characters, knownPrecedingText: typedThrough) + dispatchSpeculativeTypingGeneration(rawContext: raw, typed: typed, knownPrecedingText: typedThrough) + } + + /// The character the host will most likely insert for `event`, or nil when it cannot be told + /// from the event alone. A key event's Unicode string does not always carry Shift (synthetic + /// events report "s" for a shifted S), so a letter follows Shift XOR Caps Lock; uppercasing an + /// already-capital letter changes nothing. A shifted symbol depends on the keyboard layout + /// ("/" becomes "?" on US layouts), so it is left to the ordinary publish-gated request. + static func speculatedCharacters(for event: CapturedInputEvent) -> String? { + guard event.characters.count == 1, let character = event.characters.first else { return nil } + let shift = event.flags.contains(.maskShift) + guard character.isLetter else { return shift ? nil : event.characters } + return shift != event.flags.contains(.maskAlphaShift) ? event.characters.uppercased() : event.characters } /// Maximum wall time we'll wait for the host app to publish post-keystroke AX before giving @@ -437,6 +450,27 @@ extension SuggestionCoordinator { } } + /// Records why a publish did not match the speculated text, without logging either text: the + /// lengths, selections, and the first differing UTF-16 offset are enough to tell a lagging base + /// snapshot from a host transformation (smart punctuation, auto-pairing, a shifted key). + private func logSpeculationMiss(expected: FocusedInputContext, published: FocusedInputSnapshot) { + let lhs = Array(expected.precedingText.utf16), rhs = Array(published.precedingText.utf16) + let shared = zip(lhs, rhs).prefix { $0 == $1 }.count + CotabbyLogger.suggestion.debug( + "Speculation missed the published text", + metadata: [ + "stage": .string("speculation-missed"), + "same_session": .stringConvertible(expected.sessionIdentity == published.sessionIdentity), + "expected_utf16": .stringConvertible(lhs.count), + "published_utf16": .stringConvertible(rhs.count), + "first_difference": .stringConvertible(shared), + "expected_selection": .stringConvertible(expected.selection.location), + "published_selection": .stringConvertible(published.selection.location), + "trailing_matches": .stringConvertible(expected.trailingText == published.trailingText) + ] + ) + } + /// The AX state captured at keystroke time, against which the host-publish poll detects "the /// host has processed the key", plus the uptime anchor for folding the wait into the debounce. private struct HostPublishBaseline { @@ -506,6 +540,9 @@ extension SuggestionCoordinator { ) return } + if let expected = pendingSpeculativeContext, let published = currentContext { + logSpeculationMiss(expected: expected, published: published) + } pendingSpeculativeContext = nil schedulePrediction( consumedDelayMilliseconds: Self.elapsedMilliseconds(since: baseline.keystrokeUptimeNanoseconds) diff --git a/CotabbyTests/App/Coordinators/Suggestion/SuggestionCoordinatorInputTests.swift b/CotabbyTests/App/Coordinators/Suggestion/SuggestionCoordinatorInputTests.swift index e911f5cff..3c2a8e514 100644 --- a/CotabbyTests/App/Coordinators/Suggestion/SuggestionCoordinatorInputTests.swift +++ b/CotabbyTests/App/Coordinators/Suggestion/SuggestionCoordinatorInputTests.swift @@ -1,3 +1,4 @@ +import CoreGraphics import Foundation import XCTest @testable import Cotabby @@ -345,6 +346,22 @@ final class SuggestionCoordinatorInputTests: SuggestionCoordinatorRigTestCase { XCTAssertEqual(rig.engine.requests.first?.context.precedingText, "Thanks for the quick repl") } + /// The speculated character follows Shift XOR Caps Lock for letters, and shifted symbols are + /// not guessed because their character depends on the keyboard layout. + func test_speculatedCharactersApplyShiftAndSkipShiftedSymbols() { + func event(_ characters: String, _ flags: CGEventFlags = []) -> CapturedInputEvent { + CapturedInputEvent(kind: .textMutation, keyCode: 1, characters: characters, flags: flags) + } + XCTAssertEqual(SuggestionCoordinator.speculatedCharacters(for: event("s")), "s") + XCTAssertEqual(SuggestionCoordinator.speculatedCharacters(for: event("s", .maskShift)), "S") + XCTAssertEqual(SuggestionCoordinator.speculatedCharacters(for: event("S", .maskShift)), "S") + XCTAssertEqual(SuggestionCoordinator.speculatedCharacters(for: event("s", .maskAlphaShift)), "S") + XCTAssertEqual(SuggestionCoordinator.speculatedCharacters(for: event("s", [.maskShift, .maskAlphaShift])), "s") + XCTAssertEqual(SuggestionCoordinator.speculatedCharacters(for: event(",")), ",") + XCTAssertNil(SuggestionCoordinator.speculatedCharacters(for: event("/", .maskShift))) + XCTAssertNil(SuggestionCoordinator.speculatedCharacters(for: event("ab"))) + } + /// Deletions, multi-character input, and the speculation kill switch keep the ordinary /// publish-gated request. func test_typedSpeculationSkipsDeletionsAndHonorsKillSwitch() { From f7d0bfe3b81e36ff7c16f1999882f63392ec4c69 Mon Sep 17 00:00:00 2001 From: Senad Date: Sat, 17 Oct 2026 22:38:55 +0200 Subject: [PATCH 12/19] Pause the accept tap between suggestions instead of destroying it While typing, a ghost hides and reappears on nearly every keystroke, and each hide destroyed the active accept-key tap 50 ms later. The next appearance then re-created it, a synchronous WindowServer round trip measured at ~20-30 ms on the main thread right before the ghost was presented. The tap is now disabled when no suggestion wants it (it intercepts nothing, so typing stays off the synchronous path) and only re-enabled for the next suggestion; it is released after 10 s idle. A system-initiated disable is still undone, an intentional one is not. --- Cotabby/Services/Input/InputMonitor.swift | 53 +++++++++++++++++++---- 1 file changed, 44 insertions(+), 9 deletions(-) diff --git a/Cotabby/Services/Input/InputMonitor.swift b/Cotabby/Services/Input/InputMonitor.swift index 60956f9ca..62409a0da 100644 --- a/Cotabby/Services/Input/InputMonitor.swift +++ b/Cotabby/Services/Input/InputMonitor.swift @@ -107,6 +107,10 @@ final class InputMonitor { private var acceptTap: CFMachPort? private var acceptRunLoopSource: CFRunLoopSource? + /// Whether `acceptTap` is currently intercepting. Between suggestions the tap is disabled + /// rather than destroyed (see `updateAcceptTapState`), so this distinguishes an intentional + /// pause from the system disabling it on timeout, which the callback re-enables. + private var acceptTapEnabled = false /// Dedicated consuming tap for the global-toggle hotkey. Lives independently of the accept tap /// because it must fire even when no suggestion is visible — and even when Cotabby is globally @@ -195,6 +199,12 @@ final class InputMonitor { /// one short hop lets the synthetic event finish delivery first. This is the invariant PR #385 /// ("Fix last tab issue") protected before the two-tap ownership rewrite dropped it. private static let acceptTapTeardownDelaySeconds: TimeInterval = 0.05 + /// How long a disabled accept tap is kept for its next suggestion before it is released. + private static let acceptTapReleaseDelaySeconds: TimeInterval = 10 + + private var acceptTapWanted: Bool { + permissionProvider() && (suggestionInterceptionActive || captureInterceptionActive) + } /// Suggestion-overlay reason for the active tap. The coordinator calls this when a suggestion /// becomes visible or hidden, so Cotabby only enters the synchronous event path while there is @@ -230,12 +240,19 @@ final class InputMonitor { // drain before the tap is removed (see `acceptTapTeardownDelaySeconds`). Re-check both // reasons at fire time so a suggestion or an emoji capture that re-armed the tap during the // delay keeps it installed. + // + // While typing, a ghost hides and reappears on nearly every keystroke. Destroying the tap + // each time made the next appearance re-create it, a synchronous WindowServer round trip + // (~20-30 ms measured) on the main thread right before the ghost could be presented. A + // disabled tap intercepts nothing, so typing stays off this synchronous path exactly as + // before; re-enabling it is cheap. It is released only after a longer idle period. DispatchQueue.main.asyncAfter(deadline: .now() + Self.acceptTapTeardownDelaySeconds) { [weak self] in - guard let self else { return } - let stillWanted = self.permissionProvider() - && (self.suggestionInterceptionActive || self.captureInterceptionActive) - guard !stillWanted else { return } - self.destroyAcceptTap() + guard let self, !self.acceptTapWanted else { return } + self.disableAcceptTap() + DispatchQueue.main.asyncAfter(deadline: .now() + Self.acceptTapReleaseDelaySeconds) { [weak self] in + guard let self, !self.acceptTapWanted, !self.acceptTapEnabled else { return } + self.destroyAcceptTap() + } } } } @@ -285,8 +302,13 @@ final class InputMonitor { private func installAcceptTapIfNeeded() { // Ownership of the accept key is decided by `updateAcceptTapState`, not here, so this method - // only guarantees the tap exists. - guard acceptTap == nil else { + // only guarantees the tap exists and is intercepting. + if let acceptTap { + if !acceptTapEnabled { + CGEvent.tapEnable(tap: acceptTap, enable: true) + acceptTapEnabled = true + CotabbyLogger.app.debug("CGEvent accept tap re-enabled") + } return } @@ -327,6 +349,17 @@ final class InputMonitor { } CGEvent.tapEnable(tap: tap, enable: true) + acceptTapEnabled = true + } + + /// Stops the accept tap from intercepting without releasing it, so the next visible + /// suggestion only has to re-enable it. + private func disableAcceptTap() { + isAcceptTapOwningAcceptKeys = false + guard let acceptTap, acceptTapEnabled else { return } + CGEvent.tapEnable(tap: acceptTap, enable: false) + acceptTapEnabled = false + CotabbyLogger.app.debug("CGEvent accept tap disabled") } private func installToggleTapIfNeeded() { @@ -400,6 +433,7 @@ final class InputMonitor { CFMachPortInvalidate(tap) } acceptTap = nil + acceptTapEnabled = false CotabbyLogger.app.info("CGEvent accept tap removed") } @@ -547,8 +581,9 @@ final class InputMonitor { func handleAcceptTap(type: CGEventType, event: CGEvent) -> Unmanaged? { switch type { case .tapDisabledByTimeout, .tapDisabledByUserInput: - CotabbyLogger.app.warning("Accept tap was disabled by system, re-enabling") - if let acceptTap { + // Only undo a system-initiated disable; a tap Cotabby paused between suggestions stays off. + if let acceptTap, acceptTapEnabled { + CotabbyLogger.app.warning("Accept tap was disabled by system, re-enabling") CGEvent.tapEnable(tap: acceptTap, enable: true) } return Unmanaged.passUnretained(event) From 223a66c720332a7256b70310d185f76ffd729fd2 Mon Sep 17 00:00:00 2001 From: Senad Date: Sat, 17 Oct 2026 22:38:55 +0200 Subject: [PATCH 13/19] Keep shown history examples while they stay highly ranked Each change of the typing-history examples re-decodes that prompt section and everything after it (~100 ms on Gemma 4 E4B); in live runs these were the largest delays left in the p90 tail, at every query block. A block boundary often only reshuffles near-equal matches, so the examples already in the prompt now stay while the new ranking still puts them in its top four; a passage that drops out, or now echoes the draft, is replaced at once. The query and ranking are unchanged, so relevance is too. --- .../Services/History/TypingHistoryStore.swift | 25 +++++++++++++- .../History/TypingHistoryStoreTests.swift | 34 +++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/Cotabby/Services/History/TypingHistoryStore.swift b/Cotabby/Services/History/TypingHistoryStore.swift index e16a5e11a..476a0cc62 100644 --- a/Cotabby/Services/History/TypingHistoryStore.swift +++ b/Cotabby/Services/History/TypingHistoryStore.swift @@ -73,6 +73,9 @@ final class TypingHistoryStore: ObservableObject, SuggestionHistoryProviding { private static let maximumRecentRecordings = 64 /// Ranked example candidates for the current query block (see `historyExamples`). private var exampleCache: (key: String, candidates: [TypingHistoryIndex.Candidate])? + /// The examples most recently handed to a prompt, per field, so a new ranking that still rates + /// them highly keeps the prompt (and its KV cache) unchanged. Cleared with `exampleCache`. + private var shownExamples: (fieldKey: UInt64, examples: [String])? /// The field being typed in right now. Its raw text is kept here and only scrubbed and copied /// into `records` when saving or when focus moves on, so recording costs a string comparison @@ -128,6 +131,7 @@ final class TypingHistoryStore: ObservableObject, SuggestionHistoryProviding { preferences.isUsingHistory = enabled userDefaults.set(enabled, forKey: DefaultsKey.isUsingHistory) exampleCache = nil + shownExamples = nil } func setRecording(_ enabled: Bool) { @@ -251,6 +255,7 @@ final class TypingHistoryStore: ObservableObject, SuggestionHistoryProviding { self.index = built.0 self.phrases = built.1 self.exampleCache = nil + self.shownExamples = nil } } @@ -524,6 +529,7 @@ final class TypingHistoryStore: ObservableObject, SuggestionHistoryProviding { index = nil phrases = nil exampleCache = nil + shownExamples = nil rebuildGeneration += 1 lastImportMessage = nil deletionError = nil @@ -553,9 +559,26 @@ final class TypingHistoryStore: ObservableObject, SuggestionHistoryProviding { } // The ranking holds for a whole block of words, but the field keeps growing inside the // block, so the check against echoing the user's own draft runs on the live text each time. - return TypingHistoryIndex.examples(from: candidates, currentFieldText: context.precedingText) + let fresh = TypingHistoryIndex.examples(from: candidates, currentFieldText: context.precedingText) + // Keep the examples already in the prompt while the new ranking still rates them among its + // best few. Each change of the examples re-decodes that prompt section and everything after + // it (~100 ms on a 4B-class model), and a block boundary often only reshuffles near-equal + // matches; a passage that dropped out of the top candidates, or now echoes the draft, is + // still replaced at once. + let fieldKey = context.focusedInputIdentityKey + if let shown = shownExamples, shown.fieldKey == fieldKey, !shown.examples.isEmpty { + let stillRanked = TypingHistoryIndex.examples( + from: candidates, currentFieldText: context.precedingText, limit: Self.stickyExampleRankLimit + ) + if shown.examples.allSatisfy(stillRanked.contains) { return shown.examples } + } + shownExamples = (fieldKey, fresh) + return fresh } + /// How far down the new ranking an example already in the prompt may fall and still be kept. + private static let stickyExampleRankLimit = 4 + func phraseContinuation(for request: SuggestionRequest, engine: SuggestionEngineKind) -> String? { guard preferences.isUsingHistory, engine != .openAICompatible, let phrases else { return nil } // Text after the caret on the same line would be pushed along by an inserted phrase; leave diff --git a/CotabbyTests/Services/History/TypingHistoryStoreTests.swift b/CotabbyTests/Services/History/TypingHistoryStoreTests.swift index b85c2f2c2..59388f7ce 100644 --- a/CotabbyTests/Services/History/TypingHistoryStoreTests.swift +++ b/CotabbyTests/Services/History/TypingHistoryStoreTests.swift @@ -611,6 +611,40 @@ final class TypingHistoryStoreTests: XCTestCase { "The field now contains that writing; showing it would make the model echo the draft") } + /// Examples already in the prompt survive a new query block while the new ranking still rates + /// them among its top few, so the prompt (and its KV cache) does not change on every block. A + /// fresh store answers the same later query with a different top example, which proves the + /// first store kept its examples on purpose rather than by coincidence. + func test_shownExamplesStayWhileTheyRemainHighlyRanked() async throws { + let rows: [[String: Any]] = [ + ["appBundleIdentifier": "com.example.TestApp", "textUpToCursor": "The Imperum connector plan review moved to Monday with the steering group."], + ["appBundleIdentifier": "com.example.TestApp", "textUpToCursor": "Connector plan notes are ready for the Imperum steering group on Monday."], + ["appBundleIdentifier": "com.example.TestApp", "textUpToCursor": "Detections rollout for the SOC analysts starts after the dashboard migration."] + ] + // Exactly 8 words, then exactly 16: each is one whole query block. The second block's new + // words all belong to the third passage, so a fresh ranking puts it first. + let opening = "Notes for Monday about the Imperum connector plan " + let later = opening + "SOC analysts detections rollout after the dashboard migration " + + let sticky = makeStore() + sticky.setUsingHistory(true) + await sticky.importCotypistExport(from: try writeExport(rows)) + let first = CotabbyTestFixtures.focusedInputContext(precedingText: opening) + await waitUntil { !sticky.historyExamples(for: first, engine: .llamaOpenSource).isEmpty } + let shown = sticky.historyExamples(for: first, engine: .llamaOpenSource) + XCTAssertFalse(shown.contains { $0.contains("Detections rollout") }) + + let laterContext = CotabbyTestFixtures.focusedInputContext(precedingText: later) + XCTAssertEqual(sticky.historyExamples(for: laterContext, engine: .llamaOpenSource), shown) + + let fresh = makeStore() + fresh.setUsingHistory(true) + await fresh.importCotypistExport(from: try writeExport(rows)) + await waitUntil { !fresh.historyExamples(for: laterContext, engine: .llamaOpenSource).isEmpty } + XCTAssertNotEqual(fresh.historyExamples(for: laterContext, engine: .llamaOpenSource), shown, + "Without shown examples the later query must rank differently, or this test proves nothing") + } + func test_anExampleHiddenByTheFieldsLatestWordsComesBackWhenTheWordingMovesOn() async throws { let store = makeStore() store.setUsingHistory(true) From 5f74c244225fdb3dfbac3047cfcf0531ab9dd59d Mon Sep 17 00:00:00 2001 From: Senad Date: Sat, 17 Oct 2026 22:50:12 +0200 Subject: [PATCH 14/19] Replace only the history example that dropped out, in its own slot Live runs still changed the examples at most query blocks: with many near-equal candidates, one of two shown examples would slip just past the top four, and the replacement set also reordered the survivor, so the prompt diverged at the first example and re-decoded the whole section. Shown examples now stay while they are anywhere in the ranked candidates, and a dropped one is replaced in its own position. --- .../Services/History/TypingHistoryStore.swift | 28 ++++++++++++++++--- .../History/TypingHistoryStoreTests.swift | 10 +++++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/Cotabby/Services/History/TypingHistoryStore.swift b/Cotabby/Services/History/TypingHistoryStore.swift index 476a0cc62..35f51b4d9 100644 --- a/Cotabby/Services/History/TypingHistoryStore.swift +++ b/Cotabby/Services/History/TypingHistoryStore.swift @@ -566,18 +566,38 @@ final class TypingHistoryStore: ObservableObject, SuggestionHistoryProviding { // matches; a passage that dropped out of the top candidates, or now echoes the draft, is // still replaced at once. let fieldKey = context.focusedInputIdentityKey + var examples = fresh if let shown = shownExamples, shown.fieldKey == fieldKey, !shown.examples.isEmpty { let stillRanked = TypingHistoryIndex.examples( from: candidates, currentFieldText: context.precedingText, limit: Self.stickyExampleRankLimit ) - if shown.examples.allSatisfy(stillRanked.contains) { return shown.examples } + examples = Self.stableExamples(shown: shown.examples, ranked: stillRanked, fresh: fresh) } - shownExamples = (fieldKey, fresh) - return fresh + shownExamples = (fieldKey, examples) + return examples } /// How far down the new ranking an example already in the prompt may fall and still be kept. - private static let stickyExampleRankLimit = 4 + private static let stickyExampleRankLimit = 6 + + /// Keeps every shown example that is still ranked, in its position, and fills only the slots of + /// examples that dropped out with the best fresh ones. The prompt then diverges at the first + /// replaced example instead of at the start of the section, so less of it is re-decoded. + static func stableExamples(shown: [String], ranked: [String], fresh: [String]) -> [String] { + var replacements = fresh.filter { !shown.contains($0) } + var result: [String] = [] + for example in shown { + if ranked.contains(example) { + result.append(example) + } else if !replacements.isEmpty { + result.append(replacements.removeFirst()) + } + } + while result.count < fresh.count, !replacements.isEmpty { + result.append(replacements.removeFirst()) + } + return result + } func phraseContinuation(for request: SuggestionRequest, engine: SuggestionEngineKind) -> String? { guard preferences.isUsingHistory, engine != .openAICompatible, let phrases else { return nil } diff --git a/CotabbyTests/Services/History/TypingHistoryStoreTests.swift b/CotabbyTests/Services/History/TypingHistoryStoreTests.swift index 59388f7ce..6fc17660e 100644 --- a/CotabbyTests/Services/History/TypingHistoryStoreTests.swift +++ b/CotabbyTests/Services/History/TypingHistoryStoreTests.swift @@ -645,6 +645,16 @@ final class TypingHistoryStoreTests: XCTestCase { "Without shown examples the later query must rank differently, or this test proves nothing") } + /// A dropped example is replaced in its own slot; survivors keep their positions, so the prompt + /// diverges as late as possible. + func test_stableExamplesReplaceOnlyDroppedSlots() { + XCTAssertEqual(TypingHistoryStore.stableExamples(shown: ["a", "b"], ranked: ["b", "a", "c"], fresh: ["b", "c"]), ["a", "b"]) + XCTAssertEqual(TypingHistoryStore.stableExamples(shown: ["a", "b"], ranked: ["b", "c"], fresh: ["c", "b"]), ["c", "b"]) + XCTAssertEqual(TypingHistoryStore.stableExamples(shown: ["a", "b"], ranked: ["a", "c"], fresh: ["c", "a"]), ["a", "c"]) + XCTAssertEqual(TypingHistoryStore.stableExamples(shown: ["a"], ranked: ["a", "c"], fresh: ["c", "a"]), ["a", "c"]) + XCTAssertEqual(TypingHistoryStore.stableExamples(shown: ["a", "b"], ranked: [], fresh: []), []) + } + func test_anExampleHiddenByTheFieldsLatestWordsComesBackWhenTheWordingMovesOn() async throws { let store = makeStore() store.setUsingHistory(true) From f040acfe14ba333fc4d30004425202108dc13a8c Mon Sep 17 00:00:00 2001 From: Senad Date: Sat, 17 Oct 2026 23:11:13 +0200 Subject: [PATCH 15/19] Show a streamed word as soon as it is a known word The streaming seam guard waited for the token after a word to prove the word had ended, which cost a full decode step (~15-20 ms) before any ghost text appeared. A streamed word that is already a known dictionary word of four or more letters now shows immediately, both for a word completing the caret's fragment and for a new leading word. If it then grows (build -> building) the ghost only extends. Fragments, unknown words, short words that often grow (the, and), and words ending in a connector (does', don') still wait, and a prefix is never suppressed as a typo. This relaxes upstream's rule that no unfinished word reaches the spell checker; the coordinator caches assessments per word, so a stream adds at most one lookup per new word. Typing-session eval (Gemma 4 E4B, streaming): first useful text 41-59 ms on most traces, down from 50-69 ms; withdrawn suggestions stay at 0 and non-useful visible revisions at 102 (103 before). --- .../Output/CompletionSeamGuard.swift | 17 +++++- .../Output/CompletionSeamGuardTests.swift | 56 +++++++++++++------ 2 files changed, 53 insertions(+), 20 deletions(-) diff --git a/Cotabby/Support/Suggestion/Output/CompletionSeamGuard.swift b/Cotabby/Support/Suggestion/Output/CompletionSeamGuard.swift index 06c13a0a1..4df0a5708 100644 --- a/Cotabby/Support/Suggestion/Output/CompletionSeamGuard.swift +++ b/Cotabby/Support/Suggestion/Output/CompletionSeamGuard.swift @@ -143,7 +143,14 @@ nonisolated enum CompletionSeamGuard { } if !isFinal { switch leadingWordProbe(precedingText: precedingText, completion: completion) { - case .incomplete, .candidate(_, isComplete: false): return .wait + case .incomplete: return .wait + case let .candidate(word, isComplete: false): + // Same early show as a mid-word ending: a first word that is already a known + // dictionary word need not wait a decode step to prove it ended. A trailing + // apostrophe or hyphen (`does'`) may still join more letters, so it keeps waiting, + // and only a known word is ever shown early; nothing is suppressed on a prefix. + guard !CaretWordContext.isConnector(completion.last ?? " "), + word.count >= minimumSeamWordLength, spellingAssessment(word) == .known else { return .wait } case .notApplicable, .candidate: break } } @@ -163,7 +170,13 @@ nonisolated enum CompletionSeamGuard { let following = completion.dropFirst(ending.count).first guard following?.isNumber != true, following != "_" else { return nil } if !isFinal && (ending.count == completion.count || ending.last.map(CaretWordContext.isConnector) == true) { - return .wait + // The word may still be arriving. Waiting for the next token to prove it ended costs a + // whole decode step (~15-20 ms) before anything is visible, so a stream whose word is + // already a known dictionary word shows now; a longer word it turns into later + // (`Monday` -> `Mondays`) only extends the ghost. Fragments and connector endings wait. + guard ending.count == completion.count, !CaretWordContext.isConnector(ending.last ?? " "), + (prefix + ending).count >= minimumSeamWordLength, + spellingAssessment(prefix + ending) == .known else { return .wait } } let word = prefix + ending let assessment = spellingAssessment(word) diff --git a/CotabbyTests/Support/Suggestion/Output/CompletionSeamGuardTests.swift b/CotabbyTests/Support/Suggestion/Output/CompletionSeamGuardTests.swift index 3f83abaa3..dee4b22bb 100644 --- a/CotabbyTests/Support/Suggestion/Output/CompletionSeamGuardTests.swift +++ b/CotabbyTests/Support/Suggestion/Output/CompletionSeamGuardTests.swift @@ -441,15 +441,31 @@ final class CompletionSeamGuardTests: XCTestCase { // MARK: - Streamed leading words /// Streaming buffers a lowercase prefix because checking it before its boundary is unreliable. + /// An unfinished leading word that is not (yet) a known word keeps waiting: a prefix is never + /// suppressed as a typo, it is only withheld until its boundary arrives. func testStreamedLeadingWordWaitsUntilItsBoundaryArrives() { XCTAssertEqual( CompletionSeamGuard.streamedLeadingWordVerdict( precedingText: "Je veux ", completion: "ecrir", - spellingAssessment: { _ in - XCTFail("an incomplete streamed word must not reach the spell checker") - return .known - } + spellingAssessment: { $0 == "ecrir" ? .correctableTypo : .known } + ), + .wait + ) + } + + /// A streamed leading word that is already a known word of four or more letters shows without + /// waiting a decode step for its boundary; short words, which often grow, still wait. + func testStreamedKnownLeadingWordShowsBeforeItsBoundary() { + XCTAssertEqual( + CompletionSeamGuard.streamedLeadingWordVerdict( + precedingText: "Please send the ", completion: "report", spellingAssessment: { _ in .known } + ), + .allow + ) + XCTAssertEqual( + CompletionSeamGuard.streamedLeadingWordVerdict( + precedingText: "Please send ", completion: "the", spellingAssessment: { _ in .known } ), .wait ) @@ -520,7 +536,7 @@ final class CompletionSeamGuardTests: XCTestCase { func testBadJoinedWordNeverAppearsDuringStreaming() { XCTAssertEqual(CompletionSeamGuard.presentation(precedingText: "so gre", completion: "atf", isFinal: false, - spellingAssessment: { _ in XCTFail("Wait for the full word"); return .known }), .wait) + spellingAssessment: { _ in .correctableTypo }), .wait) for isFinal in [false, true] { XCTAssertEqual(CompletionSeamGuard.presentation(precedingText: "so gre", completion: "atful for this", isFinal: isFinal, spellingAssessment: { _ in .correctableTypo }), .suppress(.seamMisspelling(word: "greatful"))) @@ -549,23 +565,27 @@ final class CompletionSeamGuardTests: XCTestCase { } } - /// A dictionary entry is not a word boundary: `build` could still grow into `building`. - /// Buffer until the stream proves the seam word complete, then expose its phrase in one step. - func testKnownJoinedWordStillWaitsForItsStreamedBoundary() { - for completion in ["u", "uil", "uild", "uilding"] { + /// Fragments wait for the stream to prove the seam word complete. A joined word that is + /// already a known word of four or more letters shows a decode step early; if it then grows + /// (`build` -> `building`) the ghost only extends, and nothing is suppressed on a prefix. + func testJoinedWordShowsOnceItIsAKnownWord() { + let dictionary = knowing(["build", "building"]) + for fragment in ["u", "uil"] { XCTAssertEqual( CompletionSeamGuard.presentation( - precedingText: "I would like to b", - completion: completion, - isFinal: false, - spellingAssessment: { _ in - XCTFail("An unfinished joined word must not reach spelling assessment") - return .known - } + precedingText: "I would like to b", completion: fragment, isFinal: false, spellingAssessment: dictionary ), .wait ) } + for word in ["uild", "uilding"] { + XCTAssertEqual( + CompletionSeamGuard.presentation( + precedingText: "I would like to b", completion: word, isFinal: false, spellingAssessment: dictionary + ), + .show(text: word, wordOnly: false) + ) + } XCTAssertEqual( CompletionSeamGuard.presentation( precedingText: "I would like to b", @@ -601,9 +621,9 @@ final class CompletionSeamGuardTests: XCTestCase { XCTAssertEqual( CompletionSeamGuard.presentation( precedingText: "Try " + opening + "b", - completion: "uild", + completion: "uil", isFinal: false, - spellingAssessment: knowsEverything + spellingAssessment: knowing(["build"]) ), .wait ) From 96d406a3524787d83bacc56bab9f5a0ccb78a4ac Mon Sep 17 00:00:00 2001 From: Senad Date: Sun, 18 Oct 2026 01:22:20 +0200 Subject: [PATCH 16/19] Keep the accept tap enabled through a typing burst While typing, the ghost hides and reappears on nearly every keystroke, and the active accept tap was switched off 50 ms after each hide and on again at the next ghost. Live TextEdit runs dropped about one typed character in a hundred with Cotabby running, both before and after the latency work, and none with Cotabby quit. The tap now stays enabled until suggestions have been hidden for two seconds, and only the last scheduled disable runs. Accept-key ownership is still released at once, so a Tab with no visible suggestion keeps reaching the app. --- Cotabby/Services/Input/InputMonitor.swift | 38 ++++++++++++++--------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/Cotabby/Services/Input/InputMonitor.swift b/Cotabby/Services/Input/InputMonitor.swift index 62409a0da..0337d67d7 100644 --- a/Cotabby/Services/Input/InputMonitor.swift +++ b/Cotabby/Services/Input/InputMonitor.swift @@ -189,18 +189,19 @@ final class InputMonitor { private static let disabledKeyCode: CGKeyCode = CGKeyCode(UInt16.max) - /// How long the accept tap lingers (fail-open) after the overlay hides before its mach port is - /// invalidated. A final-chunk accept runs *inside* this tap's own callback: it posts the - /// synthetic insertion to `.cghidEventTap` and then hides the overlay, which routes back here to - /// tear the tap down. Invalidating the mach port in that same run-loop turn pulls the tap out of - /// the session tap chain before the just-posted synthetic keystroke drains through it, so the - /// last accepted word never reaches the host (non-final words keep the overlay visible and never - /// hit this path, which is why only the final word failed to commit). Deferring the invalidation - /// one short hop lets the synthetic event finish delivery first. This is the invariant PR #385 - /// ("Fix last tab issue") protected before the two-tap ownership rewrite dropped it. - private static let acceptTapTeardownDelaySeconds: TimeInterval = 0.05 + /// How long the accept tap stays enabled after the last suggestion hid. Switching an active tap + /// off pulls it out of the session tap chain, and an event still on its way through it is lost: + /// a final-chunk accept runs *inside* this tap's own callback, posts the synthetic insertion to + /// `.cghidEventTap`, then hides the overlay, and tearing the tap down in that same run-loop turn + /// once made the last accepted word never reach the host (PR #385, "Fix last tab issue"). The + /// user's own keystrokes are exposed the same way: toggling the tap around every keystroke of a + /// burst dropped about one typed character in a hundred in live TextEdit runs, and none with + /// Cotabby quit. Two seconds spans a typing burst, so the tap is only switched off once input + /// has gone quiet, and it also covers the synthetic-insertion drain. + private static let acceptTapIdleDisableSeconds: TimeInterval = 2 /// How long a disabled accept tap is kept for its next suggestion before it is released. private static let acceptTapReleaseDelaySeconds: TimeInterval = 10 + private var acceptTapStateGeneration: UInt64 = 0 private var acceptTapWanted: Bool { permissionProvider() && (suggestionInterceptionActive || captureInterceptionActive) @@ -234,10 +235,11 @@ final class InputMonitor { // through immediately. isAcceptTapOwningAcceptKeys = wantsTap && suggestionInterceptionActive if wantsTap { + acceptTapStateGeneration &+= 1 installAcceptTapIfNeeded() } else { - // Defer only the mach-port invalidation, so a final-chunk accept's synthetic insertion can - // drain before the tap is removed (see `acceptTapTeardownDelaySeconds`). Re-check both + // Defer the disable, so a final-chunk accept's synthetic insertion can drain before the + // tap stops intercepting (see `acceptTapIdleDisableSeconds`). Re-check both // reasons at fire time so a suggestion or an emoji capture that re-armed the tap during the // delay keeps it installed. // @@ -246,8 +248,16 @@ final class InputMonitor { // (~20-30 ms measured) on the main thread right before the ghost could be presented. A // disabled tap intercepts nothing, so typing stays off this synchronous path exactly as // before; re-enabling it is cheap. It is released only after a longer idle period. - DispatchQueue.main.asyncAfter(deadline: .now() + Self.acceptTapTeardownDelaySeconds) { [weak self] in - guard let self, !self.acceptTapWanted else { return } + // + // The tap is disabled only after a quiet period, never between keystrokes of a burst. + // Switching an active tap off while a keystroke is in flight through it can lose that + // keystroke: live runs dropped one character in roughly a hundred while the tap was + // toggled around every keystroke, and none with Cotabby quit. Every state change bumps + // `acceptTapStateGeneration`, so only a disable scheduled after the last change runs. + acceptTapStateGeneration &+= 1 + let generation = acceptTapStateGeneration + DispatchQueue.main.asyncAfter(deadline: .now() + Self.acceptTapIdleDisableSeconds) { [weak self] in + guard let self, generation == self.acceptTapStateGeneration, !self.acceptTapWanted else { return } self.disableAcceptTap() DispatchQueue.main.asyncAfter(deadline: .now() + Self.acceptTapReleaseDelaySeconds) { [weak self] in guard let self, !self.acceptTapWanted, !self.acceptTapEnabled else { return } From f6f74a24a2e00e1557dbdff6998302961e7cd0ec Mon Sep 17 00:00:00 2001 From: Senad Date: Sun, 18 Oct 2026 10:18:57 +0200 Subject: [PATCH 17/19] Capture the front-most window under the caret With several windows of one app open (two TextEdit documents, two compose windows), the screenshot for screen context took the first window ScreenCaptureKit listed as active. That list has no documented order and more than one window can report itself active, so the capture sometimes read a background document: its text entered the prompt, and the change from the focused window's text read as navigation and retired the visible suggestion. The capture now takes the front-most window containing the caret, in the window server's front-to-back order, falling back to the front-most active window and then to any window of the app. The choice is a pure function, CaptureWindowSelector, so it is tested without ScreenCaptureKit. --- Cotabby.xcodeproj/project.pbxproj | 10 ++++++ .../Visual/WindowScreenshotService.swift | 30 ++++++++++------ .../Context/CaptureWindowSelector.swift | 33 +++++++++++++++++ .../Context/CaptureWindowSelectorTests.swift | 36 +++++++++++++++++++ 4 files changed, 99 insertions(+), 10 deletions(-) create mode 100644 Cotabby/Support/Context/CaptureWindowSelector.swift create mode 100644 CotabbyTests/Support/Context/CaptureWindowSelectorTests.swift diff --git a/Cotabby.xcodeproj/project.pbxproj b/Cotabby.xcodeproj/project.pbxproj index a7e88fd82..ad180920c 100644 --- a/Cotabby.xcodeproj/project.pbxproj +++ b/Cotabby.xcodeproj/project.pbxproj @@ -19,6 +19,7 @@ 0203A1A205162551A22FAC98 /* PermissionGuidanceController.swift in Sources */ = {isa = PBXBuildFile; fileRef = B0C41A249D9538B91B7FA8A9 /* PermissionGuidanceController.swift */; }; 0209B7FDA016D12F91C1D197 /* BlockBreakAlignment.swift in Sources */ = {isa = PBXBuildFile; fileRef = 82420F9505E69AE2ADE9F583 /* BlockBreakAlignment.swift */; }; 023A1A2A26B192E244192D8D /* TrailingDuplicationFilterTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6E824A5614BC7E7FCA05AD29 /* TrailingDuplicationFilterTests.swift */; }; + 027541D3AA54FB311C3D82F3 /* CaptureWindowSelectorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 316FC5B19A79B3D16727D0BD /* CaptureWindowSelectorTests.swift */; }; 02A837BD3861FD72B3E99D0F /* BaseCompletionPromptRendererTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 612CA2988BF19FAE0114165A /* BaseCompletionPromptRendererTests.swift */; }; 02C80969B8D6A0DBDC619FCA /* FocusSessionScopedCacheTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 487D8C5521411FC6845D4A51 /* FocusSessionScopedCacheTests.swift */; }; 02DA43985CDAE6859014F14F /* SuggestionOverlayPresenter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9D598CC3134879999D567455 /* SuggestionOverlayPresenter.swift */; }; @@ -102,6 +103,7 @@ 16BD2E6ECD6A818711A799D3 /* InsertedTextAdvanceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 35AA2C8F42B510F013D86C3C /* InsertedTextAdvanceTests.swift */; }; 16D638B016CEF136AB6BCC1C /* SuggestionPhraseAcceptanceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2D520FBB6AE0E786FFAD1B11 /* SuggestionPhraseAcceptanceTests.swift */; }; 16E1A22AA7F4827E69C7F2CE /* SuggestionSettingsStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8AB17696CE79982039061F29 /* SuggestionSettingsStore.swift */; }; + 172DC6FABE7A78C5FFE17ECE /* CaptureWindowSelector.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8F130886CA3C592D3309A61C /* CaptureWindowSelector.swift */; }; 1741F4AFFEA90CD6F81959F3 /* ArithmeticEvaluatorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D381F1752705DC80E491B025 /* ArithmeticEvaluatorTests.swift */; }; 1816D3D625F561209590D45D /* TagChip.swift in Sources */ = {isa = PBXBuildFile; fileRef = A04DFC5BD7821D73DA21D7DB /* TagChip.swift */; }; 181DB7463BA225142F02DEFD /* InsertionStrategySelectorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 12BB15F7F2E595129B190E89 /* InsertionStrategySelectorTests.swift */; }; @@ -803,6 +805,7 @@ C72537496242A0A90F4442B3 /* SpeculativeAcceptanceContextTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4B98B1ABB36E600D7ECA185 /* SpeculativeAcceptanceContextTests.swift */; }; C73BCA5544F526106FE162AE /* ModelDownloadSessionDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = AEFBEB715AF043D3315B6656 /* ModelDownloadSessionDelegate.swift */; }; C7492A81906AD943E286F4B5 /* SuggestionContinuationPlanTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 790082EFCDEF94454A3B805F /* SuggestionContinuationPlanTests.swift */; }; + C762BC10845F3C5156E1EF3B /* CaptureWindowSelector.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8F130886CA3C592D3309A61C /* CaptureWindowSelector.swift */; }; C7A48F93691943EF2EEBD775 /* DownloadOutcomeClassifierTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B3AD6D4125ED99669F4EC32 /* DownloadOutcomeClassifierTests.swift */; }; C82AD8930B5C3EB06D51A75A /* SuggestionRequestFactory.swift in Sources */ = {isa = PBXBuildFile; fileRef = DD834BCFF812935889E16D5A /* SuggestionRequestFactory.swift */; }; C8434622BFCE324E7044E581 /* TickMarkSlider.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1BB42779285A1BA40B7848C8 /* TickMarkSlider.swift */; }; @@ -1160,6 +1163,7 @@ 308F8660F70ED64511BDB59F /* RandomMacroEvaluator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RandomMacroEvaluator.swift; sourceTree = ""; }; 30AB3A99BEB2EA978298FC6F /* MidWordContinuationPolicyTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MidWordContinuationPolicyTests.swift; sourceTree = ""; }; 3153FB925190B70E0502C43D /* InkCaretAnalyzerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InkCaretAnalyzerTests.swift; sourceTree = ""; }; + 316FC5B19A79B3D16727D0BD /* CaptureWindowSelectorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CaptureWindowSelectorTests.swift; sourceTree = ""; }; 31A62CE4DA3E55B5E117DBD8 /* LlamaSuggestionEvalTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LlamaSuggestionEvalTests.swift; sourceTree = ""; }; 3350EDE01ED5125520C79D53 /* SettingsCoordinator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsCoordinator.swift; sourceTree = ""; }; 338EE4D92F8BE01CF43FE876 /* OllamaPreloadWorkControllerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OllamaPreloadWorkControllerTests.swift; sourceTree = ""; }; @@ -1399,6 +1403,7 @@ 8DAD5637347E83DDCF515568 /* SuggestionCoordinator+HostMarkedText.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SuggestionCoordinator+HostMarkedText.swift"; sourceTree = ""; }; 8E542E57459488F3D39A9053 /* PhrasePredictionScoringTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PhrasePredictionScoringTests.swift; sourceTree = ""; }; 8E89746E8CE7E9487337EE6F /* InsertionSafetyGate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InsertionSafetyGate.swift; sourceTree = ""; }; + 8F130886CA3C592D3309A61C /* CaptureWindowSelector.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CaptureWindowSelector.swift; sourceTree = ""; }; 9227DE16CA5CE2E085C1A891 /* PerAppShortcutOverrideStoreTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PerAppShortcutOverrideStoreTests.swift; sourceTree = ""; }; 924714B1688E884D9AB2A79D /* Aria2LocatorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Aria2LocatorTests.swift; sourceTree = ""; }; 924CAA5E25C596A9FAB7602B /* InlinePreviewPanelController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InlinePreviewPanelController.swift; sourceTree = ""; }; @@ -2668,6 +2673,7 @@ 7FAA2090D6421D9BAC2CFD37 /* Context */ = { isa = PBXGroup; children = ( + 8F130886CA3C592D3309A61C /* CaptureWindowSelector.swift */, B2229037BCDF7EFD3D47C15F /* ClipboardContentDistiller.swift */, 2BF37C1A0F734BB940E371C2 /* ClipboardRelevanceFilter.swift */, 7736E6B390C822F91CC1DA99 /* OCRTextHygiene.swift */, @@ -3463,6 +3469,7 @@ DEC0DDC1DF558A8932276294 /* Context */ = { isa = PBXGroup; children = ( + 316FC5B19A79B3D16727D0BD /* CaptureWindowSelectorTests.swift */, 3680E1B8FA712A888F509640 /* ClipboardContentDistillerTests.swift */, 04364C3242A25358E7C85FD6 /* ClipboardRelevanceFilterTests.swift */, 64B0584FA16465453F04CA8C /* OCRTextHygieneTests.swift */, @@ -4006,6 +4013,7 @@ 24D5BE34BE7B7B72A93C6A9F /* BundledRuntimeLocator.swift in Sources */, 7291020840F2164EBB764FAF /* CalendarAccessibilityCaptureGuard.swift in Sources */, BFD3F43E9C83F3D0F7BF9ADE /* CalendarAccessibilityCapturePolicy.swift in Sources */, + C762BC10845F3C5156E1EF3B /* CaptureWindowSelector.swift in Sources */, 3744E92A7A36B0958F9DB488 /* CaretAdvanceSampleStore.swift in Sources */, 1C853F8F3BE4DFE89F90BF35 /* CaretAdvanceSampler.swift in Sources */, 3D62542845E1048A93A48749 /* CaretGeometrySelector.swift in Sources */, @@ -4348,6 +4356,7 @@ F140C1894C54E0A94EAE70BC /* BundledRuntimeLocator.swift in Sources */, 77823BCFB29F63615C514927 /* CalendarAccessibilityCaptureGuard.swift in Sources */, 51F0B064F34BA3F88196C39F /* CalendarAccessibilityCapturePolicy.swift in Sources */, + 172DC6FABE7A78C5FFE17ECE /* CaptureWindowSelector.swift in Sources */, E4B3CF7EB30D22DABFF46F33 /* CaretAdvanceSampleStore.swift in Sources */, F09B3B8B1E61818BCE539358 /* CaretAdvanceSampler.swift in Sources */, 6D088E2762DBDE195463DE47 /* CaretGeometrySelector.swift in Sources */, @@ -4679,6 +4688,7 @@ A6878832F0F19C3C70108185 /* BundleVersionTests.swift in Sources */, 85FCDC35AC29F4A89E30F559 /* BundledRuntimeLocatorTests.swift in Sources */, 715BD5081873EF5FF070D3B0 /* CalendarAccessibilityCapturePolicyTests.swift in Sources */, + 027541D3AA54FB311C3D82F3 /* CaptureWindowSelectorTests.swift in Sources */, 111905666B5F1D0B65B87D44 /* CapturedInputEventTests.swift in Sources */, 7AC032855FD65F67B93E172D /* CaretAdvanceSamplerTests.swift in Sources */, C4511DF96F8D3A88DB6161DE /* CaretGeometrySelectorTests.swift in Sources */, diff --git a/Cotabby/Services/Visual/WindowScreenshotService.swift b/Cotabby/Services/Visual/WindowScreenshotService.swift index 517aaa1dc..e86870c1d 100644 --- a/Cotabby/Services/Visual/WindowScreenshotService.swift +++ b/Cotabby/Services/Visual/WindowScreenshotService.swift @@ -77,17 +77,15 @@ struct WindowScreenshotService: WindowScreenshotCapturing { let shareableContent = try await currentShareableContent() try Task.checkCancellation() let caretCG = convertBetweenAppKitAndCG(rect: context.caretRect) - let focusedWindow = capturesEntireWindow ? shareableContent.windows.first(where: { + let appWindows = shareableContent.windows.filter { $0.owningApplication?.processID == processIdentifier && $0.isOnScreen - && $0.frame.contains(CGPoint(x: caretCG.midX, y: caretCG.midY)) && $0.isActive - }) : nil - let matchingWindow = - focusedWindow ?? shareableContent.windows.first(where: { - $0.owningApplication?.processID == processIdentifier && $0.isActive && $0.isOnScreen - }) - ?? shareableContent.windows.first(where: { - $0.owningApplication?.processID == processIdentifier && $0.isOnScreen - }) + } + let selectedWindowID = CaptureWindowSelector.select( + appWindows.map { .init(id: $0.windowID, frame: $0.frame, isActive: $0.isActive) }, + caret: CGPoint(x: caretCG.midX, y: caretCG.midY), + frontToBack: Self.frontToBackWindowIDs(for: processIdentifier) + ) + let matchingWindow = appWindows.first { $0.windowID == selectedWindowID } guard let matchingWindow else { CotabbyLogger.app.debug("No visible window for pid \(processIdentifier)") @@ -138,6 +136,18 @@ struct WindowScreenshotService: WindowScreenshotCapturing { return CapturedWindowScreenshot(image: image, windowTitle: matchingWindow.title, focusBounds: focusBounds) } + /// The process's normal-layer on-screen window numbers, front-most first. ScreenCaptureKit's + /// window list has no documented order; the window server's list is front to back. + private static func frontToBackWindowIDs(for processIdentifier: pid_t) -> [CGWindowID] { + let options: CGWindowListOption = [.optionOnScreenOnly, .excludeDesktopElements] + let windows = CGWindowListCopyWindowInfo(options, kCGNullWindowID) as? [[String: Any]] ?? [] + return windows.compactMap { info in + guard (info[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == processIdentifier, + (info[kCGWindowLayer as String] as? NSNumber)?.intValue == 0 else { return nil } + return (info[kCGWindowNumber as String] as? NSNumber)?.uint32Value + } + } + private func snapshotRect( around context: FocusedInputSnapshot, windowFrame: CGRect, diff --git a/Cotabby/Support/Context/CaptureWindowSelector.swift b/Cotabby/Support/Context/CaptureWindowSelector.swift new file mode 100644 index 000000000..613a9d595 --- /dev/null +++ b/Cotabby/Support/Context/CaptureWindowSelector.swift @@ -0,0 +1,33 @@ +import CoreGraphics + +/// Pure choice of the window to capture for the focused field. +/// +/// ScreenCaptureKit lists an app's windows in no documented order, and with several documents +/// open more than one can report itself active. Taking the first such window captured a background +/// document of the same app: its text entered the prompt, and the change from the focused +/// window's text read as navigation and retired the visible suggestion. The focused field's window +/// is the front-most one under its caret, in the window server's front-to-back order. +/// +/// Kept apart from `WindowScreenshotService` so the choice is testable without ScreenCaptureKit. +nonisolated enum CaptureWindowSelector { + struct Candidate: Equatable { + let id: CGWindowID + /// Global display points, top-left origin (ScreenCaptureKit's space). + let frame: CGRect + let isActive: Bool + } + + /// - Parameters: + /// - candidates: The focused app's on-screen windows. + /// - caret: The caret's centre in the same space as the frames. + /// - frontToBack: Window numbers front-most first. Windows missing from it rank behind all + /// listed ones and keep their list order, so an empty order falls back to list order. + static func select(_ candidates: [Candidate], caret: CGPoint, frontToBack: [CGWindowID]) -> CGWindowID? { + let rank = Dictionary(frontToBack.enumerated().map { ($1, $0) }, uniquingKeysWith: min) + func frontMost(_ windows: [Candidate]) -> Candidate? { + windows.min { (rank[$0.id] ?? .max) < (rank[$1.id] ?? .max) } + } + let underCaret = candidates.filter { $0.frame.contains(caret) } + return (frontMost(underCaret) ?? frontMost(candidates.filter(\.isActive)) ?? frontMost(candidates))?.id + } +} diff --git a/CotabbyTests/Support/Context/CaptureWindowSelectorTests.swift b/CotabbyTests/Support/Context/CaptureWindowSelectorTests.swift new file mode 100644 index 000000000..e65113e58 --- /dev/null +++ b/CotabbyTests/Support/Context/CaptureWindowSelectorTests.swift @@ -0,0 +1,36 @@ +import CoreGraphics +import XCTest +@testable import Cotabby + +final class CaptureWindowSelectorTests: XCTestCase { + private let caret = CGPoint(x: 300, y: 300) + + func test_picksTheFrontMostWindowUnderTheCaret() { + // Two cascaded TextEdit documents, both reported active, the background one listed first: + // capturing it put another document's text in the prompt and read as navigation. + let background = CaptureWindowSelector.Candidate(id: 11, frame: CGRect(x: 0, y: 0, width: 603, height: 505), isActive: true) + let focused = CaptureWindowSelector.Candidate(id: 22, frame: CGRect(x: 20, y: 20, width: 603, height: 505), isActive: true) + let selected = CaptureWindowSelector.select([background, focused], caret: caret, frontToBack: [22, 11]) + XCTAssertEqual(selected, 22) + } + + func test_aWindowUnderTheCaretBeatsAnActiveOneElsewhere() { + let elsewhere = CaptureWindowSelector.Candidate(id: 1, frame: CGRect(x: 900, y: 0, width: 400, height: 400), isActive: true) + let underCaret = CaptureWindowSelector.Candidate(id: 2, frame: CGRect(x: 0, y: 0, width: 600, height: 600), isActive: false) + XCTAssertEqual(CaptureWindowSelector.select([elsewhere, underCaret], caret: caret, frontToBack: [1, 2]), 2) + } + + func test_fallsBackToTheFrontMostActiveWindowWhenNoneContainsTheCaret() { + let back = CaptureWindowSelector.Candidate(id: 1, frame: CGRect(x: 900, y: 0, width: 400, height: 400), isActive: true) + let front = CaptureWindowSelector.Candidate(id: 2, frame: CGRect(x: 1400, y: 0, width: 400, height: 400), isActive: true) + let inactive = CaptureWindowSelector.Candidate(id: 3, frame: CGRect(x: 2000, y: 0, width: 400, height: 400), isActive: false) + XCTAssertEqual(CaptureWindowSelector.select([back, front, inactive], caret: caret, frontToBack: [3, 2, 1]), 2) + } + + func test_keepsListOrderWhenTheWindowOrderIsUnknown() { + let first = CaptureWindowSelector.Candidate(id: 1, frame: CGRect(x: 0, y: 0, width: 600, height: 600), isActive: true) + let second = CaptureWindowSelector.Candidate(id: 2, frame: CGRect(x: 0, y: 0, width: 600, height: 600), isActive: true) + XCTAssertEqual(CaptureWindowSelector.select([first, second], caret: caret, frontToBack: []), 1) + XCTAssertNil(CaptureWindowSelector.select([], caret: caret, frontToBack: [1])) + } +} From 39c9f712b8533df8ff4e2e53040c082c783aa342 Mon Sep 17 00:00:00 2001 From: Senad Date: Sun, 18 Oct 2026 10:22:19 +0200 Subject: [PATCH 18/19] Compare a screen refresh against the field as it reads when OCR lands VisualContextChangePolicy leaves out lines that echo the focused field, so the user's own typing on screen neither rewrites the prompt's screen section nor retires the visible suggestion. It was given the field text from when the capture started, though, and OCR takes 60-100 ms while the user keeps typing: in a window the draft fills, the words typed meanwhile made the draft line read as new content. The refresh now passes the field as it reads when the result is applied, which already contains everything the capture saw. --- .../Visual/VisualContextCoordinator.swift | 13 +++++--- .../VisualContextCoordinatorTests.swift | 32 +++++++++++++++++++ 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/Cotabby/Services/Visual/VisualContextCoordinator.swift b/Cotabby/Services/Visual/VisualContextCoordinator.swift index 334e2deb5..4e0175ade 100644 --- a/Cotabby/Services/Visual/VisualContextCoordinator.swift +++ b/Cotabby/Services/Visual/VisualContextCoordinator.swift @@ -202,22 +202,27 @@ final class VisualContextCoordinator { guard !Task.isCancelled else { return } + // The field as it reads now, not at capture start: everything the capture saw of + // the user's typing is already in it, so the change policy can discount all of it. + var fieldContext = snapshotContext if let provider = refreshContextProvider { let liveContext = provider() guard activeAugmentationSession?.sessionID == session.sessionID else { return } - guard screenRecordingPermissionProvider(), liveContext?.identity == snapshotContext.identity, - liveContext?.sessionIdentity == snapshotContext.sessionIdentity, - liveContext?.isSecure == false else { + guard screenRecordingPermissionProvider(), let liveContext, + liveContext.identity == snapshotContext.identity, + liveContext.sessionIdentity == snapshotContext.sessionIdentity, + !liveContext.isSecure else { cancel(resetState: true) return } + fieldContext = liveContext } applyExcerpt( excerpt, for: session.sessionID, identity: snapshotContext.identity, - fieldText: snapshotContext.precedingText + snapshotContext.trailingText, + fieldText: fieldContext.precedingText + fieldContext.trailingText, capturedAt: capturedAt ) } catch is CancellationError { diff --git a/CotabbyTests/Services/Visual/VisualContextCoordinatorTests.swift b/CotabbyTests/Services/Visual/VisualContextCoordinatorTests.swift index be0255a32..cc860daa3 100644 --- a/CotabbyTests/Services/Visual/VisualContextCoordinatorTests.swift +++ b/CotabbyTests/Services/Visual/VisualContextCoordinatorTests.swift @@ -288,6 +288,38 @@ final class VisualContextCoordinatorTests: XCTestCase { XCTAssertEqual(generator.configurations, [.local, .default]) } + /// TextEdit or a compose window: the draft fills the capture, and the user keeps typing while + /// OCR runs. Words typed after the capture started are on screen but were not in the field when + /// it started, so the comparison must use the field as it reads when the result lands. + func test_typingIntoAWindowFillingFieldIsNotANavigationSignal() async throws { + let generator = StubVisualContextGenerator() + let chrome = "Untitled 3 — Edited\nHelvetica Regular 12 B I U S\n" + generator.text = chrome + "Hello Martin, thanks for the update on the" + let coordinator = makeCoordinator(generator) + var live = CotabbyTestFixtures.focusedInputSnapshot(precedingText: "Hello Martin, thanks for the update on the") + coordinator.refreshContextProvider = { live } + var notifications = 0 + coordinator.onInjectedContextReady = { _ in notifications += 1 } + coordinator.startSessionIfNeeded(for: live, configuration: .local) + defer { coordinator.cancel(resetState: true) } + try await waitUntil { coordinator.status == .ready } + let original = coordinator.latestExcerpt + + generator.suspendNext = true + try await waitUntil { generator.pending != nil } + live = CotabbyTestFixtures.focusedInputSnapshot( + precedingText: "Hello Martin, thanks for the update on the onboarding project. I wanted to check" + ) + generator.text = chrome + "Hello Martin, thanks for the update on the onboarding project. I wanted to che" + let completedBefore = generator.completedCount + generator.pending?.resume() + generator.pending = nil + try await waitUntil { generator.completedCount > completedBefore } + + XCTAssertEqual(notifications, 1, "The user's own typing on screen must not retire the visible suggestion") + XCTAssertEqual(coordinator.latestExcerpt, original, "Typing alone must keep the prompt's screen text") + } + private func makeCoordinator( _ generator: StubVisualContextGenerator, permission: @escaping @MainActor () -> Bool = { true } From 4212ad223fba4ea02448afb4d62e7e81d847c9e2 Mon Sep 17 00:00:00 2001 From: Senad Date: Sun, 18 Oct 2026 10:24:52 +0200 Subject: [PATCH 19/19] Split the debounce choice by engine to satisfy strict SwiftLint --- .../Suggestion/Request/DebouncePolicy.swift | 39 +++++++++++++------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/Cotabby/Support/Suggestion/Request/DebouncePolicy.swift b/Cotabby/Support/Suggestion/Request/DebouncePolicy.swift index 3087eeb6d..0d6e5e2dc 100644 --- a/Cotabby/Support/Suggestion/Request/DebouncePolicy.swift +++ b/Cotabby/Support/Suggestion/Request/DebouncePolicy.swift @@ -21,21 +21,36 @@ nonisolated enum DebouncePolicy { fallback: Int, engine: SuggestionEngineKind = .llamaOpenSource ) -> Int { - // An HTTP endpoint cannot reuse Cotabby's in-process KV cache, and cancelling a request does - // not guarantee the remote server stops its tokenizer immediately. A trailing-edge pause - // therefore saves more time than it costs: it collapses a typing burst into one useful - // request instead of filling Ollama's queue with work that will be discarded. if engine == .openAICompatible { - guard let last = lastGenerationLatencyMilliseconds, last > 0 else { - return max(fallback, 180) - } - switch last { - case ...300: return 100 - case ...700: return 150 - default: return 220 - } + return endpointMilliseconds(lastGenerationLatencyMilliseconds: lastGenerationLatencyMilliseconds, fallback: fallback) } + return inProcessMilliseconds( + lastGenerationLatencyMilliseconds: lastGenerationLatencyMilliseconds, + lastFirstTextLatencyMilliseconds: lastFirstTextLatencyMilliseconds, + fallback: fallback + ) + } + /// An HTTP endpoint cannot reuse Cotabby's in-process KV cache, and cancelling a request does + /// not guarantee the remote server stops its tokenizer immediately. A trailing-edge pause + /// therefore saves more time than it costs: it collapses a typing burst into one useful + /// request instead of filling Ollama's queue with work that will be discarded. + private static func endpointMilliseconds(lastGenerationLatencyMilliseconds: Int?, fallback: Int) -> Int { + guard let last = lastGenerationLatencyMilliseconds, last > 0 else { + return max(fallback, 180) + } + switch last { + case ...300: return 100 + case ...700: return 150 + default: return 220 + } + } + + private static func inProcessMilliseconds( + lastGenerationLatencyMilliseconds: Int?, + lastFirstTextLatencyMilliseconds: Int?, + fallback: Int + ) -> Int { if let firstText = lastFirstTextLatencyMilliseconds, firstText > 0 { switch firstText { case ...60: return 0