diff --git a/Cotabby.xcodeproj/project.pbxproj b/Cotabby.xcodeproj/project.pbxproj index efe7ecb17..ad180920c 100644 --- a/Cotabby.xcodeproj/project.pbxproj +++ b/Cotabby.xcodeproj/project.pbxproj @@ -19,6 +19,7 @@ 0203A1A205162551A22FAC98 /* PermissionGuidanceController.swift in Sources */ = {isa = PBXBuildFile; fileRef = B0C41A249D9538B91B7FA8A9 /* PermissionGuidanceController.swift */; }; 0209B7FDA016D12F91C1D197 /* BlockBreakAlignment.swift in Sources */ = {isa = PBXBuildFile; fileRef = 82420F9505E69AE2ADE9F583 /* BlockBreakAlignment.swift */; }; 023A1A2A26B192E244192D8D /* TrailingDuplicationFilterTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6E824A5614BC7E7FCA05AD29 /* TrailingDuplicationFilterTests.swift */; }; + 027541D3AA54FB311C3D82F3 /* CaptureWindowSelectorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 316FC5B19A79B3D16727D0BD /* CaptureWindowSelectorTests.swift */; }; 02A837BD3861FD72B3E99D0F /* BaseCompletionPromptRendererTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 612CA2988BF19FAE0114165A /* BaseCompletionPromptRendererTests.swift */; }; 02C80969B8D6A0DBDC619FCA /* FocusSessionScopedCacheTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 487D8C5521411FC6845D4A51 /* FocusSessionScopedCacheTests.swift */; }; 02DA43985CDAE6859014F14F /* SuggestionOverlayPresenter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9D598CC3134879999D567455 /* SuggestionOverlayPresenter.swift */; }; @@ -36,6 +37,7 @@ 05CFA72F4FF44A0CDBEFED3C /* PostExhaustionAcceptanceState.swift in Sources */ = {isa = PBXBuildFile; fileRef = 22FA75F9AD318B9084B23ECF /* PostExhaustionAcceptanceState.swift */; }; 067371AD898788392F158EDE /* PhrasePredictionScreenContext.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6E6A9735EBD163A2E8AD67A5 /* PhrasePredictionScreenContext.swift */; }; 07639673B5A52D2075A9ED57 /* VisualContextModels.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4A51B1559F4CE6728D39C959 /* VisualContextModels.swift */; }; + 07C1D9A7919366C977AC1623 /* VisualContextChangePolicy.swift in Sources */ = {isa = PBXBuildFile; fileRef = 357BDB23F653C34D18233A49 /* VisualContextChangePolicy.swift */; }; 07E54011F501E14AC15DE2B7 /* OnboardingTemplateFeatureList.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7174EF88BA41ACC0A74CA582 /* OnboardingTemplateFeatureList.swift */; }; 081EB4DF65AD5736DE7FBFDE /* MirrorOverlayLayoutTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2D962AC9D24D00957562CA7B /* MirrorOverlayLayoutTests.swift */; }; 085BB87581DFFA260A630E24 /* CotabbyBrand.swift in Sources */ = {isa = PBXBuildFile; fileRef = D3C1C9A9BF1F74C16E4401EC /* CotabbyBrand.swift */; }; @@ -101,6 +103,7 @@ 16BD2E6ECD6A818711A799D3 /* InsertedTextAdvanceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 35AA2C8F42B510F013D86C3C /* InsertedTextAdvanceTests.swift */; }; 16D638B016CEF136AB6BCC1C /* SuggestionPhraseAcceptanceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2D520FBB6AE0E786FFAD1B11 /* SuggestionPhraseAcceptanceTests.swift */; }; 16E1A22AA7F4827E69C7F2CE /* SuggestionSettingsStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8AB17696CE79982039061F29 /* SuggestionSettingsStore.swift */; }; + 172DC6FABE7A78C5FFE17ECE /* CaptureWindowSelector.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8F130886CA3C592D3309A61C /* CaptureWindowSelector.swift */; }; 1741F4AFFEA90CD6F81959F3 /* ArithmeticEvaluatorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D381F1752705DC80E491B025 /* ArithmeticEvaluatorTests.swift */; }; 1816D3D625F561209590D45D /* TagChip.swift in Sources */ = {isa = PBXBuildFile; fileRef = A04DFC5BD7821D73DA21D7DB /* TagChip.swift */; }; 181DB7463BA225142F02DEFD /* InsertionStrategySelectorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 12BB15F7F2E595129B190E89 /* InsertionStrategySelectorTests.swift */; }; @@ -419,6 +422,7 @@ 67545937A4CAA6B1A1B4B86A /* AXTextGeometryResolver.swift in Sources */ = {isa = PBXBuildFile; fileRef = 98423150126CC5969C2886BA /* AXTextGeometryResolver.swift */; }; 676533C6555BAC0C4A263576 /* HuggingFaceSearchServiceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5A9DE35F3B6F0DFEB9538692 /* HuggingFaceSearchServiceTests.swift */; }; 67A5529348C03A5B97BEFFC5 /* BlockBreakAlignmentTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D681DDF8E81F868C1BC92CB4 /* BlockBreakAlignmentTests.swift */; }; + 67A8673F38C847E921CC8BEE /* VisualContextChangePolicyTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4C8F4DA76268B231B9F2C7D /* VisualContextChangePolicyTests.swift */; }; 67C36958132793C76843F875 /* LlamaTypingSessionEvalTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = A936CA937FA629F828938516 /* LlamaTypingSessionEvalTests.swift */; }; 68DA5F93B7185B4F5E6DB4C3 /* it.txt in Resources */ = {isa = PBXBuildFile; fileRef = 0397F1DACB094A0F6A66BC0E /* it.txt */; }; 6952E5B030740443C684F263 /* SettingsSearchRanker.swift in Sources */ = {isa = PBXBuildFile; fileRef = 57B29DAC600626783649475A /* SettingsSearchRanker.swift */; }; @@ -531,6 +535,7 @@ 83B25F54B74EE35787D5DBD4 /* de-100k.txt in Resources */ = {isa = PBXBuildFile; fileRef = 4B8665A5495891F9E3DDA48B /* de-100k.txt */; }; 83CFB4EA330BE0201DA9F03A /* TextLayoutCaretEstimator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1BF6C6A11D5B5A7B987AFB9C /* TextLayoutCaretEstimator.swift */; }; 840B11AFD82E64D5452CC7A3 /* WordPrefixIndex.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6428AF104998876A60AC602C /* WordPrefixIndex.swift */; }; + 845687C938075DC9C7074F9C /* VisualContextChangePolicy.swift in Sources */ = {isa = PBXBuildFile; fileRef = 357BDB23F653C34D18233A49 /* VisualContextChangePolicy.swift */; }; 8456BDA5B716AB3B0AFF19E4 /* THIRD_PARTY_LICENSES.md in Resources */ = {isa = PBXBuildFile; fileRef = 5FF4FF5C43E650FF46F4BE76 /* THIRD_PARTY_LICENSES.md */; }; 84C7C981F9243D1DB1BC90A4 /* PermissionOverlayWindowController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4213A267AE55FE56A071646F /* PermissionOverlayWindowController.swift */; }; 84E9E1F4D55CEB8A9692734E /* MenuBarPresentationObserver.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9DE86A49339BB507D197629D /* MenuBarPresentationObserver.swift */; }; @@ -800,6 +805,7 @@ C72537496242A0A90F4442B3 /* SpeculativeAcceptanceContextTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4B98B1ABB36E600D7ECA185 /* SpeculativeAcceptanceContextTests.swift */; }; C73BCA5544F526106FE162AE /* ModelDownloadSessionDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = AEFBEB715AF043D3315B6656 /* ModelDownloadSessionDelegate.swift */; }; C7492A81906AD943E286F4B5 /* SuggestionContinuationPlanTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 790082EFCDEF94454A3B805F /* SuggestionContinuationPlanTests.swift */; }; + C762BC10845F3C5156E1EF3B /* CaptureWindowSelector.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8F130886CA3C592D3309A61C /* CaptureWindowSelector.swift */; }; C7A48F93691943EF2EEBD775 /* DownloadOutcomeClassifierTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B3AD6D4125ED99669F4EC32 /* DownloadOutcomeClassifierTests.swift */; }; C82AD8930B5C3EB06D51A75A /* SuggestionRequestFactory.swift in Sources */ = {isa = PBXBuildFile; fileRef = DD834BCFF812935889E16D5A /* SuggestionRequestFactory.swift */; }; C8434622BFCE324E7044E581 /* TickMarkSlider.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1BB42779285A1BA40B7848C8 /* TickMarkSlider.swift */; }; @@ -1157,6 +1163,7 @@ 308F8660F70ED64511BDB59F /* RandomMacroEvaluator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RandomMacroEvaluator.swift; sourceTree = ""; }; 30AB3A99BEB2EA978298FC6F /* MidWordContinuationPolicyTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MidWordContinuationPolicyTests.swift; sourceTree = ""; }; 3153FB925190B70E0502C43D /* InkCaretAnalyzerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InkCaretAnalyzerTests.swift; sourceTree = ""; }; + 316FC5B19A79B3D16727D0BD /* CaptureWindowSelectorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CaptureWindowSelectorTests.swift; sourceTree = ""; }; 31A62CE4DA3E55B5E117DBD8 /* LlamaSuggestionEvalTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LlamaSuggestionEvalTests.swift; sourceTree = ""; }; 3350EDE01ED5125520C79D53 /* SettingsCoordinator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsCoordinator.swift; sourceTree = ""; }; 338EE4D92F8BE01CF43FE876 /* OllamaPreloadWorkControllerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OllamaPreloadWorkControllerTests.swift; sourceTree = ""; }; @@ -1164,6 +1171,7 @@ 341A1BBA636AEBA2491F6A26 /* FocusTrackingModelTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FocusTrackingModelTests.swift; sourceTree = ""; }; 34EF2406E7A384F3325AAF9A /* SettingsIndex.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsIndex.swift; sourceTree = ""; }; 353191D1D8A1C655E1B5F562 /* CapturedInputEventTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CapturedInputEventTests.swift; sourceTree = ""; }; + 357BDB23F653C34D18233A49 /* VisualContextChangePolicy.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VisualContextChangePolicy.swift; sourceTree = ""; }; 35AA2C8F42B510F013D86C3C /* InsertedTextAdvanceTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InsertedTextAdvanceTests.swift; sourceTree = ""; }; 35C0B587D81D87ACB952C95E /* SuggestionSettingsStoreTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SuggestionSettingsStoreTests.swift; sourceTree = ""; }; 365AE69E4E1A3DD3486D203A /* CredentialFieldDetectorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CredentialFieldDetectorTests.swift; sourceTree = ""; }; @@ -1395,6 +1403,7 @@ 8DAD5637347E83DDCF515568 /* SuggestionCoordinator+HostMarkedText.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SuggestionCoordinator+HostMarkedText.swift"; sourceTree = ""; }; 8E542E57459488F3D39A9053 /* PhrasePredictionScoringTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PhrasePredictionScoringTests.swift; sourceTree = ""; }; 8E89746E8CE7E9487337EE6F /* InsertionSafetyGate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InsertionSafetyGate.swift; sourceTree = ""; }; + 8F130886CA3C592D3309A61C /* CaptureWindowSelector.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CaptureWindowSelector.swift; sourceTree = ""; }; 9227DE16CA5CE2E085C1A891 /* PerAppShortcutOverrideStoreTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PerAppShortcutOverrideStoreTests.swift; sourceTree = ""; }; 924714B1688E884D9AB2A79D /* Aria2LocatorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Aria2LocatorTests.swift; sourceTree = ""; }; 924CAA5E25C596A9FAB7602B /* InlinePreviewPanelController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InlinePreviewPanelController.swift; sourceTree = ""; }; @@ -1662,6 +1671,7 @@ F3D991D3536ED989C89AF44D /* EmojiUsageStoreTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EmojiUsageStoreTests.swift; sourceTree = ""; }; F47CD9DF50E7192A70D1870E /* CotabbyDebugOptions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CotabbyDebugOptions.swift; sourceTree = ""; }; F4B98B1ABB36E600D7ECA185 /* SpeculativeAcceptanceContextTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SpeculativeAcceptanceContextTests.swift; sourceTree = ""; }; + F4C8F4DA76268B231B9F2C7D /* VisualContextChangePolicyTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VisualContextChangePolicyTests.swift; sourceTree = ""; }; F6809474A7C1B57F1D13AA6F /* SuggestionDismissalMemoryTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SuggestionDismissalMemoryTests.swift; sourceTree = ""; }; F7014E6E969FD4579028BAC4 /* TypingCadence.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TypingCadence.swift; sourceTree = ""; }; F7F3F983901FC3648076B23D /* MacroReferenceSheet.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MacroReferenceSheet.swift; sourceTree = ""; }; @@ -2663,11 +2673,13 @@ 7FAA2090D6421D9BAC2CFD37 /* Context */ = { isa = PBXGroup; children = ( + 8F130886CA3C592D3309A61C /* CaptureWindowSelector.swift */, B2229037BCDF7EFD3D47C15F /* ClipboardContentDistiller.swift */, 2BF37C1A0F734BB940E371C2 /* ClipboardRelevanceFilter.swift */, 7736E6B390C822F91CC1DA99 /* OCRTextHygiene.swift */, D182046E27CB170D943E8F86 /* PromptContextSanitizer.swift */, 27B03589D82D0ADCF41B9CD0 /* SurfaceContextComposer.swift */, + 357BDB23F653C34D18233A49 /* VisualContextChangePolicy.swift */, 47457FED99065966328047EB /* VisualContextExcerptSelector.swift */, DEF8F1A40EAA82773C07F8B9 /* VisualContextStartCoalescer.swift */, ); @@ -3457,11 +3469,13 @@ DEC0DDC1DF558A8932276294 /* Context */ = { isa = PBXGroup; children = ( + 316FC5B19A79B3D16727D0BD /* CaptureWindowSelectorTests.swift */, 3680E1B8FA712A888F509640 /* ClipboardContentDistillerTests.swift */, 04364C3242A25358E7C85FD6 /* ClipboardRelevanceFilterTests.swift */, 64B0584FA16465453F04CA8C /* OCRTextHygieneTests.swift */, 0DA66559D50874865032EE8C /* PromptContextSanitizerTests.swift */, 652973AF18C20D51F81F76C5 /* SurfaceContextComposerTests.swift */, + F4C8F4DA76268B231B9F2C7D /* VisualContextChangePolicyTests.swift */, 17B5B1AE537B78F54A32BC63 /* VisualContextExcerptSelectorTests.swift */, 70F68E9A9255F607C4439ED5 /* VisualContextStartCoalescerTests.swift */, ); @@ -3999,6 +4013,7 @@ 24D5BE34BE7B7B72A93C6A9F /* BundledRuntimeLocator.swift in Sources */, 7291020840F2164EBB764FAF /* CalendarAccessibilityCaptureGuard.swift in Sources */, BFD3F43E9C83F3D0F7BF9ADE /* CalendarAccessibilityCapturePolicy.swift in Sources */, + C762BC10845F3C5156E1EF3B /* CaptureWindowSelector.swift in Sources */, 3744E92A7A36B0958F9DB488 /* CaretAdvanceSampleStore.swift in Sources */, 1C853F8F3BE4DFE89F90BF35 /* CaretAdvanceSampler.swift in Sources */, 3D62542845E1048A93A48749 /* CaretGeometrySelector.swift in Sources */, @@ -4289,6 +4304,7 @@ C0468063222C5FEA7C1CFAB3 /* TypoCaseTransfer.swift in Sources */, A88F3C7039E8DDB71C5D6246 /* TypoGate.swift in Sources */, B2A6C1474B9F3A6E26F488C9 /* UnitConversionEvaluator.swift in Sources */, + 07C1D9A7919366C977AC1623 /* VisualContextChangePolicy.swift in Sources */, 0A15FAD03A93D57372D267E0 /* VisualContextCoordinator.swift in Sources */, 0DBDD71AB17F87D6558DC07D /* VisualContextExcerptSelector.swift in Sources */, 948AD3BD25C113539DF12FD7 /* VisualContextModels.swift in Sources */, @@ -4340,6 +4356,7 @@ F140C1894C54E0A94EAE70BC /* BundledRuntimeLocator.swift in Sources */, 77823BCFB29F63615C514927 /* CalendarAccessibilityCaptureGuard.swift in Sources */, 51F0B064F34BA3F88196C39F /* CalendarAccessibilityCapturePolicy.swift in Sources */, + 172DC6FABE7A78C5FFE17ECE /* CaptureWindowSelector.swift in Sources */, E4B3CF7EB30D22DABFF46F33 /* CaretAdvanceSampleStore.swift in Sources */, F09B3B8B1E61818BCE539358 /* CaretAdvanceSampler.swift in Sources */, 6D088E2762DBDE195463DE47 /* CaretGeometrySelector.swift in Sources */, @@ -4630,6 +4647,7 @@ CF391BC4A3C80C41048D64D1 /* TypoCaseTransfer.swift in Sources */, 48A2F371756723299597F5A1 /* TypoGate.swift in Sources */, 2DD6D14058C7FD05DBE1AD60 /* UnitConversionEvaluator.swift in Sources */, + 845687C938075DC9C7074F9C /* VisualContextChangePolicy.swift in Sources */, E9E4CC657771DF9F4C56183C /* VisualContextCoordinator.swift in Sources */, 540C1F9A70A5C7DB988FBDD4 /* VisualContextExcerptSelector.swift in Sources */, 07639673B5A52D2075A9ED57 /* VisualContextModels.swift in Sources */, @@ -4670,6 +4688,7 @@ A6878832F0F19C3C70108185 /* BundleVersionTests.swift in Sources */, 85FCDC35AC29F4A89E30F559 /* BundledRuntimeLocatorTests.swift in Sources */, 715BD5081873EF5FF070D3B0 /* CalendarAccessibilityCapturePolicyTests.swift in Sources */, + 027541D3AA54FB311C3D82F3 /* CaptureWindowSelectorTests.swift in Sources */, 111905666B5F1D0B65B87D44 /* CapturedInputEventTests.swift in Sources */, 7AC032855FD65F67B93E172D /* CaretAdvanceSamplerTests.swift in Sources */, C4511DF96F8D3A88DB6161DE /* CaretGeometrySelectorTests.swift in Sources */, @@ -4919,6 +4938,7 @@ F25F1800B79BF85AF07048C3 /* TypoCaseTransferTests.swift in Sources */, C3F63C89E82ED82DF24BD603 /* TypoGateTests.swift in Sources */, 9CD0DEFE83D73972985D2F1C /* UnitConversionEvaluatorTests.swift in Sources */, + 67A8673F38C847E921CC8BEE /* VisualContextChangePolicyTests.swift in Sources */, E5B8A88A1BC2F84444EABDE0 /* VisualContextCoordinatorTests.swift in Sources */, 9259C21BDBE17A814F0C6431 /* VisualContextExcerptSelectorTests.swift in Sources */, DC6FEE1602B6891CA4D0CDCA /* VisualContextStartCoalescerTests.swift in Sources */, diff --git a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Input.swift b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Input.swift index 4ec63409a..21a2a4f84 100644 --- a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Input.swift +++ b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Input.swift @@ -1,3 +1,4 @@ +import CoreGraphics import Foundation import Logging @@ -220,10 +221,19 @@ extension SuggestionCoordinator { // null the freshly primed session out from under us. await self.awaitCachedGenerationContextResetIfNeeded() let prewarmContext = FocusedInputContext(snapshot: rawContext, generation: 0) + // Build the prompt from the same context sources the first real request will read + // (`SuggestionCoordinator+Prediction`): the pinned clipboard and any ready screen + // excerpt. The prefill is only worth its decode if the next request shares its bytes; + // leaving these out made the first request diverge right after the surface line and + // re-decode everything the prewarm had already paid for. let request = SuggestionRequestFactory.buildRequest( context: prewarmContext, settings: settings, configuration: configuration, + clipboardContext: self.pinnedClipboardContext(rawContext: rawContext), + visualContextSummary: self.permissionManager.screenRecordingGranted + ? self.visualContextCoordinator.excerpt(for: prewarmContext) + : nil, historyExamples: self.historyExamples(for: prewarmContext) ).request await suggestionEngine.prewarm(for: request) @@ -334,11 +344,40 @@ extension SuggestionCoordinator { // *before* the host app processes the keystroke, so a synchronous `refreshNow()` here // reads pre-keystroke text and feeds it into generation. The result is a suggestion // that looks like the typed character was ignored — see - // `schedulePredictionAfterHostPublishDelay` for the full rationale. + // `schedulePredictionAfterHostPublishDelay` for the full rationale. A plain typed + // character can still start generating now, against the text the host is about to + // publish; the poll then validates that bet instead of gating the request. + speculateOnTypedCharacter(event) schedulePredictionAfterHostPublishDelay() } } + /// Starts the keystroke-time speculative generation for a plain typed character. The base is + /// the last captured snapshot, extended by the letters a visible suggestion was typed through + /// (`session`), which the snapshot may not include yet. When it still misses an unpublished key, + /// the speculated text simply misses and the publish poll falls back to the ordinary request. + private func speculateOnTypedCharacter(_ event: CapturedInputEvent, after session: ActiveSuggestionSession? = nil) { + guard event.kind == .textMutation, let typed = Self.speculatedCharacters(for: event), + let raw = focusModel.snapshot.context else { return } + let typedThrough = session.flatMap { session -> String? in + guard session.baseContext.elementIdentifier == raw.elementIdentifier else { return nil } + return session.baseContext.precedingText + String(session.fullText.prefix(session.consumedCharacterCount)) + } + dispatchSpeculativeTypingGeneration(rawContext: raw, typed: typed, knownPrecedingText: typedThrough) + } + + /// The character the host will most likely insert for `event`, or nil when it cannot be told + /// from the event alone. A key event's Unicode string does not always carry Shift (synthetic + /// events report "s" for a shifted S), so a letter follows Shift XOR Caps Lock; uppercasing an + /// already-capital letter changes nothing. A shifted symbol depends on the keyboard layout + /// ("/" becomes "?" on US layouts), so it is left to the ordinary publish-gated request. + static func speculatedCharacters(for event: CapturedInputEvent) -> String? { + guard event.characters.count == 1, let character = event.characters.first else { return nil } + let shift = event.flags.contains(.maskShift) + guard character.isLetter else { return shift ? nil : event.characters } + return shift != event.flags.contains(.maskAlphaShift) ? event.characters.uppercased() : event.characters + } + /// Maximum wall time we'll wait for the host app to publish post-keystroke AX before giving /// up and generating against whatever's there. Chosen empirically: long enough to cover /// Chrome's slower contenteditable publish on a busy page, short enough that the user can @@ -354,7 +393,16 @@ extension SuggestionCoordinator { /// own tap callback before the host has even processed the key and therefore always misses. A /// short first retry catches fast-publishing native apps in ~10ms instead of making them wait a /// full steady interval; slow Chromium hosts fall through to `hostPublishPollIntervalMs` below. - private static let hostPublishFirstPollIntervalMs = 10 + private static let hostPublishFirstPollIntervalMs = 6 + + /// Native hosts publish a keystroke to AX within roughly 10-40 ms, so the wait polls densely + /// during that window and only then backs off to `hostPublishPollIntervalMs` for slow Chromium + /// hosts. With a 30 ms step from the start, a publish that landed just after the first retry + /// waited a further ~30 ms (live TextEdit trace: key-to-request 50 ms instead of ~20 ms), the + /// largest remaining wait before generation. The dense phase costs a handful of ~2-3 ms AX + /// reads per keystroke and ends after `hostPublishFastPhaseMs`. + private static let hostPublishFastPollIntervalMs = 8 + private static let hostPublishFastPhaseMs = 60 /// Schedules a fresh prediction once the host app has actually published the new /// contenteditable text to AX. The previous fix waited a fixed 150ms — see PR #376 — but the @@ -402,6 +450,27 @@ extension SuggestionCoordinator { } } + /// Records why a publish did not match the speculated text, without logging either text: the + /// lengths, selections, and the first differing UTF-16 offset are enough to tell a lagging base + /// snapshot from a host transformation (smart punctuation, auto-pairing, a shifted key). + private func logSpeculationMiss(expected: FocusedInputContext, published: FocusedInputSnapshot) { + let lhs = Array(expected.precedingText.utf16), rhs = Array(published.precedingText.utf16) + let shared = zip(lhs, rhs).prefix { $0 == $1 }.count + CotabbyLogger.suggestion.debug( + "Speculation missed the published text", + metadata: [ + "stage": .string("speculation-missed"), + "same_session": .stringConvertible(expected.sessionIdentity == published.sessionIdentity), + "expected_utf16": .stringConvertible(lhs.count), + "published_utf16": .stringConvertible(rhs.count), + "first_difference": .stringConvertible(shared), + "expected_selection": .stringConvertible(expected.selection.location), + "published_selection": .stringConvertible(published.selection.location), + "trailing_matches": .stringConvertible(expected.trailingText == published.trailingText) + ] + ) + } + /// The AX state captured at keystroke time, against which the host-publish poll detects "the /// host has processed the key", plus the uptime anchor for folding the wait into the debounce. private struct HostPublishBaseline { @@ -471,6 +540,9 @@ extension SuggestionCoordinator { ) return } + if let expected = pendingSpeculativeContext, let published = currentContext { + logSpeculationMiss(expected: expected, published: published) + } pendingSpeculativeContext = nil schedulePrediction( consumedDelayMilliseconds: Self.elapsedMilliseconds(since: baseline.keystrokeUptimeNanoseconds) @@ -482,8 +554,11 @@ extension SuggestionCoordinator { // produces no AX change (rare but possible — e.g. dead-key composition) would never get // its next prediction. // We already waited the short first interval before entering this method; remaining polls - // use the steady cadence until the ceiling. - let interval = Self.hostPublishPollIntervalMs + // stay dense while native hosts usually publish, then use the steady cadence until the + // ceiling. + let interval = elapsedMs < Self.hostPublishFastPhaseMs + ? Self.hostPublishFastPollIntervalMs + : Self.hostPublishPollIntervalMs let nextElapsed = elapsedMs + interval guard nextElapsed < Self.hostPublishWaitCeilingMs else { if baseline.requiresTextChange { @@ -546,6 +621,7 @@ extension SuggestionCoordinator { clearDiagnostics: false ) if event.shouldSchedulePrediction { + speculateOnTypedCharacter(event, after: session) schedulePredictionAfterHostPublishDelay() } return false diff --git a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Prediction.swift b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Prediction.swift index 3853fd634..32c6a25dc 100644 --- a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Prediction.swift +++ b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+Prediction.swift @@ -26,12 +26,14 @@ extension SuggestionCoordinator { return } - // The debounce window adapts to the last generation latency: snappier when the model is - // fast, calmer when it is slow (fewer doomed generations to cancel). The configured value - // is the fallback until a first latency exists. + // The debounce window adapts to how quickly the engine produces text: none when a warm + // request shows its first text fast, calmer when the model is slow (fewer doomed + // generations to cancel). The configured value is the fallback until a latency exists. let debounceMilliseconds = DebouncePolicy.milliseconds( lastGenerationLatencyMilliseconds: lastLatencyByEngine[settingsSnapshot.selectedEngine], + lastFirstTextLatencyMilliseconds: + typicalFirstTextLatency(engine: settingsSnapshot.selectedEngine), fallback: settingsSnapshot.debounceMilliseconds, engine: settingsSnapshot.selectedEngine ) @@ -252,11 +254,46 @@ extension SuggestionCoordinator { guard !userDefaults.bool(forKey: Self.speculativePrefetchDisabledDefaultsKey) else { return } guard !insertionChunk.isEmpty else { return } - let optimistic = SpeculativeAcceptanceContext.optimisticSnapshot( - after: rawContext, - inserting: insertionChunk + dispatchSpeculativeGeneration( + optimistic: SpeculativeAcceptanceContext.optimisticSnapshot(after: rawContext, inserting: insertionChunk), + message: "Started the post-acceptance generation against the expected post-insert text." ) + } + /// Starts generating for a plain typed character at keystroke time, against the text the host + /// is about to publish, instead of after the host-publish poll sees it (10-50 ms in native + /// hosts, longer in Chromium). The poll that follows validates exactly as it does for + /// post-acceptance speculation: a matching publish keeps this request, anything else (smart + /// quotes, dead keys, IME, auto-pairing, a key the host had not yet applied to the snapshot) + /// schedules the ordinary request and retires this one. Results and streamed partials are only + /// shown once the live text matches, so a wrong bet costs one discarded generation, never a + /// ghost at the wrong caret. + /// + /// Endpoint requests keep their debounced path: speculative traffic must not multiply requests + /// to a configured server. + /// + /// `knownPrecedingText` is the text before the caret as Cotabby already knows it when the + /// captured snapshot lags: typing through a visible suggestion advances the session without a + /// new AX capture, so the snapshot can still miss those letters. It is used only when it + /// extends the snapshot's own text. + func dispatchSpeculativeTypingGeneration( + rawContext: FocusedInputSnapshot, typed characters: String, knownPrecedingText: String? = nil + ) { + guard settingsSnapshot.selectedEngine != .openAICompatible, + !userDefaults.bool(forKey: Self.speculativePrefetchDisabledDefaultsKey), + characters.count == 1, let typed = characters.first, !typed.isNewline, + rawContext.selection.length == 0, !rawContext.isSecure else { return } + let base = knownPrecedingText.flatMap { $0.hasPrefix(rawContext.precedingText) ? $0 : nil } + ?? rawContext.precedingText + dispatchSpeculativeGeneration( + optimistic: SpeculativeAcceptanceContext.optimisticSnapshot(after: rawContext, precedingText: base + characters), + message: "Started generating against the typed character before the host published it." + ) + } + + /// Shared body of the speculative paths: gate, build the request for `optimistic`, and run it + /// with no debounce while `pendingSpeculativeContext` lets the publish poll validate it. + private func dispatchSpeculativeGeneration(optimistic: FocusedInputSnapshot, message: String) { // Same pre-generation gates the ordinary cycle applies, minus their UI side effects: a // speculative request must not spend a decode on text the normal path would refuse (too // little text) or suppress (typo gate). The post-publish regeneration still runs the full @@ -306,7 +343,7 @@ extension SuggestionCoordinator { "speculative-generating", workID: workID, generation: context.generation, - message: "Started the post-acceptance generation against the expected post-insert text.", + message: message, prompt: requestBuildResult.promptPreview ) } @@ -576,7 +613,25 @@ extension SuggestionCoordinator { } return rebased } - return liveContext.generation == partial.generation ? partial : nil + if liveContext.generation == partial.generation { return partial } + // A speculative request was built against text the host had not published yet, so any + // read of the pre-publish snapshot moves the live generation past the partial's. Like + // `apply`, accept it by content instead: once the live field reads exactly what was + // speculated, the partial is current; before that it stays hidden. + if let expected = pendingSpeculativeContext, + expected.sessionIdentity == liveContext.sessionIdentity, + expected.contentSignature == liveContext.contentSignature, + expected.generation == partial.generation { + return SuggestionResult( + generation: liveContext.generation, + rawText: partial.rawText, + text: partial.text, + latency: partial.latency, + suppressionReason: partial.suppressionReason, + spacingIsExact: partial.spacingIsExact + ) + } + return nil } /// Runs the typo gate for the current word. Returns `true` when it handled the cycle by suppressing, diff --git a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+TypingPrediction.swift b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+TypingPrediction.swift index b56e49b25..8acdcfd41 100644 --- a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+TypingPrediction.swift +++ b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator+TypingPrediction.swift @@ -67,6 +67,10 @@ extension SuggestionCoordinator { /// presentation; the hidden buffer still lets a complete answer survive intervening typing. func receiveTypingPartial(_ result: SuggestionResult, workID: UInt64, showPartials: Bool) { guard workController.isCurrent(workID) else { return } + if firstTextRecordedWorkID != workID { + firstTextRecordedWorkID = workID + recordFirstTextLatency(Int((result.latency * 1_000).rounded()), engine: settingsSnapshot.selectedEngine) + } if var candidate = typingPrediction { guard !candidate.isFinal, result.generation == candidate.context.generation else { return } guard candidate.receive(result, final: false) else { diff --git a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator.swift b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator.swift index 298ec3811..bebf08ef9 100644 --- a/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator.swift +++ b/Cotabby/App/Coordinators/Suggestion/SuggestionCoordinator.swift @@ -77,6 +77,24 @@ final class SuggestionCoordinator: ObservableObject { /// a slow HTTP backend retain an appropriate debounce without leaking that latency into the /// in-process engines. var lastLatencyByEngine: [SuggestionEngineKind: Int] = [:] + /// Latency to the first streamed text of the most recent requests per backend: what starting a + /// request costs once the KV prefix is warm. `DebouncePolicy` prefers it, because a superseded + /// request stops at its next token while a debounce delays every keystroke. Recorded once per + /// work item (`firstTextRecordedWorkID`), only for the request that is still current, and read + /// as the median of the last few so one cache miss (a context refresh re-decoding a whole + /// section) cannot put the next keystroke behind the slow tier. + var recentFirstTextLatenciesByEngine: [SuggestionEngineKind: [Int]] = [:] + var firstTextRecordedWorkID: UInt64? + + func recordFirstTextLatency(_ milliseconds: Int, engine: SuggestionEngineKind) { + recentFirstTextLatenciesByEngine[engine, default: []].append(milliseconds) + recentFirstTextLatenciesByEngine[engine] = Array(recentFirstTextLatenciesByEngine[engine, default: []].suffix(3)) + } + + func typicalFirstTextLatency(engine: SuggestionEngineKind) -> Int? { + let recent = recentFirstTextLatenciesByEngine[engine, default: []].sorted() + return recent.isEmpty ? nil : recent[recent.count / 2] + } // Synchronous input/focus callbacks cannot directly `await`, so resets are represented as a // barrier task that the next generation must cross before it can ask the runtime for output. var cacheResetSequence: UInt64 = 0 diff --git a/Cotabby/Models/Onboarding/OnboardingTemplate.swift b/Cotabby/Models/Onboarding/OnboardingTemplate.swift index ed11e0b9d..60c3d97f7 100644 --- a/Cotabby/Models/Onboarding/OnboardingTemplate.swift +++ b/Cotabby/Models/Onboarding/OnboardingTemplate.swift @@ -86,16 +86,19 @@ enum OnboardingTemplate: String, CaseIterable, Identifiable, Equatable, Sendable } } + /// Short suggestions are both faster (fewer generated tokens) and more often exactly right, and + /// accepting one extends it with a continuation, so every tier but Powerful starts short. + /// Powerful keeps the long preset for users who explicitly choose longer suggestions. var wordCountPreset: SuggestionWordCountPreset { switch self { case .quick: - return .fourToSeven + return .twoToFour case .everyday: - return .sevenToTwelve + return .fourToSeven case .powerful: return .twelveToTwenty case .custom: - return .sevenToTwelve + return .fourToSeven } } diff --git a/Cotabby/Models/Suggestion/Request/SuggestionConfiguration.swift b/Cotabby/Models/Suggestion/Request/SuggestionConfiguration.swift index 4654b8975..7f0d447ef 100644 --- a/Cotabby/Models/Suggestion/Request/SuggestionConfiguration.swift +++ b/Cotabby/Models/Suggestion/Request/SuggestionConfiguration.swift @@ -110,8 +110,8 @@ struct SuggestionConfiguration: Equatable, Sendable { let focusPollIntervalMilliseconds: Int - /// Output reserve built into the configured llama prompt budget. It covers the default - /// single-line request (12-20 words, 26 tokens) with room to spare; a request that asks for + /// Output reserve built into the configured llama prompt budget. It covers the longest + /// single-line preset (12-20 words, 26 tokens) with room to spare; a request that asks for /// more has the excess taken out of its own prompt (`SuggestionRequestFactory.promptTokenBudget`). static let llamaPromptOutputCeilingTokens = 50 /// Margin for BOS plus token-estimator error; the estimator skews conservative, so real @@ -146,11 +146,11 @@ struct SuggestionConfiguration: Equatable, Sendable { // can hold after the preface), not by a word cap: a 150-word cap left most of a long // document out of the prompt, and the model's sense of the topic with it. The caps here // only stop pathological fields (a megabyte of log text) from being windowed each poll. - // Latency honesty: where KV prefix reuse works (dense models), the window is prefilled - // once per field and every keystroke after that decodes only the delta (measured: one - // token); the hybrid/SWA catalog models reject partial trims and re-prefill per request, - // so there the wider window costs prefill only in long-document sessions, which is - // exactly where it buys quality. + // Latency honesty: dense and sliding-window models (Gemma, via the native `swa_full` + // cache) prefill the window once per field, and every keystroke after that decodes only + // the delta. Hybrid/recurrent catalog models restore from a near-caret checkpoint and + // re-prefill when an edit lands before it, so there the wider window costs prefill only + // in long-document sessions, which is exactly where it buys quality. maxPrefixWords: 2400, maxPrefixCharacters: 14000, // Apple's on-device model has a 4096-token shared context. Even with instructions plus @@ -167,7 +167,12 @@ struct SuggestionConfiguration: Equatable, Sendable { // endpoint backend can carry that prompt off the Mac, so a real name is an explicit opt-in // and a placeholder would sign every user's mail with someone else's name. defaultUserName: nil, - defaultWordCountPreset: .twelveToTwenty, + // Short by default: at ~15-20 ms per generated token on a 4B-class model, a 12-20 word + // budget (26 tokens) spent most of each request decoding words the user rarely kept, while + // the next keystroke was already cancelling it. A 4-7 word suggestion (10 tokens) lands in + // a fraction of that, is right more often, and accepting it schedules the continuation + // that extends it. Longer presets stay one selection away in Settings. + defaultWordCountPreset: .fourToSeven, focusPollIntervalMilliseconds: 50 ) } diff --git a/Cotabby/Services/History/TypingHistoryStore.swift b/Cotabby/Services/History/TypingHistoryStore.swift index e16a5e11a..35f51b4d9 100644 --- a/Cotabby/Services/History/TypingHistoryStore.swift +++ b/Cotabby/Services/History/TypingHistoryStore.swift @@ -73,6 +73,9 @@ final class TypingHistoryStore: ObservableObject, SuggestionHistoryProviding { private static let maximumRecentRecordings = 64 /// Ranked example candidates for the current query block (see `historyExamples`). private var exampleCache: (key: String, candidates: [TypingHistoryIndex.Candidate])? + /// The examples most recently handed to a prompt, per field, so a new ranking that still rates + /// them highly keeps the prompt (and its KV cache) unchanged. Cleared with `exampleCache`. + private var shownExamples: (fieldKey: UInt64, examples: [String])? /// The field being typed in right now. Its raw text is kept here and only scrubbed and copied /// into `records` when saving or when focus moves on, so recording costs a string comparison @@ -128,6 +131,7 @@ final class TypingHistoryStore: ObservableObject, SuggestionHistoryProviding { preferences.isUsingHistory = enabled userDefaults.set(enabled, forKey: DefaultsKey.isUsingHistory) exampleCache = nil + shownExamples = nil } func setRecording(_ enabled: Bool) { @@ -251,6 +255,7 @@ final class TypingHistoryStore: ObservableObject, SuggestionHistoryProviding { self.index = built.0 self.phrases = built.1 self.exampleCache = nil + self.shownExamples = nil } } @@ -524,6 +529,7 @@ final class TypingHistoryStore: ObservableObject, SuggestionHistoryProviding { index = nil phrases = nil exampleCache = nil + shownExamples = nil rebuildGeneration += 1 lastImportMessage = nil deletionError = nil @@ -553,7 +559,44 @@ final class TypingHistoryStore: ObservableObject, SuggestionHistoryProviding { } // The ranking holds for a whole block of words, but the field keeps growing inside the // block, so the check against echoing the user's own draft runs on the live text each time. - return TypingHistoryIndex.examples(from: candidates, currentFieldText: context.precedingText) + let fresh = TypingHistoryIndex.examples(from: candidates, currentFieldText: context.precedingText) + // Keep the examples already in the prompt while the new ranking still rates them among its + // best few. Each change of the examples re-decodes that prompt section and everything after + // it (~100 ms on a 4B-class model), and a block boundary often only reshuffles near-equal + // matches; a passage that dropped out of the top candidates, or now echoes the draft, is + // still replaced at once. + let fieldKey = context.focusedInputIdentityKey + var examples = fresh + if let shown = shownExamples, shown.fieldKey == fieldKey, !shown.examples.isEmpty { + let stillRanked = TypingHistoryIndex.examples( + from: candidates, currentFieldText: context.precedingText, limit: Self.stickyExampleRankLimit + ) + examples = Self.stableExamples(shown: shown.examples, ranked: stillRanked, fresh: fresh) + } + shownExamples = (fieldKey, examples) + return examples + } + + /// How far down the new ranking an example already in the prompt may fall and still be kept. + private static let stickyExampleRankLimit = 6 + + /// Keeps every shown example that is still ranked, in its position, and fills only the slots of + /// examples that dropped out with the best fresh ones. The prompt then diverges at the first + /// replaced example instead of at the start of the section, so less of it is re-decoded. + static func stableExamples(shown: [String], ranked: [String], fresh: [String]) -> [String] { + var replacements = fresh.filter { !shown.contains($0) } + var result: [String] = [] + for example in shown { + if ranked.contains(example) { + result.append(example) + } else if !replacements.isEmpty { + result.append(replacements.removeFirst()) + } + } + while result.count < fresh.count, !replacements.isEmpty { + result.append(replacements.removeFirst()) + } + return result } func phraseContinuation(for request: SuggestionRequest, engine: SuggestionEngineKind) -> String? { diff --git a/Cotabby/Services/Input/InputMonitor.swift b/Cotabby/Services/Input/InputMonitor.swift index 60956f9ca..0337d67d7 100644 --- a/Cotabby/Services/Input/InputMonitor.swift +++ b/Cotabby/Services/Input/InputMonitor.swift @@ -107,6 +107,10 @@ final class InputMonitor { private var acceptTap: CFMachPort? private var acceptRunLoopSource: CFRunLoopSource? + /// Whether `acceptTap` is currently intercepting. Between suggestions the tap is disabled + /// rather than destroyed (see `updateAcceptTapState`), so this distinguishes an intentional + /// pause from the system disabling it on timeout, which the callback re-enables. + private var acceptTapEnabled = false /// Dedicated consuming tap for the global-toggle hotkey. Lives independently of the accept tap /// because it must fire even when no suggestion is visible — and even when Cotabby is globally @@ -185,16 +189,23 @@ final class InputMonitor { private static let disabledKeyCode: CGKeyCode = CGKeyCode(UInt16.max) - /// How long the accept tap lingers (fail-open) after the overlay hides before its mach port is - /// invalidated. A final-chunk accept runs *inside* this tap's own callback: it posts the - /// synthetic insertion to `.cghidEventTap` and then hides the overlay, which routes back here to - /// tear the tap down. Invalidating the mach port in that same run-loop turn pulls the tap out of - /// the session tap chain before the just-posted synthetic keystroke drains through it, so the - /// last accepted word never reaches the host (non-final words keep the overlay visible and never - /// hit this path, which is why only the final word failed to commit). Deferring the invalidation - /// one short hop lets the synthetic event finish delivery first. This is the invariant PR #385 - /// ("Fix last tab issue") protected before the two-tap ownership rewrite dropped it. - private static let acceptTapTeardownDelaySeconds: TimeInterval = 0.05 + /// How long the accept tap stays enabled after the last suggestion hid. Switching an active tap + /// off pulls it out of the session tap chain, and an event still on its way through it is lost: + /// a final-chunk accept runs *inside* this tap's own callback, posts the synthetic insertion to + /// `.cghidEventTap`, then hides the overlay, and tearing the tap down in that same run-loop turn + /// once made the last accepted word never reach the host (PR #385, "Fix last tab issue"). The + /// user's own keystrokes are exposed the same way: toggling the tap around every keystroke of a + /// burst dropped about one typed character in a hundred in live TextEdit runs, and none with + /// Cotabby quit. Two seconds spans a typing burst, so the tap is only switched off once input + /// has gone quiet, and it also covers the synthetic-insertion drain. + private static let acceptTapIdleDisableSeconds: TimeInterval = 2 + /// How long a disabled accept tap is kept for its next suggestion before it is released. + private static let acceptTapReleaseDelaySeconds: TimeInterval = 10 + private var acceptTapStateGeneration: UInt64 = 0 + + private var acceptTapWanted: Bool { + permissionProvider() && (suggestionInterceptionActive || captureInterceptionActive) + } /// Suggestion-overlay reason for the active tap. The coordinator calls this when a suggestion /// becomes visible or hidden, so Cotabby only enters the synchronous event path while there is @@ -224,18 +235,34 @@ final class InputMonitor { // through immediately. isAcceptTapOwningAcceptKeys = wantsTap && suggestionInterceptionActive if wantsTap { + acceptTapStateGeneration &+= 1 installAcceptTapIfNeeded() } else { - // Defer only the mach-port invalidation, so a final-chunk accept's synthetic insertion can - // drain before the tap is removed (see `acceptTapTeardownDelaySeconds`). Re-check both + // Defer the disable, so a final-chunk accept's synthetic insertion can drain before the + // tap stops intercepting (see `acceptTapIdleDisableSeconds`). Re-check both // reasons at fire time so a suggestion or an emoji capture that re-armed the tap during the // delay keeps it installed. - DispatchQueue.main.asyncAfter(deadline: .now() + Self.acceptTapTeardownDelaySeconds) { [weak self] in - guard let self else { return } - let stillWanted = self.permissionProvider() - && (self.suggestionInterceptionActive || self.captureInterceptionActive) - guard !stillWanted else { return } - self.destroyAcceptTap() + // + // While typing, a ghost hides and reappears on nearly every keystroke. Destroying the tap + // each time made the next appearance re-create it, a synchronous WindowServer round trip + // (~20-30 ms measured) on the main thread right before the ghost could be presented. A + // disabled tap intercepts nothing, so typing stays off this synchronous path exactly as + // before; re-enabling it is cheap. It is released only after a longer idle period. + // + // The tap is disabled only after a quiet period, never between keystrokes of a burst. + // Switching an active tap off while a keystroke is in flight through it can lose that + // keystroke: live runs dropped one character in roughly a hundred while the tap was + // toggled around every keystroke, and none with Cotabby quit. Every state change bumps + // `acceptTapStateGeneration`, so only a disable scheduled after the last change runs. + acceptTapStateGeneration &+= 1 + let generation = acceptTapStateGeneration + DispatchQueue.main.asyncAfter(deadline: .now() + Self.acceptTapIdleDisableSeconds) { [weak self] in + guard let self, generation == self.acceptTapStateGeneration, !self.acceptTapWanted else { return } + self.disableAcceptTap() + DispatchQueue.main.asyncAfter(deadline: .now() + Self.acceptTapReleaseDelaySeconds) { [weak self] in + guard let self, !self.acceptTapWanted, !self.acceptTapEnabled else { return } + self.destroyAcceptTap() + } } } } @@ -285,8 +312,13 @@ final class InputMonitor { private func installAcceptTapIfNeeded() { // Ownership of the accept key is decided by `updateAcceptTapState`, not here, so this method - // only guarantees the tap exists. - guard acceptTap == nil else { + // only guarantees the tap exists and is intercepting. + if let acceptTap { + if !acceptTapEnabled { + CGEvent.tapEnable(tap: acceptTap, enable: true) + acceptTapEnabled = true + CotabbyLogger.app.debug("CGEvent accept tap re-enabled") + } return } @@ -327,6 +359,17 @@ final class InputMonitor { } CGEvent.tapEnable(tap: tap, enable: true) + acceptTapEnabled = true + } + + /// Stops the accept tap from intercepting without releasing it, so the next visible + /// suggestion only has to re-enable it. + private func disableAcceptTap() { + isAcceptTapOwningAcceptKeys = false + guard let acceptTap, acceptTapEnabled else { return } + CGEvent.tapEnable(tap: acceptTap, enable: false) + acceptTapEnabled = false + CotabbyLogger.app.debug("CGEvent accept tap disabled") } private func installToggleTapIfNeeded() { @@ -400,6 +443,7 @@ final class InputMonitor { CFMachPortInvalidate(tap) } acceptTap = nil + acceptTapEnabled = false CotabbyLogger.app.info("CGEvent accept tap removed") } @@ -547,8 +591,9 @@ final class InputMonitor { func handleAcceptTap(type: CGEventType, event: CGEvent) -> Unmanaged? { switch type { case .tapDisabledByTimeout, .tapDisabledByUserInput: - CotabbyLogger.app.warning("Accept tap was disabled by system, re-enabling") - if let acceptTap { + // Only undo a system-initiated disable; a tap Cotabby paused between suggestions stays off. + if let acceptTap, acceptTapEnabled { + CotabbyLogger.app.warning("Accept tap was disabled by system, re-enabling") CGEvent.tapEnable(tap: acceptTap, enable: true) } return Unmanaged.passUnretained(event) diff --git a/Cotabby/Services/Runtime/Llama/LlamaRuntimeCore.swift b/Cotabby/Services/Runtime/Llama/LlamaRuntimeCore.swift index 1a18da0f8..1d21d5598 100644 --- a/Cotabby/Services/Runtime/Llama/LlamaRuntimeCore.swift +++ b/Cotabby/Services/Runtime/Llama/LlamaRuntimeCore.swift @@ -35,6 +35,10 @@ nonisolated final class LlamaRuntimeCore: @unchecked Sendable { private var autocompletePromptBytes: [UInt8] = [] private var autocompletePromptTokens: [Int32] = [] private var autocompleteSamplingFingerprint: SamplingFingerprint? + /// Prompt tokens the most recent generate/prefill kept from native KV instead of decoding + /// (0 after a fresh build). Diagnostics and tests read it to prove reuse actually happened; + /// equal outputs alone cannot distinguish a reused cache from a silent rebuild. + private(set) var lastReusedPromptTokens = 0 /// The sequence the in-flight autocomplete operation is decoding into, published for /// `abortInFlightGeneration` to target from the canceller's thread. Guarded by its own lock @@ -317,13 +321,13 @@ nonisolated final class LlamaRuntimeCore: @unchecked Sendable { let healingPrefix = healing.replayBytes // A byte-fallback vocabulary can replay at most one token per prefix byte. Reserve a // separate bounded allowance so healing cannot consume the user's output-token budget. + // A front-truncated prompt shifts every token position, so its token prefix no longer + // matches the cached one and reuse simply finds nothing in common; no flag is needed. let maxPromptTokens = max(1, preparedRuntime.contextWindowTokens - options.maxPredictionTokens - healingPrefix.count) - let truncated = tokens.count > maxPromptTokens return PreparedPrompt( promptBytes: promptBytes, promptTokens: Array(tokens.suffix(maxPromptTokens)), - cachedPrefixBytes: truncated ? nil : cachedPrefixBytes, healingPrefix: healingPrefix, fingerprint: SamplingFingerprint(options: options) ) @@ -332,7 +336,6 @@ nonisolated final class LlamaRuntimeCore: @unchecked Sendable { private struct PreparedPrompt { let promptBytes: [UInt8] let promptTokens: [Int32] - let cachedPrefixBytes: Int? let healingPrefix: [UInt8] let fingerprint: SamplingFingerprint } @@ -561,6 +564,12 @@ nonisolated final class LlamaRuntimeCore: @unchecked Sendable { /// retain generated tokens or a cancellation flag; restoration is mandatory before reuse, /// even when both prompt descriptions are identical. A missed checkpoint rebuilds cold. /// Must be called while holding `autocompleteLock`. + /// + /// Reuse is decided here from the token prefix alone. KV contents are a pure function of the + /// token sequence and its positions, so a shared token prefix is always safe to keep no matter + /// which field, frame, or request produced it; the caller's byte hint is diagnostic only. + /// Requiring the hint used to discard the cache whenever the hint tracker had no record + /// (every cancelled request) or saw the field's frame grow while the user typed. private func obtainAutocompleteSequence( preparation: PreparedPrompt, options: LlamaGenerationOptions @@ -568,16 +577,11 @@ nonisolated final class LlamaRuntimeCore: @unchecked Sendable { let promptTokens = preparation.promptTokens let promptBytes = preparation.promptBytes let fingerprint = preparation.fingerprint - let cachedPrefixBytes = preparation.cachedPrefixBytes let healingPrefix = preparation.healingPrefix if autocompleteSequenceID >= 0, - let cachedPrefixBytes, cachedPrefixBytes > 0, - autocompleteSamplingFingerprint == fingerprint { + adoptSamplingIfNeeded(fingerprint, options: options) { - let confirmedCommonBytes = min( - cachedPrefixBytes, - Self.commonPrefixCount(autocompletePromptBytes, promptBytes) - ) + let confirmedCommonBytes = Self.commonPrefixCount(autocompletePromptBytes, promptBytes) if confirmedCommonBytes > 0 { let commonTokenPrefix = Self.commonPrefixCount(autocompletePromptTokens, promptTokens) @@ -630,6 +634,7 @@ nonisolated final class LlamaRuntimeCore: @unchecked Sendable { return try buildFreshSequence(promptTokens: promptTokens, healingPrefix: healingPrefix, options: options) } } + lastReusedPromptTokens = reusableTokenCount CotabbyLogger.runtime.debug( "KV prefix reused", metadata: [ @@ -652,11 +657,26 @@ nonisolated final class LlamaRuntimeCore: @unchecked Sendable { return try buildFreshSequence(promptTokens: promptTokens, healingPrefix: healingPrefix, options: options) } + /// Brings the live sequence's sampler in line with this request without touching its KV. + /// A sampling change (temperature, top-k, single-line mask) only affects how the next tokens + /// are drawn, never the cached prompt, so the native sampler chain is rebuilt in place instead + /// of destroying the reusable prefix. Returns false when the swap fails; the caller then + /// rebuilds the sequence from scratch. Must be called while holding `autocompleteLock`. + private func adoptSamplingIfNeeded(_ fingerprint: SamplingFingerprint, options: LlamaGenerationOptions) -> Bool { + guard autocompleteSamplingFingerprint != fingerprint else { return true } + guard engine.updateSampling(autocompleteSequenceID, Self.samplingConfig(from: options)) else { + return false + } + autocompleteSamplingFingerprint = fingerprint + return true + } + private func buildFreshSequence( promptTokens: [Int32], healingPrefix: [UInt8], options: LlamaGenerationOptions ) throws -> Int32 { + lastReusedPromptTokens = 0 let config = Self.samplingConfig(from: options) let seqID = engine.createSequence(config) guard seqID >= 0 else { @@ -783,9 +803,11 @@ nonisolated final class LlamaRuntimeCore: @unchecked Sendable { return index } - /// Generation knobs that intentionally break KV reuse when changed. + /// The settings baked into the native sampler chain. A change swaps the chain in place + /// (`adoptSamplingIfNeeded`); it never invalidates the KV prefix. The output-token budget is + /// deliberately absent: the Swift decode loop enforces it, so performance tuning that moves the + /// budget between requests must not look like a different sampler. private struct SamplingFingerprint: Equatable { - let maxPredictionTokens: Int let temperature: Double let topK: Int let topP: Double @@ -795,7 +817,6 @@ nonisolated final class LlamaRuntimeCore: @unchecked Sendable { let singleLine: Bool init(options: LlamaGenerationOptions) { - maxPredictionTokens = options.maxPredictionTokens temperature = options.temperature topK = options.topK topP = options.topP diff --git a/Cotabby/Services/Runtime/Llama/LlamaSuggestionEngine.swift b/Cotabby/Services/Runtime/Llama/LlamaSuggestionEngine.swift index a3baa0dd1..958ae82f3 100644 --- a/Cotabby/Services/Runtime/Llama/LlamaSuggestionEngine.swift +++ b/Cotabby/Services/Runtime/Llama/LlamaSuggestionEngine.swift @@ -289,9 +289,10 @@ final class LlamaSuggestionEngine { extension LlamaSuggestionEngine: SuggestionGenerating {} -/// Tracks the last successful llama prompt so the engine can pass a conservative byte-prefix hint -/// into `LlamaRuntimeManager.generate`. This type deliberately does not own correctness: native KV -/// state is still validated by `LlamaRuntimeCore` after tokenization. +/// Tracks the last successful llama prompt so the engine can pass a byte-prefix hint into +/// `LlamaRuntimeManager.generate` for diagnostics (`cache_hint_bytes` in the logs). This type +/// deliberately does not own correctness or gate reuse: `LlamaRuntimeCore` keeps whatever token +/// prefix the new prompt shares with native KV, whether or not a hint was offered. struct LlamaPromptCacheHintTracker: Equatable { private var lastRequest: CachedRequest? @@ -387,8 +388,10 @@ private extension LlamaPromptCacheHintTracker { } } + /// Mirrors the native sampler settings. The output-token budget is excluded for the same reason + /// as in `LlamaRuntimeCore`: performance tuning moves it between requests without changing + /// anything a cached prompt depends on. struct SamplingFingerprint: Equatable { - let maxPredictionTokens: Int let temperature: Double let topK: Int let topP: Double @@ -397,7 +400,6 @@ private extension LlamaPromptCacheHintTracker { let randomSeed: UInt32? init(request: SuggestionRequest) { - maxPredictionTokens = request.maxPredictionTokens temperature = request.temperature topK = request.topK topP = request.topP diff --git a/Cotabby/Services/Visual/VisualContextCoordinator.swift b/Cotabby/Services/Visual/VisualContextCoordinator.swift index a5d09c5ae..4e0175ade 100644 --- a/Cotabby/Services/Visual/VisualContextCoordinator.swift +++ b/Cotabby/Services/Visual/VisualContextCoordinator.swift @@ -202,21 +202,27 @@ final class VisualContextCoordinator { guard !Task.isCancelled else { return } + // The field as it reads now, not at capture start: everything the capture saw of + // the user's typing is already in it, so the change policy can discount all of it. + var fieldContext = snapshotContext if let provider = refreshContextProvider { let liveContext = provider() guard activeAugmentationSession?.sessionID == session.sessionID else { return } - guard screenRecordingPermissionProvider(), liveContext?.identity == snapshotContext.identity, - liveContext?.sessionIdentity == snapshotContext.sessionIdentity, - liveContext?.isSecure == false else { + guard screenRecordingPermissionProvider(), let liveContext, + liveContext.identity == snapshotContext.identity, + liveContext.sessionIdentity == snapshotContext.sessionIdentity, + !liveContext.isSecure else { cancel(resetState: true) return } + fieldContext = liveContext } applyExcerpt( excerpt, for: session.sessionID, identity: snapshotContext.identity, + fieldText: fieldContext.precedingText + fieldContext.trailingText, capturedAt: capturedAt ) } catch is CancellationError { @@ -332,10 +338,17 @@ final class VisualContextCoordinator { } /// Commits the generated screenshot excerpt and reports readiness for the still-focused field. + /// + /// A refresh that only differs by OCR noise or by the user's own typing keeps the excerpt text + /// already in the prompt (`VisualContextChangePolicy`) and only renews its capture time. The + /// new capture still proves the screen looks the same, so expiry semantics hold, while the + /// prompt stays byte-identical: the KV cache keeps the screen section and the visible + /// suggestion is not retired. Only a material change replaces the text and notifies. private func applyExcerpt( _ excerpt: VisualContextExcerpt, for sessionID: UUID, identity: FocusedInputIdentity, + fieldText: String, capturedAt: TimeInterval ) { guard activeAugmentationSession?.sessionID == sessionID, @@ -350,14 +363,18 @@ final class VisualContextCoordinator { return } - let changed = activeAugmentationSession?.excerpt?.text != excerpt.text + let previousText = activeAugmentationSession?.excerpt?.text + let changed = VisualContextChangePolicy.isMaterialChange( + previous: previousText, next: excerpt.text, fieldText: fieldText + ) + let committed = changed ? excerpt : VisualContextExcerpt(text: previousText ?? excerpt.text) activeAugmentationSession?.status = .ready - activeAugmentationSession?.excerpt = excerpt + activeAugmentationSession?.excerpt = committed status = .ready - latestExcerpt = excerpt.text + latestExcerpt = committed.text excerptCapturedAt = capturedAt scheduleExpiry(sessionID: sessionID, capturedAt: capturedAt) - CotabbyLogger.app.debug("Visual context ready: \(excerpt.text.count) chars") + CotabbyLogger.app.debug("Visual context ready: \(committed.text.count) chars\(changed ? "" : " (unchanged; kept prompt text)")") publishState() if changed { onInjectedContextReady?(identity) } } diff --git a/Cotabby/Services/Visual/WindowScreenshotService.swift b/Cotabby/Services/Visual/WindowScreenshotService.swift index 517aaa1dc..e86870c1d 100644 --- a/Cotabby/Services/Visual/WindowScreenshotService.swift +++ b/Cotabby/Services/Visual/WindowScreenshotService.swift @@ -77,17 +77,15 @@ struct WindowScreenshotService: WindowScreenshotCapturing { let shareableContent = try await currentShareableContent() try Task.checkCancellation() let caretCG = convertBetweenAppKitAndCG(rect: context.caretRect) - let focusedWindow = capturesEntireWindow ? shareableContent.windows.first(where: { + let appWindows = shareableContent.windows.filter { $0.owningApplication?.processID == processIdentifier && $0.isOnScreen - && $0.frame.contains(CGPoint(x: caretCG.midX, y: caretCG.midY)) && $0.isActive - }) : nil - let matchingWindow = - focusedWindow ?? shareableContent.windows.first(where: { - $0.owningApplication?.processID == processIdentifier && $0.isActive && $0.isOnScreen - }) - ?? shareableContent.windows.first(where: { - $0.owningApplication?.processID == processIdentifier && $0.isOnScreen - }) + } + let selectedWindowID = CaptureWindowSelector.select( + appWindows.map { .init(id: $0.windowID, frame: $0.frame, isActive: $0.isActive) }, + caret: CGPoint(x: caretCG.midX, y: caretCG.midY), + frontToBack: Self.frontToBackWindowIDs(for: processIdentifier) + ) + let matchingWindow = appWindows.first { $0.windowID == selectedWindowID } guard let matchingWindow else { CotabbyLogger.app.debug("No visible window for pid \(processIdentifier)") @@ -138,6 +136,18 @@ struct WindowScreenshotService: WindowScreenshotCapturing { return CapturedWindowScreenshot(image: image, windowTitle: matchingWindow.title, focusBounds: focusBounds) } + /// The process's normal-layer on-screen window numbers, front-most first. ScreenCaptureKit's + /// window list has no documented order; the window server's list is front to back. + private static func frontToBackWindowIDs(for processIdentifier: pid_t) -> [CGWindowID] { + let options: CGWindowListOption = [.optionOnScreenOnly, .excludeDesktopElements] + let windows = CGWindowListCopyWindowInfo(options, kCGNullWindowID) as? [[String: Any]] ?? [] + return windows.compactMap { info in + guard (info[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == processIdentifier, + (info[kCGWindowLayer as String] as? NSNumber)?.intValue == 0 else { return nil } + return (info[kCGWindowNumber as String] as? NSNumber)?.uint32Value + } + } + private func snapshotRect( around context: FocusedInputSnapshot, windowFrame: CGRect, diff --git a/Cotabby/Support/Context/CaptureWindowSelector.swift b/Cotabby/Support/Context/CaptureWindowSelector.swift new file mode 100644 index 000000000..613a9d595 --- /dev/null +++ b/Cotabby/Support/Context/CaptureWindowSelector.swift @@ -0,0 +1,33 @@ +import CoreGraphics + +/// Pure choice of the window to capture for the focused field. +/// +/// ScreenCaptureKit lists an app's windows in no documented order, and with several documents +/// open more than one can report itself active. Taking the first such window captured a background +/// document of the same app: its text entered the prompt, and the change from the focused +/// window's text read as navigation and retired the visible suggestion. The focused field's window +/// is the front-most one under its caret, in the window server's front-to-back order. +/// +/// Kept apart from `WindowScreenshotService` so the choice is testable without ScreenCaptureKit. +nonisolated enum CaptureWindowSelector { + struct Candidate: Equatable { + let id: CGWindowID + /// Global display points, top-left origin (ScreenCaptureKit's space). + let frame: CGRect + let isActive: Bool + } + + /// - Parameters: + /// - candidates: The focused app's on-screen windows. + /// - caret: The caret's centre in the same space as the frames. + /// - frontToBack: Window numbers front-most first. Windows missing from it rank behind all + /// listed ones and keep their list order, so an empty order falls back to list order. + static func select(_ candidates: [Candidate], caret: CGPoint, frontToBack: [CGWindowID]) -> CGWindowID? { + let rank = Dictionary(frontToBack.enumerated().map { ($1, $0) }, uniquingKeysWith: min) + func frontMost(_ windows: [Candidate]) -> Candidate? { + windows.min { (rank[$0.id] ?? .max) < (rank[$1.id] ?? .max) } + } + let underCaret = candidates.filter { $0.frame.contains(caret) } + return (frontMost(underCaret) ?? frontMost(candidates.filter(\.isActive)) ?? frontMost(candidates))?.id + } +} diff --git a/Cotabby/Support/Context/VisualContextChangePolicy.swift b/Cotabby/Support/Context/VisualContextChangePolicy.swift new file mode 100644 index 000000000..514e7cd36 --- /dev/null +++ b/Cotabby/Support/Context/VisualContextChangePolicy.swift @@ -0,0 +1,87 @@ +import Foundation + +/// File overview: +/// Decides whether a refreshed screen excerpt differs from the one already in the prompt in a way +/// the model should see, or only by OCR noise and the user's own typing. +/// +/// Why this exists: `VisualContextCoordinator` re-captures the screen every few seconds, and the +/// capture usually contains the very field the user is typing into. Treating every textual +/// difference as new context made each refresh replace the screen section of the prompt, which +/// discards the KV cache from that section onward (up to ~1,200 tokens to re-decode) and retires +/// the visible suggestion through `onInjectedContextReady`, so ghost text vanished every refresh +/// while the user typed. Keeping the previous excerpt when nothing material changed holds the +/// prompt byte-stable across refreshes; real changes (a new chat message, navigation, scrolling) +/// still replace it immediately. +/// +/// The comparison is line based because OCR jitter is local: a misread letter changes a word or +/// two inside a line, while a genuine update adds or removes whole lines. +nonisolated enum VisualContextChangePolicy { + /// A refreshed line counts as the same line when it shares at least this fraction of its words + /// with some previous line. OCR misreads ("includina" for "including") stay well above it. + static let lineMatchThreshold = 0.5 + /// A line made mostly of words the user has in the focused field is their own text rendered + /// on screen, which the prompt already carries as the caret prefix. + static let fieldEchoThreshold = 0.6 + /// Lines shorter than this many words (timestamps, button labels, "Edited") are too noisy to + /// count as new content on their own. + static let minimumLineWords = 3 + /// Content that disappears without anything new appearing (a scrolled-off line) only matters + /// once enough of it is gone to make the old excerpt misleading. + static let removedLinesThreshold = 3 + + /// - Parameters: + /// - previous: The excerpt currently in the prompt, or nil when none is. + /// - next: The freshly captured excerpt. + /// - fieldText: The focused field's text around the caret, used to ignore the user's own + /// typing where it shows up in the capture. + /// - Returns: true when `next` should replace `previous` in the prompt. + static func isMaterialChange(previous: String?, next: String, fieldText: String) -> Bool { + guard let previous else { return true } + if previous == next { return false } + + let fieldWords = Set(words(in: fieldText)) + let previousLines = contentLines(previous, fieldWords: fieldWords) + let nextLines = contentLines(next, fieldWords: fieldWords) + + let added = nextLines.filter { line in + !previousLines.contains { similarity(line, $0) >= lineMatchThreshold } + } + if !added.isEmpty { return true } + + let removed = previousLines.filter { line in + !nextLines.contains { similarity(line, $0) >= lineMatchThreshold } + } + return removed.count >= removedLinesThreshold + } + + /// The word sets of the lines that carry screen content: long enough to mean something and + /// not an echo of the field the user is typing in. + private static func contentLines(_ text: String, fieldWords: Set) -> [Set] { + text.split(whereSeparator: \.isNewline).compactMap { rawLine in + let lineWords = words(in: String(rawLine)) + guard lineWords.count >= minimumLineWords else { return nil } + let lineSet = Set(lineWords) + if !fieldWords.isEmpty { + let echoed = Double(lineSet.intersection(fieldWords).count) / Double(lineSet.count) + if echoed >= fieldEchoThreshold { return nil } + } + return lineSet + } + } + + /// Jaccard similarity of two word sets. + private static func similarity(_ lhs: Set, _ rhs: Set) -> Double { + let union = lhs.union(rhs).count + guard union > 0 else { return 1 } + return Double(lhs.intersection(rhs).count) / Double(union) + } + + /// Lowercased letter/digit runs of two or more characters. Single characters are mostly OCR + /// debris (bullets, stray glyphs) and would make short lines look different for no reason. + private static func words(in text: String) -> [String] { + text.lowercased() + .split(whereSeparator: { !$0.isLetter && !$0.isNumber }) + .filter { $0.count >= 2 } + .map(String.init) + } +} diff --git a/Cotabby/Support/History/TypingHistoryIndex.swift b/Cotabby/Support/History/TypingHistoryIndex.swift index f52a0ac05..3ee7276b1 100644 --- a/Cotabby/Support/History/TypingHistoryIndex.swift +++ b/Cotabby/Support/History/TypingHistoryIndex.swift @@ -16,9 +16,20 @@ nonisolated struct TypingHistoryQuery: Equatable, Sendable { /// in that head. Querying on every keystroke would change the examples, and with them the cached /// prefix, on nearly every request. Rounding to blocks of `wordsPerBlock` words keeps the /// examples fixed while a few words are typed and refreshes them as the topic moves on. - static func stableText(from precedingText: String, wordsPerBlock: Int = 8, maxWords: Int = 48) -> String { + /// + /// Short blocks apply until `laterWordsPerBlock` words so a brief message still gets examples + /// early; after that the query moves in `laterWordsPerBlock` steps (8, 16, 24, 48, ...). Each + /// step re-decodes the examples section and everything + /// after it (~100 ms on a 4B-class model), so refreshing every 8 words throughout a long text + /// was the most frequent cache miss left while typing. + static func stableText( + from precedingText: String, wordsPerBlock: Int = 8, laterWordsPerBlock: Int = 24, maxWords: Int = 48 + ) -> String { let words = precedingText.split(whereSeparator: \.isWhitespace) - let completeBlocks = (words.count / wordsPerBlock) * wordsPerBlock + let step = max(wordsPerBlock, laterWordsPerBlock) + let completeBlocks = words.count < step + ? (words.count / wordsPerBlock) * wordsPerBlock + : (words.count / step) * step guard completeBlocks > 0 else { return "" } let window = words[..= minimumSeamWordLength, spellingAssessment(word) == .known else { return .wait } case .notApplicable, .candidate: break } } @@ -163,7 +170,13 @@ nonisolated enum CompletionSeamGuard { let following = completion.dropFirst(ending.count).first guard following?.isNumber != true, following != "_" else { return nil } if !isFinal && (ending.count == completion.count || ending.last.map(CaretWordContext.isConnector) == true) { - return .wait + // The word may still be arriving. Waiting for the next token to prove it ended costs a + // whole decode step (~15-20 ms) before anything is visible, so a stream whose word is + // already a known dictionary word shows now; a longer word it turns into later + // (`Monday` -> `Mondays`) only extends the ghost. Fragments and connector endings wait. + guard ending.count == completion.count, !CaretWordContext.isConnector(ending.last ?? " "), + (prefix + ending).count >= minimumSeamWordLength, + spellingAssessment(prefix + ending) == .known else { return .wait } } let word = prefix + ending let assessment = spellingAssessment(word) diff --git a/Cotabby/Support/Suggestion/Request/DebouncePolicy.swift b/Cotabby/Support/Suggestion/Request/DebouncePolicy.swift index 2f43e24ee..0d6e5e2dc 100644 --- a/Cotabby/Support/Suggestion/Request/DebouncePolicy.swift +++ b/Cotabby/Support/Suggestion/Request/DebouncePolicy.swift @@ -7,24 +7,55 @@ import Foundation /// that cannot keep up (each cancel still costs a decode setup and teardown). Keying the debounce /// to the most recent generation latency makes fast machines snappier and slow machines calmer, /// with no configuration. The configured value remains the fallback until a first latency exists. +/// +/// For the in-process engines the cost a debounce saves is the cost of *starting* a request, not +/// of finishing it: a superseded generation is cancelled between tokens, and with KV prefix reuse a +/// warm request decodes only the typed delta before its first text appears. So when the latency to +/// that first text is known and small, the debounce shrinks to nothing; keying it to the full +/// latency instead made every short-but-complete suggestion (100-200 ms) buy the maximum 55 ms +/// wait in front of the next keystroke, the largest single delay before visible text. nonisolated enum DebouncePolicy { static func milliseconds( lastGenerationLatencyMilliseconds: Int?, + lastFirstTextLatencyMilliseconds: Int? = nil, fallback: Int, engine: SuggestionEngineKind = .llamaOpenSource ) -> Int { - // An HTTP endpoint cannot reuse Cotabby's in-process KV cache, and cancelling a request does - // not guarantee the remote server stops its tokenizer immediately. A trailing-edge pause - // therefore saves more time than it costs: it collapses a typing burst into one useful - // request instead of filling Ollama's queue with work that will be discarded. if engine == .openAICompatible { - guard let last = lastGenerationLatencyMilliseconds, last > 0 else { - return max(fallback, 180) - } - switch last { - case ...300: return 100 - case ...700: return 150 - default: return 220 + return endpointMilliseconds(lastGenerationLatencyMilliseconds: lastGenerationLatencyMilliseconds, fallback: fallback) + } + return inProcessMilliseconds( + lastGenerationLatencyMilliseconds: lastGenerationLatencyMilliseconds, + lastFirstTextLatencyMilliseconds: lastFirstTextLatencyMilliseconds, + fallback: fallback + ) + } + + /// An HTTP endpoint cannot reuse Cotabby's in-process KV cache, and cancelling a request does + /// not guarantee the remote server stops its tokenizer immediately. A trailing-edge pause + /// therefore saves more time than it costs: it collapses a typing burst into one useful + /// request instead of filling Ollama's queue with work that will be discarded. + private static func endpointMilliseconds(lastGenerationLatencyMilliseconds: Int?, fallback: Int) -> Int { + guard let last = lastGenerationLatencyMilliseconds, last > 0 else { + return max(fallback, 180) + } + switch last { + case ...300: return 100 + case ...700: return 150 + default: return 220 + } + } + + private static func inProcessMilliseconds( + lastGenerationLatencyMilliseconds: Int?, + lastFirstTextLatencyMilliseconds: Int?, + fallback: Int + ) -> Int { + if let firstText = lastFirstTextLatencyMilliseconds, firstText > 0 { + switch firstText { + case ...60: return 0 + case ...120: return 10 + default: return 20 } } diff --git a/Cotabby/Support/Suggestion/Request/SuggestionRequestFactory.swift b/Cotabby/Support/Suggestion/Request/SuggestionRequestFactory.swift index e8711a205..087814600 100644 --- a/Cotabby/Support/Suggestion/Request/SuggestionRequestFactory.swift +++ b/Cotabby/Support/Suggestion/Request/SuggestionRequestFactory.swift @@ -297,7 +297,7 @@ enum SuggestionRequestFactory { /// The prompt's token budget for one request. The configured budget holds back /// `llamaPromptOutputCeilingTokens` of the context window for output, but a request can ask for /// more: a 50-word range reaches 100 tokens in a 2-tokens-per-word language, and multi-line - /// doubles budgets up to 120 (the default 12-20 range becomes 52). `LlamaRuntimeCore` reserves + /// doubles budgets up to 120 (the 12-20 range becomes 52). `LlamaRuntimeCore` reserves /// the whole decode before admitting the prompt, so an overflowing prompt loses its oldest raw /// tokens there: the preface is cut mid-section and the request forfeits KV prefix reuse. /// Taking the excess out of the budget here lets the renderer trim by section priority instead. diff --git a/Cotabby/Support/Suggestion/Streaming/TypingCadence.swift b/Cotabby/Support/Suggestion/Streaming/TypingCadence.swift index 17124b829..94d022724 100644 --- a/Cotabby/Support/Suggestion/Streaming/TypingCadence.swift +++ b/Cotabby/Support/Suggestion/Streaming/TypingCadence.swift @@ -34,9 +34,13 @@ nonisolated struct TypingCadence { guard self.identityKey == identityKey, let last = lastLetterAt, CaretWordContext.unfinishedWord(in: precedingText) != nil else { return 0 } let ordered = intervals.sorted() - // These are bounded presentation limits, not a new global debounce. Before enough rhythm - // exists, 80ms merely absorbs an ordinary brief hesitation; a completed word pays no delay. - let quietWindow = ordered.count >= 2 ? min(0.22, max(0.08, ordered[ordered.count / 2] * 1.2)) : 0.08 + // These are bounded presentation limits, not a new global debounce, and they only suppress + // a mid-word ghost that would flash for a fraction of a keystroke. The window runs from the + // last letter, so it overlaps the generation itself: a warm request needs ~40-60 ms to + // produce text, and by then a window of at most 50 ms has usually elapsed and costs + // nothing. The previous 80-220 ms window (1.2x the typing interval) held every mid-word + // suggestion back for a whole keystroke on top of generation. A completed word pays none. + let quietWindow = ordered.count >= 2 ? min(0.05, max(0.02, ordered[ordered.count / 2] * 0.4)) : 0.04 return max(0, quietWindow - max(0, time - last)) } } diff --git a/CotabbyTests/App/Coordinators/Suggestion/SuggestionCoordinatorInputTests.swift b/CotabbyTests/App/Coordinators/Suggestion/SuggestionCoordinatorInputTests.swift index 51087d36a..3c2a8e514 100644 --- a/CotabbyTests/App/Coordinators/Suggestion/SuggestionCoordinatorInputTests.swift +++ b/CotabbyTests/App/Coordinators/Suggestion/SuggestionCoordinatorInputTests.swift @@ -1,3 +1,4 @@ +import CoreGraphics import Foundation import XCTest @testable import Cotabby @@ -304,6 +305,77 @@ final class SuggestionCoordinatorInputTests: SuggestionCoordinatorRigTestCase { XCTAssertEqual(rig.engine.prewarmedRequests.first?.generation, 0) } + /// The focus prefill is only reusable if the first real request renders the same prompt bytes, + /// so the prewarm must carry the same pinned clipboard context a prediction would. + func test_fieldSwitchPrewarmIncludesPinnedClipboardContext() async { + let rig = retained(makeCoordinatorRig()) + _ = rig.interactionState.materializeContext(from: rig.focusProvider.snapshot.context!) + rig.clipboardFilter.filtered = "Quarterly numbers attached" + + let otherApp = CotabbyTestFixtures.focusedInputSnapshot(processIdentifier: 456, precedingText: "Hi") + rig.coordinator.handleSupportedSnapshot(FocusSnapshot( + applicationName: otherApp.applicationName, + bundleIdentifier: otherApp.bundleIdentifier, + capability: .supported, + context: otherApp + )) + + await waitUntil("Engine was never prewarmed for the new field") { + !rig.engine.prewarmedRequests.isEmpty + } + XCTAssertEqual(rig.engine.prewarmedRequests.first?.clipboardContext, "Quarterly numbers attached") + } + + // MARK: - Keystroke-time speculation + + /// A plain typed character starts generating at once against the text the host is about to + /// publish; the host-publish poll only validates it. + func test_typedCharacterStartsSpeculativeGenerationBeforeHostPublish() async { + let rig = retained(makeCoordinatorRig( + snapshot: CotabbyTestFixtures.focusedInputSnapshot(precedingText: "Thanks for the quick rep") + )) + _ = rig.coordinator.handleInputEvent( + CapturedInputEvent(kind: .textMutation, keyCode: 37, characters: "l", flags: []) + ) + + XCTAssertEqual(rig.coordinator.pendingSpeculativeContext?.precedingText, "Thanks for the quick repl") + XCTAssertEqual(rig.coordinator.state, .generating) + await waitUntil("The speculative request never reached the engine") { + !rig.engine.requests.isEmpty + } + XCTAssertEqual(rig.engine.requests.first?.context.precedingText, "Thanks for the quick repl") + } + + /// The speculated character follows Shift XOR Caps Lock for letters, and shifted symbols are + /// not guessed because their character depends on the keyboard layout. + func test_speculatedCharactersApplyShiftAndSkipShiftedSymbols() { + func event(_ characters: String, _ flags: CGEventFlags = []) -> CapturedInputEvent { + CapturedInputEvent(kind: .textMutation, keyCode: 1, characters: characters, flags: flags) + } + XCTAssertEqual(SuggestionCoordinator.speculatedCharacters(for: event("s")), "s") + XCTAssertEqual(SuggestionCoordinator.speculatedCharacters(for: event("s", .maskShift)), "S") + XCTAssertEqual(SuggestionCoordinator.speculatedCharacters(for: event("S", .maskShift)), "S") + XCTAssertEqual(SuggestionCoordinator.speculatedCharacters(for: event("s", .maskAlphaShift)), "S") + XCTAssertEqual(SuggestionCoordinator.speculatedCharacters(for: event("s", [.maskShift, .maskAlphaShift])), "s") + XCTAssertEqual(SuggestionCoordinator.speculatedCharacters(for: event(",")), ",") + XCTAssertNil(SuggestionCoordinator.speculatedCharacters(for: event("/", .maskShift))) + XCTAssertNil(SuggestionCoordinator.speculatedCharacters(for: event("ab"))) + } + + /// Deletions, multi-character input, and the speculation kill switch keep the ordinary + /// publish-gated request. + func test_typedSpeculationSkipsDeletionsAndHonorsKillSwitch() { + let rig = retained(makeCoordinatorRig( + snapshot: CotabbyTestFixtures.focusedInputSnapshot(precedingText: "Thanks for the quick rep") + )) + _ = rig.coordinator.handleInputEvent(CapturedInputEvent(kind: .textMutation, keyCode: 51, characters: "", flags: [])) + XCTAssertNil(rig.coordinator.pendingSpeculativeContext, "Backspace has no character to speculate with") + + rig.coordinator.userDefaults.set(true, forKey: SuggestionCoordinator.speculativePrefetchDisabledDefaultsKey) + _ = rig.coordinator.handleInputEvent(CapturedInputEvent(kind: .textMutation, keyCode: 37, characters: "l", flags: [])) + XCTAssertNil(rig.coordinator.pendingSpeculativeContext, "The kill switch must skip typing speculation") + } + func test_handleSupportedSnapshot_withoutContextDisablesOutright() { let rig = retained(makeCoordinatorRig()) let bareSnapshot = FocusSnapshot( diff --git a/CotabbyTests/Services/History/TypingHistoryStoreTests.swift b/CotabbyTests/Services/History/TypingHistoryStoreTests.swift index b85c2f2c2..6fc17660e 100644 --- a/CotabbyTests/Services/History/TypingHistoryStoreTests.swift +++ b/CotabbyTests/Services/History/TypingHistoryStoreTests.swift @@ -611,6 +611,50 @@ final class TypingHistoryStoreTests: XCTestCase { "The field now contains that writing; showing it would make the model echo the draft") } + /// Examples already in the prompt survive a new query block while the new ranking still rates + /// them among its top few, so the prompt (and its KV cache) does not change on every block. A + /// fresh store answers the same later query with a different top example, which proves the + /// first store kept its examples on purpose rather than by coincidence. + func test_shownExamplesStayWhileTheyRemainHighlyRanked() async throws { + let rows: [[String: Any]] = [ + ["appBundleIdentifier": "com.example.TestApp", "textUpToCursor": "The Imperum connector plan review moved to Monday with the steering group."], + ["appBundleIdentifier": "com.example.TestApp", "textUpToCursor": "Connector plan notes are ready for the Imperum steering group on Monday."], + ["appBundleIdentifier": "com.example.TestApp", "textUpToCursor": "Detections rollout for the SOC analysts starts after the dashboard migration."] + ] + // Exactly 8 words, then exactly 16: each is one whole query block. The second block's new + // words all belong to the third passage, so a fresh ranking puts it first. + let opening = "Notes for Monday about the Imperum connector plan " + let later = opening + "SOC analysts detections rollout after the dashboard migration " + + let sticky = makeStore() + sticky.setUsingHistory(true) + await sticky.importCotypistExport(from: try writeExport(rows)) + let first = CotabbyTestFixtures.focusedInputContext(precedingText: opening) + await waitUntil { !sticky.historyExamples(for: first, engine: .llamaOpenSource).isEmpty } + let shown = sticky.historyExamples(for: first, engine: .llamaOpenSource) + XCTAssertFalse(shown.contains { $0.contains("Detections rollout") }) + + let laterContext = CotabbyTestFixtures.focusedInputContext(precedingText: later) + XCTAssertEqual(sticky.historyExamples(for: laterContext, engine: .llamaOpenSource), shown) + + let fresh = makeStore() + fresh.setUsingHistory(true) + await fresh.importCotypistExport(from: try writeExport(rows)) + await waitUntil { !fresh.historyExamples(for: laterContext, engine: .llamaOpenSource).isEmpty } + XCTAssertNotEqual(fresh.historyExamples(for: laterContext, engine: .llamaOpenSource), shown, + "Without shown examples the later query must rank differently, or this test proves nothing") + } + + /// A dropped example is replaced in its own slot; survivors keep their positions, so the prompt + /// diverges as late as possible. + func test_stableExamplesReplaceOnlyDroppedSlots() { + XCTAssertEqual(TypingHistoryStore.stableExamples(shown: ["a", "b"], ranked: ["b", "a", "c"], fresh: ["b", "c"]), ["a", "b"]) + XCTAssertEqual(TypingHistoryStore.stableExamples(shown: ["a", "b"], ranked: ["b", "c"], fresh: ["c", "b"]), ["c", "b"]) + XCTAssertEqual(TypingHistoryStore.stableExamples(shown: ["a", "b"], ranked: ["a", "c"], fresh: ["c", "a"]), ["a", "c"]) + XCTAssertEqual(TypingHistoryStore.stableExamples(shown: ["a"], ranked: ["a", "c"], fresh: ["c", "a"]), ["a", "c"]) + XCTAssertEqual(TypingHistoryStore.stableExamples(shown: ["a", "b"], ranked: [], fresh: []), []) + } + func test_anExampleHiddenByTheFieldsLatestWordsComesBackWhenTheWordingMovesOn() async throws { let store = makeStore() store.setUsingHistory(true) diff --git a/CotabbyTests/Services/Runtime/Llama/LlamaPromptCacheHintTrackerTests.swift b/CotabbyTests/Services/Runtime/Llama/LlamaPromptCacheHintTrackerTests.swift index 504f46b45..27ef0928c 100644 --- a/CotabbyTests/Services/Runtime/Llama/LlamaPromptCacheHintTrackerTests.swift +++ b/CotabbyTests/Services/Runtime/Llama/LlamaPromptCacheHintTrackerTests.swift @@ -56,6 +56,15 @@ final class LlamaPromptCacheHintTrackerTests: XCTestCase { XCTAssertNil(tracker.cachedPrefixBytes(for: makeRequest(prompt: "hello!", topK: 40))) } + /// Performance tuning moves the output budget between requests; the cached prompt does not + /// depend on it, so the hint must survive the change. + func test_cacheHint_survivesOutputBudgetChange() { + var tracker = LlamaPromptCacheHintTracker() + tracker.recordSuccessfulRequest(makeRequest(prompt: "hello", maxPredictionTokens: 26)) + + XCTAssertEqual(tracker.cachedPrefixBytes(for: makeRequest(prompt: "hello!", maxPredictionTokens: 6)), 5) + } + func test_cacheHint_invalidatesWhenProcessChanges() { var tracker = LlamaPromptCacheHintTracker() tracker.recordSuccessfulRequest(makeRequest(prompt: "hello", processIdentifier: 100)) @@ -123,6 +132,7 @@ final class LlamaPromptCacheHintTrackerTests: XCTestCase { elementIdentifier: String = "field", processIdentifier: Int32 = 123, topK: Int = 20, + maxPredictionTokens: Int = 8, inputFrameRect: CGRect? = nil ) -> SuggestionRequest { let context = CotabbyTestFixtures.focusedInputContext( @@ -137,7 +147,7 @@ final class LlamaPromptCacheHintTrackerTests: XCTestCase { prefixText: prompt, prompt: prompt, generation: context.generation, - maxPredictionTokens: 8, + maxPredictionTokens: maxPredictionTokens, temperature: 0.1, topK: topK, topP: 0.7, diff --git a/CotabbyTests/Services/Runtime/Llama/LlamaRuntimeCoreIntegrationTests.swift b/CotabbyTests/Services/Runtime/Llama/LlamaRuntimeCoreIntegrationTests.swift index fb0c528b0..bb1f6c675 100644 --- a/CotabbyTests/Services/Runtime/Llama/LlamaRuntimeCoreIntegrationTests.swift +++ b/CotabbyTests/Services/Runtime/Llama/LlamaRuntimeCoreIntegrationTests.swift @@ -111,6 +111,54 @@ final class LlamaRuntimeCoreIntegrationTests: XCTestCase { } } + /// Reuse is decided from the token prefix, not from the caller's byte hint. A request without a + /// hint (the tracker had no record, e.g. after a cancelled request) must still keep the shared + /// prompt and produce exactly the fresh result. With a sliding-window model this is the case + /// `swa_full` enables: the trim lands inside the window instead of rebuilding. + func testHintlessEditReusesPromptAndMatchesFreshRequest() throws { + let core = try makeCore() + defer { core.shutdown() } + let original = "Hi Alex, thanks for sending the project update. I will review the schedule and send you my" + for edited in [original + " feedback", String(original.dropLast(2)) + "our feedback"] { + core.resetPromptCache() + _ = try core.generate(prompt: original, options: options) + let reused = try core.generate(prompt: edited, options: options) + XCTAssertGreaterThan(core.lastReusedPromptTokens, 0, "A shared token prefix must be kept without a hint: \(edited)") + core.resetPromptCache() + let fresh = try core.generate(prompt: edited, options: options) + XCTAssertEqual(core.lastReusedPromptTokens, 0) + XCTAssertEqual(reused.text, fresh.text, "Hintless reuse changed the completion for: \(edited)") + } + } + + /// A different output budget or sampler setting must not discard the cached prompt: the budget + /// lives in the Swift loop and the sampler chain is swapped in place. The reused request must + /// still sample exactly like a fresh sequence built with the new settings. + func testSamplingChangeKeepsPromptAndMatchesFreshRequest() throws { + let core = try makeCore() + defer { core.shutdown() } + let prompt = "Hi Alex, thanks for sending the project update. I will review the schedule and send you my" + for configuration in sampledConfigurations { + core.resetPromptCache() + _ = try core.generate(prompt: prompt, options: options) + let changed = LlamaGenerationOptions( + maxPredictionTokens: configuration.options.maxPredictionTokens - 4, + temperature: configuration.options.temperature, + topK: configuration.options.topK, + topP: configuration.options.topP, + minP: configuration.options.minP, + repetitionPenalty: configuration.options.repetitionPenalty, + seed: configuration.options.seed, + stopAtArgmaxEOG: false + ) + let reused = try core.generate(prompt: prompt, options: changed) + XCTAssertGreaterThan(core.lastReusedPromptTokens, 0, "\(configuration.label): sampling change must keep the prompt") + core.resetPromptCache() + let fresh = try core.generate(prompt: prompt, options: changed) + XCTAssertEqual(reused.text, fresh.text, "\(configuration.label): swapped sampler diverged from a fresh one") + } + } + func testCancelledRetainedTailCanBeRestoredByNextRequest() throws { let core = try makeCore() defer { core.shutdown() } diff --git a/CotabbyTests/Services/Visual/VisualContextCoordinatorTests.swift b/CotabbyTests/Services/Visual/VisualContextCoordinatorTests.swift index a5846f0e9..cc860daa3 100644 --- a/CotabbyTests/Services/Visual/VisualContextCoordinatorTests.swift +++ b/CotabbyTests/Services/Visual/VisualContextCoordinatorTests.swift @@ -23,6 +23,27 @@ final class VisualContextCoordinatorTests: XCTestCase { XCTAssertEqual(coordinator.latestExcerpt, generator.text) } + /// A refresh that only misreads a letter must keep the excerpt already in the prompt, so the + /// prompt stays byte-identical (KV reuse) and no prediction is retired. + func test_ocrJitterKeepsPromptTextAndDoesNotNotify() async throws { + let generator = StubVisualContextGenerator() + let coordinator = makeCoordinator(generator) + let snapshot = CotabbyTestFixtures.focusedInputSnapshot(precedingText: "Draft") + coordinator.refreshContextProvider = { snapshot } + var notifications = 0 + coordinator.onInjectedContextReady = { _ in notifications += 1 } + coordinator.startSessionIfNeeded(for: snapshot, configuration: .local) + defer { coordinator.cancel(resetState: true) } + try await waitUntil { generator.contexts.count >= 1 && coordinator.status == .ready } + let original = coordinator.latestExcerpt + generator.text = "Project agenda and deadlinc" + try await waitUntil { generator.contexts.count >= 2 } + try await waitUntil { generator.completedCount >= 2 } + XCTAssertEqual(notifications, 1) + XCTAssertEqual(coordinator.latestExcerpt, original, "OCR noise must not replace the prompt's screen text") + XCTAssertEqual(coordinator.status, .ready) + } + func test_permissionRevocationStopsRefreshAndClearsExcerpt() async throws { let generator = StubVisualContextGenerator() var allowed = true @@ -267,6 +288,38 @@ final class VisualContextCoordinatorTests: XCTestCase { XCTAssertEqual(generator.configurations, [.local, .default]) } + /// TextEdit or a compose window: the draft fills the capture, and the user keeps typing while + /// OCR runs. Words typed after the capture started are on screen but were not in the field when + /// it started, so the comparison must use the field as it reads when the result lands. + func test_typingIntoAWindowFillingFieldIsNotANavigationSignal() async throws { + let generator = StubVisualContextGenerator() + let chrome = "Untitled 3 — Edited\nHelvetica Regular 12 B I U S\n" + generator.text = chrome + "Hello Martin, thanks for the update on the" + let coordinator = makeCoordinator(generator) + var live = CotabbyTestFixtures.focusedInputSnapshot(precedingText: "Hello Martin, thanks for the update on the") + coordinator.refreshContextProvider = { live } + var notifications = 0 + coordinator.onInjectedContextReady = { _ in notifications += 1 } + coordinator.startSessionIfNeeded(for: live, configuration: .local) + defer { coordinator.cancel(resetState: true) } + try await waitUntil { coordinator.status == .ready } + let original = coordinator.latestExcerpt + + generator.suspendNext = true + try await waitUntil { generator.pending != nil } + live = CotabbyTestFixtures.focusedInputSnapshot( + precedingText: "Hello Martin, thanks for the update on the onboarding project. I wanted to check" + ) + generator.text = chrome + "Hello Martin, thanks for the update on the onboarding project. I wanted to che" + let completedBefore = generator.completedCount + generator.pending?.resume() + generator.pending = nil + try await waitUntil { generator.completedCount > completedBefore } + + XCTAssertEqual(notifications, 1, "The user's own typing on screen must not retire the visible suggestion") + XCTAssertEqual(coordinator.latestExcerpt, original, "Typing alone must keep the prompt's screen text") + } + private func makeCoordinator( _ generator: StubVisualContextGenerator, permission: @escaping @MainActor () -> Bool = { true } diff --git a/CotabbyTests/Support/Context/CaptureWindowSelectorTests.swift b/CotabbyTests/Support/Context/CaptureWindowSelectorTests.swift new file mode 100644 index 000000000..e65113e58 --- /dev/null +++ b/CotabbyTests/Support/Context/CaptureWindowSelectorTests.swift @@ -0,0 +1,36 @@ +import CoreGraphics +import XCTest +@testable import Cotabby + +final class CaptureWindowSelectorTests: XCTestCase { + private let caret = CGPoint(x: 300, y: 300) + + func test_picksTheFrontMostWindowUnderTheCaret() { + // Two cascaded TextEdit documents, both reported active, the background one listed first: + // capturing it put another document's text in the prompt and read as navigation. + let background = CaptureWindowSelector.Candidate(id: 11, frame: CGRect(x: 0, y: 0, width: 603, height: 505), isActive: true) + let focused = CaptureWindowSelector.Candidate(id: 22, frame: CGRect(x: 20, y: 20, width: 603, height: 505), isActive: true) + let selected = CaptureWindowSelector.select([background, focused], caret: caret, frontToBack: [22, 11]) + XCTAssertEqual(selected, 22) + } + + func test_aWindowUnderTheCaretBeatsAnActiveOneElsewhere() { + let elsewhere = CaptureWindowSelector.Candidate(id: 1, frame: CGRect(x: 900, y: 0, width: 400, height: 400), isActive: true) + let underCaret = CaptureWindowSelector.Candidate(id: 2, frame: CGRect(x: 0, y: 0, width: 600, height: 600), isActive: false) + XCTAssertEqual(CaptureWindowSelector.select([elsewhere, underCaret], caret: caret, frontToBack: [1, 2]), 2) + } + + func test_fallsBackToTheFrontMostActiveWindowWhenNoneContainsTheCaret() { + let back = CaptureWindowSelector.Candidate(id: 1, frame: CGRect(x: 900, y: 0, width: 400, height: 400), isActive: true) + let front = CaptureWindowSelector.Candidate(id: 2, frame: CGRect(x: 1400, y: 0, width: 400, height: 400), isActive: true) + let inactive = CaptureWindowSelector.Candidate(id: 3, frame: CGRect(x: 2000, y: 0, width: 400, height: 400), isActive: false) + XCTAssertEqual(CaptureWindowSelector.select([back, front, inactive], caret: caret, frontToBack: [3, 2, 1]), 2) + } + + func test_keepsListOrderWhenTheWindowOrderIsUnknown() { + let first = CaptureWindowSelector.Candidate(id: 1, frame: CGRect(x: 0, y: 0, width: 600, height: 600), isActive: true) + let second = CaptureWindowSelector.Candidate(id: 2, frame: CGRect(x: 0, y: 0, width: 600, height: 600), isActive: true) + XCTAssertEqual(CaptureWindowSelector.select([first, second], caret: caret, frontToBack: []), 1) + XCTAssertNil(CaptureWindowSelector.select([], caret: caret, frontToBack: [1])) + } +} diff --git a/CotabbyTests/Support/Context/VisualContextChangePolicyTests.swift b/CotabbyTests/Support/Context/VisualContextChangePolicyTests.swift new file mode 100644 index 000000000..ee20aee03 --- /dev/null +++ b/CotabbyTests/Support/Context/VisualContextChangePolicyTests.swift @@ -0,0 +1,66 @@ +import XCTest +@testable import Cotabby + +/// Pure-function tests for deciding when a refreshed screen excerpt should replace the one in the +/// prompt. The cases mirror what a refresh actually sees while someone types: OCR misreads, the +/// user's own text growing inside the captured field, and genuine screen updates. +final class VisualContextChangePolicyTests: XCTestCase { + private let chat = """ + Alex Morgan 10:42 + Can you send the updated proposal before Friday? + The pricing section still lists last quarter's numbers. + Sam Lee 10:44 + I will check with finance and get back to you today. + """ + + func test_firstExcerptIsAlwaysMaterial() { + XCTAssertTrue(VisualContextChangePolicy.isMaterialChange(previous: nil, next: chat, fieldText: "")) + } + + func test_identicalExcerptIsNotMaterial() { + XCTAssertFalse(VisualContextChangePolicy.isMaterialChange(previous: chat, next: chat, fieldText: "")) + } + + func test_ocrMisreadsAreNotMaterial() { + let jittered = chat + .replacingOccurrences(of: "updated proposal", with: "updatcd proposal") + .replacingOccurrences(of: "quarter's", with: "quarters") + .replacingOccurrences(of: "10:44", with: "10.44") + XCTAssertFalse(VisualContextChangePolicy.isMaterialChange(previous: chat, next: jittered, fieldText: "")) + } + + /// The capture usually includes the field being typed into. Its growing text must not count + /// as new screen content: the prompt already carries it as the caret prefix. + func test_userTypingInsideCapturedFieldIsNotMaterial() { + let typed = "Thanks Alex, I will update the pricing section and send the proposal" + let previous = chat + "\nThanks Alex, I will update" + let next = chat + "\n" + typed + XCTAssertFalse(VisualContextChangePolicy.isMaterialChange(previous: previous, next: next, fieldText: typed)) + } + + func test_newIncomingMessageIsMaterial() { + let next = chat + "\nAlex Morgan 10:47\nAlso please include the revised delivery timeline." + XCTAssertTrue(VisualContextChangePolicy.isMaterialChange(previous: chat, next: next, fieldText: "")) + } + + func test_navigationToAnotherConversationIsMaterial() { + let other = """ + Priya Shah 09:15 + The staging cluster is down again after the last deploy. + Could someone from platform take a look this morning? + """ + XCTAssertTrue(VisualContextChangePolicy.isMaterialChange(previous: chat, next: other, fieldText: "")) + } + + func test_shortChromeChangesAreNotMaterial() { + let next = chat.replacingOccurrences(of: "Alex Morgan 10:42", with: "Alex Morgan 10:42 Edited") + XCTAssertFalse(VisualContextChangePolicy.isMaterialChange(previous: chat, next: next, fieldText: "")) + } + + func test_contentDisappearingIsMaterialOnlyPastThreshold() { + let lines = chat.split(separator: "\n").map(String.init) + let oneGone = lines.dropFirst(2).joined(separator: "\n") + XCTAssertFalse(VisualContextChangePolicy.isMaterialChange(previous: chat, next: oneGone, fieldText: "")) + XCTAssertTrue(VisualContextChangePolicy.isMaterialChange(previous: chat, next: "", fieldText: "")) + } +} diff --git a/CotabbyTests/Support/History/TypingHistoryIndexTests.swift b/CotabbyTests/Support/History/TypingHistoryIndexTests.swift index 31b917bdb..453802c36 100644 --- a/CotabbyTests/Support/History/TypingHistoryIndexTests.swift +++ b/CotabbyTests/Support/History/TypingHistoryIndexTests.swift @@ -77,6 +77,20 @@ final class TypingHistoryIndexTests: XCTestCase { XCTAssertEqual(examples, ["Pipeline status for the Imperum connector build is green again today!"]) } + /// After the first short block the query moves in larger steps, so long texts refresh their + /// examples (and re-decode that prompt section) less often. + func test_stableTextUsesLargerBlocksAfterTheFirst() { + let words = (1...60).map { "w\($0)" } + let text = { (count: Int) in words.prefix(count).joined(separator: " ") } + XCTAssertEqual(TypingHistoryQuery.stableText(from: text(7)), "") + XCTAssertEqual(TypingHistoryQuery.stableText(from: text(8)), text(8)) + XCTAssertEqual(TypingHistoryQuery.stableText(from: text(15)), text(8)) + XCTAssertEqual(TypingHistoryQuery.stableText(from: text(23)), text(16)) + XCTAssertEqual(TypingHistoryQuery.stableText(from: text(24)), text(24)) + XCTAssertEqual(TypingHistoryQuery.stableText(from: text(47)), text(24)) + XCTAssertEqual(TypingHistoryQuery.stableText(from: text(48)), text(48)) + } + func test_stableTextOnlyChangesOnWholeWordBlocks() { XCTAssertEqual(TypingHistoryQuery.stableText(from: "one two three", wordsPerBlock: 4), "") XCTAssertEqual(TypingHistoryQuery.stableText(from: "a b c d e f", wordsPerBlock: 4), "a b c d") diff --git a/CotabbyTests/Support/Onboarding/OnboardingTemplateRecommenderTests.swift b/CotabbyTests/Support/Onboarding/OnboardingTemplateRecommenderTests.swift index c1cc902c9..5564976a6 100644 --- a/CotabbyTests/Support/Onboarding/OnboardingTemplateRecommenderTests.swift +++ b/CotabbyTests/Support/Onboarding/OnboardingTemplateRecommenderTests.swift @@ -40,12 +40,13 @@ final class OnboardingTemplateRecommenderTests: XCTestCase { func testAppleIntelligenceStillCarriesTierBehaviorFlags() { let quick = OnboardingTemplateRecommender.resolvePlan(for: .quick, engine: .appleIntelligence) - XCTAssertEqual(quick.wordCountPreset, .fourToSeven) + XCTAssertEqual(quick.wordCountPreset, .twoToFour) XCTAssertFalse(quick.enablesFastMode) XCTAssertFalse(quick.enablesMultiLine) XCTAssertFalse(quick.enablesClipboardContext) let everyday = OnboardingTemplateRecommender.resolvePlan(for: .everyday, engine: .appleIntelligence) + XCTAssertEqual(everyday.wordCountPreset, .fourToSeven) XCTAssertFalse(everyday.enablesFastMode) XCTAssertFalse(everyday.enablesMultiLine) XCTAssertTrue(everyday.enablesClipboardContext) diff --git a/CotabbyTests/Support/Prompting/BaseCompletionPromptRendererTests.swift b/CotabbyTests/Support/Prompting/BaseCompletionPromptRendererTests.swift index 3db10bbb9..fb462d566 100644 --- a/CotabbyTests/Support/Prompting/BaseCompletionPromptRendererTests.swift +++ b/CotabbyTests/Support/Prompting/BaseCompletionPromptRendererTests.swift @@ -90,6 +90,30 @@ final class BaseCompletionPromptRendererTests: XCTestCase { ) } + /// Context runs from least to most volatile so a change re-decodes as little cached prompt as + /// possible: pinned clipboard, then the screen excerpt, then history examples (which move + /// every few words), then the following text and the caret prefix. + func test_historyExamplesFollowClipboardAndScreenText() { + let prompt = BaseCompletionPromptRenderer.prompt( + prefixText: "the meeting is at ", + applicationName: "Mail", + userName: nil, + trailingText: " on Friday.", + clipboardContext: "zoom link", + visualContextSummary: "Calendar: Q3 planning 3pm", + historyExamples: ["See you at the standup."] + ) + XCTAssertEqual( + prompt, + [ + "On the clipboard: zoom link", + "Nearby on screen: Calendar: Q3 planning 3pm", + "Earlier writing by the same author:\n“See you at the standup.”", + "Later in the same passage:\n“ on Friday.”" + ].joined(separator: "\n") + "\n\nthe meeting is at " + ) + } + /// The following text is cut to `maxSuffixCharacters` and quoted, keeping its whitespace. func test_followingTextIsBoundedQuotedContextBeforeExactPrefix() { let prefix = "We are meeting " diff --git a/CotabbyTests/Support/Suggestion/Output/CompletionSeamGuardTests.swift b/CotabbyTests/Support/Suggestion/Output/CompletionSeamGuardTests.swift index 3f83abaa3..dee4b22bb 100644 --- a/CotabbyTests/Support/Suggestion/Output/CompletionSeamGuardTests.swift +++ b/CotabbyTests/Support/Suggestion/Output/CompletionSeamGuardTests.swift @@ -441,15 +441,31 @@ final class CompletionSeamGuardTests: XCTestCase { // MARK: - Streamed leading words /// Streaming buffers a lowercase prefix because checking it before its boundary is unreliable. + /// An unfinished leading word that is not (yet) a known word keeps waiting: a prefix is never + /// suppressed as a typo, it is only withheld until its boundary arrives. func testStreamedLeadingWordWaitsUntilItsBoundaryArrives() { XCTAssertEqual( CompletionSeamGuard.streamedLeadingWordVerdict( precedingText: "Je veux ", completion: "ecrir", - spellingAssessment: { _ in - XCTFail("an incomplete streamed word must not reach the spell checker") - return .known - } + spellingAssessment: { $0 == "ecrir" ? .correctableTypo : .known } + ), + .wait + ) + } + + /// A streamed leading word that is already a known word of four or more letters shows without + /// waiting a decode step for its boundary; short words, which often grow, still wait. + func testStreamedKnownLeadingWordShowsBeforeItsBoundary() { + XCTAssertEqual( + CompletionSeamGuard.streamedLeadingWordVerdict( + precedingText: "Please send the ", completion: "report", spellingAssessment: { _ in .known } + ), + .allow + ) + XCTAssertEqual( + CompletionSeamGuard.streamedLeadingWordVerdict( + precedingText: "Please send ", completion: "the", spellingAssessment: { _ in .known } ), .wait ) @@ -520,7 +536,7 @@ final class CompletionSeamGuardTests: XCTestCase { func testBadJoinedWordNeverAppearsDuringStreaming() { XCTAssertEqual(CompletionSeamGuard.presentation(precedingText: "so gre", completion: "atf", isFinal: false, - spellingAssessment: { _ in XCTFail("Wait for the full word"); return .known }), .wait) + spellingAssessment: { _ in .correctableTypo }), .wait) for isFinal in [false, true] { XCTAssertEqual(CompletionSeamGuard.presentation(precedingText: "so gre", completion: "atful for this", isFinal: isFinal, spellingAssessment: { _ in .correctableTypo }), .suppress(.seamMisspelling(word: "greatful"))) @@ -549,23 +565,27 @@ final class CompletionSeamGuardTests: XCTestCase { } } - /// A dictionary entry is not a word boundary: `build` could still grow into `building`. - /// Buffer until the stream proves the seam word complete, then expose its phrase in one step. - func testKnownJoinedWordStillWaitsForItsStreamedBoundary() { - for completion in ["u", "uil", "uild", "uilding"] { + /// Fragments wait for the stream to prove the seam word complete. A joined word that is + /// already a known word of four or more letters shows a decode step early; if it then grows + /// (`build` -> `building`) the ghost only extends, and nothing is suppressed on a prefix. + func testJoinedWordShowsOnceItIsAKnownWord() { + let dictionary = knowing(["build", "building"]) + for fragment in ["u", "uil"] { XCTAssertEqual( CompletionSeamGuard.presentation( - precedingText: "I would like to b", - completion: completion, - isFinal: false, - spellingAssessment: { _ in - XCTFail("An unfinished joined word must not reach spelling assessment") - return .known - } + precedingText: "I would like to b", completion: fragment, isFinal: false, spellingAssessment: dictionary ), .wait ) } + for word in ["uild", "uilding"] { + XCTAssertEqual( + CompletionSeamGuard.presentation( + precedingText: "I would like to b", completion: word, isFinal: false, spellingAssessment: dictionary + ), + .show(text: word, wordOnly: false) + ) + } XCTAssertEqual( CompletionSeamGuard.presentation( precedingText: "I would like to b", @@ -601,9 +621,9 @@ final class CompletionSeamGuardTests: XCTestCase { XCTAssertEqual( CompletionSeamGuard.presentation( precedingText: "Try " + opening + "b", - completion: "uild", + completion: "uil", isFinal: false, - spellingAssessment: knowsEverything + spellingAssessment: knowing(["build"]) ), .wait ) diff --git a/CotabbyTests/Support/Suggestion/Request/DebouncePolicyTests.swift b/CotabbyTests/Support/Suggestion/Request/DebouncePolicyTests.swift index f184073dd..9a04705aa 100644 --- a/CotabbyTests/Support/Suggestion/Request/DebouncePolicyTests.swift +++ b/CotabbyTests/Support/Suggestion/Request/DebouncePolicyTests.swift @@ -40,6 +40,43 @@ final class DebouncePolicyTests: XCTestCase { assertDebounce(cases, fallback: 20, engine: .appleIntelligence) } + /// A fast time to first text removes the debounce for in-process engines even when the full + /// suggestion took long; a slow first text keeps a short 20 ms pause instead of the 55 ms + /// full-latency tier, since cancelling a superseded local request costs at most one token. + func test_localEngines_firstTextLatencyTiers() { + let cases: [(firstText: Int?, full: Int?, expected: Int)] = [ + (1, 900, 0), (60, 900, 0), + (61, 900, 10), (120, 900, 10), + (121, 900, 20), (121, 100, 20), + (0, 45, 15), (nil, 45, 15) + ] + for engine in [SuggestionEngineKind.llamaOpenSource, .appleIntelligence] { + for testCase in cases { + XCTAssertEqual( + DebouncePolicy.milliseconds( + lastGenerationLatencyMilliseconds: testCase.full, + lastFirstTextLatencyMilliseconds: testCase.firstText, + fallback: 20, + engine: engine + ), + testCase.expected, + "\(engine) first \(String(describing: testCase.firstText)) full \(String(describing: testCase.full))" + ) + } + } + } + + /// The endpoint keeps its trailing-edge pause regardless of first-text latency. + func test_endpoint_ignoresFirstTextLatency() { + XCTAssertEqual( + DebouncePolicy.milliseconds( + lastGenerationLatencyMilliseconds: 200, lastFirstTextLatencyMilliseconds: 20, + fallback: 20, engine: .openAICompatible + ), + 100 + ) + } + func test_defaultEngineIsLocal() { XCTAssertEqual(DebouncePolicy.milliseconds(lastGenerationLatencyMilliseconds: 45, fallback: 20), 15) } diff --git a/CotabbyTests/Support/Suggestion/Streaming/TypingCadenceTests.swift b/CotabbyTests/Support/Suggestion/Streaming/TypingCadenceTests.swift index 7286048f1..d5aeca54c 100644 --- a/CotabbyTests/Support/Suggestion/Streaming/TypingCadenceTests.swift +++ b/CotabbyTests/Support/Suggestion/Streaming/TypingCadenceTests.swift @@ -7,7 +7,7 @@ final class TypingCadenceTests: XCTestCase { for (index, letter) in ["s", "c", "h", "e"].enumerated() { cadence.record(identityKey: 1, characters: letter, at: Double(index) * 0.1) } - XCTAssertEqual(cadence.remainingDelay(identityKey: 1, precedingText: "sche", at: 0.32), 0.1, accuracy: 0.001) + XCTAssertEqual(cadence.remainingDelay(identityKey: 1, precedingText: "sche", at: 0.32), 0.02, accuracy: 0.001) XCTAssertEqual(cadence.remainingDelay(identityKey: 1, precedingText: "sche", at: 0.5), 0) XCTAssertEqual(cadence.remainingDelay(identityKey: 2, precedingText: "sche", at: 0.32), 0) cadence.record(identityKey: 1, characters: " ", at: 0.33) @@ -15,19 +15,19 @@ final class TypingCadenceTests: XCTestCase { } func testQuietWindowTracksMedianRhythmWithinItsBounds() { - // Slow but steady typing (500ms gaps) would ask for 600ms; the window is capped at 220ms. + // Slow but steady typing (500ms gaps) would ask for 200ms; the window is capped at 50ms. var slow = TypingCadence() for (index, letter) in ["a", "b", "c"].enumerated() { slow.record(identityKey: 1, characters: letter, at: Double(index) * 0.5) } - XCTAssertEqual(slow.remainingDelay(identityKey: 1, precedingText: "abc", at: 1.0), 0.22, accuracy: 0.001) + XCTAssertEqual(slow.remainingDelay(identityKey: 1, precedingText: "abc", at: 1.0), 0.05, accuracy: 0.001) - // Very fast typing (30ms gaps) would ask for 36ms; the window never drops below 80ms. + // Very fast typing (30ms gaps) would ask for 12ms; the window never drops below 20ms. var fast = TypingCadence() for (index, letter) in ["a", "b", "c"].enumerated() { fast.record(identityKey: 1, characters: letter, at: Double(index) * 0.03) } - XCTAssertEqual(fast.remainingDelay(identityKey: 1, precedingText: "abc", at: 0.06), 0.08, accuracy: 0.001) + XCTAssertEqual(fast.remainingDelay(identityKey: 1, precedingText: "abc", at: 0.06), 0.02, accuracy: 0.001) } func testSwitchingFieldsDiscardsThePreviousFieldsRhythm() { @@ -38,7 +38,7 @@ final class TypingCadenceTests: XCTestCase { cadence.record(identityKey: 2, characters: "x", at: 1.1) XCTAssertEqual(cadence.remainingDelay(identityKey: 1, precedingText: "abc", at: 1.1), 0) // The new field starts without intervals, so only the default brief-hesitation window applies. - XCTAssertEqual(cadence.remainingDelay(identityKey: 2, precedingText: "x", at: 1.1), 0.08, accuracy: 0.001) + XCTAssertEqual(cadence.remainingDelay(identityKey: 2, precedingText: "x", at: 1.1), 0.04, accuracy: 0.001) } func testNonFiniteTimestampsAreIgnoredAndNonWordPrefixesPayNoDelay() { @@ -56,7 +56,7 @@ final class TypingCadenceTests: XCTestCase { var cadence = TypingCadence() cadence.record(identityKey: 1, characters: "s", at: 0) cadence.record(identityKey: 1, characters: "c", at: 5) - XCTAssertEqual(cadence.remainingDelay(identityKey: 1, precedingText: "sc", at: 5), 0.08, accuracy: 0.001) + XCTAssertEqual(cadence.remainingDelay(identityKey: 1, precedingText: "sc", at: 5), 0.04, accuracy: 0.001) cadence.record(identityKey: 1, characters: "pasted", at: 5.01) XCTAssertEqual(cadence.remainingDelay(identityKey: 1, precedingText: "pasted", at: 5.02), 0) } diff --git a/patches/cotabbyinference-upstream-pending.patch b/patches/cotabbyinference-upstream-pending.patch index d2de3d353..7f3446b1a 100644 --- a/patches/cotabbyinference-upstream-pending.patch +++ b/patches/cotabbyinference-upstream-pending.patch @@ -1,5 +1,5 @@ diff --git a/README.md b/README.md -index 1b2aca3..673e8de 100644 +index 1b2aca3..eac988c 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Cotabby LlamaRuntimeCore @@ -47,7 +47,7 @@ index 1b2aca3..673e8de 100644 ## Generation Semantics -@@ -119,16 +122,55 @@ Cotabby controls the maximum token count in Swift. The engine controls token sel +@@ -119,16 +122,60 @@ Cotabby controls the maximum token count in Swift. The engine controls token sel sampling selected visible text; - `logprob`: the selected token's raw-model log-probability when enabled. @@ -72,7 +72,10 @@ index 1b2aca3..673e8de 100644 -pending feedback token. Cotabby independently validates request continuity, UTF-8 prefix, token -prefix, and sampling compatibility before calling it. +`trimKV` restores a prefix in slot zero and invalidates its pending seed/feedback token. Ordinary -+attention uses direct suffix removal. Recurrent, hybrid, and sliding-window models use one ++and sliding-window attention use direct suffix removal: the context is created with `swa_full`, ++so the SWA cache has one cell per context position and never overwrites a row a later trim ++would need (a default-sized SWA cache keeps only `n_swa + n_ubatch` cells). Recurrent and hybrid ++models, whose recurrent state cannot be rewound by removing KV, use one +`PARTIAL_ONLY | ON_DEVICE` checkpoint near the prompt tail, retaining full attention KV in place. +The checkpoint's tensor memory is capped at 128 MiB and restoration replays at most eight prompt +tokens without sampling. Device storage belongs to the llama context's slot-zero checkpoint and @@ -81,9 +84,11 @@ index 1b2aca3..673e8de 100644 +A request that edits before the saved checkpoint, a very short prompt without a nonempty checkpoint, +or a model exceeding the cap returns a cache miss; callers rebuild that request. A miss must not +permanently disable reuse for the model. Saving a newer near-caret checkpoint intentionally gives -+up deeper backspace history. Gemma's sliding-window cache supports suffix removal, but restoring -+its saved window also protects rows evicted during prediction. No model-family-name heuristics -+are used: llama model metadata selects the partial-state path. ++up deeper backspace history. No model-family-name heuristics are used: llama model metadata ++selects the partial-state path. ++ ++`updateSampling(sequence, config)` swaps the sampler chain of the live sequence without touching ++its KV, so a request whose sampling settings differ from the cached one still reuses the prompt. + +`decodePrompt` resets the sampler and accepts only committed prompt tokens. Discarded predictions +therefore never pollute penalties or RNG state; `llama_sampler_sample` already accepts its sample, @@ -108,7 +113,7 @@ index 1b2aca3..673e8de 100644 ## Testing -@@ -144,10 +186,31 @@ Run the full native path with a local GGUF: +@@ -144,10 +191,31 @@ Run the full native path with a local GGUF: COTABBY_TEST_MODEL_PATH=/absolute/path/model.gguf swift test ~~~ @@ -142,7 +147,7 @@ index 1b2aca3..673e8de 100644 ## Requirements diff --git a/Sources/CotabbyInferenceEngine/CotabbyInferenceEngine.cpp b/Sources/CotabbyInferenceEngine/CotabbyInferenceEngine.cpp -index 5fc867f..b9b6aa6 100644 +index 5fc867f..7ff725a 100644 --- a/Sources/CotabbyInferenceEngine/CotabbyInferenceEngine.cpp +++ b/Sources/CotabbyInferenceEngine/CotabbyInferenceEngine.cpp @@ -1,4 +1,5 @@ @@ -337,16 +342,34 @@ index 5fc867f..b9b6aa6 100644 } // Masks every "starts a new word" token (decoded text begins with whitespace) in the logits -@@ -424,6 +555,8 @@ EngineStatus CotabbyInferenceEngine::loadModel(const char* path, int gpu_layers, +@@ -424,6 +555,13 @@ EngineStatus CotabbyInferenceEngine::loadModel(const char* path, int gpu_layers, impl_->context_window_tokens = context_window_tokens; impl_->batch_size = batch_size; impl_->gpu_layer_count = gpu_layers; ++ // Only recurrent state cannot be rewound by removing a KV suffix. Sliding-window attention ++ // can, provided its cache keeps every position (`swa_full` below); a default-sized SWA cache ++ // holds only n_swa + n_ubatch cells and overwrites older ones, which is what used to force ++ // SWA models such as Gemma onto the eight-token checkpoint path and a full prompt re-decode ++ // after nearly every keystroke. + impl_->needs_prompt_checkpoint = llama_model_is_recurrent(impl_->model) || -+ llama_model_is_hybrid(impl_->model) || llama_model_n_swa(impl_->model) > 0; ++ llama_model_is_hybrid(impl_->model); // Performance cores only — see resolveDecodeThreadCount. hardware_concurrency() counted // every logical core including efficiency cores, which both slows barriered matmuls and // burns extra package power for nothing when layers are Metal-offloaded anyway. -@@ -471,6 +604,7 @@ void CotabbyInferenceEngine::unloadModel() { +@@ -439,6 +577,12 @@ EngineStatus CotabbyInferenceEngine::loadModel(const char* path, int gpu_layers, + ctx_params.n_threads = static_cast(impl_->thread_count); + ctx_params.n_threads_batch = static_cast(impl_->thread_count); + ctx_params.offload_kqv = true; ++ // Size the sliding-window cache like the full-attention cache so no position inside the ++ // context window is ever overwritten. That keeps `llama_memory_seq_rm` suffix trims valid for ++ // SWA models, which is what lets each keystroke decode only its changed tail. The cost is KV ++ // memory proportional to n_ctx for the SWA layers; Cotabby's single 4K sequence keeps that ++ // bounded. Models without SWA layers ignore the flag. ++ ctx_params.swa_full = true; + + impl_->shared_ctx = llama_init_from_model(impl_->model, ctx_params); + if (!impl_->shared_ctx) { +@@ -471,6 +615,7 @@ void CotabbyInferenceEngine::unloadModel() { } impl_->vocab = nullptr; impl_->model_path.clear(); @@ -354,7 +377,7 @@ index 5fc867f..b9b6aa6 100644 if (impl_->backend_initialized) { llama_backend_free(); -@@ -495,6 +629,7 @@ int32_t CotabbyInferenceEngine::createSequence(SamplingConfig config) { +@@ -495,10 +640,31 @@ int32_t CotabbyInferenceEngine::createSequence(SamplingConfig config) { auto state = std::make_unique(); state->external_id = id; state->sampler = sampler; @@ -362,7 +385,31 @@ index 5fc867f..b9b6aa6 100644 impl_->sequence = std::move(state); return id; } -@@ -574,6 +709,9 @@ EngineStatus CotabbyInferenceEngine::decodePrompt(int32_t sequence_id, + ++bool CotabbyInferenceEngine::updateSampling(int32_t sequence_id, SamplingConfig config) { ++ if (!impl_ || !impl_->model || !impl_->shared_ctx) return false; ++ llama_sampler* sampler = impl_->buildSampler(config); ++ if (!sampler) return false; ++ ++ // Hold the context lock as well: sampleNext reads the sampler under decode_mutex, so swapping ++ // it here can never race a token being sampled by an in-flight generation. ++ std::lock_guard sequence_lock(impl_->sequence_mutex); ++ std::lock_guard decode_lock(impl_->decode_mutex); ++ if (!impl_->sequence || impl_->sequence->external_id != sequence_id) { ++ llama_sampler_free(sampler); ++ return false; ++ } ++ SequenceState& seq = *impl_->sequence; ++ if (seq.sampler) llama_sampler_free(seq.sampler); ++ seq.sampler = sampler; ++ seq.single_line = config.single_line; ++ return true; ++} ++ + void CotabbyInferenceEngine::destroySequence(int32_t sequence_id) { + if (!impl_) return; + +@@ -574,6 +740,9 @@ EngineStatus CotabbyInferenceEngine::decodePrompt(int32_t sequence_id, SequenceState* seq = impl_->findSequence(sequence_id); if (!seq) return EngineStatus::error; @@ -372,7 +419,7 @@ index 5fc867f..b9b6aa6 100644 if (seq->cancelled.load(std::memory_order_acquire)) { return EngineStatus::cancelled; -@@ -587,6 +725,12 @@ EngineStatus CotabbyInferenceEngine::decodePrompt(int32_t sequence_id, +@@ -587,6 +756,12 @@ EngineStatus CotabbyInferenceEngine::decodePrompt(int32_t sequence_id, int cursor = 0; int end = token_count; int total_end_position = start_position + token_count; @@ -385,7 +432,7 @@ index 5fc867f..b9b6aa6 100644 while (cursor < end) { if (seq->cancelled.load(std::memory_order_acquire)) { -@@ -595,6 +739,10 @@ EngineStatus CotabbyInferenceEngine::decodePrompt(int32_t sequence_id, +@@ -595,6 +770,10 @@ EngineStatus CotabbyInferenceEngine::decodePrompt(int32_t sequence_id, } int chunk_end = std::min(cursor + batch_cap, end); @@ -396,7 +443,7 @@ index 5fc867f..b9b6aa6 100644 int chunk_size = chunk_end - cursor; batch.n_tokens = static_cast(chunk_size); -@@ -612,20 +760,37 @@ EngineStatus CotabbyInferenceEngine::decodePrompt(int32_t sequence_id, +@@ -612,20 +791,37 @@ EngineStatus CotabbyInferenceEngine::decodePrompt(int32_t sequence_id, } if (llama_decode(impl_->shared_ctx, batch) != 0) { @@ -435,7 +482,7 @@ index 5fc867f..b9b6aa6 100644 impl_->maskNewWordStarts(-1); seq->force_word_continuation = false; } -@@ -633,10 +798,10 @@ EngineStatus CotabbyInferenceEngine::decodePrompt(int32_t sequence_id, +@@ -633,10 +829,10 @@ EngineStatus CotabbyInferenceEngine::decodePrompt(int32_t sequence_id, // Seed sample: take one token from the prompt's final logits row. The seed will be returned by // the next sampleNext call as-is and feedback-decoded by the call after that. llama_token seed = llama_sampler_sample(seq->sampler, impl_->shared_ctx, -1); @@ -448,7 +495,7 @@ index 5fc867f..b9b6aa6 100644 seq->has_seed_token = true; seq->has_pending_input = false; -@@ -668,6 +833,10 @@ SampleResult CotabbyInferenceEngine::sampleNext(int32_t sequence_id) { +@@ -668,6 +864,10 @@ SampleResult CotabbyInferenceEngine::sampleNext(int32_t sequence_id) { result.is_eos = true; return result; } @@ -459,7 +506,7 @@ index 5fc867f..b9b6aa6 100644 if (seq->cancelled.load(std::memory_order_acquire)) { result.was_cancelled = true; -@@ -732,20 +901,37 @@ SampleResult CotabbyInferenceEngine::sampleNext(int32_t sequence_id) { +@@ -732,20 +932,37 @@ SampleResult CotabbyInferenceEngine::sampleNext(int32_t sequence_id) { batch.logits[0] = 1; const int status = llama_decode(impl_->shared_ctx, batch); @@ -499,7 +546,7 @@ index 5fc867f..b9b6aa6 100644 seq->last_piece.resize(64); while (true) { const int written = llama_token_to_piece( -@@ -775,7 +961,6 @@ SampleResult CotabbyInferenceEngine::sampleNext(int32_t sequence_id) { +@@ -775,7 +992,6 @@ SampleResult CotabbyInferenceEngine::sampleNext(int32_t sequence_id) { // Feedback decode advanced KV by one position; record the just-sampled // token as input for the next call. @@ -507,7 +554,7 @@ index 5fc867f..b9b6aa6 100644 seq->pending_input_token = result.token; seq->has_pending_input = true; return result; -@@ -789,6 +974,10 @@ bool CotabbyInferenceEngine::trimKV(int32_t sequence_id, int keep_positions) { +@@ -789,6 +1005,10 @@ bool CotabbyInferenceEngine::trimKV(int32_t sequence_id, int keep_positions) { if (!impl_->shared_ctx) return false; SequenceState* seq = impl_->findSequence(sequence_id); if (!seq) return false; @@ -518,7 +565,7 @@ index 5fc867f..b9b6aa6 100644 llama_memory_t memory = llama_get_memory(impl_->shared_ctx); if (!memory) return false; -@@ -796,20 +985,34 @@ bool CotabbyInferenceEngine::trimKV(int32_t sequence_id, int keep_positions) { +@@ -796,20 +1016,34 @@ bool CotabbyInferenceEngine::trimKV(int32_t sequence_id, int keep_positions) { // Serialize with prompt and feedback decode; never remove KV while llama is mutating it. std::lock_guard lock(impl_->decode_mutex); @@ -559,7 +606,7 @@ index 5fc867f..b9b6aa6 100644 } return ok; } -@@ -822,6 +1025,23 @@ void CotabbyInferenceEngine::setForceWordContinuation(int32_t sequence_id, bool +@@ -822,6 +1056,23 @@ void CotabbyInferenceEngine::setForceWordContinuation(int32_t sequence_id, bool } } @@ -583,7 +630,7 @@ index 5fc867f..b9b6aa6 100644 void CotabbyInferenceEngine::setComputeLogprob(int32_t sequence_id, bool enabled) { if (!impl_) return; SequenceState* seq = impl_->findSequence(sequence_id); -@@ -835,9 +1055,12 @@ void CotabbyInferenceEngine::setComputeLogprob(int32_t sequence_id, bool enabled +@@ -835,9 +1086,12 @@ void CotabbyInferenceEngine::setComputeLogprob(int32_t sequence_id, bool enabled // --------------------------------------------------------------------------- void CotabbyInferenceEngine::cancelSequence(int32_t sequence_id) { @@ -599,7 +646,7 @@ index 5fc867f..b9b6aa6 100644 } } -@@ -860,3 +1083,16 @@ int CotabbyInferenceEngine::getThreadCount() const { +@@ -860,3 +1114,16 @@ int CotabbyInferenceEngine::getThreadCount() const { int CotabbyInferenceEngine::getGPULayerCount() const { return impl_->gpu_layer_count; } @@ -772,7 +819,7 @@ index 0000000..15712fb + +} // namespace cotabby diff --git a/Sources/CotabbyInferenceEngine/include/CotabbyInferenceEngine.h b/Sources/CotabbyInferenceEngine/include/CotabbyInferenceEngine.h -index 383bcaf..d15151e 100644 +index 383bcaf..c35f0d9 100644 --- a/Sources/CotabbyInferenceEngine/include/CotabbyInferenceEngine.h +++ b/Sources/CotabbyInferenceEngine/include/CotabbyInferenceEngine.h @@ -40,6 +40,15 @@ enum class EngineStatus : int { @@ -791,7 +838,15 @@ index 383bcaf..d15151e 100644 class CotabbyInferenceEngine { public: CotabbyInferenceEngine(); -@@ -64,6 +73,11 @@ public: +@@ -61,9 +70,19 @@ public: + // alive; destroy the current sequence before creating its replacement. + int32_t createSequence(SamplingConfig config); + void destroySequence(int32_t sequence_id); ++ // Replaces the live sequence's sampler chain while keeping its KV cache. Sampler state never ++ // depends on KV contents: decodePrompt resets it and re-accepts the prompt tokens, so a ++ // changed temperature/top-k/single-line request can reuse the cached prompt instead of ++ // destroying it. Returns false when the sequence is unknown or the chain cannot be built. ++ bool updateSampling(int32_t sequence_id, SamplingConfig config); // Tokenization (thread-safe, read-only on vocab) std::vector tokenize(const char* text, int text_length) const; @@ -803,7 +858,7 @@ index 383bcaf..d15151e 100644 // Prompt decoding EngineStatus decodePrompt(int32_t sequence_id, const int32_t* tokens, int token_count, -@@ -94,6 +108,7 @@ public: +@@ -94,6 +113,7 @@ public: int getBatchSize() const; int getThreadCount() const; int getGPULayerCount() const;