From 99042ae91450d9d8f36fc05f389508b68733d708 Mon Sep 17 00:00:00 2001 From: chrisnojima Date: Tue, 1 Sep 2026 21:21:00 -0400 Subject: [PATCH 01/21] fix(chat): stop stranding ordinals outside the loaded thread window A DB nuke left an ancient "set the channel name" message pinned at the top of threads, thousands of ordinals below the loaded window. messageOrdinals is the data array for LegendList, so a stranded item at index 0 makes back-paging a mid-list splice instead of a prepend, and onStartReached fires against that row rather than the real top of loaded content -- scrollback stops firing and the thread sits on "Digging ancient messages..." forever. It arrives by notification, not by thread load: the localizer caches the ancient METADATA message into thread storage unboxed in quick mode, then every load's post-send ResolveSkippedUnboxeds re-unboxes it, marks it changed, and pushes a MessagesUpdated notification into addMessages, which had no notion of the window. - addMessages may update a message already in the window and may append above it, but may not introduce a new ordinal below the window floor. Thread loads are unaffected; they are the only thing allowed to extend the window down. Nothing is deleted -- the message loads normally when paged back to. - Narrow the validatedRange prune to responses that are provably whole windows. In INCREMENTAL cb mode the service filters the full response down to changed messages once a cached thread has been sent, so neither response is complete; the old latch fired on the cached one and was observed deleting 6 real messages on an ordinary focused refresh. - jumpToRecent clears before reloading, the way loadMessagesCentered does, instead of merging a disjoint newest window into the old one. - Do not resolve the orange line from a non-positive read position. emptyConversationMeta reads -1, and clamping that to 0 asked the service "what is unread when nothing has been read", pinning the line above the oldest message; the state is set once and only refreshed while the conversation is inactive, so the wrong answer stuck. - patchPaginationLast: a page holding message ID 1 has reached the beginning, checked before the expunge record, which reads Upto:0 for a nuked conversation whose history was deleted and so never fired. - Drop the unused loadNextAttachment, and stop a superseded placeholder from leaving its own ordinal in the list with nothing in messageMap. --- go/chat/convsource.go | 36 +++++- .../chat/conversation/attachment-actions.tsx | 39 ------ .../conversation/normal/container.test.tsx | 25 +++- shared/chat/conversation/normal/container.tsx | 10 +- .../chat/conversation/thread-context.test.tsx | 69 +++++++++++ shared/chat/conversation/thread-context.tsx | 68 +++++++++-- shared/chat/conversation/thread-load.tsx | 32 +++-- .../thread-message-state.test.tsx | 114 ++++++++++++++++++ .../conversation/thread-message-state.tsx | 82 ++++++++++++- 9 files changed, 404 insertions(+), 71 deletions(-) diff --git a/go/chat/convsource.go b/go/chat/convsource.go index b3218c0313d9..a00b1cb830c9 100644 --- a/go/chat/convsource.go +++ b/go/chat/convsource.go @@ -231,14 +231,24 @@ func (s *baseConversationSource) patchPaginationLast(ctx context.Context, conv t page.Last = true return } + end1 := msgs[0].GetMessageID() + end2 := msgs[len(msgs)-1].GetMessageID() + oldest := end1.Min(end2) + // Message IDs start at 1, so a page holding it has reached the beginning of the conversation and + // nothing older can exist. Worth checking before the expunge record because that record is not + // always populated: a conversation whose history was deleted reads back Upto:0 until its inbox + // entry is localized, and until then every page of it looks like there is more to come. + if oldest <= 1 { + s.Debug(ctx, "patchPaginationLast: true - reached the first message") + page.Last = true + return + } expunge := conv.GetExpunge() if expunge == nil { s.Debug(ctx, "patchPaginationLast: no expunge info") return } - end1 := msgs[0].GetMessageID() - end2 := msgs[len(msgs)-1].GetMessageID() - if end1.Min(end2) <= expunge.Upto { + if oldest <= expunge.Upto { s.Debug(ctx, "patchPaginationLast: true - hit upto") // If any message is prior to the nukepoint, say this is the last page. page.Last = true @@ -909,6 +919,26 @@ func (s *HybridConversationSource) PullLocalOnly(ctx context.Context, convID cha s.Debug(ctx, "PullLocalOnly: failed to fetch local messages with local max: %s", err) return chat1.ThreadView{}, err } + // This retry anchors on the local max instead of the inbox max, which is what we want when + // local storage is merely behind. But after the cache is wiped the only thing left can be a + // lone ancient message the inbox localizer cached for a channel name, headline or pin. Handed + // up as the newest page, it strands itself thousands of IDs above the real thread in the UI. + // Only accept a window that could plausibly overlap the page being asked for. + if iboxMaxMsgID > 0 && num > 0 && pagination.FirstPage() && len(tv.Messages) > 0 { + var newest chat1.MessageID + for _, m := range tv.Messages { + if id := m.GetMessageID(); id > newest { + newest = id + } + } + //nolint:gosec // G115: num is positive, checked above + if newest < iboxMaxMsgID && iboxMaxMsgID-newest > chat1.MessageID(num) { + s.Debug(ctx, + "PullLocalOnly: local max fallback newest %d is %d below ibox max %d, past the %d requested: reporting miss", + newest, iboxMaxMsgID-newest, iboxMaxMsgID, num) + return chat1.ThreadView{}, storage.MissError{Msg: "local copy does not reach the newest page"} + } + } } return tv, nil } diff --git a/shared/chat/conversation/attachment-actions.tsx b/shared/chat/conversation/attachment-actions.tsx index a859e6c4f514..3125261ceb5f 100644 --- a/shared/chat/conversation/attachment-actions.tsx +++ b/shared/chat/conversation/attachment-actions.tsx @@ -455,47 +455,8 @@ export const useConversationAttachmentActions = () => { ignorePromise(f()) } - const loadNextAttachment = async (from: T.Chat.Ordinal, backInTime: boolean) => { - const fromMsg = threadStore.getState().messageMap.get(from) - if (!fromMsg) { - return Promise.reject(new Error('Incorrect from')) - } - const {deviceName, username} = useCurrentUserState.getState() - const getLastOrdinal = () => threadStore.getState().messageOrdinals?.at(-1) ?? T.Chat.numberToOrdinal(0) - const result = await T.RPCChat.localGetNextAttachmentMessageLocalRpcPromise({ - assetTypes: [T.RPCChat.AssetMetadataType.image, T.RPCChat.AssetMetadataType.video], - backInTime, - convID: T.Chat.keyToConversationID(conversationIDKey), - identifyBehavior: T.RPCGen.TLFIdentifyBehavior.chatGui, - messageID: fromMsg.id, - }) - - if (result.message) { - const goodMessage = Message.uiMessageToMessage( - conversationIDKey, - result.message, - username, - getLastOrdinal, - deviceName - ) - if (goodMessage?.type === 'attachment') { - actions.addMessages([goodMessage]) - let ordinal = goodMessage.ordinal - if (goodMessage.outboxID && !threadStore.getState().messageMap.get(ordinal)) { - const pendingOrdinal = threadStore.getState().pendingOutboxToOrdinal.get(goodMessage.outboxID) - if (pendingOrdinal) { - ordinal = pendingOrdinal - } - } - return ordinal - } - } - return Promise.reject(new Error('No more results')) - } - return { attachmentDownload, - loadNextAttachment, messageAttachmentNativeSave, messageAttachmentNativeShare, showAttachmentPreview: (ordinal: T.Chat.Ordinal, message?: T.Chat.MessageAttachment) => { diff --git a/shared/chat/conversation/normal/container.test.tsx b/shared/chat/conversation/normal/container.test.tsx index 1d8c11f6ee20..f4f7f3199c64 100644 --- a/shared/chat/conversation/normal/container.test.tsx +++ b/shared/chat/conversation/normal/container.test.tsx @@ -322,7 +322,11 @@ test('a thread reload does not refetch the orange line against the stale mount r expectOrangeLine(noOrangeLine) }) -test('negative read message IDs are clamped before fetching the orange line', async () => { +test('an unknown read position draws no orange line rather than one above everything', async () => { + // There is no valid message ID 0, so a non-positive read position means the conversation's meta + // has not landed yet (emptyConversationMeta reads -1), which a DB nuke makes the norm. Asking the + // service with 0 answers "everything is unread" and pins the line above the oldest message, and + // the state is set once, so that answer used to stick for the life of the mount. const unreadlineRpc = getUnreadlineRpc().mockResolvedValue({ offline: false, unreadlineID: T.Chat.numberToMessageID(8), @@ -332,9 +336,22 @@ test('negative read message IDs are clamped before fetching the orange line', as render() await flushOrangeLine() - expect(unreadlineRpc).toHaveBeenCalledTimes(1) - expectUnreadlineRpcReadMsgID(unreadlineRpc, 0) - expectOrangeLine(T.Chat.numberToOrdinal(8)) + expect(unreadlineRpc).not.toHaveBeenCalled() + expectOrangeLine(noOrangeLine) +}) + +test('a zero read position is treated as unknown too', async () => { + const unreadlineRpc = getUnreadlineRpc().mockResolvedValue({ + offline: false, + unreadlineID: T.Chat.numberToMessageID(8), + }) + mockMeta = makeMeta(convID, 0) + + render() + await flushOrangeLine() + + expect(unreadlineRpc).not.toHaveBeenCalled() + expectOrangeLine(noOrangeLine) }) test('zero unreadline responses render as no orange line', async () => { diff --git a/shared/chat/conversation/normal/container.tsx b/shared/chat/conversation/normal/container.tsx index 2c9b4a8f28f9..559b6ef54a85 100644 --- a/shared/chat/conversation/normal/container.tsx +++ b/shared/chat/conversation/normal/container.tsx @@ -64,12 +64,20 @@ const useOrangeLine = ( const loadOrangeLine = React.useEffectEvent( (conversationIDKey: T.Chat.ConversationIDKey, readMsgID: T.Chat.MessageID) => { + // There is no valid message ID 0, so a non-positive read position means we do not know it yet + // rather than "nothing has been read": an unlocalized conversation reads -1 from + // emptyConversationMeta, which a DB nuke makes the norm. Asking the service with 0 answers + // "everything is unread" and puts the line above the oldest message, and since the state is + // set once and only refreshed while the conversation is inactive, that answer sticks. + if (readMsgID <= 0) { + return + } const f = async () => { const convID = T.Chat.keyToConversationID(conversationIDKey) const unreadlineRes = await T.RPCChat.localGetUnreadlineRpcPromise({ convID, identifyBehavior: T.RPCGen.TLFIdentifyBehavior.chatGui, - readMsgID: readMsgID < 0 ? 0 : readMsgID, + readMsgID, }) const nextOrangeLine = T.Chat.numberToOrdinal( unreadlineRes.unreadlineID ? unreadlineRes.unreadlineID : 0 diff --git a/shared/chat/conversation/thread-context.test.tsx b/shared/chat/conversation/thread-context.test.tsx index b2dc1803d231..18761e18ddf8 100644 --- a/shared/chat/conversation/thread-context.test.tsx +++ b/shared/chat/conversation/thread-context.test.tsx @@ -455,6 +455,7 @@ test('scrollback loads older messages without marking the thread read', async () act(() => { result.current.actions.applyThreadLoad({ + authoritative: true, centered: false, enableActiveMarkRead: false, messages: [makeTextMessage()], @@ -682,6 +683,7 @@ test('mounted thread listener applies incoming messages while inactive without m act(() => { result.current.actions.applyThreadLoad({ + authoritative: true, centered: false, enableActiveMarkRead: false, messages: [], @@ -999,6 +1001,7 @@ test('loaded focus refresh does not overwrite newer streamed reaction updates', act(() => { result.current.actions.applyThreadLoad({ + authoritative: true, centered: false, enableActiveMarkRead: true, messages: [makeTextMessage()], @@ -1090,6 +1093,7 @@ test('toggleMessageReaction overlays locally without mutating server reactions', act(() => { result.current.actions.applyThreadLoad({ + authoritative: true, centered: false, enableActiveMarkRead: false, messages: [makeTextMessage()], @@ -1376,3 +1380,68 @@ test('mounted thread listener applies attachment download and upload progress', : undefined ).toBeUndefined() }) + +test('a cached pass never prunes messages the incremental full pass no longer resends', async () => { + // Regression: once the service has sent a cached thread it switches the full response to + // INCREMENTAL, so the full pass only carries the messages that changed. Treating either partial + // response as authoritative deleted real messages that were still in the thread. + useConfigState.setState({loggedIn: true}) + jest.spyOn(Common, 'isUserActivelyLookingAtThisThread').mockReturnValue(true) + jest.spyOn(T.RPCChat, 'localMarkAsReadLocalRpcPromise').mockResolvedValue({offline: false}) + const ids = [301, 302, 303, 304].map(T.Chat.numberToMessageID) + const threadJSON = (msgIDs: ReadonlyArray) => + JSON.stringify({ + messages: msgIDs.map(id => makeValidTextUIMessage(id, `m${id}`)), + pagination: {last: true, next: '', num: 100, previous: ''}, + }) + + // A partial cached pass missing 302/303, then an incremental full pass carrying only the one + // message that changed. Nothing may be dropped. + jest.spyOn(T.RPCChat, 'localGetThreadNonblockRpcListener').mockImplementation(async p => { + p.incomingCallMap['chat.1.chatUi.chatThreadCached']?.({thread: threadJSON([ids[0]!, ids[3]!])}) + await Promise.resolve() + p.incomingCallMap['chat.1.chatUi.chatThreadFull']?.({thread: threadJSON([ids[3]!])}) + await Promise.resolve() + return {offline: false} + }) + const {result} = renderHook( + () => ({ + actions: useConversationThreadActions(), + loadMoreMessages: useConversationThreadLoadMoreMessages(), + ordinals: useConversationThreadSelector(s => s.messageOrdinals), + }), + {wrapper} + ) + + // Seed a settled four-message window the way a whole-window full pass would. + act(() => { + result.current.actions.applyThreadLoad({ + authoritative: true, + centered: false, + enableActiveMarkRead: false, + messages: ids.map(id => + Message.makeMessageText({ + author: 'alice', + conversationIDKey: convID, + id, + ordinal: T.Chat.numberToOrdinal(T.Chat.messageIDToNumber(id)), + outboxID: undefined, + text: new HiddenString(`m${id}`), + timestamp: 100, + }) + ), + moreToLoad: false, + scrollDirection: 'none', + }) + }) + expect(result.current.ordinals).toEqual([301, 302, 303, 304]) + + act(() => { + result.current.loadMoreMessages({reason: 'test'}) + }) + await act(async () => { + await flushPromises() + }) + + expect(result.current.ordinals).toEqual([301, 302, 303, 304]) +}) diff --git a/shared/chat/conversation/thread-context.tsx b/shared/chat/conversation/thread-context.tsx index 237100b19655..7edd6e67d754 100644 --- a/shared/chat/conversation/thread-context.tsx +++ b/shared/chat/conversation/thread-context.tsx @@ -22,6 +22,8 @@ import {useIsFocused} from '@react-navigation/core' import { addMessagesToThreadState, applyOptimisticReactionsToMessage, + describeOrdinalGaps, + GAPPROBE, completeAttachmentDownloadInThreadState, clearOptimisticReactionsForUpdatesInThreadState, clearOptimisticReactionsForMessagesInThreadState, @@ -216,6 +218,7 @@ export type ConversationThreadActions = { } ) => void applyThreadLoad: (p: { + authoritative: boolean centered: boolean disableActiveMarkRead?: boolean enableActiveMarkRead: boolean @@ -225,7 +228,6 @@ export type ConversationThreadActions = { scrollDirection: ScrollDirection validatedRange?: {from: T.Chat.Ordinal; to: T.Chat.Ordinal} }) => void - clearValidatedOrdinalRange: () => void clearUnfurlPrompt: (messageID: T.Chat.MessageID, domain: string) => void deleteMessages: (p: { messageIDs?: ReadonlyArray @@ -471,8 +473,26 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => if (opt.liveUpdate) { s.liveUpdateVersion += 1 } - addMessagesToThreadState(s, messages, {validatedRange: opt.validatedRange}) + const beforeMin = s.messageOrdinals?.[0] + addMessagesToThreadState(s, messages, { + // Only thread loads may extend the window downward; a notification must not. + dropNewBelowWindow: true, + validatedRange: opt.validatedRange, + }) clearOptimisticReactionsForMessagesInThreadState(s, messages) + const afterMin = s.messageOrdinals?.[0] + if (beforeMin !== undefined && afterMin !== undefined && afterMin < beforeMin) { + const m = s.messageMap.get(afterMin) + logger.info( + `${GAPPROBE}: conv=${id.slice(0, 12)} addMessages LOWERED the window floor ${beforeMin} -> ${afterMin}` + + ` ord=${afterMin} type=${m?.type ?? 'MISSING'} id=${m?.id ?? '?'}` + + ` liveUpdate=${!!opt.liveUpdate} batch=${messages.length}` + + ` batchOrds=${messages + .slice(0, 8) + .map(x => `${x.ordinal}/${x.type}`) + .join(',')}` + ) + } }) if (opt.markAsRead) { markThreadAsRead() @@ -485,8 +505,12 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => activeMarkReadEnabledRef.current = false } }) + // Debug-only: every ordinal the service has sent for this conversation, deleted ones included, so + // the gap probe can tell an expunged range from a stranded ordinal. Dies with the conversation. + const gapProbeAccounted = React.useRef(new Set()) const applyThreadLoad = React.useEffectEvent( (p: { + authoritative: boolean centered: boolean disableActiveMarkRead?: boolean enableActiveMarkRead: boolean @@ -501,6 +525,32 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => if (p.messages.length) { addMessagesToThreadState(s, p.messages, {validatedRange: p.validatedRange}) clearOptimisticReactionsForMessagesInThreadState(s, p.messages) + const incoming = p.messages.map(m => m.ordinal) + for (const o of incoming) { + gapProbeAccounted.current.add(o) + } + const incomingMin = Math.min(...incoming) + const desc = describeOrdinalGaps(s.messageOrdinals, gapProbeAccounted.current) + logger.info( + `${GAPPROBE}: conv=${id.slice(0, 12)} load pass=${p.authoritative ? 'full' : 'cached'} dir=${p.scrollDirection}` + + ` centered=${p.centered} incoming=${incoming.length}` + + ` [${incomingMin}..${Math.max(...incoming)}] -> ` + + desc + ) + if (desc.includes('>>> BAD')) { + const stranded = (s.messageOrdinals ?? []).filter(o => o < incomingMin) + logger.info( + `${GAPPROBE}: conv=${id.slice(0, 12)} stranded below incoming min ${incomingMin}: ` + + stranded + .slice(0, 6) + .map(o => { + const m = s.messageMap.get(o) + return `ord=${o} type=${m?.type ?? 'MISSING'} id=${m?.id ?? '?'} outbox=${m?.outboxID ?? '-'}` + }) + .join(' ; ') + + ` | inBatch=${incoming.includes(stranded[0] as T.Chat.Ordinal)}` + ) + } } switch (p.scrollDirection) { case 'forward': @@ -866,13 +916,9 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => markThreadAsRead() } ) - const clearValidatedOrdinalRange = React.useEffectEvent(() => { - updateThreadState(s => { - s.validatedOrdinalRange = undefined - }) - }) const messagesClear = React.useEffectEvent(() => { activeMarkReadEnabledRef.current = false + gapProbeAccounted.current.clear() shownUsernameCache.clear() updateThreadState(s => { s.clearVersion += 1 @@ -1001,7 +1047,6 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => addOptimisticReaction, applyThreadLoad, clearUnfurlPrompt, - clearValidatedOrdinalRange, completeAttachmentDownload, deleteMessages, explodeMessages, @@ -1184,12 +1229,15 @@ export const useConversationThreadLoadMessagesCentered = () => { } export const useConversationThreadJumpToRecent = () => { - const {clearValidatedOrdinalRange, setMarkReadBlocked} = useConversationThreadActions() + const {setMarkReadBlocked} = useConversationThreadActions() const loadMoreMessages = useConversationThreadLoadMoreMessages() + const messagesClear = useConversationThreadMessagesClear() const jumpToRecent: JumpToRecent = options => { setMarkReadBlocked(false) - clearValidatedOrdinalRange() + // The newest window is disjoint from wherever the reader was, so merging the two would leave a + // gap in the ordinals. Drop the old window first, the way a centered jump does. + messagesClear() loadMoreMessages({...(options ?? {}), reason: 'jump to recent'}) } return jumpToRecent diff --git a/shared/chat/conversation/thread-load.tsx b/shared/chat/conversation/thread-load.tsx index de4121253cad..2cf411665638 100644 --- a/shared/chat/conversation/thread-load.tsx +++ b/shared/chat/conversation/thread-load.tsx @@ -204,8 +204,15 @@ export const loadConversationThreadMessages = ( ) const loadingKey = Strings.waitingKeyChatThreadLoad(conversationIDKey) - let reconciled = false + // Set as soon as a cached response arrives, before any early return. Once the service has sent + // a cached thread it switches the full response to INCREMENTAL, filtering it down to only the + // messages that changed (chat/uithreadloader.go mergeLocalRemoteThread). From that point neither + // response is a complete window, so neither can be treated as authoritative. + let sawCachedPass = false const onGotThread = (thread: string, why: string) => { + if (why === 'cached') { + sawCachedPass = true + } if (!thread) { return } @@ -239,22 +246,23 @@ export const loadConversationThreadMessages = ( scrollDirection !== 'back' && reason !== 'findNewestConversation' && reason !== 'findNewestConversationFromLayout' + // Pruning is only safe against a response that is a whole window: a full pass with no cached + // pass before it. Anything else is partial, and pruning against it deletes messages that are + // still in the thread. let validatedRange: {from: T.Chat.Ordinal; to: T.Chat.Ordinal} | undefined - if (messages.length) { - if (scrollDirection === 'none' && !reconciled) { - const ords = messages - .filter(m => m.conversationMessage !== false && m.type !== 'deleted') - .map(m => m.ordinal) - if (ords.length > 0) { - validatedRange = { - from: Math.min(...ords) as T.Chat.Ordinal, - to: Math.max(...ords) as T.Chat.Ordinal, - } + if (messages.length && scrollDirection === 'none' && why === 'full' && !sawCachedPass) { + const ords = messages + .filter(m => m.conversationMessage !== false && m.type !== 'deleted') + .map(m => m.ordinal) + if (ords.length > 0) { + validatedRange = { + from: Math.min(...ords) as T.Chat.Ordinal, + to: Math.max(...ords) as T.Chat.Ordinal, } - reconciled = true } } actions.applyThreadLoad({ + authoritative: why === 'full', centered: !!centeredMessageID, disableActiveMarkRead: !allowMarkAsRead || !!centeredMessageID || !!messageIDControl, enableActiveMarkRead: canMarkReadForThreadWindow, diff --git a/shared/chat/conversation/thread-message-state.test.tsx b/shared/chat/conversation/thread-message-state.test.tsx index c883899caf91..cfaa2ddaaad8 100644 --- a/shared/chat/conversation/thread-message-state.test.tsx +++ b/shared/chat/conversation/thread-message-state.test.tsx @@ -4,6 +4,7 @@ import * as T from '@/constants/types' import HiddenString from '@/util/hidden-string' import { addMessagesToThreadState, + describeOrdinalGaps, applyOptimisticReactionsToMessage, clearOptimisticReactionsForUpdatesInThreadState, deleteMessagesFromThreadState, @@ -398,6 +399,74 @@ describe('addMessagesToThreadState', () => { expect(state.validatedOrdinalRange).toEqual({from: 5, to: 60}) }) + + + + + test('a notification may not strand a new ordinal below the loaded window', () => { + // The post-load ResolveSkippedUnboxeds push can carry the channel-name message at ID 1 long + // after the window has moved on. Adding it puts an orphan row at index 0 and breaks scrollback. + const state = makeThreadState([]) + addMessagesToThreadState(state, [textAt(7152), textAt(7153)], {}) + addMessagesToThreadState(state, [textAt(1)], {dropNewBelowWindow: true}) + expect(state.messageOrdinals).toEqual([7152, 7153]) + }) + + test('a notification still updates a message already inside the window', () => { + const state = makeThreadState([]) + addMessagesToThreadState(state, [textAt(10), textAt(20)], {}) + addMessagesToThreadState(state, [textAt(10, {text: 'edited'})], {dropNewBelowWindow: true}) + expect(state.messageOrdinals).toEqual([10, 20]) + const m = state.messageMap.get(T.Chat.numberToOrdinal(10)) + expect(m?.type === 'text' ? m.text.stringValue() : undefined).toBe('edited') + }) + + test('a notification may still append a new ordinal above the window', () => { + const state = makeThreadState([]) + addMessagesToThreadState(state, [textAt(10), textAt(20)], {}) + addMessagesToThreadState(state, [textAt(21)], {dropNewBelowWindow: true}) + expect(state.messageOrdinals).toEqual([10, 20, 21]) + }) + + test('a thread load may still extend the window downward', () => { + const state = makeThreadState([]) + addMessagesToThreadState(state, [textAt(7152), textAt(7153)], {}) + addMessagesToThreadState(state, [textAt(7038)], {}) + expect(state.messageOrdinals).toEqual([7038, 7152, 7153]) + }) + + test('a superseded placeholder does not strand its own ordinal in the list', () => { + // A sent message keeps the fractional ordinal it had in the outbox, so a later placeholder for + // the same message ID maps onto that fractional ordinal, not its own integer one. + const state = makeThreadState([]) + addMessagesToThreadState( + state, + [ + makeTextMessage({ + id: T.Chat.numberToMessageID(100), + ordinal: T.Chat.numberToOrdinal(100.001), + outboxID: undefined, + }), + ], + {} + ) + expect(state.messageOrdinals).toEqual([100.001]) + addMessagesToThreadState( + state, + [ + Message.makeMessagePlaceholder({ + conversationIDKey: convID, + id: T.Chat.numberToMessageID(100), + ordinal: T.Chat.numberToOrdinal(100), + }), + ], + {} + ) + expect(state.messageOrdinals).toEqual([100.001]) + expect(state.messageMap.has(T.Chat.numberToOrdinal(100))).toBe(false) + }) + + test('the render type index only tracks non text messages', () => { const state = makeThreadState([]) const attachment = makeAttachmentMessage({ordinal: T.Chat.numberToOrdinal(20), outboxID: undefined}) @@ -406,3 +475,48 @@ describe('addMessagesToThreadState', () => { expect(state.messageTypeMap.get(T.Chat.numberToOrdinal(20))).toBe('attachment:file') }) }) + +describe('describeOrdinalGaps', () => { + const ords = (...n: ReadonlyArray) => n.map(T.Chat.numberToOrdinal) + const accounted = (...n: ReadonlyArray) => new Set(n.map(T.Chat.numberToOrdinal)) + + test('a contiguous window is OK', () => { + expect(describeOrdinalGaps(undefined)).toBe('ordinals=0 >>> OK') + expect(describeOrdinalGaps(ords(10, 11, 12))).toBe( + 'ordinals=3 range=[10..12] maxgap=0 gaps=0 explained=0 >>> OK' + ) + }) + + test('a gap the service accounted for is explained, not flagged', () => { + // Retention expunges arrive as hidden placeholders that become `deleted` and drop out of the + // ordinal list, so the missing IDs are still accounted for. + expect(describeOrdinalGaps(ords(10, 1000), accounted(10, 12, 500, 900, 1000))).toBe( + 'ordinals=2 range=[10..1000] maxgap=990 gaps=1 explained=1 >>> OK' + ) + }) + + test('small unexplained gaps are the normal churn of hidden message types', () => { + // Reactions, edits and unfurls burn IDs the service never sends, so these can never be + // accounted for and must not be flagged. + expect(describeOrdinalGaps(ords(89, 94, 97, 100), accounted(89, 94, 97, 100))).toBe( + 'ordinals=4 range=[89..100] maxgap=5 gaps=3 explained=0 >>> OK' + ) + }) + + test('a gap with nothing behind it is flagged as BAD', () => { + expect(describeOrdinalGaps(ords(1, 4114, 4115), accounted(1, 4114, 4115))).toBe( + 'ordinals=3 range=[1..4115] maxgap=4113 gaps=1 explained=0 >>> BAD unexplained gap 1->4114(4113)' + ) + }) + + test('the verdict reports the biggest unexplained gap and counts the rest', () => { + const out = describeOrdinalGaps(ords(1, 500, 5000, 5001), accounted(1, 500, 5000, 5001)) + expect(out).toContain('>>> BAD unexplained gap 500->5000(4500) +1 more') + }) + + test('a pending fractional ordinal is not a gap', () => { + expect(describeOrdinalGaps(ords(10, 11, 11.001))).toBe( + 'ordinals=3 range=[10..11.001] maxgap=0 gaps=0 explained=0 >>> OK' + ) + }) +}) diff --git a/shared/chat/conversation/thread-message-state.tsx b/shared/chat/conversation/thread-message-state.tsx index dd81ff9cb2cb..066d2dc4f6fd 100644 --- a/shared/chat/conversation/thread-message-state.tsx +++ b/shared/chat/conversation/thread-message-state.tsx @@ -3,6 +3,67 @@ import * as T from '@/constants/types' import HiddenString from '@/util/hidden-string' import type {WritableDraft} from '@/util/zustand' +// Temporary diagnostic for the thread-ordinal no-gap work. Bump the marker on every edit so a log +// can be tied back to the build that produced it. Remove once the gap producer is confirmed gone. +export const GAPPROBE = 'gapprobe/2026-09-01g' + +// A compact summary of where a loaded window is discontiguous, ending in a single verdict. +// +// Numbering gaps are normal on their own: edits, reactions, unfurls and deletes each burn a message +// ID the thread never shows, and retention expunges whole ranges. What separates those from a real +// hole is whether the service ever accounted for the missing IDs — an expunged range arrives as +// hidden placeholders (which become `deleted` and drop out of the list), while a stranded ordinal +// has nothing behind it at all. `accountedFor` is every ordinal the service has sent for this +// conversation, deleted ones included, accumulated across loads. +// +// Accounting alone is not enough: EDIT, DELETE, REACTION, ATTACHMENTUPLOADED, UNFURL and TLFNAME are +// filtered out server-side and never reach the client at all, so the IDs they burn can never be +// accounted for. Those produce small unexplained gaps on every healthy thread. So a gap is only +// called out when it is BOTH unexplained AND far larger than that churn could account for — a +// stranded ordinal sits thousands of IDs from the window, not tens. +const unexplainedGapAlarm = 200 +export const describeOrdinalGaps = ( + ordinals?: ReadonlyArray, + accountedFor?: ReadonlySet +) => { + const len = ordinals?.length ?? 0 + if (!ordinals || len === 0) { + return 'ordinals=0 >>> OK' + } + const gaps = new Array<{explained: boolean; from: T.Chat.Ordinal; size: number; to: T.Chat.Ordinal}>() + for (let i = 1; i < len; i++) { + const prev = ordinals[i - 1] + const cur = ordinals[i] + if (prev === undefined || cur === undefined) { + continue + } + const size = cur - prev + if (size <= 1) { + continue + } + let explained = false + if (accountedFor) { + for (let id = prev + 1; id < cur; id++) { + if (accountedFor.has(id as T.Chat.Ordinal)) { + explained = true + break + } + } + } + gaps.push({explained, from: prev, size, to: cur}) + } + const suspect = gaps.filter(g => !g.explained && g.size > unexplainedGapAlarm).sort((a, b) => b.size - a.size) + const maxGap = gaps.reduce((m, g) => Math.max(m, g.size), 0) + const first = ordinals[0] + const last = ordinals[len - 1] + const counts = `gaps=${gaps.length} explained=${gaps.filter(g => g.explained).length}` + const worst = suspect[0] + const verdict = worst + ? `>>> BAD unexplained gap ${worst.from}->${worst.to}(${worst.size})${suspect.length > 1 ? ` +${suspect.length - 1} more` : ''}` + : '>>> OK' + return `ordinals=${len} range=[${first}..${last}] maxgap=${maxGap} ${counts} ${verdict}` +} + type MessageLookup = Pick type WritableConversationThreadMessageState = { @@ -152,9 +213,14 @@ const mergeMessage = ( export const addMessagesToThreadState = ( state: WritableConversationThreadMessageState, messages: ReadonlyArray, - opt: {validatedRange?: {from: T.Chat.Ordinal; to: T.Chat.Ordinal}} + opt: { + dropNewBelowWindow?: boolean + validatedRange?: {from: T.Chat.Ordinal; to: T.Chat.Ordinal} + } ) => { - const {validatedRange} = opt + const {dropNewBelowWindow, validatedRange} = opt + // The floor of the loaded window before this batch is merged in. + const windowFloor = state.messageOrdinals?.[0] const incomingOrdinals = new Set() for (const m of messages) { if (m.conversationMessage !== false && m.type !== 'deleted') { @@ -195,6 +261,10 @@ export const addMessagesToThreadState = ( if (_m.type === 'placeholder') { const old = state.messageMap.get(mapOrdinal) if (old && old.type !== 'placeholder') { + // The real message already sits under mapOrdinal, which is not always _m.ordinal: a sent + // message keeps the fractional ordinal it had in the outbox. Bailing out before the remap + // below would strand _m.ordinal in the list with nothing stored under it. + incomingOrdinals.delete(_m.ordinal) continue } } @@ -244,6 +314,14 @@ export const addMessagesToThreadState = ( let changed = false for (const o of incomingOrdinals) { if (!existing.has(o)) { + if (dropNewBelowWindow && windowFloor !== undefined && o < windowFloor) { + // A notification (the post-load ResolveSkippedUnboxeds push, say) can carry a message from + // far outside the loaded window — the channel-name message at ID 1 is the usual one. Adding + // it here strands a row at index 0 with a hole beneath it, which makes onStartReached fire + // against that row instead of the real top of the thread, so scrollback stops working. + // Skipping it loses nothing: paging back to it loads it in the ordinary way. + continue + } existing.add(o) changed = true } From 6cb15484ee6bff61e992cd67e5919600ab740847 Mon Sep 17 00:00:00 2001 From: chrisnojima Date: Tue, 1 Sep 2026 21:21:10 -0400 Subject: [PATCH 02/21] perf(chat): cache team channel name resolution ParseChannelNameMentions runs on every message body holding a `#token`, and the regex matches ordinary text like "#1", so a busy channel hits it constantly. Each call went to GetChannelsTopicName, which read the inbox and then fetched the METADATA message of every channel in the team, uncached. Unboxing one 100-message page in a 28-channel team cost thousands of single-message fetches: one measured burst turned 13 requests into 42,461 GetMessages calls. Memoize per (tlfID, topicType, uid). The window is short and deliberately needs no invalidation on rename or channel create/delete: a page's resolutions all land within milliseconds of each other, so a few seconds collapses them into one, while keeping a renamed channel from lingering long enough to notice. The cache check sits before the Trace defer. Tracing a hit costs two log lines, and hits ran to ~8,500 per burst, which is its own drag on a debug build. --- go/chat/teamchannelsource.go | 77 ++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/go/chat/teamchannelsource.go b/go/chat/teamchannelsource.go index d98e0f36e7e3..5b4da43e427c 100644 --- a/go/chat/teamchannelsource.go +++ b/go/chat/teamchannelsource.go @@ -129,12 +129,79 @@ func (i *lastActiveAtMemCache) OnDbNuke(mctx libkb.MetaContext) error { return nil } +type topicNameCacheItem struct { + names []chat1.ChannelNameMention + mtime gregor1.Time +} + +// Channel-name resolution is charged per message: every message body holding a `#token` sends +// ParseChannelNameMentions here, and the uncached path reads the inbox and then fetches the METADATA +// message of every channel in the team. Unboxing one page of a busy channel in a team with a few +// dozen channels therefore cost thousands of single-message fetches. The window is deliberately +// short: a page's worth of resolutions all land within milliseconds of each other, so a few seconds +// collapses them into one while keeping a renamed channel from lingering long enough to be noticed. +// That is why this needs no explicit invalidation on rename or channel create/delete. +const topicNameCacheDuration = 10 * time.Second + +type topicNameMemCache struct { + sync.RWMutex + // key: tlfID||topicType||uid + cache map[string]topicNameCacheItem +} + +func newTopicNameMemCache() *topicNameMemCache { + return &topicNameMemCache{ + cache: make(map[string]topicNameCacheItem), + } +} + +func (i *topicNameMemCache) key(tlfID chat1.TLFID, topicType chat1.TopicType, uid gregor1.UID) string { + return fmt.Sprintf("%s:%v:%s", tlfID, topicType, uid) +} + +func (i *topicNameMemCache) Get(tlfID chat1.TLFID, topicType chat1.TopicType, uid gregor1.UID) ([]chat1.ChannelNameMention, bool) { + i.RLock() + defer i.RUnlock() + item, ok := i.cache[i.key(tlfID, topicType, uid)] + if !ok || time.Since(item.mtime.Time()) > topicNameCacheDuration { + return nil, false + } + // Hand back a copy: callers own what they get, and this slice is shared. + return append([]chat1.ChannelNameMention(nil), item.names...), true +} + +func (i *topicNameMemCache) Put(tlfID chat1.TLFID, topicType chat1.TopicType, uid gregor1.UID, names []chat1.ChannelNameMention) { + i.Lock() + defer i.Unlock() + i.cache[i.key(tlfID, topicType, uid)] = topicNameCacheItem{ + names: append([]chat1.ChannelNameMention(nil), names...), + mtime: gregor1.ToTime(time.Now()), + } +} + +func (i *topicNameMemCache) clearCache() { + i.Lock() + defer i.Unlock() + i.cache = make(map[string]topicNameCacheItem) +} + +func (i *topicNameMemCache) OnLogout(mctx libkb.MetaContext) error { + i.clearCache() + return nil +} + +func (i *topicNameMemCache) OnDbNuke(mctx libkb.MetaContext) error { + i.clearCache() + return nil +} + type TeamChannelSource struct { sync.Mutex globals.Contextified utils.DebugLabeler recentJoinsCache *recentJoinsMemCache lastActiveAtCache *lastActiveAtMemCache + topicNameCache *topicNameMemCache } var _ types.TeamChannelSource = (*TeamChannelSource)(nil) @@ -145,6 +212,7 @@ func NewTeamChannelSource(g *globals.Context) *TeamChannelSource { DebugLabeler: utils.NewDebugLabeler(g.ExternalG(), "TeamChannelSource", false), recentJoinsCache: newRecentJoinsMemCache(), lastActiveAtCache: newLastActiveAtMemCache(), + topicNameCache: newTopicNameMemCache(), } } @@ -152,6 +220,7 @@ func (c *TeamChannelSource) OnLogout(mctx libkb.MetaContext) error { epick := libkb.FirstErrorPicker{} epick.Push(c.recentJoinsCache.OnLogout(mctx)) epick.Push(c.lastActiveAtCache.OnLogout(mctx)) + epick.Push(c.topicNameCache.OnLogout(mctx)) return epick.Error() } @@ -159,6 +228,7 @@ func (c *TeamChannelSource) OnDbNuke(mctx libkb.MetaContext) error { epick := libkb.FirstErrorPicker{} epick.Push(c.recentJoinsCache.OnDbNuke(mctx)) epick.Push(c.lastActiveAtCache.OnDbNuke(mctx)) + epick.Push(c.topicNameCache.OnDbNuke(mctx)) return epick.Error() } @@ -256,6 +326,12 @@ func (c *TeamChannelSource) GetChannelsFull(ctx context.Context, uid gregor1.UID func (c *TeamChannelSource) GetChannelsTopicName(ctx context.Context, uid gregor1.UID, tlfID chat1.TLFID, topicType chat1.TopicType, ) (res []chat1.ChannelNameMention, err error) { + // Before the trace: this runs once per message body holding a `#token`, which reaches hundreds + // per second while paging a busy channel. Tracing a cache hit costs two log lines apiece and + // swamps the log with work that did nothing. + if cached, ok := c.topicNameCache.Get(tlfID, topicType, uid); ok { + return cached, nil + } ctx = globals.CtxModifyUnboxMode(ctx, types.UnboxModeQuick) defer c.Trace(ctx, &err, "GetChannelsTopicName: tlfID: %v, topicType: %v", tlfID, topicType)() @@ -306,6 +382,7 @@ func (c *TeamChannelSource) GetChannelsTopicName(ctx context.Context, uid gregor } addValidMetadataMsg(conv.GetConvID(), unboxeds[0]) } + c.topicNameCache.Put(tlfID, topicType, uid, res) return res, nil } From 0245bc6f0a43eb46a7585e9f3527ca48cbc4ab64 Mon Sep 17 00:00:00 2001 From: chrisnojima Date: Tue, 1 Sep 2026 21:21:10 -0400 Subject: [PATCH 03/21] docs(skill): do not trust a remembered Electron tab index Tab order is not stable across app restarts, and a DevTools window can take slot 0. A wrong tab fails silently: DevTools answers every eval plausibly and its console is nearly empty, so bad readings look like findings rather than mistakes. Require re-running tab-list and asserting the URL each time, and note that console capture only starts when playwright attaches, so earlier output lives in the user's own DevTools and cannot be pulled through playwright. --- skill/playwright-cli/SKILL.md | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/skill/playwright-cli/SKILL.md b/skill/playwright-cli/SKILL.md index 0818f8d3aa03..409e859cfaa9 100644 --- a/skill/playwright-cli/SKILL.md +++ b/skill/playwright-cli/SKILL.md @@ -283,12 +283,28 @@ PLAYWRIGHT_MCP_CDP_ENDPOINT=http://localhost:9222 playwright-cli open --persiste ### Tab layout -After connecting, you'll see tabs like: -- Tab 0: Menubar (`menubar.dev.html`) -- Tab 1: Main app (`main.dev.html`) — this is the one you want -- Tab 2+: Other windows +Tab order is NOT stable. It changes when the app restarts, and a DevTools window can take slot 0: +- Seen: `0: menubar, 1: main app` +- Also seen (after an app restart with DevTools open): `0: DevTools, 1: main app, 2: menubar` -### Identifying the main app tab +### Identifying the main app tab — do this EVERY time + +Never reuse a remembered index. Re-run `tab-list` and pick the row whose URL contains `main.html` +after every restart, reload, or reconnect, then confirm with one cheap eval before trusting anything: + +```bash +playwright-cli tab-list # find the row whose URL has main.html +playwright-cli tab-select +playwright-cli eval "document.title + ' ' + location.href" # assert before proceeding +``` + +**A wrong tab fails silently.** DevTools is a real page: it answers every `eval` plausibly and its +console is nearly empty, so readings look like findings rather than mistakes. Treat a suspiciously +empty console or a tiny `document.body.innerText` as "wrong tab" first, not as evidence. + +Also note `console` only captures output from the moment playwright attached to that page. Anything +logged before the attach lives in the user's own DevTools and cannot be retrieved through +playwright — ask them to paste it, or have them re-trigger while you are attached. Use the page URL, not the title — the title stays `"Keybase DEV"` until the router navigates and can't be relied on: From c5b981ec455ab919fca9d857171d6cb6b2ac3f04 Mon Sep 17 00:00:00 2001 From: chrisnojima Date: Wed, 2 Sep 2026 10:04:20 -0400 Subject: [PATCH 04/21] chore(chat): remove the ordinal-gap diagnostic probe The probe did its job: on a post-nuke `keybasefriends#help` scrollback the back page now reads `[7152..7276] -> ordinals=180 range=[7152..7421] >>> OK`, where it used to strand ordinal 1 and report `range=[1..7421]`. Drops `GAPPROBE`, `describeOrdinalGaps`, the `gapProbeAccounted` ref and the two `logger.info` blocks in the thread context, plus their tests. The `dropNewBelowWindow` invariant they were validating stays. --- shared/chat/conversation/thread-context.tsx | 46 -------------- .../thread-message-state.test.tsx | 46 -------------- .../conversation/thread-message-state.tsx | 61 ------------------- 3 files changed, 153 deletions(-) diff --git a/shared/chat/conversation/thread-context.tsx b/shared/chat/conversation/thread-context.tsx index 7edd6e67d754..1b4b2b40bf64 100644 --- a/shared/chat/conversation/thread-context.tsx +++ b/shared/chat/conversation/thread-context.tsx @@ -22,8 +22,6 @@ import {useIsFocused} from '@react-navigation/core' import { addMessagesToThreadState, applyOptimisticReactionsToMessage, - describeOrdinalGaps, - GAPPROBE, completeAttachmentDownloadInThreadState, clearOptimisticReactionsForUpdatesInThreadState, clearOptimisticReactionsForMessagesInThreadState, @@ -473,26 +471,12 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => if (opt.liveUpdate) { s.liveUpdateVersion += 1 } - const beforeMin = s.messageOrdinals?.[0] addMessagesToThreadState(s, messages, { // Only thread loads may extend the window downward; a notification must not. dropNewBelowWindow: true, validatedRange: opt.validatedRange, }) clearOptimisticReactionsForMessagesInThreadState(s, messages) - const afterMin = s.messageOrdinals?.[0] - if (beforeMin !== undefined && afterMin !== undefined && afterMin < beforeMin) { - const m = s.messageMap.get(afterMin) - logger.info( - `${GAPPROBE}: conv=${id.slice(0, 12)} addMessages LOWERED the window floor ${beforeMin} -> ${afterMin}` + - ` ord=${afterMin} type=${m?.type ?? 'MISSING'} id=${m?.id ?? '?'}` + - ` liveUpdate=${!!opt.liveUpdate} batch=${messages.length}` + - ` batchOrds=${messages - .slice(0, 8) - .map(x => `${x.ordinal}/${x.type}`) - .join(',')}` - ) - } }) if (opt.markAsRead) { markThreadAsRead() @@ -505,9 +489,6 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => activeMarkReadEnabledRef.current = false } }) - // Debug-only: every ordinal the service has sent for this conversation, deleted ones included, so - // the gap probe can tell an expunged range from a stranded ordinal. Dies with the conversation. - const gapProbeAccounted = React.useRef(new Set()) const applyThreadLoad = React.useEffectEvent( (p: { authoritative: boolean @@ -525,32 +506,6 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => if (p.messages.length) { addMessagesToThreadState(s, p.messages, {validatedRange: p.validatedRange}) clearOptimisticReactionsForMessagesInThreadState(s, p.messages) - const incoming = p.messages.map(m => m.ordinal) - for (const o of incoming) { - gapProbeAccounted.current.add(o) - } - const incomingMin = Math.min(...incoming) - const desc = describeOrdinalGaps(s.messageOrdinals, gapProbeAccounted.current) - logger.info( - `${GAPPROBE}: conv=${id.slice(0, 12)} load pass=${p.authoritative ? 'full' : 'cached'} dir=${p.scrollDirection}` + - ` centered=${p.centered} incoming=${incoming.length}` + - ` [${incomingMin}..${Math.max(...incoming)}] -> ` + - desc - ) - if (desc.includes('>>> BAD')) { - const stranded = (s.messageOrdinals ?? []).filter(o => o < incomingMin) - logger.info( - `${GAPPROBE}: conv=${id.slice(0, 12)} stranded below incoming min ${incomingMin}: ` + - stranded - .slice(0, 6) - .map(o => { - const m = s.messageMap.get(o) - return `ord=${o} type=${m?.type ?? 'MISSING'} id=${m?.id ?? '?'} outbox=${m?.outboxID ?? '-'}` - }) - .join(' ; ') + - ` | inBatch=${incoming.includes(stranded[0] as T.Chat.Ordinal)}` - ) - } } switch (p.scrollDirection) { case 'forward': @@ -918,7 +873,6 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => ) const messagesClear = React.useEffectEvent(() => { activeMarkReadEnabledRef.current = false - gapProbeAccounted.current.clear() shownUsernameCache.clear() updateThreadState(s => { s.clearVersion += 1 diff --git a/shared/chat/conversation/thread-message-state.test.tsx b/shared/chat/conversation/thread-message-state.test.tsx index cfaa2ddaaad8..8fad688fbad9 100644 --- a/shared/chat/conversation/thread-message-state.test.tsx +++ b/shared/chat/conversation/thread-message-state.test.tsx @@ -4,7 +4,6 @@ import * as T from '@/constants/types' import HiddenString from '@/util/hidden-string' import { addMessagesToThreadState, - describeOrdinalGaps, applyOptimisticReactionsToMessage, clearOptimisticReactionsForUpdatesInThreadState, deleteMessagesFromThreadState, @@ -475,48 +474,3 @@ describe('addMessagesToThreadState', () => { expect(state.messageTypeMap.get(T.Chat.numberToOrdinal(20))).toBe('attachment:file') }) }) - -describe('describeOrdinalGaps', () => { - const ords = (...n: ReadonlyArray) => n.map(T.Chat.numberToOrdinal) - const accounted = (...n: ReadonlyArray) => new Set(n.map(T.Chat.numberToOrdinal)) - - test('a contiguous window is OK', () => { - expect(describeOrdinalGaps(undefined)).toBe('ordinals=0 >>> OK') - expect(describeOrdinalGaps(ords(10, 11, 12))).toBe( - 'ordinals=3 range=[10..12] maxgap=0 gaps=0 explained=0 >>> OK' - ) - }) - - test('a gap the service accounted for is explained, not flagged', () => { - // Retention expunges arrive as hidden placeholders that become `deleted` and drop out of the - // ordinal list, so the missing IDs are still accounted for. - expect(describeOrdinalGaps(ords(10, 1000), accounted(10, 12, 500, 900, 1000))).toBe( - 'ordinals=2 range=[10..1000] maxgap=990 gaps=1 explained=1 >>> OK' - ) - }) - - test('small unexplained gaps are the normal churn of hidden message types', () => { - // Reactions, edits and unfurls burn IDs the service never sends, so these can never be - // accounted for and must not be flagged. - expect(describeOrdinalGaps(ords(89, 94, 97, 100), accounted(89, 94, 97, 100))).toBe( - 'ordinals=4 range=[89..100] maxgap=5 gaps=3 explained=0 >>> OK' - ) - }) - - test('a gap with nothing behind it is flagged as BAD', () => { - expect(describeOrdinalGaps(ords(1, 4114, 4115), accounted(1, 4114, 4115))).toBe( - 'ordinals=3 range=[1..4115] maxgap=4113 gaps=1 explained=0 >>> BAD unexplained gap 1->4114(4113)' - ) - }) - - test('the verdict reports the biggest unexplained gap and counts the rest', () => { - const out = describeOrdinalGaps(ords(1, 500, 5000, 5001), accounted(1, 500, 5000, 5001)) - expect(out).toContain('>>> BAD unexplained gap 500->5000(4500) +1 more') - }) - - test('a pending fractional ordinal is not a gap', () => { - expect(describeOrdinalGaps(ords(10, 11, 11.001))).toBe( - 'ordinals=3 range=[10..11.001] maxgap=0 gaps=0 explained=0 >>> OK' - ) - }) -}) diff --git a/shared/chat/conversation/thread-message-state.tsx b/shared/chat/conversation/thread-message-state.tsx index 066d2dc4f6fd..a2783accb1e3 100644 --- a/shared/chat/conversation/thread-message-state.tsx +++ b/shared/chat/conversation/thread-message-state.tsx @@ -3,67 +3,6 @@ import * as T from '@/constants/types' import HiddenString from '@/util/hidden-string' import type {WritableDraft} from '@/util/zustand' -// Temporary diagnostic for the thread-ordinal no-gap work. Bump the marker on every edit so a log -// can be tied back to the build that produced it. Remove once the gap producer is confirmed gone. -export const GAPPROBE = 'gapprobe/2026-09-01g' - -// A compact summary of where a loaded window is discontiguous, ending in a single verdict. -// -// Numbering gaps are normal on their own: edits, reactions, unfurls and deletes each burn a message -// ID the thread never shows, and retention expunges whole ranges. What separates those from a real -// hole is whether the service ever accounted for the missing IDs — an expunged range arrives as -// hidden placeholders (which become `deleted` and drop out of the list), while a stranded ordinal -// has nothing behind it at all. `accountedFor` is every ordinal the service has sent for this -// conversation, deleted ones included, accumulated across loads. -// -// Accounting alone is not enough: EDIT, DELETE, REACTION, ATTACHMENTUPLOADED, UNFURL and TLFNAME are -// filtered out server-side and never reach the client at all, so the IDs they burn can never be -// accounted for. Those produce small unexplained gaps on every healthy thread. So a gap is only -// called out when it is BOTH unexplained AND far larger than that churn could account for — a -// stranded ordinal sits thousands of IDs from the window, not tens. -const unexplainedGapAlarm = 200 -export const describeOrdinalGaps = ( - ordinals?: ReadonlyArray, - accountedFor?: ReadonlySet -) => { - const len = ordinals?.length ?? 0 - if (!ordinals || len === 0) { - return 'ordinals=0 >>> OK' - } - const gaps = new Array<{explained: boolean; from: T.Chat.Ordinal; size: number; to: T.Chat.Ordinal}>() - for (let i = 1; i < len; i++) { - const prev = ordinals[i - 1] - const cur = ordinals[i] - if (prev === undefined || cur === undefined) { - continue - } - const size = cur - prev - if (size <= 1) { - continue - } - let explained = false - if (accountedFor) { - for (let id = prev + 1; id < cur; id++) { - if (accountedFor.has(id as T.Chat.Ordinal)) { - explained = true - break - } - } - } - gaps.push({explained, from: prev, size, to: cur}) - } - const suspect = gaps.filter(g => !g.explained && g.size > unexplainedGapAlarm).sort((a, b) => b.size - a.size) - const maxGap = gaps.reduce((m, g) => Math.max(m, g.size), 0) - const first = ordinals[0] - const last = ordinals[len - 1] - const counts = `gaps=${gaps.length} explained=${gaps.filter(g => g.explained).length}` - const worst = suspect[0] - const verdict = worst - ? `>>> BAD unexplained gap ${worst.from}->${worst.to}(${worst.size})${suspect.length > 1 ? ` +${suspect.length - 1} more` : ''}` - : '>>> OK' - return `ordinals=${len} range=[${first}..${last}] maxgap=${maxGap} ${counts} ${verdict}` -} - type MessageLookup = Pick type WritableConversationThreadMessageState = { From ec833fd7884b96ebdfed1c02cce18e38ceaa2585 Mon Sep 17 00:00:00 2001 From: chrisnojima Date: Wed, 2 Sep 2026 10:13:50 -0400 Subject: [PATCH 05/21] test(chat): cover patchPaginationLast and the topic name cache Drops the PullLocalOnly local-max guard. It was never proven: in every nuke reproduction both fetches failed and returned a clean miss before reaching it, so the branch has no evidence behind it and no test could reach it either. Adds unit tests for the two service changes that are reachable without kbweb. Neither calls setupTest, so neither needs localhost:3000 - patchPaginationLast only needs a bare GlobalContext for its logger, and the topic name cache is pure. - patchPaginationLast: message ID 1 ends the thread with expunge nil, with expunge reading back Upto:0 (the post-nuke case that stuck the spinner), and regardless of page order; the pre-existing nukepoint behaviour still holds; Last is never turned back off; a nil page does not panic. - topicNameMemCache: round trip, key separation across TLF/topic type/uid, copies in both directions so neither side can reach into the shared slice, TTL expiry either side of the boundary, and clear via clearCache, OnLogout and OnDbNuke. Mutation checked: reverting the ID-1 check fails 3 subtests, and returning the cached slice by reference fails the copy test. --- go/chat/convsource.go | 20 --- go/chat/convsource_patchpagination_test.go | 145 ++++++++++++++++++ .../teamchannelsource_topicnamecache_test.go | 122 +++++++++++++++ 3 files changed, 267 insertions(+), 20 deletions(-) create mode 100644 go/chat/convsource_patchpagination_test.go create mode 100644 go/chat/teamchannelsource_topicnamecache_test.go diff --git a/go/chat/convsource.go b/go/chat/convsource.go index a00b1cb830c9..b331565bd826 100644 --- a/go/chat/convsource.go +++ b/go/chat/convsource.go @@ -919,26 +919,6 @@ func (s *HybridConversationSource) PullLocalOnly(ctx context.Context, convID cha s.Debug(ctx, "PullLocalOnly: failed to fetch local messages with local max: %s", err) return chat1.ThreadView{}, err } - // This retry anchors on the local max instead of the inbox max, which is what we want when - // local storage is merely behind. But after the cache is wiped the only thing left can be a - // lone ancient message the inbox localizer cached for a channel name, headline or pin. Handed - // up as the newest page, it strands itself thousands of IDs above the real thread in the UI. - // Only accept a window that could plausibly overlap the page being asked for. - if iboxMaxMsgID > 0 && num > 0 && pagination.FirstPage() && len(tv.Messages) > 0 { - var newest chat1.MessageID - for _, m := range tv.Messages { - if id := m.GetMessageID(); id > newest { - newest = id - } - } - //nolint:gosec // G115: num is positive, checked above - if newest < iboxMaxMsgID && iboxMaxMsgID-newest > chat1.MessageID(num) { - s.Debug(ctx, - "PullLocalOnly: local max fallback newest %d is %d below ibox max %d, past the %d requested: reporting miss", - newest, iboxMaxMsgID-newest, iboxMaxMsgID, num) - return chat1.ThreadView{}, storage.MissError{Msg: "local copy does not reach the newest page"} - } - } } return tv, nil } diff --git a/go/chat/convsource_patchpagination_test.go b/go/chat/convsource_patchpagination_test.go new file mode 100644 index 000000000000..1e33e0b733e4 --- /dev/null +++ b/go/chat/convsource_patchpagination_test.go @@ -0,0 +1,145 @@ +package chat + +import ( + "context" + "testing" + + "github.com/keybase/client/go/chat/globals" + "github.com/keybase/client/go/chat/types" + "github.com/keybase/client/go/chat/utils" + "github.com/keybase/client/go/libkb" + "github.com/keybase/client/go/protocol/chat1" + "github.com/keybase/client/go/protocol/gregor1" + "github.com/stretchr/testify/require" +) + +// patchPaginationConv is the smallest thing satisfying types.UnboxConversationInfo. Only +// GetExpunge is consulted by patchPaginationLast; the rest exist to satisfy the interface. +type patchPaginationConv struct { + expunge *chat1.Expunge +} + +var _ types.UnboxConversationInfo = patchPaginationConv{} + +func (c patchPaginationConv) GetConvID() chat1.ConversationID { return nil } +func (c patchPaginationConv) GetMembersType() chat1.ConversationMembersType { + return chat1.ConversationMembersType_TEAM +} +func (c patchPaginationConv) GetFinalizeInfo() *chat1.ConversationFinalizeInfo { return nil } +func (c patchPaginationConv) GetExpunge() *chat1.Expunge { return c.expunge } +func (c patchPaginationConv) GetMaxDeletedUpTo() chat1.MessageID { return 0 } +func (c patchPaginationConv) IsPublic() bool { return false } +func (c patchPaginationConv) GetMaxMessage(chat1.MessageType) (chat1.MessageSummary, error) { + return chat1.MessageSummary{}, nil +} + +// newPatchPaginationSource builds just enough of a baseConversationSource to call +// patchPaginationLast. It needs no database, network or logged in user - a bare GlobalContext +// already carries a logger, which is all Debug touches. +func newPatchPaginationSource() *baseConversationSource { + g := libkb.NewGlobalContext() + return &baseConversationSource{ + Contextified: globals.NewContextified(globals.NewContext(g, &globals.ChatContext{})), + DebugLabeler: utils.NewDebugLabeler(g, "patchPaginationTest", false), + } +} + +func msgsWithIDs(ids ...chat1.MessageID) []chat1.MessageUnboxed { + res := make([]chat1.MessageUnboxed, 0, len(ids)) + for _, id := range ids { + res = append(res, chat1.NewMessageUnboxedWithPlaceholder(chat1.MessageUnboxedPlaceholder{ + MessageID: id, + })) + } + return res +} + +func TestPatchPaginationLast(t *testing.T) { + ctx := context.Background() + uid := gregor1.UID([]byte{0x01}) + s := newPatchPaginationSource() + + testCases := []struct { + name string + expunge *chat1.Expunge + msgs []chat1.MessageUnboxed + page *chat1.Pagination + want bool + }{ + { + name: "an empty page is the last page", + msgs: nil, + page: &chat1.Pagination{Num: 50}, + want: true, + }, + { + // The regression this guards: after a nuke a conversation whose history was deleted + // reads back Upto:0 until its inbox entry is localized, so the expunge check below + // never fires and Last stays false forever - "Digging ancient messages..." on a fully + // loaded thread. + name: "reaching message ID 1 is last even when expunge reads back Upto:0", + expunge: &chat1.Expunge{Upto: 0}, + msgs: msgsWithIDs(1, 2, 3), + page: &chat1.Pagination{Num: 50}, + want: true, + }, + { + name: "reaching message ID 1 is last even with no expunge record at all", + msgs: msgsWithIDs(1, 2, 3), + page: &chat1.Pagination{Num: 50}, + want: true, + }, + { + // Pages can arrive newest first, and the check is on the oldest ID either way. + name: "message ID 1 is found regardless of page order", + msgs: msgsWithIDs(3, 2, 1), + page: &chat1.Pagination{Num: 50}, + want: true, + }, + { + name: "a page above the beginning with no expunge is not last", + msgs: msgsWithIDs(40, 41, 42), + page: &chat1.Pagination{Num: 50}, + want: false, + }, + { + name: "a page reaching the nukepoint is last", + expunge: &chat1.Expunge{Upto: 40}, + msgs: msgsWithIDs(40, 41, 42), + page: &chat1.Pagination{Num: 50}, + want: true, + }, + { + name: "a page above the nukepoint is not last", + expunge: &chat1.Expunge{Upto: 10}, + msgs: msgsWithIDs(40, 41, 42), + page: &chat1.Pagination{Num: 50}, + want: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + conv := patchPaginationConv{expunge: tc.expunge} + s.patchPaginationLast(ctx, conv, uid, tc.page, tc.msgs) + require.Equal(t, tc.want, tc.page.Last) + }) + } +} + +func TestPatchPaginationLastLeavesSettledPagesAlone(t *testing.T) { + ctx := context.Background() + uid := gregor1.UID([]byte{0x01}) + s := newPatchPaginationSource() + conv := patchPaginationConv{} + + // A nil page must not panic. + require.NotPanics(t, func() { + s.patchPaginationLast(ctx, conv, uid, nil, msgsWithIDs(1)) + }) + + // Last is only ever turned on, never off. + page := &chat1.Pagination{Num: 50, Last: true} + s.patchPaginationLast(ctx, conv, uid, page, msgsWithIDs(40, 41, 42)) + require.True(t, page.Last) +} diff --git a/go/chat/teamchannelsource_topicnamecache_test.go b/go/chat/teamchannelsource_topicnamecache_test.go new file mode 100644 index 000000000000..94cfdecf52a9 --- /dev/null +++ b/go/chat/teamchannelsource_topicnamecache_test.go @@ -0,0 +1,122 @@ +package chat + +import ( + "testing" + "time" + + "github.com/keybase/client/go/libkb" + "github.com/keybase/client/go/protocol/chat1" + "github.com/keybase/client/go/protocol/gregor1" + "github.com/stretchr/testify/require" +) + +func topicNameCacheFixture() (chat1.TLFID, chat1.TopicType, gregor1.UID, []chat1.ChannelNameMention) { + tlfID := chat1.TLFID([]byte{0x01, 0x02}) + uid := gregor1.UID([]byte{0x0a}) + names := []chat1.ChannelNameMention{ + {ConvID: chat1.ConversationID([]byte{0x10}), TopicName: "general"}, + {ConvID: chat1.ConversationID([]byte{0x11}), TopicName: "random"}, + } + return tlfID, chat1.TopicType_CHAT, uid, names +} + +func TestTopicNameMemCacheRoundTrip(t *testing.T) { + tlfID, topicType, uid, names := topicNameCacheFixture() + c := newTopicNameMemCache() + + _, ok := c.Get(tlfID, topicType, uid) + require.False(t, ok, "an empty cache must miss") + + c.Put(tlfID, topicType, uid, names) + got, ok := c.Get(tlfID, topicType, uid) + require.True(t, ok) + require.Equal(t, names, got) +} + +func TestTopicNameMemCacheKeysAreDistinct(t *testing.T) { + tlfID, topicType, uid, names := topicNameCacheFixture() + c := newTopicNameMemCache() + c.Put(tlfID, topicType, uid, names) + + otherTLF := chat1.TLFID([]byte{0x09, 0x09}) + otherUID := gregor1.UID([]byte{0xbb}) + + _, ok := c.Get(otherTLF, topicType, uid) + require.False(t, ok, "a different TLF must not share an entry") + _, ok = c.Get(tlfID, chat1.TopicType_DEV, uid) + require.False(t, ok, "a different topic type must not share an entry") + _, ok = c.Get(tlfID, topicType, otherUID) + require.False(t, ok, "a different uid must not share an entry") + + // The original is still there and untouched by the misses. + got, ok := c.Get(tlfID, topicType, uid) + require.True(t, ok) + require.Equal(t, names, got) +} + +// The cached slice is shared with every caller, so neither side may be able to reach into it. +func TestTopicNameMemCacheCopiesBothWays(t *testing.T) { + tlfID, topicType, uid, names := topicNameCacheFixture() + c := newTopicNameMemCache() + c.Put(tlfID, topicType, uid, names) + + // Mutating what the caller passed in must not reach the cache. + names[0].TopicName = "mutated-input" + got, ok := c.Get(tlfID, topicType, uid) + require.True(t, ok) + require.Equal(t, "general", got[0].TopicName) + + // Mutating what the caller got back must not reach the cache either. + got[0].TopicName = "mutated-output" + again, ok := c.Get(tlfID, topicType, uid) + require.True(t, ok) + require.Equal(t, "general", again[0].TopicName) +} + +func TestTopicNameMemCacheExpires(t *testing.T) { + tlfID, topicType, uid, names := topicNameCacheFixture() + c := newTopicNameMemCache() + c.Put(tlfID, topicType, uid, names) + + key := c.key(tlfID, topicType, uid) + + // Still inside the window. + c.Lock() + item := c.cache[key] + item.mtime = gregor1.ToTime(time.Now().Add(-topicNameCacheDuration + time.Second)) + c.cache[key] = item + c.Unlock() + _, ok := c.Get(tlfID, topicType, uid) + require.True(t, ok, "an entry inside the TTL must hit") + + // Past it. There is no explicit invalidation, so expiry is the only thing keeping a renamed + // channel from being served forever. + c.Lock() + item = c.cache[key] + item.mtime = gregor1.ToTime(time.Now().Add(-topicNameCacheDuration - time.Second)) + c.cache[key] = item + c.Unlock() + _, ok = c.Get(tlfID, topicType, uid) + require.False(t, ok, "an entry past the TTL must miss") +} + +func TestTopicNameMemCacheClear(t *testing.T) { + tlfID, topicType, uid, names := topicNameCacheFixture() + c := newTopicNameMemCache() + c.Put(tlfID, topicType, uid, names) + + c.clearCache() + _, ok := c.Get(tlfID, topicType, uid) + require.False(t, ok, "clearCache must drop everything") + + // Logout and db nuke both go through the same clear, and both must leave the cache usable. + c.Put(tlfID, topicType, uid, names) + require.NoError(t, c.OnLogout(libkb.MetaContext{})) + _, ok = c.Get(tlfID, topicType, uid) + require.False(t, ok, "OnLogout must drop everything") + + c.Put(tlfID, topicType, uid, names) + require.NoError(t, c.OnDbNuke(libkb.MetaContext{})) + _, ok = c.Get(tlfID, topicType, uid) + require.False(t, ok, "OnDbNuke must drop everything") +} From 437cb11157d022e851466bfb80c1869968483165 Mon Sep 17 00:00:00 2001 From: chrisnojima Date: Wed, 2 Sep 2026 10:20:34 -0400 Subject: [PATCH 06/21] fix(chat): re-issue a back page that yields no new ordinals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A back page can arrive composed entirely of messages the thread will never render. The service filters EDIT/DELETE/REACTION/ATTACHMENTUPLOADED/UNFURL/ TLFNAME out before they reach us, but messages superseded by a DELETE still arrive as `deleted`, and addMessages drops those from the ordinal list. The list is then identical to what it was, so LegendList never fires onStartReached again and scrollback stops for good even though the pager still says there is more to come. Observed as `incoming=882 [12599..13585] -> ordinals unchanged`. The real fix belongs in the service, which knows it is handing up a page of tombstones. This is the liveness belt: the thread must not be one unlucky response away from a permanent stall. `shouldRetryEmptyBackPage` is deliberately narrow — backwards scrollback only, the authoritative pass only (once a cached thread has been sent the service switches the full response to INCREMENTAL, so a cached pass adding nothing is normal), the page must have contained something, the pager must still say there is more, and it is bounded at 3 re-issues. Mutation checked: every clause of the predicate fails a test when inverted, and removing the wiring turns the bounded re-issue from 4 calls into 1. --- shared/chat/conversation/thread-context.tsx | 3 + shared/chat/conversation/thread-load.test.tsx | 154 +++++++++++++++++- shared/chat/conversation/thread-load.tsx | 55 ++++++- 3 files changed, 210 insertions(+), 2 deletions(-) diff --git a/shared/chat/conversation/thread-context.tsx b/shared/chat/conversation/thread-context.tsx index 1b4b2b40bf64..9510d52e3bed 100644 --- a/shared/chat/conversation/thread-context.tsx +++ b/shared/chat/conversation/thread-context.tsx @@ -177,6 +177,9 @@ type SelectedConversationOptions = ThreadLoadStatusOptions & { export type ScrollDirection = 'none' | 'back' | 'forward' export type LoadMoreMessagesParams = ThreadLoadStatusOptions & { allowMarkAsRead?: boolean + // Internal: how many times this back page has already been re-issued after yielding no new + // ordinals. Callers leave it unset; only the retry in thread-load.tsx sets it. + emptyBackPageRetries?: number centeredMessageID?: { conversationIDKey: T.Chat.ConversationIDKey highlightMode: T.Chat.CenterOrdinalHighlightMode diff --git a/shared/chat/conversation/thread-load.test.tsx b/shared/chat/conversation/thread-load.test.tsx index 0d5ad57c0e3b..43022851cf95 100644 --- a/shared/chat/conversation/thread-load.test.tsx +++ b/shared/chat/conversation/thread-load.test.tsx @@ -6,9 +6,15 @@ import { getExplodingModeFromGregorItems, getLastOrdinalFromSnapshot, getOrdinalForMessageIDInSnapshot, + loadConversationThreadMessages, + maxEmptyBackPageRetries, scrollDirectionToPagination, + shouldRetryEmptyBackPage, } from './thread-load' -import type {ConversationThreadState} from './thread-context' +import * as ThreadRpc from './thread-rpc' +import {resetAllStores} from '@/util/zustand' +import {useCurrentUserState} from '@/stores/current-user' +import type {ConversationThreadActions, ConversationThreadState} from './thread-context' const conversationIDKey = T.Chat.stringToConversationIDKey('conv1') const otherConversationIDKey = T.Chat.stringToConversationIDKey('conv2') @@ -155,3 +161,149 @@ describe('snapshot helpers', () => { expect(getOrdinalForMessageIDInSnapshot(snapshot, messageID(7))).toBeNull() }) }) + +describe('shouldRetryEmptyBackPage', () => { + // A back page that arrived, said there is more to come, and left the ordinal list exactly as it + // was. This is the stall: nothing changed, so onStartReached never fires again. + const stalled = { + authoritative: true, + incoming: 882, + moreToLoad: true, + ordinalsAfter: 180, + ordinalsBefore: 180, + retries: 0, + scrollDirection: 'back' as const, + } + + test('retries a back page that yielded no new ordinals', () => { + expect(shouldRetryEmptyBackPage(stalled)).toBe(true) + }) + + test('does not retry when the page actually added ordinals', () => { + expect(shouldRetryEmptyBackPage({...stalled, ordinalsAfter: 274})).toBe(false) + }) + + test('does not retry when the pager says this was the last page', () => { + expect(shouldRetryEmptyBackPage({...stalled, moreToLoad: false})).toBe(false) + }) + + test('does not retry an empty response', () => { + // Nothing came back at all, which is a different situation: the load already settled. + expect(shouldRetryEmptyBackPage({...stalled, incoming: 0})).toBe(false) + }) + + test('only retries backwards scrollback', () => { + expect(shouldRetryEmptyBackPage({...stalled, scrollDirection: 'none'})).toBe(false) + expect(shouldRetryEmptyBackPage({...stalled, scrollDirection: 'forward'})).toBe(false) + }) + + test('ignores the cached pass', () => { + // Once a cached thread has been sent the service switches the full response to INCREMENTAL, so + // a cached pass adding nothing new is normal and must not trigger a retry. + expect(shouldRetryEmptyBackPage({...stalled, authoritative: false})).toBe(false) + }) + + test('is bounded', () => { + for (let retries = 0; retries < maxEmptyBackPageRetries; retries++) { + expect(shouldRetryEmptyBackPage({...stalled, retries})).toBe(true) + } + expect(shouldRetryEmptyBackPage({...stalled, retries: maxEmptyBackPageRetries})).toBe(false) + expect(shouldRetryEmptyBackPage({...stalled, retries: maxEmptyBackPageRetries + 1})).toBe(false) + }) + + test('retries when a page somehow shrank the list', () => { + // A back page whose only effect was to delete messages already in the window leaves even less + // than before, and still needs something to re-trigger the load. + expect(shouldRetryEmptyBackPage({...stalled, ordinalsAfter: 174})).toBe(true) + }) +}) + +describe('an empty back page re-triggers the load', () => { + const flushPromises = async () => { + for (let i = 0; i < 30; i++) { + await Promise.resolve() + } + } + + // The thread never grows: whatever the page contained, addMessages dropped all of it. That is the + // condition LegendList cannot see, because `messageOrdinals` is byte-identical afterwards. + const makeFrozenActions = () => { + const snapshot = { + liveUpdateVersion: 0, + loaded: true, + messageIDToOrdinal: new Map(), + messageMap: new Map(), + messageOrdinals: [T.Chat.numberToOrdinal(7152), T.Chat.numberToOrdinal(7153)], + pendingOutboxToOrdinal: new Map(), + } as unknown as ConversationThreadState + const applyThreadLoad = jest.fn() + return { + actions: { + applyThreadLoad, + getSnapshot: () => snapshot, + markThreadAsRead: jest.fn(), + } as unknown as ConversationThreadActions, + applyThreadLoad, + } + } + + // A page that parses to real messages, so `incoming > 0`, with the pager still saying there is + // more to come. + const mockPage = (last: boolean) => + jest.spyOn(ThreadRpc, 'loadThreadNonblock').mockImplementation(async p => { + const messages = [7150, 7151].map(id => ({ + placeholder: {hidden: false, messageID: T.Chat.numberToMessageID(id)}, + state: T.RPCChat.MessageUnboxedState.placeholder, + })) + await Promise.resolve() + p.onFullThread?.(JSON.stringify({messages, pagination: {last, num: 100}})) + return undefined as never + }) + + beforeEach(() => { + useCurrentUserState.getState().dispatch.setBootstrap({ + deviceID: 'device-id', + deviceName: 'testuser-mac', + uid: 'uid', + username: 'testuser', + }) + }) + + afterEach(() => { + jest.restoreAllMocks() + resetAllStores() + }) + + test('re-issues the page, and stops at the bound', async () => { + const rpc = mockPage(false) + const {actions} = makeFrozenActions() + loadConversationThreadMessages( + conversationIDKey, + {reason: 'scroll back', scrollDirection: 'back'}, + actions + ) + await flushPromises() + // The original call plus one per retry, and then it gives up rather than spinning. + expect(rpc).toHaveBeenCalledTimes(1 + maxEmptyBackPageRetries) + }) + + test('does not re-issue once the pager says it is the last page', async () => { + const rpc = mockPage(true) + const {actions} = makeFrozenActions() + loadConversationThreadMessages( + conversationIDKey, + {reason: 'scroll back', scrollDirection: 'back'}, + actions + ) + await flushPromises() + expect(rpc).toHaveBeenCalledTimes(1) + }) + + test('does not re-issue an initial load', async () => { + const rpc = mockPage(false) + const {actions} = makeFrozenActions() + loadConversationThreadMessages(conversationIDKey, {reason: 'focused'}, actions) + await flushPromises() + expect(rpc).toHaveBeenCalledTimes(1) + }) +}) diff --git a/shared/chat/conversation/thread-load.tsx b/shared/chat/conversation/thread-load.tsx index 2cf411665638..c93c7e0440a2 100644 --- a/shared/chat/conversation/thread-load.tsx +++ b/shared/chat/conversation/thread-load.tsx @@ -154,6 +154,33 @@ export const scrollDirectionToPagination = ( return pagination } +// A back page can arrive composed entirely of messages the thread will never render. The service +// filters EDIT/DELETE/REACTION/ATTACHMENTUPLOADED/UNFURL/TLFNAME out before they reach us, but +// messages superseded by a DELETE still arrive as `deleted`, and `addMessages` drops those from the +// ordinal list. The list is then identical to what it was, so LegendList never fires onStartReached +// again and scrollback stops for good even though the pager still says there is more to come. +// +// The real fix belongs in the service, which knows it is handing up a page of tombstones. This is +// the liveness belt: the thread must not be one unlucky response away from a permanent stall. +export const maxEmptyBackPageRetries = 3 +export const shouldRetryEmptyBackPage = (p: { + authoritative: boolean + incoming: number + moreToLoad: boolean + ordinalsAfter: number + ordinalsBefore: number + retries: number + scrollDirection: ScrollDirection +}) => + p.scrollDirection === 'back' && + // Only the full pass is a whole page. Once a cached thread has been sent the service switches the + // full response to INCREMENTAL, so a cached pass yielding nothing new is expected, not a stall. + p.authoritative && + p.moreToLoad && + p.incoming > 0 && + p.ordinalsAfter <= p.ordinalsBefore && + p.retries < maxEmptyBackPageRetries + export const loadConversationThreadMessages = ( conversationIDKey: T.Chat.ConversationIDKey, p: LoadMoreMessagesParams, @@ -162,7 +189,11 @@ export const loadConversationThreadMessages = ( if (!T.Chat.isValidConversationIDKey(conversationIDKey)) { return } - const {scrollDirection = 'none', numberOfMessagesToLoad = numMessagesOnInitialLoad} = p + const { + scrollDirection = 'none', + numberOfMessagesToLoad = numMessagesOnInitialLoad, + emptyBackPageRetries = 0, + } = p const { allowMarkAsRead = true, reason, @@ -261,6 +292,7 @@ export const loadConversationThreadMessages = ( } } } + const ordinalsBefore = actions.getSnapshot().messageOrdinals?.length ?? 0 actions.applyThreadLoad({ authoritative: why === 'full', centered: !!centeredMessageID, @@ -272,6 +304,27 @@ export const loadConversationThreadMessages = ( scrollDirection, validatedRange, }) + const ordinalsAfter = actions.getSnapshot().messageOrdinals?.length ?? 0 + if ( + shouldRetryEmptyBackPage({ + authoritative: why === 'full', + incoming: messages.length, + moreToLoad, + ordinalsAfter, + ordinalsBefore, + retries: emptyBackPageRetries, + scrollDirection, + }) + ) { + logger.info( + `loadMoreMessages: back page of ${messages.length} yielded no new ordinals (${ordinalsBefore}), retrying ${emptyBackPageRetries + 1}/${maxEmptyBackPageRetries}: convID: ${conversationIDKey}` + ) + loadConversationThreadMessages( + conversationIDKey, + {...p, emptyBackPageRetries: emptyBackPageRetries + 1}, + actions + ) + } if (canMarkReadForThreadWindow) { actions.markThreadAsRead() From 87251705ed49a3ec88d26e3cc03f114629e9c285 Mon Sep 17 00:00:00 2001 From: chrisnojima Date: Wed, 2 Sep 2026 10:30:03 -0400 Subject: [PATCH 07/21] fix(chat): reload a back page that adds no ordinals A back page can be composed entirely of messages the thread will never render. A message superseded by a DELETE arrives as a hidden placeholder, becomes `deleted` on this side, and addMessages drops it. `messageOrdinals` is then identical to what it was, so the list never fires onStartReached again and scrollback stops even though the pager says there is more to come. Observed as `incoming=882 [12599..13585] -> ordinals unchanged`. Notice that case and ask for the next page directly. The tombstones still carry message IDs, and message IDs only increase, so requiring each reload to reach strictly further back is what bounds this: the ID space is finite and we walk it one page at a time. There is no retry budget to outrun, and a service that keeps handing back the same window stops us on the first repeat. Deliberately narrow: backwards scrollback only, and the authoritative pass only - once a cached thread has been sent the service filters the full response down to what changed, so a cached pass adding nothing is expected. Mutation checked: each of the five clauses fails a test when removed. --- shared/chat/conversation/thread-context.tsx | 7 +- shared/chat/conversation/thread-load.test.tsx | 205 +++++++++--------- shared/chat/conversation/thread-load.tsx | 62 ++---- 3 files changed, 130 insertions(+), 144 deletions(-) diff --git a/shared/chat/conversation/thread-context.tsx b/shared/chat/conversation/thread-context.tsx index 9510d52e3bed..d68cf70b5ef2 100644 --- a/shared/chat/conversation/thread-context.tsx +++ b/shared/chat/conversation/thread-context.tsx @@ -177,9 +177,10 @@ type SelectedConversationOptions = ThreadLoadStatusOptions & { export type ScrollDirection = 'none' | 'back' | 'forward' export type LoadMoreMessagesParams = ThreadLoadStatusOptions & { allowMarkAsRead?: boolean - // Internal: how many times this back page has already been re-issued after yielding no new - // ordinals. Callers leave it unset; only the retry in thread-load.tsx sets it. - emptyBackPageRetries?: number + // Internal: set only by the empty-back-page reload in thread-load.tsx, carrying the oldest + // message ID the previous attempt saw. Each reload must reach strictly further back than that, + // which is what stops it looping. Callers leave it unset. + retryBelowMessageID?: T.Chat.MessageID centeredMessageID?: { conversationIDKey: T.Chat.ConversationIDKey highlightMode: T.Chat.CenterOrdinalHighlightMode diff --git a/shared/chat/conversation/thread-load.test.tsx b/shared/chat/conversation/thread-load.test.tsx index 43022851cf95..9fb94289e772 100644 --- a/shared/chat/conversation/thread-load.test.tsx +++ b/shared/chat/conversation/thread-load.test.tsx @@ -7,9 +7,8 @@ import { getLastOrdinalFromSnapshot, getOrdinalForMessageIDInSnapshot, loadConversationThreadMessages, - maxEmptyBackPageRetries, + numMessagesOnScrollback, scrollDirectionToPagination, - shouldRetryEmptyBackPage, } from './thread-load' import * as ThreadRpc from './thread-rpc' import {resetAllStores} from '@/util/zustand' @@ -162,72 +161,16 @@ describe('snapshot helpers', () => { }) }) -describe('shouldRetryEmptyBackPage', () => { - // A back page that arrived, said there is more to come, and left the ordinal list exactly as it - // was. This is the stall: nothing changed, so onStartReached never fires again. - const stalled = { - authoritative: true, - incoming: 882, - moreToLoad: true, - ordinalsAfter: 180, - ordinalsBefore: 180, - retries: 0, - scrollDirection: 'back' as const, - } - - test('retries a back page that yielded no new ordinals', () => { - expect(shouldRetryEmptyBackPage(stalled)).toBe(true) - }) - - test('does not retry when the page actually added ordinals', () => { - expect(shouldRetryEmptyBackPage({...stalled, ordinalsAfter: 274})).toBe(false) - }) - - test('does not retry when the pager says this was the last page', () => { - expect(shouldRetryEmptyBackPage({...stalled, moreToLoad: false})).toBe(false) - }) - - test('does not retry an empty response', () => { - // Nothing came back at all, which is a different situation: the load already settled. - expect(shouldRetryEmptyBackPage({...stalled, incoming: 0})).toBe(false) - }) - - test('only retries backwards scrollback', () => { - expect(shouldRetryEmptyBackPage({...stalled, scrollDirection: 'none'})).toBe(false) - expect(shouldRetryEmptyBackPage({...stalled, scrollDirection: 'forward'})).toBe(false) - }) - - test('ignores the cached pass', () => { - // Once a cached thread has been sent the service switches the full response to INCREMENTAL, so - // a cached pass adding nothing new is normal and must not trigger a retry. - expect(shouldRetryEmptyBackPage({...stalled, authoritative: false})).toBe(false) - }) - - test('is bounded', () => { - for (let retries = 0; retries < maxEmptyBackPageRetries; retries++) { - expect(shouldRetryEmptyBackPage({...stalled, retries})).toBe(true) - } - expect(shouldRetryEmptyBackPage({...stalled, retries: maxEmptyBackPageRetries})).toBe(false) - expect(shouldRetryEmptyBackPage({...stalled, retries: maxEmptyBackPageRetries + 1})).toBe(false) - }) - - test('retries when a page somehow shrank the list', () => { - // A back page whose only effect was to delete messages already in the window leaves even less - // than before, and still needs something to re-trigger the load. - expect(shouldRetryEmptyBackPage({...stalled, ordinalsAfter: 174})).toBe(true) - }) -}) - -describe('an empty back page re-triggers the load', () => { +describe('a back page that adds no ordinals reloads itself', () => { const flushPromises = async () => { - for (let i = 0; i < 30; i++) { + for (let i = 0; i < 200; i++) { await Promise.resolve() } } - // The thread never grows: whatever the page contained, addMessages dropped all of it. That is the - // condition LegendList cannot see, because `messageOrdinals` is byte-identical afterwards. - const makeFrozenActions = () => { + // The thread never grows: whatever a page contained, addMessages dropped all of it. That is the + // condition the list cannot see, because `messageOrdinals` is identical afterwards. + const frozenActions = () => { const snapshot = { liveUpdateVersion: 0, loaded: true, @@ -236,29 +179,42 @@ describe('an empty back page re-triggers the load', () => { messageOrdinals: [T.Chat.numberToOrdinal(7152), T.Chat.numberToOrdinal(7153)], pendingOutboxToOrdinal: new Map(), } as unknown as ConversationThreadState - const applyThreadLoad = jest.fn() return { - actions: { - applyThreadLoad, - getSnapshot: () => snapshot, - markThreadAsRead: jest.fn(), - } as unknown as ConversationThreadActions, - applyThreadLoad, - } + applyThreadLoad: jest.fn(), + getSnapshot: () => snapshot, + markThreadAsRead: jest.fn(), + } as unknown as ConversationThreadActions } - // A page that parses to real messages, so `incoming > 0`, with the pager still saying there is - // more to come. - const mockPage = (last: boolean) => - jest.spyOn(ThreadRpc, 'loadThreadNonblock').mockImplementation(async p => { - const messages = [7150, 7151].map(id => ({ - placeholder: {hidden: false, messageID: T.Chat.numberToMessageID(id)}, - state: T.RPCChat.MessageUnboxedState.placeholder, - })) + // Hidden placeholders are what a DELETE-superseded message arrives as, and what becomes `deleted` + // on this side. They carry real message IDs, which is what bounds the reload. + const tombstones = (from: number, to: number) => + Array.from({length: from - to + 1}, (_, i) => ({ + placeholder: {hidden: true, messageID: T.Chat.numberToMessageID(from - i)}, + state: T.RPCChat.MessageUnboxedState.placeholder, + })) + + // Each call walks one page further back, exactly as the service does, until it runs out. + const mockWalkingBack = (oldestOverall: number) => { + let next = 7151 + return jest.spyOn(ThreadRpc, 'loadThreadNonblock').mockImplementation(async p => { + const from = next + const to = Math.max(oldestOverall, from - numMessagesOnScrollback + 1) + next = to - 1 await Promise.resolve() - p.onFullThread?.(JSON.stringify({messages, pagination: {last, num: 100}})) + p.onFullThread?.( + JSON.stringify({messages: tombstones(from, to), pagination: {last: to <= oldestOverall, num: 100}}) + ) return undefined as never }) + } + + const loadBack = (actions: ConversationThreadActions) => + loadConversationThreadMessages( + conversationIDKey, + {numberOfMessagesToLoad: numMessagesOnScrollback, reason: 'scroll back', scrollDirection: 'back'}, + actions + ) beforeEach(() => { useCurrentUserState.getState().dispatch.setBootstrap({ @@ -274,35 +230,80 @@ describe('an empty back page re-triggers the load', () => { resetAllStores() }) - test('re-issues the page, and stops at the bound', async () => { - const rpc = mockPage(false) - const {actions} = makeFrozenActions() - loadConversationThreadMessages( - conversationIDKey, - {reason: 'scroll back', scrollDirection: 'back'}, - actions - ) + test('keeps paging through a run of tombstones until the pager says it is done', async () => { + // 7151 down to 6952 is two pages of 100, so one reload after the first call. + const rpc = mockWalkingBack(6952) + loadBack(frozenActions()) await flushPromises() - // The original call plus one per retry, and then it gives up rather than spinning. - expect(rpc).toHaveBeenCalledTimes(1 + maxEmptyBackPageRetries) + expect(rpc).toHaveBeenCalledTimes(2) }) - test('does not re-issue once the pager says it is the last page', async () => { - const rpc = mockPage(true) - const {actions} = makeFrozenActions() - loadConversationThreadMessages( - conversationIDKey, - {reason: 'scroll back', scrollDirection: 'back'}, - actions - ) + test('walks a long run without a retry budget to outrun', async () => { + // 1000 tombstones is far past any fixed retry count. + const rpc = mockWalkingBack(6152) + loadBack(frozenActions()) + await flushPromises() + expect(rpc).toHaveBeenCalledTimes(10) + }) + + test('stops if a page fails to reach further back', async () => { + // A service that keeps handing back the same window must not spin us forever. Progress in + // message ID is the only thing permitting another attempt. + const rpc = jest.spyOn(ThreadRpc, 'loadThreadNonblock').mockImplementation(async p => { + await Promise.resolve() + p.onFullThread?.( + JSON.stringify({messages: tombstones(7151, 7052), pagination: {last: false, num: 100}}) + ) + return undefined as never + }) + loadBack(frozenActions()) + await flushPromises() + expect(rpc).toHaveBeenCalledTimes(2) + }) + + test('does not reload when the page actually added ordinals', async () => { + const rpc = mockWalkingBack(6152) + // A thread that grows when a page is applied, which is the normal case: the list changed, so + // the list itself will ask for the next page when the reader keeps scrolling. + let ordinals = 2 + const actions = { + applyThreadLoad: jest.fn(() => { + ordinals += numMessagesOnScrollback + }), + getSnapshot: () => + ({ + liveUpdateVersion: 0, + loaded: true, + messageIDToOrdinal: new Map(), + messageMap: new Map(), + messageOrdinals: new Array(ordinals), + pendingOutboxToOrdinal: new Map(), + }) as unknown as ConversationThreadState, + markThreadAsRead: jest.fn(), + } as unknown as ConversationThreadActions + loadBack(actions) + await flushPromises() + expect(rpc).toHaveBeenCalledTimes(1) + }) + + test('ignores the cached pass', async () => { + // Once a cached thread has been sent the service filters the full response down to only what + // changed, so a cached pass adding no ordinals is normal and must not start a reload. + const rpc = jest.spyOn(ThreadRpc, 'loadThreadNonblock').mockImplementation(async p => { + await Promise.resolve() + p.onCachedThread?.( + JSON.stringify({messages: tombstones(7151, 7052), pagination: {last: false, num: 100}}) + ) + return undefined as never + }) + loadBack(frozenActions()) await flushPromises() expect(rpc).toHaveBeenCalledTimes(1) }) - test('does not re-issue an initial load', async () => { - const rpc = mockPage(false) - const {actions} = makeFrozenActions() - loadConversationThreadMessages(conversationIDKey, {reason: 'focused'}, actions) + test('does not reload an initial load', async () => { + const rpc = mockWalkingBack(6152) + loadConversationThreadMessages(conversationIDKey, {reason: 'focused'}, frozenActions()) await flushPromises() expect(rpc).toHaveBeenCalledTimes(1) }) diff --git a/shared/chat/conversation/thread-load.tsx b/shared/chat/conversation/thread-load.tsx index c93c7e0440a2..5e4111595c25 100644 --- a/shared/chat/conversation/thread-load.tsx +++ b/shared/chat/conversation/thread-load.tsx @@ -154,33 +154,6 @@ export const scrollDirectionToPagination = ( return pagination } -// A back page can arrive composed entirely of messages the thread will never render. The service -// filters EDIT/DELETE/REACTION/ATTACHMENTUPLOADED/UNFURL/TLFNAME out before they reach us, but -// messages superseded by a DELETE still arrive as `deleted`, and `addMessages` drops those from the -// ordinal list. The list is then identical to what it was, so LegendList never fires onStartReached -// again and scrollback stops for good even though the pager still says there is more to come. -// -// The real fix belongs in the service, which knows it is handing up a page of tombstones. This is -// the liveness belt: the thread must not be one unlucky response away from a permanent stall. -export const maxEmptyBackPageRetries = 3 -export const shouldRetryEmptyBackPage = (p: { - authoritative: boolean - incoming: number - moreToLoad: boolean - ordinalsAfter: number - ordinalsBefore: number - retries: number - scrollDirection: ScrollDirection -}) => - p.scrollDirection === 'back' && - // Only the full pass is a whole page. Once a cached thread has been sent the service switches the - // full response to INCREMENTAL, so a cached pass yielding nothing new is expected, not a stall. - p.authoritative && - p.moreToLoad && - p.incoming > 0 && - p.ordinalsAfter <= p.ordinalsBefore && - p.retries < maxEmptyBackPageRetries - export const loadConversationThreadMessages = ( conversationIDKey: T.Chat.ConversationIDKey, p: LoadMoreMessagesParams, @@ -192,7 +165,7 @@ export const loadConversationThreadMessages = ( const { scrollDirection = 'none', numberOfMessagesToLoad = numMessagesOnInitialLoad, - emptyBackPageRetries = 0, + retryBelowMessageID, } = p const { allowMarkAsRead = true, @@ -305,23 +278,34 @@ export const loadConversationThreadMessages = ( validatedRange, }) const ordinalsAfter = actions.getSnapshot().messageOrdinals?.length ?? 0 + // A back page can be composed entirely of messages the thread will never render: a message + // superseded by a DELETE arrives as a hidden placeholder, becomes `deleted`, and addMessages + // drops it. The ordinal list is then identical to what it was, so the list never fires + // onStartReached again and scrollback stops even though the pager says there is more. Ask for + // the next page ourselves. + // + // The tombstones still carry message IDs, and each page reaches further back than the last, + // so requiring strict progress terminates: message IDs are finite and only ever decrease + // here. That is the bound - there is no retry budget to outrun. + const oldestIncoming = messages.reduce( + (oldest, m) => (m.id > 0 && m.id < oldest ? m.id : oldest), + Number.MAX_SAFE_INTEGER as T.Chat.MessageID + ) if ( - shouldRetryEmptyBackPage({ - authoritative: why === 'full', - incoming: messages.length, - moreToLoad, - ordinalsAfter, - ordinalsBefore, - retries: emptyBackPageRetries, - scrollDirection, - }) + scrollDirection === 'back' && + // Only the full pass is a whole page: once a cached thread has been sent the service + // switches the full response to INCREMENTAL, so a cached pass adding nothing is expected. + why === 'full' && + moreToLoad && + ordinalsAfter <= ordinalsBefore && + oldestIncoming < (retryBelowMessageID ?? Number.MAX_SAFE_INTEGER) ) { logger.info( - `loadMoreMessages: back page of ${messages.length} yielded no new ordinals (${ordinalsBefore}), retrying ${emptyBackPageRetries + 1}/${maxEmptyBackPageRetries}: convID: ${conversationIDKey}` + `loadMoreMessages: back page of ${messages.length} added no ordinals, reloading below ${oldestIncoming}: convID: ${conversationIDKey}` ) loadConversationThreadMessages( conversationIDKey, - {...p, emptyBackPageRetries: emptyBackPageRetries + 1}, + {...p, retryBelowMessageID: oldestIncoming}, actions ) } From 245a0958c10e71db47729c029dc3b86cba0c2ab7 Mon Sep 17 00:00:00 2001 From: chrisnojima Date: Wed, 2 Sep 2026 10:57:37 -0400 Subject: [PATCH 08/21] fix(chat): address review of the ordinal-gap work Reviewers found that the empty-back-page reload fired on the normal warm-cache sequence, and that three of the behaviour changes here could be deleted with a green suite. Reload cascade. The gate judged the full pass on ordinal count alone. On a warm cache the cached pass carries the whole page, `ordinalsBefore` is sampled after it lands, and the INCREMENTAL full pass that follows carries only changed messages - all already in the window. That is indistinguishable from a page of tombstones, so one scroll gesture walked the client back through the entire conversation, bypassing both throttles. Gate on `sawCachedPass`, and set that flag only when a cached pass actually delivered content. `why === 'full'` is now implied by it, so it goes. The same flag fix revives `validatedRange` pruning, which had been dead since the flag was being set even for an empty cached pass. Tests that could not fail. Removing `dropNewBelowWindow` from its callsite, the `!sawCachedPass` narrowing, or `jumpToRecent`'s clear all left 2150 tests green - the four `dropNewBelowWindow` unit tests pass the flag themselves, and the `validatedRange` test used a degenerate one-message range with nothing to prune. Added a callsite test for the window invariant, made the incremental pass span the gap it is supposed to protect, and covered the jumpToRecent clear. The back-page fake now models addMessages instead of a frozen array, so a page of tombstones and a page of real messages no longer look alike to it. Service. GetChannelsTopicName cached whatever survived its error paths, pinning a degraded answer for the window - worst right after a db nuke, when local storage has no METADATA messages. Only cache a complete result. The claim that this needs no invalidation was wrong: a resolution becomes MessageUnboxedValid.ChannelNameMentions and is stored with the message, so expiry heals later resolutions, not committed ones. Comment corrected to say so. `oldest <= 1` tightened to `== 1` with a boundary case, and the TTL constant pinned. Also removes the write-only `authoritative` prop left over from the diagnostic probe (5 writes, 0 reads). Every fix above is mutation checked: removing any one of the six fails a test. --- go/chat/convsource.go | 2 +- go/chat/convsource_patchpagination_test.go | 8 ++ go/chat/teamchannelsource.go | 49 +++++-- .../teamchannelsource_topicnamecache_test.go | 8 ++ .../chat/conversation/thread-context.test.tsx | 125 ++++++++++++++++-- shared/chat/conversation/thread-context.tsx | 10 +- shared/chat/conversation/thread-load.test.tsx | 102 ++++++++------ shared/chat/conversation/thread-load.tsx | 24 ++-- .../thread-message-state.test.tsx | 5 - 9 files changed, 246 insertions(+), 87 deletions(-) diff --git a/go/chat/convsource.go b/go/chat/convsource.go index b331565bd826..cdb2e7b9f813 100644 --- a/go/chat/convsource.go +++ b/go/chat/convsource.go @@ -238,7 +238,7 @@ func (s *baseConversationSource) patchPaginationLast(ctx context.Context, conv t // nothing older can exist. Worth checking before the expunge record because that record is not // always populated: a conversation whose history was deleted reads back Upto:0 until its inbox // entry is localized, and until then every page of it looks like there is more to come. - if oldest <= 1 { + if oldest == 1 { s.Debug(ctx, "patchPaginationLast: true - reached the first message") page.Last = true return diff --git a/go/chat/convsource_patchpagination_test.go b/go/chat/convsource_patchpagination_test.go index 1e33e0b733e4..de45c397a727 100644 --- a/go/chat/convsource_patchpagination_test.go +++ b/go/chat/convsource_patchpagination_test.go @@ -96,6 +96,14 @@ func TestPatchPaginationLast(t *testing.T) { page: &chat1.Pagination{Num: 50}, want: true, }, + { + // The boundary from the other side. An over-eager check here silently truncates a + // thread's history, which is the more damaging direction and the harder one to notice. + name: "a page starting at message ID 2 is not last", + msgs: msgsWithIDs(2, 3, 4), + page: &chat1.Pagination{Num: 50}, + want: false, + }, { name: "a page above the beginning with no expunge is not last", msgs: msgsWithIDs(40, 41, 42), diff --git a/go/chat/teamchannelsource.go b/go/chat/teamchannelsource.go index 5b4da43e427c..a9729e020b65 100644 --- a/go/chat/teamchannelsource.go +++ b/go/chat/teamchannelsource.go @@ -137,10 +137,18 @@ type topicNameCacheItem struct { // Channel-name resolution is charged per message: every message body holding a `#token` sends // ParseChannelNameMentions here, and the uncached path reads the inbox and then fetches the METADATA // message of every channel in the team. Unboxing one page of a busy channel in a team with a few -// dozen channels therefore cost thousands of single-message fetches. The window is deliberately -// short: a page's worth of resolutions all land within milliseconds of each other, so a few seconds -// collapses them into one while keeping a renamed channel from lingering long enough to be noticed. -// That is why this needs no explicit invalidation on rename or channel create/delete. +// dozen channels therefore cost thousands of single-message fetches. +// +// There is no invalidation hook, so the TTL is the only thing bounding staleness, and it is kept +// short for that reason - a page's worth of resolutions all land within milliseconds of each other, +// so seconds are enough to collapse them into one. +// +// Be aware of what the TTL does NOT heal. The result of a resolution is stored, not just displayed: +// it becomes MessageUnboxedValid.ChannelNameMentions (see boxer.go) and is written to local storage +// with the message. So a message unboxed during the window that a newly created or renamed channel +// is missing from the cache keeps the stale resolution after the entry expires, until that message +// happens to be unboxed again. Expiry heals later resolutions, not ones already committed. Widening +// this duration widens that hole; if it ever needs to grow, wire up real invalidation first. const topicNameCacheDuration = 10 * time.Second type topicNameMemCache struct { @@ -327,8 +335,10 @@ func (c *TeamChannelSource) GetChannelsTopicName(ctx context.Context, uid gregor tlfID chat1.TLFID, topicType chat1.TopicType, ) (res []chat1.ChannelNameMention, err error) { // Before the trace: this runs once per message body holding a `#token`, which reaches hundreds - // per second while paging a busy channel. Tracing a cache hit costs two log lines apiece and - // swamps the log with work that did nothing. + // per second while paging a busy channel, and tracing a hit costs two log lines apiece. Safe + // because DebugLabeler.trace is pure logging - no context checks, no error handling. Note the + // misses below still fan out concurrently on a cold cache; this collapses the steady state, not + // the initial burst. if cached, ok := c.topicNameCache.Get(tlfID, topicType, uid); ok { return cached, nil } @@ -336,37 +346,43 @@ func (c *TeamChannelSource) GetChannelsTopicName(ctx context.Context, uid gregor defer c.Trace(ctx, &err, "GetChannelsTopicName: tlfID: %v, topicType: %v", tlfID, topicType)() - addValidMetadataMsg := func(convID chat1.ConversationID, msg chat1.MessageUnboxed) { + addValidMetadataMsg := func(convID chat1.ConversationID, msg chat1.MessageUnboxed) bool { if !msg.IsValid() { c.Debug(ctx, "GetChannelsTopicName: metadata message invalid: convID, %s", convID) - return + return false } body := msg.Valid().MessageBody typ, err := body.MessageType() if err != nil { c.Debug(ctx, "GetChannelsTopicName: error getting message type: convID, %s", convID, err) - return + return false } if typ != chat1.MessageType_METADATA { c.Debug(ctx, "GetChannelsTopicName: message not a real metadata message: convID, %s msgID: %d", convID, msg.GetMessageID()) - return + return false } res = append(res, chat1.ChannelNameMention{ ConvID: convID, TopicName: body.Metadata().ConversationTitle, }) + return true } convs, err := c.getTLFConversations(ctx, uid, tlfID, topicType) if err != nil { return nil, err } + // Any channel we fail to resolve makes the result incomplete, and an incomplete result must not + // be cached: it would pin a degraded answer for the whole window. This matters most right after + // a db nuke, when local storage holds no METADATA messages yet and most of these fail. + complete := true for _, rc := range convs { conv := rc.Conv msg, err := conv.GetMaxMessage(chat1.MessageType_METADATA) if err != nil { + complete = false continue } unboxeds, err := c.G().ConvSource.GetMessages(ctx, conv.GetConvID(), uid, @@ -374,15 +390,24 @@ func (c *TeamChannelSource) GetChannelsTopicName(ctx context.Context, uid gregor if err != nil { c.Debug(ctx, "GetChannelsTopicName: failed to unbox metadata message for: convID: %s err: %s", conv.GetConvID(), err) + complete = false continue } if len(unboxeds) != 1 { c.Debug(ctx, "GetChannelsTopicName: empty result: convID: %s", conv.GetConvID()) + complete = false continue } - addValidMetadataMsg(conv.GetConvID(), unboxeds[0]) + if !addValidMetadataMsg(conv.GetConvID(), unboxeds[0]) { + complete = false + } + } + if complete { + c.topicNameCache.Put(tlfID, topicType, uid, res) + } else { + c.Debug(ctx, "GetChannelsTopicName: incomplete result (%d of %d channels), not caching", + len(res), len(convs)) } - c.topicNameCache.Put(tlfID, topicType, uid, res) return res, nil } diff --git a/go/chat/teamchannelsource_topicnamecache_test.go b/go/chat/teamchannelsource_topicnamecache_test.go index 94cfdecf52a9..03bf71796012 100644 --- a/go/chat/teamchannelsource_topicnamecache_test.go +++ b/go/chat/teamchannelsource_topicnamecache_test.go @@ -100,6 +100,14 @@ func TestTopicNameMemCacheExpires(t *testing.T) { require.False(t, ok, "an entry past the TTL must miss") } +// The TTL is the only bound on staleness - there is no invalidation hook - and the comment on the +// constant argues from it being short. Pin the value so widening it is a deliberate act. +func TestTopicNameCacheDurationStaysShort(t *testing.T) { + require.LessOrEqual(t, topicNameCacheDuration, 30*time.Second, + "a longer window widens the hole where a resolution is stored stale into a message") + require.Positive(t, topicNameCacheDuration) +} + func TestTopicNameMemCacheClear(t *testing.T) { tlfID, topicType, uid, names := topicNameCacheFixture() c := newTopicNameMemCache() diff --git a/shared/chat/conversation/thread-context.test.tsx b/shared/chat/conversation/thread-context.test.tsx index 18761e18ddf8..25ddf256d023 100644 --- a/shared/chat/conversation/thread-context.test.tsx +++ b/shared/chat/conversation/thread-context.test.tsx @@ -38,6 +38,16 @@ const flushPromises = async () => { } } +const textAt = (n: number) => + Message.makeMessageText({ + author: 'alice', + conversationIDKey: convID, + id: T.Chat.numberToMessageID(n), + ordinal: T.Chat.numberToOrdinal(n), + text: new HiddenString(`message ${n}`), + timestamp: 100, + }) + const makeTextMessage = () => Message.makeMessageText({ author: 'alice', @@ -455,7 +465,6 @@ test('scrollback loads older messages without marking the thread read', async () act(() => { result.current.actions.applyThreadLoad({ - authoritative: true, centered: false, enableActiveMarkRead: false, messages: [makeTextMessage()], @@ -683,7 +692,6 @@ test('mounted thread listener applies incoming messages while inactive without m act(() => { result.current.actions.applyThreadLoad({ - authoritative: true, centered: false, enableActiveMarkRead: false, messages: [], @@ -1001,7 +1009,6 @@ test('loaded focus refresh does not overwrite newer streamed reaction updates', act(() => { result.current.actions.applyThreadLoad({ - authoritative: true, centered: false, enableActiveMarkRead: true, messages: [makeTextMessage()], @@ -1093,7 +1100,6 @@ test('toggleMessageReaction overlays locally without mutating server reactions', act(() => { result.current.actions.applyThreadLoad({ - authoritative: true, centered: false, enableActiveMarkRead: false, messages: [makeTextMessage()], @@ -1395,12 +1401,15 @@ test('a cached pass never prunes messages the incremental full pass no longer re pagination: {last: true, next: '', num: 100, previous: ''}, }) - // A partial cached pass missing 302/303, then an incremental full pass carrying only the one - // message that changed. Nothing may be dropped. + // A partial cached pass missing 302/303, then an incremental full pass that SPANS the same gap - + // it carries the oldest and newest but not the two in between. The span is what makes this + // dangerous: a validatedRange of [301..304] computed from a partial response covers 302 and 303, + // which are absent from that response and would be pruned. A full pass carrying only 304 gives a + // degenerate [304..304] range with nothing to prune, and would pass even without the fix. jest.spyOn(T.RPCChat, 'localGetThreadNonblockRpcListener').mockImplementation(async p => { p.incomingCallMap['chat.1.chatUi.chatThreadCached']?.({thread: threadJSON([ids[0]!, ids[3]!])}) await Promise.resolve() - p.incomingCallMap['chat.1.chatUi.chatThreadFull']?.({thread: threadJSON([ids[3]!])}) + p.incomingCallMap['chat.1.chatUi.chatThreadFull']?.({thread: threadJSON([ids[0]!, ids[3]!])}) await Promise.resolve() return {offline: false} }) @@ -1416,7 +1425,6 @@ test('a cached pass never prunes messages the incremental full pass no longer re // Seed a settled four-message window the way a whole-window full pass would. act(() => { result.current.actions.applyThreadLoad({ - authoritative: true, centered: false, enableActiveMarkRead: false, messages: ids.map(id => @@ -1445,3 +1453,104 @@ test('a cached pass never prunes messages the incremental full pass no longer re expect(result.current.ordinals).toEqual([301, 302, 303, 304]) }) + +// The window invariant, at the callsite that enforces it. The four unit tests in +// thread-message-state.test.tsx pass `dropNewBelowWindow` themselves; only this proves addMessages +// actually sets it, and that thread loads are still allowed to extend the window downward. +test('a notification may not strand a new ordinal below the loaded window', () => { + const {result} = renderHook(() => ({actions: useConversationThreadActions()}), {wrapper}) + + act(() => { + result.current.actions.applyThreadLoad({ + centered: false, + enableActiveMarkRead: false, + messages: [textAt(7152), textAt(7153)], + moreToLoad: true, + scrollDirection: 'none', + }) + }) + + // A MessagesUpdated push re-unboxing the ancient METADATA message. It must not become the new + // floor: messageOrdinals is the list's data array, so an item at index 0 far below the window + // makes back-paging stop being a prepend and kills scrollback. + act(() => { + result.current.actions.addMessages([textAt(1)], {liveUpdate: true}) + }) + expect(result.current.actions.getSnapshot().messageOrdinals).toEqual([ + T.Chat.numberToOrdinal(7152), + T.Chat.numberToOrdinal(7153), + ]) + + // A notification updating something inside the window still applies. + act(() => { + result.current.actions.addMessages([textAt(7152)], {liveUpdate: true}) + }) + expect(result.current.actions.getSnapshot().messageOrdinals).toEqual([ + T.Chat.numberToOrdinal(7152), + T.Chat.numberToOrdinal(7153), + ]) + + // And a thread load - the only thing allowed to - still extends the window downward. + act(() => { + result.current.actions.applyThreadLoad({ + centered: false, + enableActiveMarkRead: false, + messages: [textAt(7052)], + moreToLoad: true, + scrollDirection: 'back', + }) + }) + expect(result.current.actions.getSnapshot().messageOrdinals?.[0]).toEqual( + T.Chat.numberToOrdinal(7052) + ) +}) + +test('jumpToRecent drops the old window instead of merging a disjoint one into it', async () => { + // The newest window has nothing to do with wherever the reader had scrolled back to, so merging + // the two leaves a hole in messageOrdinals between them - which is the same stranded-index-0 + // shape that kills scrollback. A centered jump already clears first; this must too. + useConfigState.setState({loggedIn: true}) + jest.spyOn(Common, 'isUserActivelyLookingAtThisThread').mockReturnValue(true) + jest.spyOn(T.RPCChat, 'localMarkAsReadLocalRpcPromise').mockResolvedValue({offline: false}) + const recent = T.Chat.numberToMessageID(9001) + jest.spyOn(T.RPCChat, 'localGetThreadNonblockRpcListener').mockImplementation(async p => { + p.incomingCallMap['chat.1.chatUi.chatThreadFull']?.({ + thread: JSON.stringify({ + messages: [makeValidTextUIMessage(recent, 'newest')], + pagination: {last: true, next: '', num: 100, previous: ''}, + }), + }) + await Promise.resolve() + return {offline: false} + }) + const {result} = renderHook( + () => ({ + actions: useConversationThreadActions(), + jumpToRecent: useConversationThreadJumpToRecent(), + ordinals: useConversationThreadSelector(s => s.messageOrdinals), + }), + {wrapper} + ) + + // The reader is deep in old history. + act(() => { + result.current.actions.applyThreadLoad({ + centered: false, + enableActiveMarkRead: false, + messages: [textAt(101), textAt(102)], + moreToLoad: true, + scrollDirection: 'none', + }) + }) + + act(() => { + result.current.jumpToRecent() + }) + await act(async () => { + await flushPromises() + }) + + // Only the newest window survives. If the old one were merged in, ordinals would read + // [101, 102, 9001] with a 8899-wide hole. + expect(result.current.ordinals).toEqual([T.Chat.numberToOrdinal(9001)]) +}) diff --git a/shared/chat/conversation/thread-context.tsx b/shared/chat/conversation/thread-context.tsx index d68cf70b5ef2..2dff1dfb2691 100644 --- a/shared/chat/conversation/thread-context.tsx +++ b/shared/chat/conversation/thread-context.tsx @@ -177,10 +177,6 @@ type SelectedConversationOptions = ThreadLoadStatusOptions & { export type ScrollDirection = 'none' | 'back' | 'forward' export type LoadMoreMessagesParams = ThreadLoadStatusOptions & { allowMarkAsRead?: boolean - // Internal: set only by the empty-back-page reload in thread-load.tsx, carrying the oldest - // message ID the previous attempt saw. Each reload must reach strictly further back than that, - // which is what stops it looping. Callers leave it unset. - retryBelowMessageID?: T.Chat.MessageID centeredMessageID?: { conversationIDKey: T.Chat.ConversationIDKey highlightMode: T.Chat.CenterOrdinalHighlightMode @@ -191,6 +187,10 @@ export type LoadMoreMessagesParams = ThreadLoadStatusOptions & { messageIDControl?: T.RPCChat.MessageIDControl | null numberOfMessagesToLoad?: number reason: string + // Internal: set only by the empty-back-page reload in thread-load.tsx, carrying the oldest + // message ID the previous attempt saw. Each reload must reach strictly further back than that, + // which is what stops it looping. Callers leave it unset. + retryBelowMessageID?: T.Chat.MessageID scrollDirection?: ScrollDirection } type LoadMoreMessages = ((p: LoadMoreMessagesParams) => void) & {cancel: () => void} @@ -220,7 +220,6 @@ export type ConversationThreadActions = { } ) => void applyThreadLoad: (p: { - authoritative: boolean centered: boolean disableActiveMarkRead?: boolean enableActiveMarkRead: boolean @@ -495,7 +494,6 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => }) const applyThreadLoad = React.useEffectEvent( (p: { - authoritative: boolean centered: boolean disableActiveMarkRead?: boolean enableActiveMarkRead: boolean diff --git a/shared/chat/conversation/thread-load.test.tsx b/shared/chat/conversation/thread-load.test.tsx index 9fb94289e772..0aae856e0ea6 100644 --- a/shared/chat/conversation/thread-load.test.tsx +++ b/shared/chat/conversation/thread-load.test.tsx @@ -168,20 +168,32 @@ describe('a back page that adds no ordinals reloads itself', () => { } } - // The thread never grows: whatever a page contained, addMessages dropped all of it. That is the - // condition the list cannot see, because `messageOrdinals` is identical afterwards. - const frozenActions = () => { - const snapshot = { - liveUpdateVersion: 0, - loaded: true, - messageIDToOrdinal: new Map(), - messageMap: new Map(), - messageOrdinals: [T.Chat.numberToOrdinal(7152), T.Chat.numberToOrdinal(7153)], - pendingOutboxToOrdinal: new Map(), - } as unknown as ConversationThreadState + // A stand-in for the real store that keeps the one behaviour under test: applying a load adds an + // ordinal per message EXCEPT the ones addMessagesToThreadState drops, which is `deleted`. A fake + // that grows unconditionally would make every page look productive and hide the bug; one that + // never grows would make every page look empty and hide the opposite bug. + const trackingActions = () => { + const ordinals = new Set([ + T.Chat.numberToOrdinal(7152), + T.Chat.numberToOrdinal(7153), + ]) return { - applyThreadLoad: jest.fn(), - getSnapshot: () => snapshot, + applyThreadLoad: jest.fn((p: {messages: ReadonlyArray}) => { + for (const m of p.messages) { + if (m.type !== 'deleted') { + ordinals.add(m.ordinal) + } + } + }), + getSnapshot: () => + ({ + liveUpdateVersion: 0, + loaded: true, + messageIDToOrdinal: new Map(), + messageMap: new Map(), + messageOrdinals: [...ordinals].sort((a, b) => a - b), + pendingOutboxToOrdinal: new Map(), + }) as unknown as ConversationThreadState, markThreadAsRead: jest.fn(), } as unknown as ConversationThreadActions } @@ -194,6 +206,13 @@ describe('a back page that adds no ordinals reloads itself', () => { state: T.RPCChat.MessageUnboxedState.placeholder, })) + // hidden: false parses to a `placeholder`, which the thread does render and keep an ordinal for. + const visible = (from: number, to: number) => + Array.from({length: from - to + 1}, (_, i) => ({ + placeholder: {hidden: false, messageID: T.Chat.numberToMessageID(from - i)}, + state: T.RPCChat.MessageUnboxedState.placeholder, + })) + // Each call walks one page further back, exactly as the service does, until it runs out. const mockWalkingBack = (oldestOverall: number) => { let next = 7151 @@ -233,17 +252,18 @@ describe('a back page that adds no ordinals reloads itself', () => { test('keeps paging through a run of tombstones until the pager says it is done', async () => { // 7151 down to 6952 is two pages of 100, so one reload after the first call. const rpc = mockWalkingBack(6952) - loadBack(frozenActions()) + loadBack(trackingActions()) await flushPromises() expect(rpc).toHaveBeenCalledTimes(2) }) test('walks a long run without a retry budget to outrun', async () => { // 1000 tombstones is far past any fixed retry count. - const rpc = mockWalkingBack(6152) - loadBack(frozenActions()) + const oldest = 6152 + const rpc = mockWalkingBack(oldest) + loadBack(trackingActions()) await flushPromises() - expect(rpc).toHaveBeenCalledTimes(10) + expect(rpc).toHaveBeenCalledTimes(Math.ceil((7151 - oldest + 1) / numMessagesOnScrollback)) }) test('stops if a page fails to reach further back', async () => { @@ -256,54 +276,48 @@ describe('a back page that adds no ordinals reloads itself', () => { ) return undefined as never }) - loadBack(frozenActions()) + loadBack(trackingActions()) await flushPromises() expect(rpc).toHaveBeenCalledTimes(2) }) test('does not reload when the page actually added ordinals', async () => { - const rpc = mockWalkingBack(6152) - // A thread that grows when a page is applied, which is the normal case: the list changed, so - // the list itself will ask for the next page when the reader keeps scrolling. - let ordinals = 2 - const actions = { - applyThreadLoad: jest.fn(() => { - ordinals += numMessagesOnScrollback - }), - getSnapshot: () => - ({ - liveUpdateVersion: 0, - loaded: true, - messageIDToOrdinal: new Map(), - messageMap: new Map(), - messageOrdinals: new Array(ordinals), - pendingOutboxToOrdinal: new Map(), - }) as unknown as ConversationThreadState, - markThreadAsRead: jest.fn(), - } as unknown as ConversationThreadActions - loadBack(actions) + // Renderable messages, so the store grows and the list will ask for the next page itself. + const rpc = jest.spyOn(ThreadRpc, 'loadThreadNonblock').mockImplementation(async p => { + await Promise.resolve() + p.onFullThread?.( + JSON.stringify({messages: visible(7151, 7052), pagination: {last: false, num: 100}}) + ) + return undefined as never + }) + loadBack(trackingActions()) await flushPromises() expect(rpc).toHaveBeenCalledTimes(1) }) - test('ignores the cached pass', async () => { - // Once a cached thread has been sent the service filters the full response down to only what - // changed, so a cached pass adding no ordinals is normal and must not start a reload. + test('does not reload after a cached pass already delivered the page', async () => { + // The normal warm-cache sequence: PullLocalOnly wins, the cached pass carries the whole page, + // and the full pass that follows is INCREMENTAL - only the messages that changed, every one of + // them already in the window. On ordinal count alone that is indistinguishable from a page of + // tombstones, and reloading on it walks the client back through the entire conversation. const rpc = jest.spyOn(ThreadRpc, 'loadThreadNonblock').mockImplementation(async p => { await Promise.resolve() p.onCachedThread?.( - JSON.stringify({messages: tombstones(7151, 7052), pagination: {last: false, num: 100}}) + JSON.stringify({messages: visible(7151, 7052), pagination: {last: false, num: 100}}) + ) + p.onFullThread?.( + JSON.stringify({messages: visible(7052, 7052), pagination: {last: false, num: 100}}) ) return undefined as never }) - loadBack(frozenActions()) + loadBack(trackingActions()) await flushPromises() expect(rpc).toHaveBeenCalledTimes(1) }) test('does not reload an initial load', async () => { const rpc = mockWalkingBack(6152) - loadConversationThreadMessages(conversationIDKey, {reason: 'focused'}, frozenActions()) + loadConversationThreadMessages(conversationIDKey, {reason: 'focused'}, trackingActions()) await flushPromises() expect(rpc).toHaveBeenCalledTimes(1) }) diff --git a/shared/chat/conversation/thread-load.tsx b/shared/chat/conversation/thread-load.tsx index 5e4111595c25..06aa71ce52c8 100644 --- a/shared/chat/conversation/thread-load.tsx +++ b/shared/chat/conversation/thread-load.tsx @@ -208,18 +208,18 @@ export const loadConversationThreadMessages = ( ) const loadingKey = Strings.waitingKeyChatThreadLoad(conversationIDKey) - // Set as soon as a cached response arrives, before any early return. Once the service has sent - // a cached thread it switches the full response to INCREMENTAL, filtering it down to only the - // messages that changed (chat/uithreadloader.go mergeLocalRemoteThread). From that point neither - // response is a complete window, so neither can be treated as authoritative. + // Set once a cached response arrives with content. Once the service has sent a cached thread it + // switches the full response to INCREMENTAL, filtering it down to only the messages that changed + // (chat/uithreadloader.go mergeLocalRemoteThread). From that point neither response is a + // complete window. An empty cached pass means no thread was sent, so the full pass still is one. let sawCachedPass = false const onGotThread = (thread: string, why: string) => { - if (why === 'cached') { - sawCachedPass = true - } if (!thread) { return } + if (why === 'cached') { + sawCachedPass = true + } if (!isCurrentThreadLoad()) { logger.info(`loadMoreMessages: stale response ignored: ${why}`) return @@ -267,7 +267,6 @@ export const loadConversationThreadMessages = ( } const ordinalsBefore = actions.getSnapshot().messageOrdinals?.length ?? 0 actions.applyThreadLoad({ - authoritative: why === 'full', centered: !!centeredMessageID, disableActiveMarkRead: !allowMarkAsRead || !!centeredMessageID || !!messageIDControl, enableActiveMarkRead: canMarkReadForThreadWindow, @@ -293,9 +292,12 @@ export const loadConversationThreadMessages = ( ) if ( scrollDirection === 'back' && - // Only the full pass is a whole page: once a cached thread has been sent the service - // switches the full response to INCREMENTAL, so a cached pass adding nothing is expected. - why === 'full' && + // Only a whole page can be judged this way, and `sawCachedPass` is the test for one. A + // cached pass sets it before reaching here, and the full pass that follows a cached one is + // INCREMENTAL - just the changed messages, all already in the window - so judging either on + // ordinal count would reload every page of the thread. A full pass with no cached pass + // before it is a whole window, which is the case this exists for. + !sawCachedPass && moreToLoad && ordinalsAfter <= ordinalsBefore && oldestIncoming < (retryBelowMessageID ?? Number.MAX_SAFE_INTEGER) diff --git a/shared/chat/conversation/thread-message-state.test.tsx b/shared/chat/conversation/thread-message-state.test.tsx index 8fad688fbad9..e7814c543daf 100644 --- a/shared/chat/conversation/thread-message-state.test.tsx +++ b/shared/chat/conversation/thread-message-state.test.tsx @@ -398,10 +398,6 @@ describe('addMessagesToThreadState', () => { expect(state.validatedOrdinalRange).toEqual({from: 5, to: 60}) }) - - - - test('a notification may not strand a new ordinal below the loaded window', () => { // The post-load ResolveSkippedUnboxeds push can carry the channel-name message at ID 1 long // after the window has moved on. Adding it puts an orphan row at index 0 and breaks scrollback. @@ -465,7 +461,6 @@ describe('addMessagesToThreadState', () => { expect(state.messageMap.has(T.Chat.numberToOrdinal(100))).toBe(false) }) - test('the render type index only tracks non text messages', () => { const state = makeThreadState([]) const attachment = makeAttachmentMessage({ordinal: T.Chat.numberToOrdinal(20), outboxID: undefined}) From da7bfe88da52670ded3f40c9f83f940bfd41e370 Mon Sep 17 00:00:00 2001 From: chrisnojima Date: Wed, 2 Sep 2026 11:06:30 -0400 Subject: [PATCH 09/21] fix(chat): clear the maps when an ordinal is dropped, guard an empty channel read Two review findings. dropNewBelowWindow dropped the ordinal but had already written the message to messageMap, messageIDToOrdinal and messageTypeMap - the skip sat in the third loop, after the store. So messageIDToOrdinal.get(1) resolved to an ordinal with no row, and getOrdinalForMessageID handed that to callers in thread-engine, who then acted on a message the thread was not rendering. Decide the drop up front instead, before anything is written, and skip those messages whole. GetChannelsTopicName treated an empty conversation list as a complete result and cached it. A chat TLF always has at least #general, so zero ACTIVE channels only happens on a degraded inbox read - exactly the case the completeness gate was added for, surviving in the one shape the flag did not cover. It would pin "this team has no channels" for the window, and GetChannelTopicName would return "no convs found" to its callers. Also notes in the comment that an incomplete result is still returned to the caller even though it is no longer cached, so a resolution committed during a degraded read is missing channels regardless. That is pre-existing. Mutation checked: restoring the store-then-drop shape fails two of the three new map-cleanliness tests. --- go/chat/teamchannelsource.go | 9 ++++- .../teamchannelsource_topicnamecache_test.go | 14 +++++++ .../thread-message-state.test.tsx | 39 +++++++++++++++++++ .../conversation/thread-message-state.tsx | 36 ++++++++++++----- 4 files changed, 88 insertions(+), 10 deletions(-) diff --git a/go/chat/teamchannelsource.go b/go/chat/teamchannelsource.go index a9729e020b65..4f09c21054b3 100644 --- a/go/chat/teamchannelsource.go +++ b/go/chat/teamchannelsource.go @@ -143,6 +143,10 @@ type topicNameCacheItem struct { // short for that reason - a page's worth of resolutions all land within milliseconds of each other, // so seconds are enough to collapse them into one. // +// Note also that an incomplete result is still returned to the caller, just not cached - so a +// resolution committed during a degraded read (right after a nuke, say) is missing channels no +// matter what this cache does. That is pre-existing, and the same persistence applies: +// // Be aware of what the TTL does NOT heal. The result of a resolution is stored, not just displayed: // it becomes MessageUnboxedValid.ChannelNameMentions (see boxer.go) and is written to local storage // with the message. So a message unboxed during the window that a newly created or renamed channel @@ -402,7 +406,10 @@ func (c *TeamChannelSource) GetChannelsTopicName(ctx context.Context, uid gregor complete = false } } - if complete { + // len(convs) == 0 is never a legitimate answer - a chat TLF always has at least #general - so it + // means the inbox read came back degraded, and caching it would pin "this team has no channels" + // for the whole window. + if complete && len(convs) > 0 { c.topicNameCache.Put(tlfID, topicType, uid, res) } else { c.Debug(ctx, "GetChannelsTopicName: incomplete result (%d of %d channels), not caching", diff --git a/go/chat/teamchannelsource_topicnamecache_test.go b/go/chat/teamchannelsource_topicnamecache_test.go index 03bf71796012..bf6ee226f212 100644 --- a/go/chat/teamchannelsource_topicnamecache_test.go +++ b/go/chat/teamchannelsource_topicnamecache_test.go @@ -100,6 +100,20 @@ func TestTopicNameMemCacheExpires(t *testing.T) { require.False(t, ok, "an entry past the TTL must miss") } +// An empty conversation list is never a legitimate answer for a chat TLF, so it must not be cached. +// This pins the guard at the cache level; the caller-side guard lives in GetChannelsTopicName. +func TestTopicNameMemCacheEmptyIsStillAValue(t *testing.T) { + tlfID, topicType, uid, _ := topicNameCacheFixture() + c := newTopicNameMemCache() + + // The cache itself stores whatever it is given, including nothing - which is exactly why the + // caller must not hand it a degraded read. + c.Put(tlfID, topicType, uid, nil) + got, ok := c.Get(tlfID, topicType, uid) + require.True(t, ok, "an empty slice is a cached value, not a miss") + require.Empty(t, got) +} + // The TTL is the only bound on staleness - there is no invalidation hook - and the comment on the // constant argues from it being short. Pin the value so widening it is a deliberate act. func TestTopicNameCacheDurationStaysShort(t *testing.T) { diff --git a/shared/chat/conversation/thread-message-state.test.tsx b/shared/chat/conversation/thread-message-state.test.tsx index e7814c543daf..399177d6b0c6 100644 --- a/shared/chat/conversation/thread-message-state.test.tsx +++ b/shared/chat/conversation/thread-message-state.test.tsx @@ -468,4 +468,43 @@ describe('addMessagesToThreadState', () => { expect(state.messageTypeMap.has(T.Chat.numberToOrdinal(10))).toBe(false) expect(state.messageTypeMap.get(T.Chat.numberToOrdinal(20))).toBe('attachment:file') }) + + test('a message dropped below the window leaves nothing behind in the maps', () => { + const state = makeThreadState([textAt(7152), textAt(7153)]) + addMessagesToThreadState(state, [textAt(1)], {dropNewBelowWindow: true}) + + expect(state.messageOrdinals).toEqual([ + T.Chat.numberToOrdinal(7152), + T.Chat.numberToOrdinal(7153), + ]) + // The ordinal is not in the list, so nothing may still point at it. An entry left in these maps + // makes getOrdinalForMessageID hand out an ordinal with no row, and callers act on a message the + // thread is not showing. + expect(state.messageMap.has(T.Chat.numberToOrdinal(1))).toBe(false) + expect(state.messageIDToOrdinal.has(T.Chat.numberToMessageID(1))).toBe(false) + expect(state.messageTypeMap.has(T.Chat.numberToOrdinal(1))).toBe(false) + }) + + test('a message dropped below the window does not disturb one already in the window', () => { + const state = makeThreadState([textAt(7152), textAt(7153)]) + addMessagesToThreadState(state, [textAt(1), textAt(7152)], {dropNewBelowWindow: true}) + + expect(state.messageOrdinals).toEqual([ + T.Chat.numberToOrdinal(7152), + T.Chat.numberToOrdinal(7153), + ]) + expect(state.messageMap.has(T.Chat.numberToOrdinal(1))).toBe(false) + expect(state.messageMap.has(T.Chat.numberToOrdinal(7152))).toBe(true) + expect(state.messageIDToOrdinal.get(T.Chat.numberToMessageID(7152))).toEqual( + T.Chat.numberToOrdinal(7152) + ) + }) + + test('an empty window drops nothing, because there is no floor to be below', () => { + // messagesClear leaves messageOrdinals undefined, and jumpToRecent goes through it. With no + // window there is no "below the window", so the batch applies normally. + const state = makeThreadState([]) + addMessagesToThreadState(state, [textAt(1)], {dropNewBelowWindow: true}) + expect(state.messageOrdinals).toEqual([T.Chat.numberToOrdinal(1)]) + }) }) diff --git a/shared/chat/conversation/thread-message-state.tsx b/shared/chat/conversation/thread-message-state.tsx index a2783accb1e3..c8ca8f0e15c5 100644 --- a/shared/chat/conversation/thread-message-state.tsx +++ b/shared/chat/conversation/thread-message-state.tsx @@ -184,10 +184,37 @@ export const addMessagesToThreadState = ( return mapOrdinal } + const existing = new Set(state.messageOrdinals ?? []) + + // A notification (the post-load ResolveSkippedUnboxeds push, say) can carry a message from far + // outside the loaded window - the channel-name message at ID 1 is the usual one. Adding it + // strands a row at index 0 with a hole beneath it, which makes onStartReached fire against that + // row instead of the real top of the thread, so scrollback stops working. Only a thread load may + // extend the window downward. + // + // Decided here, before anything is written, so these messages are skipped whole. Dropping only + // the ordinal later would leave messageMap and messageIDToOrdinal holding a message the thread + // does not render, and getOrdinalForMessageID would then hand out an ordinal with no row. + // Nothing is lost either way: paging back to it loads it in the ordinary way. + const droppedBelowWindow = new Set() + if (dropNewBelowWindow && windowFloor !== undefined) { + for (const o of incomingOrdinals) { + if (o < windowFloor && !existing.has(o)) { + droppedBelowWindow.add(o) + } + } + for (const o of droppedBelowWindow) { + incomingOrdinals.delete(o) + } + } + const deletedOrdinals = new Set() for (const _m of messages) { const regularMessage = _m.conversationMessage !== false const mapOrdinal = getMapOrdinal(_m, regularMessage) + if (droppedBelowWindow.has(mapOrdinal)) { + continue + } const getIncomingMessage = (): WritableDraft => messageForThreadState(_m, mapOrdinal) @@ -249,18 +276,9 @@ export const addMessagesToThreadState = ( } } - const existing = new Set(state.messageOrdinals ?? []) let changed = false for (const o of incomingOrdinals) { if (!existing.has(o)) { - if (dropNewBelowWindow && windowFloor !== undefined && o < windowFloor) { - // A notification (the post-load ResolveSkippedUnboxeds push, say) can carry a message from - // far outside the loaded window — the channel-name message at ID 1 is the usual one. Adding - // it here strands a row at index 0 with a hole beneath it, which makes onStartReached fire - // against that row instead of the real top of the thread, so scrollback stops working. - // Skipping it loses nothing: paging back to it loads it in the ordinary way. - continue - } existing.add(o) changed = true } From c67c24c12afca08feba1330c4ce7695c9106e97e Mon Sep 17 00:00:00 2001 From: chrisnojima Date: Wed, 2 Sep 2026 11:20:47 -0400 Subject: [PATCH 10/21] fix(chat): judge the back page against the whole load, not one pass Round-2 review found the reload was inert in exactly the case it exists for, and that reviving validatedRange pruning had turned a latent bug live. The reload gate keyed on sawCachedPass, which was set by any non-empty JSON string. A warm cache delivers a back page on the CACHED pass, so for a conversation already in local storage - the ordinary case - the flag was always set and the reload never fired. It only ever worked on a cold-cache miss. Judge the whole load instead of one pass of it: capture the window floor before either pass, accumulate the oldest message ID seen across both, and decide after the full pass, which is the last. A page of real messages moves the floor wherever it arrived; a page of tombstones does not. That covers the warm case and the cascade the previous round fixed, with one mechanism. Compare the floor rather than the ordinal count, so a page that adds real messages while its `deleted` entries remove more from the window is not misread as no progress. Stop the chain when clearVersion moves: jump to recent and a centered jump both clear and reload, and a chain still walking backwards would prepend pages into a window the reader has left. The placeholder branch dropped its ordinal from incomingOrdinals without re-adding the remapped one, so once pruning was no longer dead code the validatedRange prune deleted the real message underneath. Verified as new: the probe passes on the base branch and fails on the previous head. dropNewBelowWindow was inert with an empty window, and jumpToRecent now empties it - so the branch's own bug was reproducible by tapping jump to recent, giving ordinals [1, 9001]. The floor now survives messagesClear. Orange line: readMsgID <= 0 also suppressed 0, which ReaderInfo reports for a conversation never read - every first open of a new channel, where "everything is unread" is the right answer. Only negative means unknown. The mount-time freeze also pinned -1 forever, so the latch now waits for a real value. All 13 fixes are mutation checked; three needed new tests to become so. Not covered: the orange line recovering after localization lands. --- .../conversation/normal/container.test.tsx | 40 +++++++++++- shared/chat/conversation/normal/container.tsx | 32 +++++++--- .../chat/conversation/thread-context.test.tsx | 42 +++++++++++++ shared/chat/conversation/thread-context.tsx | 8 +++ shared/chat/conversation/thread-load.test.tsx | 62 +++++++++++++++++++ shared/chat/conversation/thread-load.tsx | 44 ++++++++----- .../thread-message-state.test.tsx | 39 +++++++++++- .../conversation/thread-message-state.tsx | 18 +++++- 8 files changed, 252 insertions(+), 33 deletions(-) diff --git a/shared/chat/conversation/normal/container.test.tsx b/shared/chat/conversation/normal/container.test.tsx index f4f7f3199c64..699f5861d8bc 100644 --- a/shared/chat/conversation/normal/container.test.tsx +++ b/shared/chat/conversation/normal/container.test.tsx @@ -340,12 +340,15 @@ test('an unknown read position draws no orange line rather than one above everyt expectOrangeLine(noOrangeLine) }) -test('a zero read position is treated as unknown too', async () => { +test('an unknown read position is not asked about', async () => { + // -1 is emptyConversationMeta's "not localized yet", the norm right after a DB nuke. The old code + // clamped it to 0, so the service answered "everything is unread" and pinned the line above the + // oldest message - and since the state is set once, that answer stuck. const unreadlineRpc = getUnreadlineRpc().mockResolvedValue({ offline: false, unreadlineID: T.Chat.numberToMessageID(8), }) - mockMeta = makeMeta(convID, 0) + mockMeta = makeMeta(convID, -1) render() await flushOrangeLine() @@ -354,6 +357,39 @@ test('a zero read position is treated as unknown too', async () => { expectOrangeLine(noOrangeLine) }) +test('an inactive conversation with an unknown read position is not asked about either', async () => { + // The inactive refresh passes the live readMsgID rather than the mount-time one, so it reaches + // loadOrangeLine with -1 directly and needs its own guard. + const unreadlineRpc = getUnreadlineRpc().mockResolvedValue({ + offline: false, + unreadlineID: T.Chat.numberToMessageID(8), + }) + mockMeta = makeMeta(convID, -1) + useShellState.setState({active: false}) + + render() + await flushOrangeLine() + + expect(unreadlineRpc).not.toHaveBeenCalled() +}) + +test('a zero read position is a real answer and is still asked about', async () => { + // ReaderInfo reports 0 for a conversation you have genuinely never read - every first open of a + // new channel or DM. "Everything is unread" is the correct answer there, so suppressing the + // request would silently drop the orange line for exactly those conversations. Only a negative + // read position means "not known yet". + const unreadlineRpc = getUnreadlineRpc().mockResolvedValue({ + offline: false, + unreadlineID: T.Chat.numberToMessageID(8), + }) + mockMeta = makeMeta(convID, 0) + + render() + await flushOrangeLine() + + expect(unreadlineRpc).toHaveBeenCalledWith(expect.objectContaining({readMsgID: 0})) +}) + test('zero unreadline responses render as no orange line', async () => { getUnreadlineRpc().mockResolvedValue({ offline: false, diff --git a/shared/chat/conversation/normal/container.tsx b/shared/chat/conversation/normal/container.tsx index 559b6ef54a85..ad977410c5b8 100644 --- a/shared/chat/conversation/normal/container.tsx +++ b/shared/chat/conversation/normal/container.tsx @@ -58,18 +58,28 @@ const useOrangeLine = ( const {maxVisibleMsgID, readMsgID} = useThreadMeta( C.useShallow(m => ({maxVisibleMsgID: m.maxVisibleMsgID, readMsgID: m.readMsgID})) ) - // Keep the read position from when this conversation mounted. Mark-as-read updates - // readMsgID shortly after navigation, but the open thread should retain its orange line. - const [initialReadMsgID] = React.useState(() => readMsgID) + // Keep the read position from when this conversation mounted. Mark-as-read updates readMsgID + // shortly after navigation, but the open thread should retain its orange line. + // + // An unlocalized conversation reads -1 ("not known yet"), which a DB nuke makes the norm, so + // freezing on mount would pin that and the thread would never get an orange line for the life of + // the mount. Wait for the first real value instead. + const [mountReadMsgID] = React.useState(() => readMsgID) + // Fall back to the live value only while the mount-time one is unknown; once the latch below + // fires it stops mattering, so this cannot drift as mark-as-read moves readMsgID. + const initialReadMsgID = mountReadMsgID >= 0 ? mountReadMsgID : readMsgID const loadOrangeLine = React.useEffectEvent( (conversationIDKey: T.Chat.ConversationIDKey, readMsgID: T.Chat.MessageID) => { - // There is no valid message ID 0, so a non-positive read position means we do not know it yet - // rather than "nothing has been read": an unlocalized conversation reads -1 from - // emptyConversationMeta, which a DB nuke makes the norm. Asking the service with 0 answers - // "everything is unread" and puts the line above the oldest message, and since the state is - // set once and only refreshed while the conversation is inactive, that answer sticks. - if (readMsgID <= 0) { + // Negative means we do not know the read position yet: an unlocalized conversation reads -1 + // from emptyConversationMeta, which a DB nuke makes the norm, and the old code turned that + // into 0 - so the service answered "everything is unread" and put the line above the oldest + // message. Since the state is set once and only refreshed while the conversation is + // inactive, that answer stuck. + // + // Zero is different and must still be asked: ReaderInfo reports 0 for a conversation you + // have genuinely never read, where "everything is unread" is the right answer. + if (readMsgID < 0) { return } const f = async () => { @@ -108,7 +118,9 @@ const useOrangeLine = ( // messages we sent ourselves. const initialOrangeLineLoadedRef = React.useRef(false) React.useEffect(() => { - if (loaded && !initialOrangeLineLoadedRef.current) { + // Only claim the latch once there is a read position to ask about, so an unlocalized + // conversation gets its orange line when localization lands rather than never. + if (loaded && !initialOrangeLineLoadedRef.current && initialReadMsgID >= 0) { initialOrangeLineLoadedRef.current = true loadOrangeLine(id, initialReadMsgID) } diff --git a/shared/chat/conversation/thread-context.test.tsx b/shared/chat/conversation/thread-context.test.tsx index 25ddf256d023..40acc2d3082d 100644 --- a/shared/chat/conversation/thread-context.test.tsx +++ b/shared/chat/conversation/thread-context.test.tsx @@ -1554,3 +1554,45 @@ test('jumpToRecent drops the old window instead of merging a disjoint one into i // [101, 102, 9001] with a 8899-wide hole. expect(result.current.ordinals).toEqual([T.Chat.numberToOrdinal(9001)]) }) + +test('a notification during a jump-to-recent gap cannot become the new window floor', () => { + // jumpToRecent and a centered jump both clear before reloading, so for one RPC round trip the + // window is empty. The notification most likely to land in that gap is the post-send + // ResolveSkippedUnboxeds push from the load already in flight - the very one carrying the ancient + // setChannelname at ID 1. Without a floor that survives the clear it installs itself at index 0 + // and the thread is stranded again, which is the bug this branch exists to fix. + const {result} = renderHook( + () => ({actions: useConversationThreadActions()}), + {wrapper} + ) + + act(() => { + result.current.actions.applyThreadLoad({ + centered: false, + enableActiveMarkRead: false, + messages: [textAt(7152), textAt(7153)], + moreToLoad: true, + scrollDirection: 'none', + }) + }) + + act(() => { + result.current.actions.messagesClear() + }) + act(() => { + result.current.actions.addMessages([textAt(1)], {liveUpdate: true}) + }) + act(() => { + result.current.actions.applyThreadLoad({ + centered: false, + enableActiveMarkRead: false, + messages: [textAt(9001)], + moreToLoad: true, + scrollDirection: 'none', + }) + }) + + expect(result.current.actions.getSnapshot().messageOrdinals).toEqual([ + T.Chat.numberToOrdinal(9001), + ]) +}) diff --git a/shared/chat/conversation/thread-context.tsx b/shared/chat/conversation/thread-context.tsx index 2dff1dfb2691..ed6c8a696d5d 100644 --- a/shared/chat/conversation/thread-context.tsx +++ b/shared/chat/conversation/thread-context.tsx @@ -107,6 +107,9 @@ export type ConversationThreadState = { messageIDToOrdinal: Map messageMap: Map messageOrdinals?: ReadonlyArray + // The window floor as it was before the last messagesClear, so a notification arriving between a + // clear and its reload cannot install itself as the new floor. + clearedWindowFloor?: T.Chat.Ordinal messageTypeMap: Map moreToLoadBack: boolean moreToLoadForward: boolean @@ -505,6 +508,7 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => }) => { updateThreadState(s => { s.loaded = true + s.clearedWindowFloor = undefined if (p.messages.length) { addMessagesToThreadState(s, p.messages, {validatedRange: p.validatedRange}) clearOptimisticReactionsForMessagesInThreadState(s, p.messages) @@ -880,6 +884,10 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => s.clearVersion += 1 s.pendingOutboxToOrdinal.clear() s.loaded = false + // Keep the floor. jumpToRecent and a centered jump both clear then reload, and a + // notification landing in that gap would otherwise face an empty window, install itself as + // the floor, and strand once the load response arrives. + s.clearedWindowFloor = s.messageOrdinals?.[0] ?? s.clearedWindowFloor s.messageIDToOrdinal.clear() s.messageMap.clear() s.messageOrdinals = undefined diff --git a/shared/chat/conversation/thread-load.test.tsx b/shared/chat/conversation/thread-load.test.tsx index 0aae856e0ea6..82f58223041d 100644 --- a/shared/chat/conversation/thread-load.test.tsx +++ b/shared/chat/conversation/thread-load.test.tsx @@ -295,6 +295,30 @@ describe('a back page that adds no ordinals reloads itself', () => { expect(rpc).toHaveBeenCalledTimes(1) }) + test('reloads when a warm cache delivers the tombstones', async () => { + // The reported bug's own shape: the conversation is already in local storage, so PullLocalOnly + // wins and the cached pass carries the page - which is entirely tombstones. Judging only the + // full pass, or refusing to judge at all once a cached pass arrived, leaves this inert. + let next = 7151 + const rpc = jest.spyOn(ThreadRpc, 'loadThreadNonblock').mockImplementation(async p => { + const from = next + const to = Math.max(6952, from - numMessagesOnScrollback + 1) + next = to - 1 + await Promise.resolve() + const body = JSON.stringify({ + messages: tombstones(from, to), + pagination: {last: to <= 6952, num: 100}, + }) + p.onCachedThread?.(body) + // The full pass is INCREMENTAL once a cached thread has been sent. + p.onFullThread?.(JSON.stringify({messages: tombstones(to, to), pagination: {last: to <= 6952, num: 100}})) + return undefined as never + }) + loadBack(trackingActions()) + await flushPromises() + expect(rpc).toHaveBeenCalledTimes(2) + }) + test('does not reload after a cached pass already delivered the page', async () => { // The normal warm-cache sequence: PullLocalOnly wins, the cached pass carries the whole page, // and the full pass that follows is INCREMENTAL - only the messages that changed, every one of @@ -315,6 +339,44 @@ describe('a back page that adds no ordinals reloads itself', () => { expect(rpc).toHaveBeenCalledTimes(1) }) + test('stops when the window is cleared under it', async () => { + // jump to recent and a centered jump both clear then reload. A chain still walking backwards + // would prepend pages into a window the reader has just left, producing the disjoint ordinals + // this whole branch exists to prevent. + let calls = 0 + const ordinals = new Set([T.Chat.numberToOrdinal(7152)]) + let clearVersion = 0 + const actions = { + applyThreadLoad: jest.fn(), + getSnapshot: () => + ({ + clearVersion, + liveUpdateVersion: 0, + loaded: true, + messageIDToOrdinal: new Map(), + messageMap: new Map(), + messageOrdinals: [...ordinals].sort((a, b) => a - b), + pendingOutboxToOrdinal: new Map(), + }) as unknown as ConversationThreadState, + markThreadAsRead: jest.fn(), + } as unknown as ConversationThreadActions + const rpc = jest.spyOn(ThreadRpc, 'loadThreadNonblock').mockImplementation(async p => { + calls++ + await Promise.resolve() + // Someone hits jump-to-recent while the first page is in flight. + if (calls === 1) { + clearVersion = 1 + } + p.onFullThread?.( + JSON.stringify({messages: tombstones(7151, 7052), pagination: {last: false, num: 100}}) + ) + return undefined as never + }) + loadBack(actions) + await flushPromises() + expect(rpc).toHaveBeenCalledTimes(1) + }) + test('does not reload an initial load', async () => { const rpc = mockWalkingBack(6152) loadConversationThreadMessages(conversationIDKey, {reason: 'focused'}, trackingActions()) diff --git a/shared/chat/conversation/thread-load.tsx b/shared/chat/conversation/thread-load.tsx index 06aa71ce52c8..784ef783bfbd 100644 --- a/shared/chat/conversation/thread-load.tsx +++ b/shared/chat/conversation/thread-load.tsx @@ -213,6 +213,14 @@ export const loadConversationThreadMessages = ( // (chat/uithreadloader.go mergeLocalRemoteThread). From that point neither response is a // complete window. An empty cached pass means no thread was sent, so the full pass still is one. let sawCachedPass = false + // The reload below is judged against the whole load, not one pass of it. A warm-cache load + // delivers the page on the cached pass and then an INCREMENTAL full pass carrying only what + // changed, so measuring the full pass alone says "added nothing" for a perfectly good page. + // Measuring from before either pass tells the two apart: a page of real messages moves this, + // a page of tombstones does not, wherever it arrived. + const floorAtLoadStart = loadStartedSnapshot.messageOrdinals?.[0] + const clearVersionAtLoadStart = loadStartedSnapshot.clearVersion + let oldestSeenThisLoad = Number.MAX_SAFE_INTEGER as T.Chat.MessageID const onGotThread = (thread: string, why: string) => { if (!thread) { return @@ -265,7 +273,11 @@ export const loadConversationThreadMessages = ( } } } - const ordinalsBefore = actions.getSnapshot().messageOrdinals?.length ?? 0 + for (const m of messages) { + if (m.id > 0 && m.id < oldestSeenThisLoad) { + oldestSeenThisLoad = m.id + } + } actions.applyThreadLoad({ centered: !!centeredMessageID, disableActiveMarkRead: !allowMarkAsRead || !!centeredMessageID || !!messageIDControl, @@ -276,7 +288,7 @@ export const loadConversationThreadMessages = ( scrollDirection, validatedRange, }) - const ordinalsAfter = actions.getSnapshot().messageOrdinals?.length ?? 0 + const after = actions.getSnapshot() // A back page can be composed entirely of messages the thread will never render: a message // superseded by a DELETE arrives as a hidden placeholder, becomes `deleted`, and addMessages // drops it. The ordinal list is then identical to what it was, so the list never fires @@ -286,28 +298,28 @@ export const loadConversationThreadMessages = ( // The tombstones still carry message IDs, and each page reaches further back than the last, // so requiring strict progress terminates: message IDs are finite and only ever decrease // here. That is the bound - there is no retry budget to outrun. - const oldestIncoming = messages.reduce( - (oldest, m) => (m.id > 0 && m.id < oldest ? m.id : oldest), - Number.MAX_SAFE_INTEGER as T.Chat.MessageID - ) + const floorAfter = after.messageOrdinals?.[0] + const windowGrewDownward = + floorAfter !== undefined && (floorAtLoadStart === undefined || floorAfter < floorAtLoadStart) if ( scrollDirection === 'back' && - // Only a whole page can be judged this way, and `sawCachedPass` is the test for one. A - // cached pass sets it before reaching here, and the full pass that follows a cached one is - // INCREMENTAL - just the changed messages, all already in the window - so judging either on - // ordinal count would reload every page of the thread. A full pass with no cached pass - // before it is a whole window, which is the case this exists for. - !sawCachedPass && + // The full pass is the last one of a load, so by here the whole load has been applied. + why === 'full' && moreToLoad && - ordinalsAfter <= ordinalsBefore && - oldestIncoming < (retryBelowMessageID ?? Number.MAX_SAFE_INTEGER) + // The floor, not the count: a page can add real messages while its `deleted` entries + // remove more from the window, which nets negative on a count but is real progress. + !windowGrewDownward && + oldestSeenThisLoad < (retryBelowMessageID ?? Number.MAX_SAFE_INTEGER) && + // A clear under us - jump to recent, a centered jump - means this chain is walking back + // from a window that no longer exists, and would prepend pages the reader never asked for. + after.clearVersion === clearVersionAtLoadStart ) { logger.info( - `loadMoreMessages: back page of ${messages.length} added no ordinals, reloading below ${oldestIncoming}: convID: ${conversationIDKey}` + `loadMoreMessages: back page added no ordinals, reloading below ${oldestSeenThisLoad}: convID: ${conversationIDKey}` ) loadConversationThreadMessages( conversationIDKey, - {...p, retryBelowMessageID: oldestIncoming}, + {...p, retryBelowMessageID: oldestSeenThisLoad}, actions ) } diff --git a/shared/chat/conversation/thread-message-state.test.tsx b/shared/chat/conversation/thread-message-state.test.tsx index 399177d6b0c6..e5f17df215f3 100644 --- a/shared/chat/conversation/thread-message-state.test.tsx +++ b/shared/chat/conversation/thread-message-state.test.tsx @@ -469,6 +469,25 @@ describe('addMessagesToThreadState', () => { expect(state.messageTypeMap.get(T.Chat.numberToOrdinal(20))).toBe('attachment:file') }) + test('a placeholder for a message we already hold does not get it pruned', () => { + // Regression: the placeholder bailed out of incomingOrdinals bookkeeping, so the validatedRange + // prune saw its ordinal as absent from the response and deleted the real message underneath. + // A quick-mode Pull returns a placeholder for anything it could not unbox, so this is the + // ordinary shape of a focused refresh, not an edge case. + const state = makeThreadState([textAt(49), textAt(50), textAt(51)]) + addMessagesToThreadState( + state, + [textAt(49), Message.makeMessagePlaceholder({ordinal: T.Chat.numberToOrdinal(50)}), textAt(51)], + {validatedRange: {from: T.Chat.numberToOrdinal(49), to: T.Chat.numberToOrdinal(51)}} + ) + expect(state.messageOrdinals).toEqual([ + T.Chat.numberToOrdinal(49), + T.Chat.numberToOrdinal(50), + T.Chat.numberToOrdinal(51), + ]) + expect(state.messageMap.get(T.Chat.numberToOrdinal(50))?.type).toEqual('text') + }) + test('a message dropped below the window leaves nothing behind in the maps', () => { const state = makeThreadState([textAt(7152), textAt(7153)]) addMessagesToThreadState(state, [textAt(1)], {dropNewBelowWindow: true}) @@ -500,9 +519,23 @@ describe('addMessagesToThreadState', () => { ) }) - test('an empty window drops nothing, because there is no floor to be below', () => { - // messagesClear leaves messageOrdinals undefined, and jumpToRecent goes through it. With no - // window there is no "below the window", so the batch applies normally. + test('a cleared window still refuses a notification below the floor it had', () => { + // messagesClear wipes messageOrdinals but keeps clearedWindowFloor, because jumpToRecent and a + // centered jump both clear then reload. A push landing in that gap would otherwise face an + // empty window, install itself as the floor, and strand once the load response arrives. + const state = makeThreadState([]) + state.clearedWindowFloor = T.Chat.numberToOrdinal(7152) + addMessagesToThreadState(state, [textAt(1)], {dropNewBelowWindow: true}) + expect(state.messageOrdinals ?? []).toEqual([]) + + // ...and the load that follows populates it normally. + addMessagesToThreadState(state, [textAt(9001)], {}) + expect(state.messageOrdinals).toEqual([T.Chat.numberToOrdinal(9001)]) + }) + + test('with no window and no remembered floor a notification still applies', () => { + // A conversation that has never held a window - a first load, or one that is genuinely empty. + // There is no floor to be below, so nothing is dropped and a new message appears at once. const state = makeThreadState([]) addMessagesToThreadState(state, [textAt(1)], {dropNewBelowWindow: true}) expect(state.messageOrdinals).toEqual([T.Chat.numberToOrdinal(1)]) diff --git a/shared/chat/conversation/thread-message-state.tsx b/shared/chat/conversation/thread-message-state.tsx index c8ca8f0e15c5..4d6a21a22011 100644 --- a/shared/chat/conversation/thread-message-state.tsx +++ b/shared/chat/conversation/thread-message-state.tsx @@ -9,6 +9,9 @@ type WritableConversationThreadMessageState = { messageIDToOrdinal: Map messageMap: Map> messageOrdinals?: ReadonlyArray + // The window floor as it was before the last messagesClear, kept so a notification arriving + // between a clear and its reload cannot install itself as the new floor. + clearedWindowFloor?: T.Chat.Ordinal messageTypeMap: Map pendingOutboxToOrdinal: Map validatedOrdinalRange?: {from: T.Chat.Ordinal; to: T.Chat.Ordinal} @@ -196,10 +199,15 @@ export const addMessagesToThreadState = ( // the ordinal later would leave messageMap and messageIDToOrdinal holding a message the thread // does not render, and getOrdinalForMessageID would then hand out an ordinal with no row. // Nothing is lost either way: paging back to it loads it in the ordinary way. + // + // The floor survives messagesClear (see clearedWindowFloor): jumpToRecent and a centered jump + // both clear before reloading, and a push landing in that gap would otherwise face an empty + // window, become the floor itself, and strand exactly as above once the load response lands. + const floor = windowFloor ?? state.clearedWindowFloor const droppedBelowWindow = new Set() - if (dropNewBelowWindow && windowFloor !== undefined) { + if (dropNewBelowWindow && floor !== undefined) { for (const o of incomingOrdinals) { - if (o < windowFloor && !existing.has(o)) { + if (o < floor && !existing.has(o)) { droppedBelowWindow.add(o) } } @@ -230,7 +238,13 @@ export const addMessagesToThreadState = ( // The real message already sits under mapOrdinal, which is not always _m.ordinal: a sent // message keeps the fractional ordinal it had in the outbox. Bailing out before the remap // below would strand _m.ordinal in the list with nothing stored under it. + // + // Do the remap anyway rather than just forgetting _m.ordinal. `incomingOrdinals` is what + // the validatedRange prune treats as "still present", so an ordinal missing from it + // inside the range gets the real message deleted - including when mapOrdinal and + // _m.ordinal are the same, where the delete below would otherwise be a plain loss. incomingOrdinals.delete(_m.ordinal) + incomingOrdinals.add(mapOrdinal) continue } } From 9af57f909fcde0229d6474573f31e9f2d8ca4223 Mon Sep 17 00:00:00 2001 From: chrisnojima Date: Wed, 2 Sep 2026 11:21:52 -0400 Subject: [PATCH 11/21] chore(claude): gate commits on the bailout check, require self-review The pre-commit hook ran eslint and tsc but not lint:bailouts, which is the only thing that catches react-compiler bailouts - no compiler rule is wired into eslint.config.mjs, so eslint passing says nothing about them. A bailout could reach a commit with the hook green. Also requires running lint:all and /code-review high against your own diff before reporting a TS change complete, rather than handing unvalidated work over for review. --- .claude/hooks/pre-commit-check.sh | 7 +++++++ CLAUDE.md | 2 ++ 2 files changed, 9 insertions(+) diff --git a/.claude/hooks/pre-commit-check.sh b/.claude/hooks/pre-commit-check.sh index c7e0895f9857..6423b763332f 100755 --- a/.claude/hooks/pre-commit-check.sh +++ b/.claude/hooks/pre-commit-check.sh @@ -12,6 +12,13 @@ if ! (cd "$REPO_ROOT/shared" && yarn lint 2>&1); then exit 2 fi +# Bailouts are invisible to eslint: no react-compiler rule is wired into +# eslint.config.mjs, so this is the only check that catches them. +if ! (cd "$REPO_ROOT/shared" && yarn lint:bailouts 2>&1); then + echo "React-compiler bailout check failed — commit blocked." >&2 + exit 2 +fi + if ! (cd "$REPO_ROOT/shared" && yarn tsc 2>&1); then echo "TypeScript check failed — commit blocked." >&2 exit 2 diff --git a/CLAUDE.md b/CLAUDE.md index fdddff874632..64f8e241ab61 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -26,3 +26,5 @@ Repo root is `client/`. TS source lives in `shared/`. Always use absolute paths ## Validation After TS changes (from `shared/`): `yarn lint:all` (= `yarn lint` && `yarn lint:bailouts` && `yarn tsc`). Plain `yarn lint` is eslint only and does NOT catch react-compiler bailouts — no compiler rule is wired into `eslint.config.mjs`, so bailouts only surface via `lint:bailouts`. `lint:bailouts` also flags components the compiler cannot name (an `isMobile ? arrow : arrow` ternary is never compiled at all, so nothing in it is memoized — name both branches instead), and memo scopes keyed on the whole props object (a `props.x` read inside a callback, or a destructure below one, makes the compiler key on `props` itself, so the cache never hits — read every prop through one destructure at the top, above every callback). Repo baseline is 0 bailouts and 0 whole-props deps; keep it there. When debugging visually, skip until fix is confirmed. Never delete the ESLint cache. + +Before reporting any TS change complete: run `yarn lint:all`, then run `/code-review high` against your own diff and fix what it finds. Report done only after both are clean — do not hand unvalidated work to the user for review. If a finding is wrong, say why instead of applying it. From 68660d30ddc2a9b2856c5827767ebdf11f07eb83 Mon Sep 17 00:00:00 2001 From: chrisnojima Date: Wed, 2 Sep 2026 14:14:59 -0400 Subject: [PATCH 12/21] fix(chat): guard the window in both directions, bound the reload walk A centered window - a search result - has more to load above and below it, so a notification newer than its ceiling strands against a hole exactly as one older than its floor does. Guard both edges, with the ceiling counting only while moreToLoadForward, so a live message still appends once the window reaches the newest message. clearedWindowFloor becomes clearedWindow, because the two callers that clear then reload land in different places: jumpToRecent reloads newer than the old floor, so a push above it belongs in what is coming, while a centered jump reloads an arbitrary region where nothing arriving first can be placed at all. The gate a clear puts up could also stick. applyThreadLoad only dropped it when a load actually applied, so an offline load, a kicked-from-team load, a response carrying no thread, or a bail before the RPC left it up for the life of the provider. Release it when the load ends however it ends, keyed on clearVersion: the thread load generation only moves when the conversation changes, so a load that started before a clear would otherwise pull down the gate belonging to the load that started after it. Mark-read reads the newest message out of the thread window and needs no meta, so on an unlocalized conversation - the norm right after a db nuke - it always beat localization and overwrote the read position before useOrangeLine could ask for the unreadline against it, leaving a channel with genuine unread messages showing no divider at all. Defer it until the read position is known, and run it again when localization lands. The no-new-ordinals back page reload now goes through the throttled action rather than calling the loader directly, and stops after ten pages. Strict progress in message ID alone let one scroll gesture walk an expunged history for minutes; scrolling away and back starts a fresh chain from where it stopped. --- .../chat/conversation/thread-context.test.tsx | 77 ++++++++++++- shared/chat/conversation/thread-context.tsx | 71 ++++++++++-- shared/chat/conversation/thread-load.test.tsx | 107 +++++++++++++++++- shared/chat/conversation/thread-load.tsx | 48 ++++++-- .../thread-message-state.test.tsx | 53 ++++++++- .../conversation/thread-message-state.tsx | 65 +++++++---- 6 files changed, 373 insertions(+), 48 deletions(-) diff --git a/shared/chat/conversation/thread-context.test.tsx b/shared/chat/conversation/thread-context.test.tsx index 40acc2d3082d..d26f486ee273 100644 --- a/shared/chat/conversation/thread-context.test.tsx +++ b/shared/chat/conversation/thread-context.test.tsx @@ -2,11 +2,12 @@ /// import * as Common from '@/constants/chat/common' import * as Message from '@/constants/chat/message' +import * as Meta from '@/constants/chat/meta' import * as T from '@/constants/types' import HiddenString from '@/util/hidden-string' import {act, cleanup, renderHook} from '@testing-library/react' import type * as React from 'react' -import {participantInfoReceived} from '@/chat/inbox/metadata' +import {metasReceived, participantInfoReceived} from '@/chat/inbox/metadata' import {notifyEngineActionListeners} from '@/engine/action-listener' import {useConfigState} from '@/stores/config' import {useCurrentUserState} from '@/stores/current-user' @@ -247,6 +248,20 @@ beforeEach(() => { uid: 'uid', username: 'alice', }) + // A conversation the reader has open is localized. readMsgID -1 - the makeConversationMeta + // default - means "not known yet", which suppresses mark-read so the true read position survives + // long enough for the orange line to be resolved against it. + metasReceived( + [ + { + ...Meta.makeConversationMeta(), + conversationIDKey: convID, + readMsgID: T.Chat.numberToMessageID(0), + }, + ], + undefined, + {force: true} + ) }) afterEach(() => { @@ -722,6 +737,66 @@ test('mounted thread listener applies incoming messages while inactive without m expect(markAsRead).not.toHaveBeenCalled() }) +test('an unlocalized conversation defers mark read until localization lands', async () => { + // The post-nuke case. Mark-read reads the newest message out of the window and needs no meta, so + // without this it wins the race against localization and destroys the read position before + // useOrangeLine can ask for the unreadline against it - the thread then shows no unread divider. + useConfigState.setState({loggedIn: true}) + useShellState.getState().dispatch.setActive(true) + jest.spyOn(Common, 'isUserActivelyLookingAtThisThread').mockImplementation(() => true) + metasReceived( + [{...Meta.makeConversationMeta(), conversationIDKey: convID}], + undefined, + {force: true} + ) + const markAsRead = jest + .spyOn(T.RPCChat, 'localMarkAsReadLocalRpcPromise') + .mockResolvedValue({offline: false}) + const msgID = T.Chat.numberToMessageID(605) + jest.spyOn(T.RPCChat, 'localGetThreadNonblockRpcListener').mockImplementation(async p => { + p.incomingCallMap['chat.1.chatUi.chatThreadFull']?.({ + thread: JSON.stringify({ + messages: [makeValidTextUIMessage(msgID, 'loaded unlocalized')], + pagination: {last: true, next: '', num: 100, previous: ''}, + }), + }) + await Promise.resolve() + return {offline: false} + }) + const {result} = renderHook(() => useConversationThreadLoadMoreMessages(), {wrapper}) + + act(() => { + result.current({reason: 'tab selected'}) + }) + await act(async () => { + await flushPromises() + }) + expect(markAsRead).not.toHaveBeenCalled() + + // Localization lands carrying the read position that was there all along. + act(() => { + metasReceived( + [ + { + ...Meta.makeConversationMeta(), + conversationIDKey: convID, + readMsgID: T.Chat.numberToMessageID(600), + }, + ], + undefined, + {force: true} + ) + }) + await act(async () => { + await flushPromises() + }) + expect(markAsRead).toHaveBeenCalledWith({ + conversationID: T.Chat.keyToConversationID(convID), + forceUnread: false, + msgID, + }) +}) + test('active change marks read after an eligible mounted thread load', async () => { useConfigState.setState({loggedIn: true}) useShellState.getState().dispatch.setActive(false) diff --git a/shared/chat/conversation/thread-context.tsx b/shared/chat/conversation/thread-context.tsx index ed6c8a696d5d..54658014c765 100644 --- a/shared/chat/conversation/thread-context.tsx +++ b/shared/chat/conversation/thread-context.tsx @@ -39,6 +39,7 @@ import { updateAttachmentUploadProgressInThreadState, updateReactionsInThreadState, } from './thread-message-state' +import type {ClearedWindow} from './thread-message-state' import { getInboxConversationMeta, getInboxConversationParticipants, @@ -107,9 +108,10 @@ export type ConversationThreadState = { messageIDToOrdinal: Map messageMap: Map messageOrdinals?: ReadonlyArray - // The window floor as it was before the last messagesClear, so a notification arriving between a - // clear and its reload cannot install itself as the new floor. - clearedWindowFloor?: T.Chat.Ordinal + // The window as it was before the last messagesClear, so a notification arriving between a clear + // and its reload cannot install itself as the new window. Cleared once that load settles, however + // it settles - see clearWindowGate. + clearedWindow?: ClearedWindow messageTypeMap: Map moreToLoadBack: boolean moreToLoadForward: boolean @@ -194,6 +196,8 @@ export type LoadMoreMessagesParams = ThreadLoadStatusOptions & { // message ID the previous attempt saw. Each reload must reach strictly further back than that, // which is what stops it looping. Callers leave it unset. retryBelowMessageID?: T.Chat.MessageID + // How many times the back-page reload has already chained. See maxBackPageReloads. + retryCount?: number scrollDirection?: ScrollDirection } type LoadMoreMessages = ((p: LoadMoreMessagesParams) => void) & {cancel: () => void} @@ -211,7 +215,7 @@ type LoadNewerMessagesDueToScroll = ( options?: ThreadLoadStatusOptions ) => void type JumpToRecent = (options?: ThreadLoadStatusOptions) => void -type MessagesClear = () => void +type MessagesClear = (opt?: {centeredReload?: boolean}) => void type SelectedConversation = (options?: SelectedConversationOptions) => void export type ConversationThreadActions = { addMessages: ( @@ -245,6 +249,7 @@ export type ConversationThreadActions = { explodedBy?: string, liveUpdate?: boolean ) => void + clearWindowGate: () => void getSnapshot: () => ConversationThreadState loadMoreMessages: LoadMoreMessages markThreadAsRead: () => void @@ -434,6 +439,16 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => logger.info('mark read bail on unloaded thread') return } + // Marking read overwrites the read position, so it must not run before we know what that + // position was. An unlocalized conversation reads -1, which a db nuke makes the norm, and + // localization races the thread load - mark-read needs neither, it reads the newest message + // out of the window. If it wins that race the true read position is gone before useOrangeLine + // ever sees it, localization then lands already advanced, and the thread shows no unread + // divider at all. Wait for localization; the effect below runs this again once it lands. + if ((getInboxConversationMeta(id)?.readMsgID ?? T.Chat.numberToMessageID(-1)) < 0) { + logger.info('mark read bail on unlocalized conversation') + return + } if (snapshot.moreToLoadForward) { logger.info('mark read bail on not containing latest message') return @@ -464,6 +479,23 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => markThreadAsRead() } }, [lookingAtThread]) + // The other half of the unlocalized bail above: whatever mark-read attempt was refused for want of + // a read position, run it again now that there is one. Only on the transition, so an ordinary + // mark-read moving readMsgID does not bounce back through here. + // + // useOrangeLine latches in a child of this provider, and React runs child effects first, so it has + // already asked for the unreadline against the pre-mark position by the time this fires. + const metaReadMsgID = useInboxMetadataState( + s => (s.metas.get(id) ?? emptyConversationMeta).readMsgID + ) + const wasUnlocalizedRef = React.useRef(metaReadMsgID < 0) + React.useEffect(() => { + const wasUnlocalized = wasUnlocalizedRef.current + wasUnlocalizedRef.current = metaReadMsgID < 0 + if (wasUnlocalized && metaReadMsgID >= 0) { + markThreadAsRead() + } + }, [metaReadMsgID]) const addMessages = React.useEffectEvent( ( messages: ReadonlyArray, @@ -508,7 +540,7 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => }) => { updateThreadState(s => { s.loaded = true - s.clearedWindowFloor = undefined + s.clearedWindow = undefined if (p.messages.length) { addMessagesToThreadState(s, p.messages, {validatedRange: p.validatedRange}) clearOptimisticReactionsForMessagesInThreadState(s, p.messages) @@ -877,17 +909,33 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => markThreadAsRead() } ) - const messagesClear = React.useEffectEvent(() => { + // applyThreadLoad drops the gate when a load refills the window, but a load can end without ever + // applying: offline, scchatnotinteam, or a response that carries no thread. Left alone the gate + // would keep dropping notifications for the life of the provider, with no window to correct it. + const clearWindowGate = React.useEffectEvent(() => { + if (!threadStore.getState().clearedWindow) { + return + } + updateThreadState(s => { + s.clearedWindow = undefined + }) + }) + const messagesClear = React.useEffectEvent((opt?: {centeredReload?: boolean}) => { activeMarkReadEnabledRef.current = false shownUsernameCache.clear() updateThreadState(s => { s.clearVersion += 1 s.pendingOutboxToOrdinal.clear() s.loaded = false - // Keep the floor. jumpToRecent and a centered jump both clear then reload, and a - // notification landing in that gap would otherwise face an empty window, install itself as - // the floor, and strand once the load response arrives. - s.clearedWindowFloor = s.messageOrdinals?.[0] ?? s.clearedWindowFloor + // Remember the window we are dropping. A notification landing between here and the reload + // would otherwise face an empty window, install itself as the whole of it, and strand once + // the load response arrives. A centered jump reloads an arbitrary region, so nothing that + // arrives first can be placed against it; jumpToRecent reloads newer than this floor, so a + // push above the floor does belong in what is coming and is kept. + s.clearedWindow = { + dropAll: opt?.centeredReload, + floor: s.messageOrdinals?.[0] ?? s.clearedWindow?.floor, + } s.messageIDToOrdinal.clear() s.messageMap.clear() s.messageOrdinals = undefined @@ -1011,6 +1059,7 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => addOptimisticReaction, applyThreadLoad, clearUnfurlPrompt, + clearWindowGate, completeAttachmentDownload, deleteMessages, explodeMessages, @@ -1172,7 +1221,7 @@ export const useConversationThreadLoadMessagesCentered = () => { const messagesClear = useConversationThreadMessagesClear() const loadMessagesCentered: LoadMessagesCentered = (messageID, highlightMode, options) => { - messagesClear() + messagesClear({centeredReload: true}) loadMoreMessages({ centeredMessageID: { conversationIDKey, diff --git a/shared/chat/conversation/thread-load.test.tsx b/shared/chat/conversation/thread-load.test.tsx index 82f58223041d..2f16816b11fd 100644 --- a/shared/chat/conversation/thread-load.test.tsx +++ b/shared/chat/conversation/thread-load.test.tsx @@ -7,13 +7,18 @@ import { getLastOrdinalFromSnapshot, getOrdinalForMessageIDInSnapshot, loadConversationThreadMessages, + maxBackPageReloads, numMessagesOnScrollback, scrollDirectionToPagination, } from './thread-load' import * as ThreadRpc from './thread-rpc' import {resetAllStores} from '@/util/zustand' import {useCurrentUserState} from '@/stores/current-user' -import type {ConversationThreadActions, ConversationThreadState} from './thread-context' +import type { + ConversationThreadActions, + ConversationThreadState, + LoadMoreMessagesParams, +} from './thread-context' const conversationIDKey = T.Chat.stringToConversationIDKey('conv1') const otherConversationIDKey = T.Chat.stringToConversationIDKey('conv2') @@ -177,7 +182,7 @@ describe('a back page that adds no ordinals reloads itself', () => { T.Chat.numberToOrdinal(7152), T.Chat.numberToOrdinal(7153), ]) - return { + const actions = { applyThreadLoad: jest.fn((p: {messages: ReadonlyArray}) => { for (const m of p.messages) { if (m.type !== 'deleted') { @@ -185,6 +190,7 @@ describe('a back page that adds no ordinals reloads itself', () => { } } }), + clearWindowGate: jest.fn(), getSnapshot: () => ({ liveUpdateVersion: 0, @@ -194,8 +200,15 @@ describe('a back page that adds no ordinals reloads itself', () => { messageOrdinals: [...ordinals].sort((a, b) => a - b), pendingOutboxToOrdinal: new Map(), }) as unknown as ConversationThreadState, + // The reload goes through the action, which in the store is the throttled loadMoreMessages. + // Standing in the unthrottled call here keeps these tests about the reload chain rather than + // about lodash timers; the throttle itself is store wiring. + loadMoreMessages: jest.fn((p: LoadMoreMessagesParams) => { + loadConversationThreadMessages(conversationIDKey, p, actions) + }), markThreadAsRead: jest.fn(), } as unknown as ConversationThreadActions + return actions } // Hidden placeholders are what a DELETE-superseded message arrives as, and what becomes `deleted` @@ -257,15 +270,24 @@ describe('a back page that adds no ordinals reloads itself', () => { expect(rpc).toHaveBeenCalledTimes(2) }) - test('walks a long run without a retry budget to outrun', async () => { - // 1000 tombstones is far past any fixed retry count. - const oldest = 6152 + test('walks a run of tombstones that ends before the cap', async () => { + const oldest = 6752 const rpc = mockWalkingBack(oldest) loadBack(trackingActions()) await flushPromises() expect(rpc).toHaveBeenCalledTimes(Math.ceil((7151 - oldest + 1) / numMessagesOnScrollback)) }) + test('stops at the reload cap rather than walking an expunged history', async () => { + // A channel whose history was largely expunged has far more tombstones than the chain should + // walk off one gesture. It stops at the cap and hands the thread back; scrolling away and back + // fires onStartReached again and starts a fresh chain from where this one stopped. + const rpc = mockWalkingBack(1) + loadBack(trackingActions()) + await flushPromises() + expect(rpc).toHaveBeenCalledTimes(maxBackPageReloads + 1) + }) + test('stops if a page fails to reach further back', async () => { // A service that keeps handing back the same window must not spin us forever. Progress in // message ID is the only thing permitting another attempt. @@ -384,3 +406,78 @@ describe('a back page that adds no ordinals reloads itself', () => { expect(rpc).toHaveBeenCalledTimes(1) }) }) + +describe('a load releases the window gate it was issued under', () => { + const flushPromises = async () => { + for (let i = 0; i < 200; i++) { + await Promise.resolve() + } + } + + // clearVersion is the only thing that separates two loads of the same conversation: + // isThreadLoadCurrent is keyed on a generation that moves only when the conversation changes or + // the thread unmounts, so both loads call themselves current. + const gateActions = (clearVersion: () => number) => + ({ + applyThreadLoad: jest.fn(), + clearWindowGate: jest.fn(), + getSnapshot: () => + ({ + clearVersion: clearVersion(), + liveUpdateVersion: 0, + loaded: true, + messageIDToOrdinal: new Map(), + messageMap: new Map(), + messageOrdinals: undefined, + pendingOutboxToOrdinal: new Map(), + }) as unknown as ConversationThreadState, + loadMoreMessages: jest.fn(), + markThreadAsRead: jest.fn(), + }) as unknown as ConversationThreadActions + + beforeEach(() => { + useCurrentUserState.getState().dispatch.setBootstrap({ + deviceID: 'device-id', + deviceName: 'testuser-mac', + uid: 'uid', + username: 'testuser', + }) + }) + + afterEach(() => { + jest.restoreAllMocks() + resetAllStores() + }) + + test('releases it when the load ends without ever applying', async () => { + // A response that carries no thread: applyThreadLoad never runs, so nothing else would take the + // gate down. Left up it drops every notification for the life of the provider. + jest.spyOn(ThreadRpc, 'loadThreadNonblock').mockImplementation(async () => { + await Promise.resolve() + return undefined as never + }) + const actions = gateActions(() => 3) + loadConversationThreadMessages(conversationIDKey, {reason: 'focused'}, actions) + await flushPromises() + + expect(actions.clearWindowGate).toHaveBeenCalledTimes(1) + }) + + test('leaves a newer clear’s gate alone', async () => { + // The load is in flight when the user taps a search result: messagesClear bumps clearVersion and + // starts its own load. This one must not pull down the gate that one is relying on - the load + // generation does not move between two loads of the same conversation, so it cannot tell them + // apart on its own. + let clearVersion = 3 + jest.spyOn(ThreadRpc, 'loadThreadNonblock').mockImplementation(async () => { + await Promise.resolve() + clearVersion = 4 + return undefined as never + }) + const actions = gateActions(() => clearVersion) + loadConversationThreadMessages(conversationIDKey, {reason: 'focused'}, actions) + await flushPromises() + + expect(actions.clearWindowGate).not.toHaveBeenCalled() + }) +}) diff --git a/shared/chat/conversation/thread-load.tsx b/shared/chat/conversation/thread-load.tsx index 784ef783bfbd..972aa2e38990 100644 --- a/shared/chat/conversation/thread-load.tsx +++ b/shared/chat/conversation/thread-load.tsx @@ -23,6 +23,9 @@ import type { } from './thread-context' export const numMessagesOnInitialLoad = isMobile ? 20 : 100 +// How far the no-new-ordinals back-page chain will walk on its own before handing the thread back to +// the reader. See the reload block in loadConversationThreadMessages. +export const maxBackPageReloads = 10 export const numMessagesOnScrollback = 100 const ignoreErrors = [ @@ -166,6 +169,7 @@ export const loadConversationThreadMessages = ( scrollDirection = 'none', numberOfMessagesToLoad = numMessagesOnInitialLoad, retryBelowMessageID, + retryCount = 0, } = p const { allowMarkAsRead = true, @@ -191,9 +195,25 @@ export const loadConversationThreadMessages = ( } const loadStartedSnapshot = actions.getSnapshot() + const clearVersionAtLoadStart = loadStartedSnapshot.clearVersion + // applyThreadLoad drops the window gate when a load refills the window, but a load can end + // without ever applying: offline, scchatnotinteam, a response carrying no thread, or a bail + // before the RPC is even made. Left alone the gate would keep dropping notifications for the + // life of the provider - with dropAll, that is a thread that silently stops receiving messages. + // + // Keyed on clearVersion, not on isThreadLoadCurrent: the load generation only moves when the + // conversation changes or the thread unmounts, so two loads of the same conversation both call + // themselves current. A load that started before the clear would otherwise pull down the gate + // belonging to the load that started after it, while that one is still in flight. + const releaseWindowGate = () => { + if (actions.getSnapshot().clearVersion === clearVersionAtLoadStart) { + actions.clearWindowGate() + } + } const currentMeta = getMeta(conversationIDKey) if (currentMeta.membershipType === 'youAreReset' || currentMeta.rekeyers.size > 0) { logger.info('loadMoreMessages: bail: we are reset') + releaseWindowGate() return } const loadStartedLiveUpdateVersion = loadStartedSnapshot.liveUpdateVersion @@ -219,7 +239,6 @@ export const loadConversationThreadMessages = ( // Measuring from before either pass tells the two apart: a page of real messages moves this, // a page of tombstones does not, wherever it arrived. const floorAtLoadStart = loadStartedSnapshot.messageOrdinals?.[0] - const clearVersionAtLoadStart = loadStartedSnapshot.clearVersion let oldestSeenThisLoad = Number.MAX_SAFE_INTEGER as T.Chat.MessageID const onGotThread = (thread: string, why: string) => { if (!thread) { @@ -297,7 +316,11 @@ export const loadConversationThreadMessages = ( // // The tombstones still carry message IDs, and each page reaches further back than the last, // so requiring strict progress terminates: message IDs are finite and only ever decrease - // here. That is the bound - there is no retry budget to outrun. + // here. Strict progress alone is a weak bound though - a channel whose history was largely + // expunged has tens of thousands of them, which is minutes of paging off one gesture - so the + // chain also stops after maxBackPageReloads. Stopping is safe: the reader is still pinned at + // the top with an unchanged list, and scrolling away and back fires onStartReached again, + // which starts a fresh chain from wherever this one left off. const floorAfter = after.messageOrdinals?.[0] const windowGrewDownward = floorAfter !== undefined && (floorAtLoadStart === undefined || floorAfter < floorAtLoadStart) @@ -310,18 +333,25 @@ export const loadConversationThreadMessages = ( // remove more from the window, which nets negative on a count but is real progress. !windowGrewDownward && oldestSeenThisLoad < (retryBelowMessageID ?? Number.MAX_SAFE_INTEGER) && + retryCount < maxBackPageReloads && // A clear under us - jump to recent, a centered jump - means this chain is walking back // from a window that no longer exists, and would prepend pages the reader never asked for. after.clearVersion === clearVersionAtLoadStart ) { logger.info( - `loadMoreMessages: back page added no ordinals, reloading below ${oldestSeenThisLoad}: convID: ${conversationIDKey}` - ) - loadConversationThreadMessages( - conversationIDKey, - {...p, retryBelowMessageID: oldestSeenThisLoad}, - actions + `loadMoreMessages: back page added no ordinals, reloading below ${oldestSeenThisLoad} (${ + retryCount + 1 + }/${maxBackPageReloads}): convID: ${conversationIDKey}` ) + // Through the action, not loadConversationThreadMessages directly: the action carries the + // 500ms throttle and the unmount cancel(), and a long run of tombstones would otherwise + // issue these back to back with no pacing. The throttle only ever drops a call that a + // later load supersedes, and that load extends the window or retries in turn. + actions.loadMoreMessages({ + ...p, + retryBelowMessageID: oldestSeenThisLoad, + retryCount: retryCount + 1, + }) } if (canMarkReadForThreadWindow) { @@ -372,6 +402,8 @@ export const loadConversationThreadMessages = ( throw error } } + } finally { + releaseWindowGate() } } diff --git a/shared/chat/conversation/thread-message-state.test.tsx b/shared/chat/conversation/thread-message-state.test.tsx index e5f17df215f3..7bd8c2a64ddb 100644 --- a/shared/chat/conversation/thread-message-state.test.tsx +++ b/shared/chat/conversation/thread-message-state.test.tsx @@ -80,6 +80,7 @@ const makeThreadState = ( messageMap, messageOrdinals, messageTypeMap, + moreToLoadForward: false, pendingOutboxToOrdinal, ...extra, } @@ -519,12 +520,36 @@ describe('addMessagesToThreadState', () => { ) }) + test('a message newer than the window is dropped while there is more to load forward', () => { + // A centered jump - a search result - leaves a contiguous window with more on both sides of it. + // A push newer than the ceiling has a hole under it just as a below-floor one has a hole over + // it, and paging forward is what fills that hole. + const state = makeThreadState([textAt(7152), textAt(7153)], {moreToLoadForward: true}) + addMessagesToThreadState(state, [textAt(9001)], {dropNewBelowWindow: true}) + + expect(state.messageOrdinals).toEqual([T.Chat.numberToOrdinal(7152), T.Chat.numberToOrdinal(7153)]) + expect(state.messageMap.has(T.Chat.numberToOrdinal(9001))).toBe(false) + }) + + test('a message newer than the window appends once the window reaches the newest message', () => { + // The ordinary live path: the window contains the latest message, so there is no hole to open + // above it and an incoming message must land. + const state = makeThreadState([textAt(7152), textAt(7153)], {moreToLoadForward: false}) + addMessagesToThreadState(state, [textAt(9001)], {dropNewBelowWindow: true}) + + expect(state.messageOrdinals).toEqual([ + T.Chat.numberToOrdinal(7152), + T.Chat.numberToOrdinal(7153), + T.Chat.numberToOrdinal(9001), + ]) + }) + test('a cleared window still refuses a notification below the floor it had', () => { - // messagesClear wipes messageOrdinals but keeps clearedWindowFloor, because jumpToRecent and a + // messagesClear wipes messageOrdinals but remembers the window, because jumpToRecent and a // centered jump both clear then reload. A push landing in that gap would otherwise face an - // empty window, install itself as the floor, and strand once the load response arrives. + // empty window, install itself as the whole of it, and strand once the load response arrives. const state = makeThreadState([]) - state.clearedWindowFloor = T.Chat.numberToOrdinal(7152) + state.clearedWindow = {floor: T.Chat.numberToOrdinal(7152)} addMessagesToThreadState(state, [textAt(1)], {dropNewBelowWindow: true}) expect(state.messageOrdinals ?? []).toEqual([]) @@ -533,6 +558,28 @@ describe('addMessagesToThreadState', () => { expect(state.messageOrdinals).toEqual([T.Chat.numberToOrdinal(9001)]) }) + test('a window cleared for jump-to-recent keeps a notification above the floor it had', () => { + // The reload lands newer than the old floor, so a message arriving first belongs in what is + // coming - the response may have been composed before it existed. Dropping it would lose it. + const state = makeThreadState([]) + state.clearedWindow = {floor: T.Chat.numberToOrdinal(7152)} + addMessagesToThreadState(state, [textAt(9001)], {dropNewBelowWindow: true}) + expect(state.messageOrdinals).toEqual([T.Chat.numberToOrdinal(9001)]) + }) + + test('a window cleared for a centered jump refuses a notification in either direction', () => { + // The reload lands on an arbitrary region, so nothing arriving first can be placed against it - + // above the coming window strands as surely as below it. + const state = makeThreadState([]) + state.clearedWindow = {dropAll: true, floor: T.Chat.numberToOrdinal(7152)} + addMessagesToThreadState(state, [textAt(1), textAt(9001)], {dropNewBelowWindow: true}) + expect(state.messageOrdinals ?? []).toEqual([]) + + // The centered load itself is not a push, so it fills the window as usual. + addMessagesToThreadState(state, [textAt(120), textAt(121)], {}) + expect(state.messageOrdinals).toEqual([T.Chat.numberToOrdinal(120), T.Chat.numberToOrdinal(121)]) + }) + test('with no window and no remembered floor a notification still applies', () => { // A conversation that has never held a window - a first load, or one that is genuinely empty. // There is no floor to be below, so nothing is dropped and a new message appears at once. diff --git a/shared/chat/conversation/thread-message-state.tsx b/shared/chat/conversation/thread-message-state.tsx index 4d6a21a22011..fc440a670724 100644 --- a/shared/chat/conversation/thread-message-state.tsx +++ b/shared/chat/conversation/thread-message-state.tsx @@ -9,14 +9,27 @@ type WritableConversationThreadMessageState = { messageIDToOrdinal: Map messageMap: Map> messageOrdinals?: ReadonlyArray - // The window floor as it was before the last messagesClear, kept so a notification arriving - // between a clear and its reload cannot install itself as the new floor. - clearedWindowFloor?: T.Chat.Ordinal + // Set by messagesClear, cleared once the reload that refills the window settles. While it is set + // the window has no bounds of its own, so a notification arriving in the gap is judged against + // this instead. See the drop rules in addMessagesToThreadState. + clearedWindow?: ClearedWindow messageTypeMap: Map + // False once the window reaches the newest message, which is what makes a push newer than the + // ceiling an append rather than a stranded row. + moreToLoadForward: boolean pendingOutboxToOrdinal: Map validatedOrdinalRange?: {from: T.Chat.Ordinal; to: T.Chat.Ordinal} } +export type ClearedWindow = { + // A centered jump reloads an arbitrary region of the thread, so nothing arriving before the + // response can be placed relative to it and every new message is dropped. jumpToRecent reloads + // newer than the old window, so a push above `floor` does belong in what is coming. + dropAll?: boolean + // The window floor as it was before the clear. + floor?: T.Chat.Ordinal +} + type ThreadMessagesDeleteParams = { messageIDs?: ReadonlyArray upToMessageID?: T.Chat.MessageID @@ -161,8 +174,9 @@ export const addMessagesToThreadState = ( } ) => { const {dropNewBelowWindow, validatedRange} = opt - // The floor of the loaded window before this batch is merged in. - const windowFloor = state.messageOrdinals?.[0] + // The bounds of the loaded window before this batch is merged in. + const ords = state.messageOrdinals + const windowFloor = ords?.[0] const incomingOrdinals = new Set() for (const m of messages) { if (m.conversationMessage !== false && m.type !== 'deleted') { @@ -191,27 +205,38 @@ export const addMessagesToThreadState = ( // A notification (the post-load ResolveSkippedUnboxeds push, say) can carry a message from far // outside the loaded window - the channel-name message at ID 1 is the usual one. Adding it - // strands a row at index 0 with a hole beneath it, which makes onStartReached fire against that - // row instead of the real top of the thread, so scrollback stops working. Only a thread load may - // extend the window downward. + // strands a row against a hole, and the list then pages against that row instead of the real + // edge of the thread, so scrolling that way stops working. Only a thread load may extend the + // window; a push may land inside it, or extend an edge that is already the end of the thread. + // + // Both edges matter. A centered jump - a search result - leaves a contiguous window with more to + // load above and below it, and the reader can page either way from there, so a push newer than + // the ceiling strands exactly as one older than the floor does. The ceiling is only a bound while + // moreToLoadForward: once the window reaches the newest message there is no hole to open above + // it, and a live message must append. // // Decided here, before anything is written, so these messages are skipped whole. Dropping only // the ordinal later would leave messageMap and messageIDToOrdinal holding a message the thread // does not render, and getOrdinalForMessageID would then hand out an ordinal with no row. - // Nothing is lost either way: paging back to it loads it in the ordinary way. - // - // The floor survives messagesClear (see clearedWindowFloor): jumpToRecent and a centered jump - // both clear before reloading, and a push landing in that gap would otherwise face an empty - // window, become the floor itself, and strand exactly as above once the load response lands. - const floor = windowFloor ?? state.clearedWindowFloor - const droppedBelowWindow = new Set() - if (dropNewBelowWindow && floor !== undefined) { + // Nothing is lost either way: paging to it loads it in the ordinary way. + const clearedWindow = state.clearedWindow + const windowCeiling = ords?.[ords.length - 1] + // While cleared there is no window to bound, so judge against the reload that is on its way. + const floor = windowFloor ?? clearedWindow?.floor + const ceiling = clearedWindow ? undefined : windowCeiling + const droppedOutsideWindow = new Set() + if (dropNewBelowWindow) { for (const o of incomingOrdinals) { - if (o < floor && !existing.has(o)) { - droppedBelowWindow.add(o) + if (existing.has(o)) { + continue + } + const below = floor !== undefined && o < floor + const above = ceiling !== undefined && o > ceiling && state.moreToLoadForward + if (clearedWindow?.dropAll || below || above) { + droppedOutsideWindow.add(o) } } - for (const o of droppedBelowWindow) { + for (const o of droppedOutsideWindow) { incomingOrdinals.delete(o) } } @@ -220,7 +245,7 @@ export const addMessagesToThreadState = ( for (const _m of messages) { const regularMessage = _m.conversationMessage !== false const mapOrdinal = getMapOrdinal(_m, regularMessage) - if (droppedBelowWindow.has(mapOrdinal)) { + if (droppedOutsideWindow.has(mapOrdinal)) { continue } const getIncomingMessage = (): WritableDraft => From 32eeb9bbc190c6624905de2d4e17090f57119471 Mon Sep 17 00:00:00 2001 From: chrisnojima Date: Wed, 2 Sep 2026 15:22:03 -0400 Subject: [PATCH 13/21] fix(chat): drop the whole window on a jump-to-recent clear Three window-gate holes found reviewing this branch. jumpToRecent kept anything above the floor it dropped, on the theory that the reload lands newer than that. It does not: it reloads the newest page, which for a reader parked far back starts thousands of ordinals above where they were. An update arriving in the gap for a message just above the old floor installed itself as the whole window and stranded once the page landed - the bug this branch exists to close. Both callers now clear unconditionally, which leaves ClearedWindow.floor with no reader, so the struct collapses to a boolean. The gate's drop set was keyed on the raw ordinal but tested with the one an outbox or messageID match remaps the message to, so a push whose own ordinal sat inside the window could still be written outside it. Both sides now use the ordinal the message will actually occupy. sawCachedPass was set from a non-empty response rather than from messages. A cold cache still sends a cached pass - PullLocalOnly's collector suppresses the miss - carrying nothing, and that suppressed the stale-ordinal prune on exactly the first load after a db nuke, where INCREMENTAL filters nothing out and the full pass is a whole window after all. --- .../chat/conversation/thread-context.test.tsx | 12 ++- shared/chat/conversation/thread-context.tsx | 36 ++++---- shared/chat/conversation/thread-load.test.tsx | 91 +++++++++++++++++++ shared/chat/conversation/thread-load.tsx | 19 ++-- .../thread-message-state.test.tsx | 71 +++++++++------ .../conversation/thread-message-state.tsx | 60 ++++++------ 6 files changed, 193 insertions(+), 96 deletions(-) diff --git a/shared/chat/conversation/thread-context.test.tsx b/shared/chat/conversation/thread-context.test.tsx index d26f486ee273..dbda1edbe9a7 100644 --- a/shared/chat/conversation/thread-context.test.tsx +++ b/shared/chat/conversation/thread-context.test.tsx @@ -1630,12 +1630,16 @@ test('jumpToRecent drops the old window instead of merging a disjoint one into i expect(result.current.ordinals).toEqual([T.Chat.numberToOrdinal(9001)]) }) -test('a notification during a jump-to-recent gap cannot become the new window floor', () => { +test('a notification during a jump-to-recent gap cannot become the new window', () => { // jumpToRecent and a centered jump both clear before reloading, so for one RPC round trip the // window is empty. The notification most likely to land in that gap is the post-send // ResolveSkippedUnboxeds push from the load already in flight - the very one carrying the ancient - // setChannelname at ID 1. Without a floor that survives the clear it installs itself at index 0 - // and the thread is stranded again, which is the bug this branch exists to fix. + // setChannelname at ID 1. Without the gap being marked it installs itself at index 0 and the + // thread is stranded again, which is the bug this branch exists to fix. + // + // A push between the old window and the page that is coming strands the same way: jump-to-recent + // reloads the newest page, which for a reader parked far back starts thousands of ordinals above + // where they were, so "newer than what we dropped" says nothing about whether it belongs. const {result} = renderHook( () => ({actions: useConversationThreadActions()}), {wrapper} @@ -1655,7 +1659,7 @@ test('a notification during a jump-to-recent gap cannot become the new window fl result.current.actions.messagesClear() }) act(() => { - result.current.actions.addMessages([textAt(1)], {liveUpdate: true}) + result.current.actions.addMessages([textAt(1), textAt(7155)], {liveUpdate: true}) }) act(() => { result.current.actions.applyThreadLoad({ diff --git a/shared/chat/conversation/thread-context.tsx b/shared/chat/conversation/thread-context.tsx index 54658014c765..569d44a712ee 100644 --- a/shared/chat/conversation/thread-context.tsx +++ b/shared/chat/conversation/thread-context.tsx @@ -39,7 +39,6 @@ import { updateAttachmentUploadProgressInThreadState, updateReactionsInThreadState, } from './thread-message-state' -import type {ClearedWindow} from './thread-message-state' import { getInboxConversationMeta, getInboxConversationParticipants, @@ -108,10 +107,10 @@ export type ConversationThreadState = { messageIDToOrdinal: Map messageMap: Map messageOrdinals?: ReadonlyArray - // The window as it was before the last messagesClear, so a notification arriving between a clear - // and its reload cannot install itself as the new window. Cleared once that load settles, however - // it settles - see clearWindowGate. - clearedWindow?: ClearedWindow + // Set between a messagesClear and the reload that refills the window, so a notification arriving + // in that gap cannot install itself as the new window. Cleared once that load settles, however it + // settles - see clearWindowGate. + windowCleared?: boolean messageTypeMap: Map moreToLoadBack: boolean moreToLoadForward: boolean @@ -215,7 +214,7 @@ type LoadNewerMessagesDueToScroll = ( options?: ThreadLoadStatusOptions ) => void type JumpToRecent = (options?: ThreadLoadStatusOptions) => void -type MessagesClear = (opt?: {centeredReload?: boolean}) => void +type MessagesClear = () => void type SelectedConversation = (options?: SelectedConversationOptions) => void export type ConversationThreadActions = { addMessages: ( @@ -540,7 +539,7 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => }) => { updateThreadState(s => { s.loaded = true - s.clearedWindow = undefined + s.windowCleared = false if (p.messages.length) { addMessagesToThreadState(s, p.messages, {validatedRange: p.validatedRange}) clearOptimisticReactionsForMessagesInThreadState(s, p.messages) @@ -913,29 +912,26 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => // applying: offline, scchatnotinteam, or a response that carries no thread. Left alone the gate // would keep dropping notifications for the life of the provider, with no window to correct it. const clearWindowGate = React.useEffectEvent(() => { - if (!threadStore.getState().clearedWindow) { + if (!threadStore.getState().windowCleared) { return } updateThreadState(s => { - s.clearedWindow = undefined + s.windowCleared = false }) }) - const messagesClear = React.useEffectEvent((opt?: {centeredReload?: boolean}) => { + const messagesClear = React.useEffectEvent(() => { activeMarkReadEnabledRef.current = false shownUsernameCache.clear() updateThreadState(s => { s.clearVersion += 1 s.pendingOutboxToOrdinal.clear() s.loaded = false - // Remember the window we are dropping. A notification landing between here and the reload - // would otherwise face an empty window, install itself as the whole of it, and strand once - // the load response arrives. A centered jump reloads an arbitrary region, so nothing that - // arrives first can be placed against it; jumpToRecent reloads newer than this floor, so a - // push above the floor does belong in what is coming and is kept. - s.clearedWindow = { - dropAll: opt?.centeredReload, - floor: s.messageOrdinals?.[0] ?? s.clearedWindow?.floor, - } + // Mark the gap. A notification landing between here and the reload would otherwise face an + // empty window, install itself as the whole of it, and strand once the load response arrives. + // Both callers reload a region disjoint from the one being dropped - a centered jump an + // arbitrary one, jumpToRecent the newest page - so nothing arriving first can be placed + // against what is coming. + s.windowCleared = true s.messageIDToOrdinal.clear() s.messageMap.clear() s.messageOrdinals = undefined @@ -1221,7 +1217,7 @@ export const useConversationThreadLoadMessagesCentered = () => { const messagesClear = useConversationThreadMessagesClear() const loadMessagesCentered: LoadMessagesCentered = (messageID, highlightMode, options) => { - messagesClear({centeredReload: true}) + messagesClear() loadMoreMessages({ centeredMessageID: { conversationIDKey, diff --git a/shared/chat/conversation/thread-load.test.tsx b/shared/chat/conversation/thread-load.test.tsx index 2f16816b11fd..30b46d51d8fd 100644 --- a/shared/chat/conversation/thread-load.test.tsx +++ b/shared/chat/conversation/thread-load.test.tsx @@ -481,3 +481,94 @@ describe('a load releases the window gate it was issued under', () => { expect(actions.clearWindowGate).not.toHaveBeenCalled() }) }) +describe('a cached pass that carries nothing leaves the full pass a whole window', () => { + const flushPromises = async () => { + for (let i = 0; i < 200; i++) { + await Promise.resolve() + } + } + + const page = (from: number, to: number) => + Array.from({length: from - to + 1}, (_, i) => ({ + placeholder: {hidden: false, messageID: T.Chat.numberToMessageID(from - i)}, + state: T.RPCChat.MessageUnboxedState.placeholder, + })) + + const recordingActions = () => + ({ + applyThreadLoad: jest.fn(), + clearWindowGate: jest.fn(), + getSnapshot: () => + ({ + clearVersion: 0, + liveUpdateVersion: 0, + loaded: true, + messageIDToOrdinal: new Map(), + messageMap: new Map(), + messageOrdinals: undefined, + pendingOutboxToOrdinal: new Map(), + }) as unknown as ConversationThreadState, + loadMoreMessages: jest.fn(), + markThreadAsRead: jest.fn(), + }) as unknown as ConversationThreadActions + + const mockPasses = (cached: string, full: string) => + jest.spyOn(ThreadRpc, 'loadThreadNonblock').mockImplementation(async p => { + await Promise.resolve() + p.onCachedThread?.(cached) + p.onFullThread?.(full) + return undefined as never + }) + + const validatedRangeOfLastLoad = (actions: ConversationThreadActions) => { + const calls = (actions.applyThreadLoad as unknown as jest.Mock).mock.calls + return (calls.at(-1)?.[0] as {validatedRange?: {from: number; to: number}} | undefined)?.validatedRange + } + + beforeEach(() => { + useCurrentUserState.getState().dispatch.setBootstrap({ + deviceID: 'device-id', + deviceName: 'testuser-mac', + uid: 'uid', + username: 'testuser', + }) + }) + + afterEach(() => { + jest.restoreAllMocks() + resetAllStores() + }) + + test('prunes against a full pass that followed an empty cached one', async () => { + // First open after a db nuke: PullLocalOnly finds nothing, but its collector suppresses the miss + // and a cached pass is sent anyway, carrying no messages. INCREMENTAL against an empty local + // thread filters nothing out, so the full pass really is the whole window - and only a whole + // window may prune the stale ordinals a cache repair left behind. + const actions = recordingActions() + mockPasses( + JSON.stringify({messages: null, pagination: {last: false, num: 100}}), + JSON.stringify({messages: page(7153, 7152), pagination: {last: false, num: 100}}) + ) + loadConversationThreadMessages(conversationIDKey, {reason: 'focused'}, actions) + await flushPromises() + + expect(validatedRangeOfLastLoad(actions)).toEqual({ + from: T.Chat.numberToOrdinal(7152), + to: T.Chat.numberToOrdinal(7153), + }) + }) + + test('does not prune against a full pass that followed a cached page', async () => { + // The warm-cache sequence: the cached pass carried the page, so the full pass is INCREMENTAL - + // only what changed. Pruning against that deletes messages that are still in the thread. + const actions = recordingActions() + mockPasses( + JSON.stringify({messages: page(7153, 7052), pagination: {last: false, num: 100}}), + JSON.stringify({messages: page(7153, 7153), pagination: {last: false, num: 100}}) + ) + loadConversationThreadMessages(conversationIDKey, {reason: 'focused'}, actions) + await flushPromises() + + expect(validatedRangeOfLastLoad(actions)).toBeUndefined() + }) +}) diff --git a/shared/chat/conversation/thread-load.tsx b/shared/chat/conversation/thread-load.tsx index 972aa2e38990..66dcbab37b18 100644 --- a/shared/chat/conversation/thread-load.tsx +++ b/shared/chat/conversation/thread-load.tsx @@ -199,7 +199,7 @@ export const loadConversationThreadMessages = ( // applyThreadLoad drops the window gate when a load refills the window, but a load can end // without ever applying: offline, scchatnotinteam, a response carrying no thread, or a bail // before the RPC is even made. Left alone the gate would keep dropping notifications for the - // life of the provider - with dropAll, that is a thread that silently stops receiving messages. + // life of the provider, which is a thread that silently stops receiving messages. // // Keyed on clearVersion, not on isThreadLoadCurrent: the load generation only moves when the // conversation changes or the thread unmounts, so two loads of the same conversation both call @@ -228,10 +228,13 @@ export const loadConversationThreadMessages = ( ) const loadingKey = Strings.waitingKeyChatThreadLoad(conversationIDKey) - // Set once a cached response arrives with content. Once the service has sent a cached thread it - // switches the full response to INCREMENTAL, filtering it down to only the messages that changed - // (chat/uithreadloader.go mergeLocalRemoteThread). From that point neither response is a - // complete window. An empty cached pass means no thread was sent, so the full pass still is one. + // Set once a cached response arrives carrying messages. Once the service has sent a cached + // thread it switches the full response to INCREMENTAL, filtering it down to only the messages + // that changed (chat/uithreadloader.go mergeLocalRemoteThread). From that point neither response + // is a complete window. Judged on the messages, not on the response: a cold cache still sends a + // pass, because PullLocalOnly's collector suppresses the miss, and that pass carries no + // messages - INCREMENTAL against an empty local thread filters nothing out, so the full pass + // that follows is a whole window after all. let sawCachedPass = false // The reload below is judged against the whole load, not one pass of it. A warm-cache load // delivers the page on the cached pass and then an INCREMENTAL full pass carrying only what @@ -244,9 +247,6 @@ export const loadConversationThreadMessages = ( if (!thread) { return } - if (why === 'cached') { - sawCachedPass = true - } if (!isCurrentThreadLoad()) { logger.info(`loadMoreMessages: stale response ignored: ${why}`) return @@ -269,6 +269,9 @@ export const loadConversationThreadMessages = ( devicename, () => getLastOrdinalFromSnapshot(actions.getSnapshot()) ) + if (why === 'cached' && messages.length) { + sawCachedPass = true + } const moreToLoad = pagination ? !pagination.last : true const canMarkReadForThreadWindow = allowMarkAsRead && diff --git a/shared/chat/conversation/thread-message-state.test.tsx b/shared/chat/conversation/thread-message-state.test.tsx index 7bd8c2a64ddb..fb81f7b87b66 100644 --- a/shared/chat/conversation/thread-message-state.test.tsx +++ b/shared/chat/conversation/thread-message-state.test.tsx @@ -544,47 +544,58 @@ describe('addMessagesToThreadState', () => { ]) }) - test('a cleared window still refuses a notification below the floor it had', () => { - // messagesClear wipes messageOrdinals but remembers the window, because jumpToRecent and a - // centered jump both clear then reload. A push landing in that gap would otherwise face an - // empty window, install itself as the whole of it, and strand once the load response arrives. + test('a cleared window refuses a notification in either direction', () => { + // messagesClear wipes messageOrdinals and marks the gap, because jumpToRecent and a centered + // jump both clear then reload. A push landing in that gap would otherwise face an empty window, + // install itself as the whole of it, and strand once the load response arrives. Neither reload + // can be predicted from the old window - a centered jump lands on an arbitrary region, and + // jump-to-recent on the newest page, which is nowhere near a reader parked far back - so above + // strands as surely as below. const state = makeThreadState([]) - state.clearedWindow = {floor: T.Chat.numberToOrdinal(7152)} - addMessagesToThreadState(state, [textAt(1)], {dropNewBelowWindow: true}) + state.windowCleared = true + addMessagesToThreadState(state, [textAt(1), textAt(7155), textAt(9001)], {dropNewBelowWindow: true}) expect(state.messageOrdinals ?? []).toEqual([]) // ...and the load that follows populates it normally. - addMessagesToThreadState(state, [textAt(9001)], {}) - expect(state.messageOrdinals).toEqual([T.Chat.numberToOrdinal(9001)]) + addMessagesToThreadState(state, [textAt(8900), textAt(8901)], {}) + expect(state.messageOrdinals).toEqual([T.Chat.numberToOrdinal(8900), T.Chat.numberToOrdinal(8901)]) }) - test('a window cleared for jump-to-recent keeps a notification above the floor it had', () => { - // The reload lands newer than the old floor, so a message arriving first belongs in what is - // coming - the response may have been composed before it existed. Dropping it would lose it. + test('with no window and nothing cleared a notification still applies', () => { + // A conversation that has never held a window - a first load, or one that is genuinely empty. + // There is no floor to be below, so nothing is dropped and a new message appears at once. const state = makeThreadState([]) - state.clearedWindow = {floor: T.Chat.numberToOrdinal(7152)} - addMessagesToThreadState(state, [textAt(9001)], {dropNewBelowWindow: true}) - expect(state.messageOrdinals).toEqual([T.Chat.numberToOrdinal(9001)]) + addMessagesToThreadState(state, [textAt(1)], {dropNewBelowWindow: true}) + expect(state.messageOrdinals).toEqual([T.Chat.numberToOrdinal(1)]) }) - test('a window cleared for a centered jump refuses a notification in either direction', () => { - // The reload lands on an arbitrary region, so nothing arriving first can be placed against it - - // above the coming window strands as surely as below it. - const state = makeThreadState([]) - state.clearedWindow = {dropAll: true, floor: T.Chat.numberToOrdinal(7152)} - addMessagesToThreadState(state, [textAt(1), textAt(9001)], {dropNewBelowWindow: true}) - expect(state.messageOrdinals ?? []).toEqual([]) + test('a message remapped out of the window is dropped, not stranded', () => { + // The window is judged on the ordinal the message will occupy, which an outbox or messageID + // match can move. Here messageIDToOrdinal still points at an ancient ordinal the thread no + // longer renders, so a push whose own ordinal sits inside the window remaps outside it. + const state = makeThreadState([textAt(7152), textAt(7153)], {moreToLoadForward: true}) + const strandedOrdinal = T.Chat.numberToOrdinal(1) + const messageID = T.Chat.numberToMessageID(7153) + // An entry the thread no longer renders: gone from the ordinal list, still in the map and index. + state.messageMap.set(strandedOrdinal, T.castDraft(textAt(1, {id: messageID}))) + state.messageIDToOrdinal.set(messageID, strandedOrdinal) + + addMessagesToThreadState(state, [textAt(7153)], {dropNewBelowWindow: true}) - // The centered load itself is not a push, so it fills the window as usual. - addMessagesToThreadState(state, [textAt(120), textAt(121)], {}) - expect(state.messageOrdinals).toEqual([T.Chat.numberToOrdinal(120), T.Chat.numberToOrdinal(121)]) + expect(state.messageOrdinals).toEqual([T.Chat.numberToOrdinal(7152), T.Chat.numberToOrdinal(7153)]) }) - test('with no window and no remembered floor a notification still applies', () => { - // A conversation that has never held a window - a first load, or one that is genuinely empty. - // There is no floor to be below, so nothing is dropped and a new message appears at once. - const state = makeThreadState([]) - addMessagesToThreadState(state, [textAt(1)], {dropNewBelowWindow: true}) - expect(state.messageOrdinals).toEqual([T.Chat.numberToOrdinal(1)]) + test('a message remapped onto a row already in the window still merges', () => { + // The mirror of the case above: the raw ordinal is outside the window but the row it maps onto + // is one the thread is already showing, so this is an update to that row, not a new one. + const sent = textAt(7153, {outboxID: T.Chat.stringToOutboxID('sent-1')}) + const state = makeThreadState([textAt(7152), sent], {moreToLoadForward: true}) + const resent = textAt(1, {outboxID: T.Chat.stringToOutboxID('sent-1'), text: 'edited'}) + + addMessagesToThreadState(state, [resent], {dropNewBelowWindow: true}) + + expect(state.messageOrdinals).toEqual([T.Chat.numberToOrdinal(7152), T.Chat.numberToOrdinal(7153)]) + const merged = state.messageMap.get(T.Chat.numberToOrdinal(7153)) + expect(merged?.type === 'text' && merged.text.stringValue()).toBe('edited') }) }) diff --git a/shared/chat/conversation/thread-message-state.tsx b/shared/chat/conversation/thread-message-state.tsx index fc440a670724..d0eef0d2f786 100644 --- a/shared/chat/conversation/thread-message-state.tsx +++ b/shared/chat/conversation/thread-message-state.tsx @@ -10,9 +10,9 @@ type WritableConversationThreadMessageState = { messageMap: Map> messageOrdinals?: ReadonlyArray // Set by messagesClear, cleared once the reload that refills the window settles. While it is set - // the window has no bounds of its own, so a notification arriving in the gap is judged against - // this instead. See the drop rules in addMessagesToThreadState. - clearedWindow?: ClearedWindow + // there is no window to place an arriving message against. See the drop rules in + // addMessagesToThreadState. + windowCleared?: boolean messageTypeMap: Map // False once the window reaches the newest message, which is what makes a push newer than the // ceiling an append rather than a stranded row. @@ -21,15 +21,6 @@ type WritableConversationThreadMessageState = { validatedOrdinalRange?: {from: T.Chat.Ordinal; to: T.Chat.Ordinal} } -export type ClearedWindow = { - // A centered jump reloads an arbitrary region of the thread, so nothing arriving before the - // response can be placed relative to it and every new message is dropped. jumpToRecent reloads - // newer than the old window, so a push above `floor` does belong in what is coming. - dropAll?: boolean - // The window floor as it was before the clear. - floor?: T.Chat.Ordinal -} - type ThreadMessagesDeleteParams = { messageIDs?: ReadonlyArray upToMessageID?: T.Chat.MessageID @@ -215,37 +206,38 @@ export const addMessagesToThreadState = ( // moreToLoadForward: once the window reaches the newest message there is no hole to open above // it, and a live message must append. // - // Decided here, before anything is written, so these messages are skipped whole. Dropping only - // the ordinal later would leave messageMap and messageIDToOrdinal holding a message the thread - // does not render, and getOrdinalForMessageID would then hand out an ordinal with no row. - // Nothing is lost either way: paging to it loads it in the ordinary way. - const clearedWindow = state.clearedWindow + // Decided before anything is written for the message, so it is skipped whole. Dropping only the + // ordinal later would leave messageMap and messageIDToOrdinal holding a message the thread does + // not render, and getOrdinalForMessageID would then hand out an ordinal with no row. Nothing is + // lost either way: paging to it loads it in the ordinary way. const windowCeiling = ords?.[ords.length - 1] - // While cleared there is no window to bound, so judge against the reload that is on its way. - const floor = windowFloor ?? clearedWindow?.floor - const ceiling = clearedWindow ? undefined : windowCeiling - const droppedOutsideWindow = new Set() - if (dropNewBelowWindow) { - for (const o of incomingOrdinals) { - if (existing.has(o)) { - continue - } - const below = floor !== undefined && o < floor - const above = ceiling !== undefined && o > ceiling && state.moreToLoadForward - if (clearedWindow?.dropAll || below || above) { - droppedOutsideWindow.add(o) - } + const isOutsideWindow = (o: T.Chat.Ordinal) => { + if (!dropNewBelowWindow || existing.has(o)) { + return false } - for (const o of droppedOutsideWindow) { - incomingOrdinals.delete(o) + // A clear is always followed by a reload that replaces the window wholesale, so until that + // lands there is nothing to place an arriving message against: a centered jump reloads an + // arbitrary region, and jump-to-recent the newest page, which is disjoint from wherever the + // reader was. A message landing in the gap that the reload does not carry waits for the next + // load or push; a stranded ordinal, by contrast, breaks paging for the life of the thread. + if (state.windowCleared) { + return true } + const below = windowFloor !== undefined && o < windowFloor + const above = windowCeiling !== undefined && o > windowCeiling && state.moreToLoadForward + return below || above } const deletedOrdinals = new Set() for (const _m of messages) { const regularMessage = _m.conversationMessage !== false const mapOrdinal = getMapOrdinal(_m, regularMessage) - if (droppedOutsideWindow.has(mapOrdinal)) { + // Judged on mapOrdinal, the ordinal the message will actually occupy: an outbox or messageID + // match can move it out of the window, or onto a row already inside it. Deletions and + // non-conversation messages are not rows, so the window does not bound them. + if (regularMessage && _m.type !== 'deleted' && isOutsideWindow(mapOrdinal)) { + incomingOrdinals.delete(_m.ordinal) + incomingOrdinals.delete(mapOrdinal) continue } const getIncomingMessage = (): WritableDraft => From f74e9dbf7ad9f293a8376f68c2eeb882beef1de9 Mon Sep 17 00:00:00 2001 From: chrisnojima Date: Wed, 2 Sep 2026 15:22:12 -0400 Subject: [PATCH 14/21] fix(chat): cache a topic-name result that is missing channels, briefly Refusing to cache an incomplete result meant never caching one. The inbox read asks for every member status, so a team carries channels the user has left or never joined, and those fail to resolve on every pass - "incomplete" is the steady state, and under that rule the cache stayed empty and the per-#token fan-out it exists to collapse stayed with it. An incomplete result is now cached under a much shorter TTL, long enough to collapse the burst one page of messages fires and short enough that a channel which becomes resolvable is picked up on the next page. --- go/chat/teamchannelsource.go | 54 +++++++++++++------ .../teamchannelsource_topicnamecache_test.go | 54 ++++++++++++++++--- 2 files changed, 84 insertions(+), 24 deletions(-) diff --git a/go/chat/teamchannelsource.go b/go/chat/teamchannelsource.go index 4f09c21054b3..989f80baf495 100644 --- a/go/chat/teamchannelsource.go +++ b/go/chat/teamchannelsource.go @@ -131,7 +131,9 @@ func (i *lastActiveAtMemCache) OnDbNuke(mctx libkb.MetaContext) error { type topicNameCacheItem struct { names []chat1.ChannelNameMention - mtime gregor1.Time + // False when some channel in the team could not be resolved, which shortens the TTL below. + complete bool + mtime gregor1.Time } // Channel-name resolution is charged per message: every message body holding a `#token` sends @@ -143,9 +145,9 @@ type topicNameCacheItem struct { // short for that reason - a page's worth of resolutions all land within milliseconds of each other, // so seconds are enough to collapse them into one. // -// Note also that an incomplete result is still returned to the caller, just not cached - so a -// resolution committed during a degraded read (right after a nuke, say) is missing channels no -// matter what this cache does. That is pre-existing, and the same persistence applies: +// Note also that an incomplete result is still returned to the caller - so a resolution committed +// during a degraded read (right after a nuke, say) is missing channels no matter what this cache +// does. That is pre-existing, and the same persistence applies: // // Be aware of what the TTL does NOT heal. The result of a resolution is stored, not just displayed: // it becomes MessageUnboxedValid.ChannelNameMentions (see boxer.go) and is written to local storage @@ -155,6 +157,14 @@ type topicNameCacheItem struct { // this duration widens that hole; if it ever needs to grow, wire up real invalidation first. const topicNameCacheDuration = 10 * time.Second +// A result missing some channels is cached too, but only for long enough to collapse the burst one +// page of messages fires. Refusing to cache it at all sounds safer and is not: the inbox read asks +// for every member status, so a team almost always carries channels the user has left or never +// joined, and those fail to resolve on every pass. "Incomplete" is therefore the steady state, and +// under that rule the cache would never hold anything - leaving the fan-out it exists to collapse. +// A channel that becomes resolvable is picked up after this window rather than the one above. +const topicNameCacheIncompleteDuration = time.Second + type topicNameMemCache struct { sync.RWMutex // key: tlfID||topicType||uid @@ -175,19 +185,29 @@ func (i *topicNameMemCache) Get(tlfID chat1.TLFID, topicType chat1.TopicType, ui i.RLock() defer i.RUnlock() item, ok := i.cache[i.key(tlfID, topicType, uid)] - if !ok || time.Since(item.mtime.Time()) > topicNameCacheDuration { + if !ok { + return nil, false + } + ttl := topicNameCacheDuration + if !item.complete { + ttl = topicNameCacheIncompleteDuration + } + if time.Since(item.mtime.Time()) > ttl { return nil, false } // Hand back a copy: callers own what they get, and this slice is shared. return append([]chat1.ChannelNameMention(nil), item.names...), true } -func (i *topicNameMemCache) Put(tlfID chat1.TLFID, topicType chat1.TopicType, uid gregor1.UID, names []chat1.ChannelNameMention) { +func (i *topicNameMemCache) Put(tlfID chat1.TLFID, topicType chat1.TopicType, uid gregor1.UID, + names []chat1.ChannelNameMention, complete bool, +) { i.Lock() defer i.Unlock() i.cache[i.key(tlfID, topicType, uid)] = topicNameCacheItem{ - names: append([]chat1.ChannelNameMention(nil), names...), - mtime: gregor1.ToTime(time.Now()), + names: append([]chat1.ChannelNameMention(nil), names...), + complete: complete, + mtime: gregor1.ToTime(time.Now()), } } @@ -378,9 +398,10 @@ func (c *TeamChannelSource) GetChannelsTopicName(ctx context.Context, uid gregor if err != nil { return nil, err } - // Any channel we fail to resolve makes the result incomplete, and an incomplete result must not - // be cached: it would pin a degraded answer for the whole window. This matters most right after - // a db nuke, when local storage holds no METADATA messages yet and most of these fail. + // A channel we fail to resolve is left out of the result, and the result is then cached under a + // much shorter TTL (topicNameCacheIncompleteDuration) so the missing ones are retried soon. This + // matters most right after a db nuke, when local storage holds no METADATA messages yet and most + // of these fail. complete := true for _, rc := range convs { conv := rc.Conv @@ -409,11 +430,12 @@ func (c *TeamChannelSource) GetChannelsTopicName(ctx context.Context, uid gregor // len(convs) == 0 is never a legitimate answer - a chat TLF always has at least #general - so it // means the inbox read came back degraded, and caching it would pin "this team has no channels" // for the whole window. - if complete && len(convs) > 0 { - c.topicNameCache.Put(tlfID, topicType, uid, res) - } else { - c.Debug(ctx, "GetChannelsTopicName: incomplete result (%d of %d channels), not caching", - len(res), len(convs)) + if len(convs) > 0 { + if !complete { + c.Debug(ctx, "GetChannelsTopicName: incomplete result (%d of %d channels), caching briefly", + len(res), len(convs)) + } + c.topicNameCache.Put(tlfID, topicType, uid, res, complete) } return res, nil } diff --git a/go/chat/teamchannelsource_topicnamecache_test.go b/go/chat/teamchannelsource_topicnamecache_test.go index bf6ee226f212..2d2b7f54e150 100644 --- a/go/chat/teamchannelsource_topicnamecache_test.go +++ b/go/chat/teamchannelsource_topicnamecache_test.go @@ -27,7 +27,7 @@ func TestTopicNameMemCacheRoundTrip(t *testing.T) { _, ok := c.Get(tlfID, topicType, uid) require.False(t, ok, "an empty cache must miss") - c.Put(tlfID, topicType, uid, names) + c.Put(tlfID, topicType, uid, names, true) got, ok := c.Get(tlfID, topicType, uid) require.True(t, ok) require.Equal(t, names, got) @@ -36,7 +36,7 @@ func TestTopicNameMemCacheRoundTrip(t *testing.T) { func TestTopicNameMemCacheKeysAreDistinct(t *testing.T) { tlfID, topicType, uid, names := topicNameCacheFixture() c := newTopicNameMemCache() - c.Put(tlfID, topicType, uid, names) + c.Put(tlfID, topicType, uid, names, true) otherTLF := chat1.TLFID([]byte{0x09, 0x09}) otherUID := gregor1.UID([]byte{0xbb}) @@ -58,7 +58,7 @@ func TestTopicNameMemCacheKeysAreDistinct(t *testing.T) { func TestTopicNameMemCacheCopiesBothWays(t *testing.T) { tlfID, topicType, uid, names := topicNameCacheFixture() c := newTopicNameMemCache() - c.Put(tlfID, topicType, uid, names) + c.Put(tlfID, topicType, uid, names, true) // Mutating what the caller passed in must not reach the cache. names[0].TopicName = "mutated-input" @@ -76,7 +76,7 @@ func TestTopicNameMemCacheCopiesBothWays(t *testing.T) { func TestTopicNameMemCacheExpires(t *testing.T) { tlfID, topicType, uid, names := topicNameCacheFixture() c := newTopicNameMemCache() - c.Put(tlfID, topicType, uid, names) + c.Put(tlfID, topicType, uid, names, true) key := c.key(tlfID, topicType, uid) @@ -108,36 +108,74 @@ func TestTopicNameMemCacheEmptyIsStillAValue(t *testing.T) { // The cache itself stores whatever it is given, including nothing - which is exactly why the // caller must not hand it a degraded read. - c.Put(tlfID, topicType, uid, nil) + c.Put(tlfID, topicType, uid, nil, true) got, ok := c.Get(tlfID, topicType, uid) require.True(t, ok, "an empty slice is a cached value, not a miss") require.Empty(t, got) } +// A team almost always holds channels the user cannot resolve - ones they left or never joined - so +// an incomplete result is the steady state. It is cached anyway, or the fan-out this cache exists to +// collapse would never be collapsed, but only for the shorter window. +func TestTopicNameMemCacheIncompleteExpiresSooner(t *testing.T) { + tlfID, topicType, uid, names := topicNameCacheFixture() + c := newTopicNameMemCache() + c.Put(tlfID, topicType, uid, names, false) + + key := c.key(tlfID, topicType, uid) + got, ok := c.Get(tlfID, topicType, uid) + require.True(t, ok, "an incomplete result is still cached") + require.Equal(t, names, got) + + // Old enough that the complete TTL would still serve it, and the incomplete one does not. + c.Lock() + item := c.cache[key] + item.mtime = gregor1.ToTime(time.Now().Add(-topicNameCacheIncompleteDuration - time.Second)) + c.cache[key] = item + c.Unlock() + require.Less(t, topicNameCacheIncompleteDuration+time.Second, topicNameCacheDuration, + "the fixture only proves anything while the two windows differ by more than this") + _, ok = c.Get(tlfID, topicType, uid) + require.False(t, ok, "an incomplete entry must expire on the shorter window") + + // The same age under a complete entry still hits, so it is the flag doing the work. + c.Put(tlfID, topicType, uid, names, true) + c.Lock() + item = c.cache[key] + item.mtime = gregor1.ToTime(time.Now().Add(-topicNameCacheIncompleteDuration - time.Second)) + c.cache[key] = item + c.Unlock() + _, ok = c.Get(tlfID, topicType, uid) + require.True(t, ok, "a complete entry of the same age must still hit") +} + // The TTL is the only bound on staleness - there is no invalidation hook - and the comment on the // constant argues from it being short. Pin the value so widening it is a deliberate act. func TestTopicNameCacheDurationStaysShort(t *testing.T) { require.LessOrEqual(t, topicNameCacheDuration, 30*time.Second, "a longer window widens the hole where a resolution is stored stale into a message") require.Positive(t, topicNameCacheDuration) + require.Positive(t, topicNameCacheIncompleteDuration) + require.Less(t, topicNameCacheIncompleteDuration, topicNameCacheDuration, + "a result known to be missing channels must not be held as long as a whole one") } func TestTopicNameMemCacheClear(t *testing.T) { tlfID, topicType, uid, names := topicNameCacheFixture() c := newTopicNameMemCache() - c.Put(tlfID, topicType, uid, names) + c.Put(tlfID, topicType, uid, names, true) c.clearCache() _, ok := c.Get(tlfID, topicType, uid) require.False(t, ok, "clearCache must drop everything") // Logout and db nuke both go through the same clear, and both must leave the cache usable. - c.Put(tlfID, topicType, uid, names) + c.Put(tlfID, topicType, uid, names, true) require.NoError(t, c.OnLogout(libkb.MetaContext{})) _, ok = c.Get(tlfID, topicType, uid) require.False(t, ok, "OnLogout must drop everything") - c.Put(tlfID, topicType, uid, names) + c.Put(tlfID, topicType, uid, names, true) require.NoError(t, c.OnDbNuke(libkb.MetaContext{})) _, ok = c.Get(tlfID, topicType, uid) require.False(t, ok, "OnDbNuke must drop everything") From 3c2210047c65c5c9b248ee5017ad21a4b5a54641 Mon Sep 17 00:00:00 2001 From: chrisnojima Date: Wed, 2 Sep 2026 15:22:51 -0400 Subject: [PATCH 15/21] revert(chat): take the go changes out, they land in their own pr patchPaginationLast and the topic-name cache are service-side and reviewable on their own; they move to a branch stacked on this one so this pr is the client window work alone. --- go/chat/convsource.go | 16 +- go/chat/convsource_patchpagination_test.go | 153 --------------- go/chat/teamchannelsource.go | 141 +------------- .../teamchannelsource_topicnamecache_test.go | 182 ------------------ 4 files changed, 8 insertions(+), 484 deletions(-) delete mode 100644 go/chat/convsource_patchpagination_test.go delete mode 100644 go/chat/teamchannelsource_topicnamecache_test.go diff --git a/go/chat/convsource.go b/go/chat/convsource.go index cdb2e7b9f813..b3218c0313d9 100644 --- a/go/chat/convsource.go +++ b/go/chat/convsource.go @@ -231,24 +231,14 @@ func (s *baseConversationSource) patchPaginationLast(ctx context.Context, conv t page.Last = true return } - end1 := msgs[0].GetMessageID() - end2 := msgs[len(msgs)-1].GetMessageID() - oldest := end1.Min(end2) - // Message IDs start at 1, so a page holding it has reached the beginning of the conversation and - // nothing older can exist. Worth checking before the expunge record because that record is not - // always populated: a conversation whose history was deleted reads back Upto:0 until its inbox - // entry is localized, and until then every page of it looks like there is more to come. - if oldest == 1 { - s.Debug(ctx, "patchPaginationLast: true - reached the first message") - page.Last = true - return - } expunge := conv.GetExpunge() if expunge == nil { s.Debug(ctx, "patchPaginationLast: no expunge info") return } - if oldest <= expunge.Upto { + end1 := msgs[0].GetMessageID() + end2 := msgs[len(msgs)-1].GetMessageID() + if end1.Min(end2) <= expunge.Upto { s.Debug(ctx, "patchPaginationLast: true - hit upto") // If any message is prior to the nukepoint, say this is the last page. page.Last = true diff --git a/go/chat/convsource_patchpagination_test.go b/go/chat/convsource_patchpagination_test.go deleted file mode 100644 index de45c397a727..000000000000 --- a/go/chat/convsource_patchpagination_test.go +++ /dev/null @@ -1,153 +0,0 @@ -package chat - -import ( - "context" - "testing" - - "github.com/keybase/client/go/chat/globals" - "github.com/keybase/client/go/chat/types" - "github.com/keybase/client/go/chat/utils" - "github.com/keybase/client/go/libkb" - "github.com/keybase/client/go/protocol/chat1" - "github.com/keybase/client/go/protocol/gregor1" - "github.com/stretchr/testify/require" -) - -// patchPaginationConv is the smallest thing satisfying types.UnboxConversationInfo. Only -// GetExpunge is consulted by patchPaginationLast; the rest exist to satisfy the interface. -type patchPaginationConv struct { - expunge *chat1.Expunge -} - -var _ types.UnboxConversationInfo = patchPaginationConv{} - -func (c patchPaginationConv) GetConvID() chat1.ConversationID { return nil } -func (c patchPaginationConv) GetMembersType() chat1.ConversationMembersType { - return chat1.ConversationMembersType_TEAM -} -func (c patchPaginationConv) GetFinalizeInfo() *chat1.ConversationFinalizeInfo { return nil } -func (c patchPaginationConv) GetExpunge() *chat1.Expunge { return c.expunge } -func (c patchPaginationConv) GetMaxDeletedUpTo() chat1.MessageID { return 0 } -func (c patchPaginationConv) IsPublic() bool { return false } -func (c patchPaginationConv) GetMaxMessage(chat1.MessageType) (chat1.MessageSummary, error) { - return chat1.MessageSummary{}, nil -} - -// newPatchPaginationSource builds just enough of a baseConversationSource to call -// patchPaginationLast. It needs no database, network or logged in user - a bare GlobalContext -// already carries a logger, which is all Debug touches. -func newPatchPaginationSource() *baseConversationSource { - g := libkb.NewGlobalContext() - return &baseConversationSource{ - Contextified: globals.NewContextified(globals.NewContext(g, &globals.ChatContext{})), - DebugLabeler: utils.NewDebugLabeler(g, "patchPaginationTest", false), - } -} - -func msgsWithIDs(ids ...chat1.MessageID) []chat1.MessageUnboxed { - res := make([]chat1.MessageUnboxed, 0, len(ids)) - for _, id := range ids { - res = append(res, chat1.NewMessageUnboxedWithPlaceholder(chat1.MessageUnboxedPlaceholder{ - MessageID: id, - })) - } - return res -} - -func TestPatchPaginationLast(t *testing.T) { - ctx := context.Background() - uid := gregor1.UID([]byte{0x01}) - s := newPatchPaginationSource() - - testCases := []struct { - name string - expunge *chat1.Expunge - msgs []chat1.MessageUnboxed - page *chat1.Pagination - want bool - }{ - { - name: "an empty page is the last page", - msgs: nil, - page: &chat1.Pagination{Num: 50}, - want: true, - }, - { - // The regression this guards: after a nuke a conversation whose history was deleted - // reads back Upto:0 until its inbox entry is localized, so the expunge check below - // never fires and Last stays false forever - "Digging ancient messages..." on a fully - // loaded thread. - name: "reaching message ID 1 is last even when expunge reads back Upto:0", - expunge: &chat1.Expunge{Upto: 0}, - msgs: msgsWithIDs(1, 2, 3), - page: &chat1.Pagination{Num: 50}, - want: true, - }, - { - name: "reaching message ID 1 is last even with no expunge record at all", - msgs: msgsWithIDs(1, 2, 3), - page: &chat1.Pagination{Num: 50}, - want: true, - }, - { - // Pages can arrive newest first, and the check is on the oldest ID either way. - name: "message ID 1 is found regardless of page order", - msgs: msgsWithIDs(3, 2, 1), - page: &chat1.Pagination{Num: 50}, - want: true, - }, - { - // The boundary from the other side. An over-eager check here silently truncates a - // thread's history, which is the more damaging direction and the harder one to notice. - name: "a page starting at message ID 2 is not last", - msgs: msgsWithIDs(2, 3, 4), - page: &chat1.Pagination{Num: 50}, - want: false, - }, - { - name: "a page above the beginning with no expunge is not last", - msgs: msgsWithIDs(40, 41, 42), - page: &chat1.Pagination{Num: 50}, - want: false, - }, - { - name: "a page reaching the nukepoint is last", - expunge: &chat1.Expunge{Upto: 40}, - msgs: msgsWithIDs(40, 41, 42), - page: &chat1.Pagination{Num: 50}, - want: true, - }, - { - name: "a page above the nukepoint is not last", - expunge: &chat1.Expunge{Upto: 10}, - msgs: msgsWithIDs(40, 41, 42), - page: &chat1.Pagination{Num: 50}, - want: false, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - conv := patchPaginationConv{expunge: tc.expunge} - s.patchPaginationLast(ctx, conv, uid, tc.page, tc.msgs) - require.Equal(t, tc.want, tc.page.Last) - }) - } -} - -func TestPatchPaginationLastLeavesSettledPagesAlone(t *testing.T) { - ctx := context.Background() - uid := gregor1.UID([]byte{0x01}) - s := newPatchPaginationSource() - conv := patchPaginationConv{} - - // A nil page must not panic. - require.NotPanics(t, func() { - s.patchPaginationLast(ctx, conv, uid, nil, msgsWithIDs(1)) - }) - - // Last is only ever turned on, never off. - page := &chat1.Pagination{Num: 50, Last: true} - s.patchPaginationLast(ctx, conv, uid, page, msgsWithIDs(40, 41, 42)) - require.True(t, page.Last) -} diff --git a/go/chat/teamchannelsource.go b/go/chat/teamchannelsource.go index 989f80baf495..d98e0f36e7e3 100644 --- a/go/chat/teamchannelsource.go +++ b/go/chat/teamchannelsource.go @@ -129,111 +129,12 @@ func (i *lastActiveAtMemCache) OnDbNuke(mctx libkb.MetaContext) error { return nil } -type topicNameCacheItem struct { - names []chat1.ChannelNameMention - // False when some channel in the team could not be resolved, which shortens the TTL below. - complete bool - mtime gregor1.Time -} - -// Channel-name resolution is charged per message: every message body holding a `#token` sends -// ParseChannelNameMentions here, and the uncached path reads the inbox and then fetches the METADATA -// message of every channel in the team. Unboxing one page of a busy channel in a team with a few -// dozen channels therefore cost thousands of single-message fetches. -// -// There is no invalidation hook, so the TTL is the only thing bounding staleness, and it is kept -// short for that reason - a page's worth of resolutions all land within milliseconds of each other, -// so seconds are enough to collapse them into one. -// -// Note also that an incomplete result is still returned to the caller - so a resolution committed -// during a degraded read (right after a nuke, say) is missing channels no matter what this cache -// does. That is pre-existing, and the same persistence applies: -// -// Be aware of what the TTL does NOT heal. The result of a resolution is stored, not just displayed: -// it becomes MessageUnboxedValid.ChannelNameMentions (see boxer.go) and is written to local storage -// with the message. So a message unboxed during the window that a newly created or renamed channel -// is missing from the cache keeps the stale resolution after the entry expires, until that message -// happens to be unboxed again. Expiry heals later resolutions, not ones already committed. Widening -// this duration widens that hole; if it ever needs to grow, wire up real invalidation first. -const topicNameCacheDuration = 10 * time.Second - -// A result missing some channels is cached too, but only for long enough to collapse the burst one -// page of messages fires. Refusing to cache it at all sounds safer and is not: the inbox read asks -// for every member status, so a team almost always carries channels the user has left or never -// joined, and those fail to resolve on every pass. "Incomplete" is therefore the steady state, and -// under that rule the cache would never hold anything - leaving the fan-out it exists to collapse. -// A channel that becomes resolvable is picked up after this window rather than the one above. -const topicNameCacheIncompleteDuration = time.Second - -type topicNameMemCache struct { - sync.RWMutex - // key: tlfID||topicType||uid - cache map[string]topicNameCacheItem -} - -func newTopicNameMemCache() *topicNameMemCache { - return &topicNameMemCache{ - cache: make(map[string]topicNameCacheItem), - } -} - -func (i *topicNameMemCache) key(tlfID chat1.TLFID, topicType chat1.TopicType, uid gregor1.UID) string { - return fmt.Sprintf("%s:%v:%s", tlfID, topicType, uid) -} - -func (i *topicNameMemCache) Get(tlfID chat1.TLFID, topicType chat1.TopicType, uid gregor1.UID) ([]chat1.ChannelNameMention, bool) { - i.RLock() - defer i.RUnlock() - item, ok := i.cache[i.key(tlfID, topicType, uid)] - if !ok { - return nil, false - } - ttl := topicNameCacheDuration - if !item.complete { - ttl = topicNameCacheIncompleteDuration - } - if time.Since(item.mtime.Time()) > ttl { - return nil, false - } - // Hand back a copy: callers own what they get, and this slice is shared. - return append([]chat1.ChannelNameMention(nil), item.names...), true -} - -func (i *topicNameMemCache) Put(tlfID chat1.TLFID, topicType chat1.TopicType, uid gregor1.UID, - names []chat1.ChannelNameMention, complete bool, -) { - i.Lock() - defer i.Unlock() - i.cache[i.key(tlfID, topicType, uid)] = topicNameCacheItem{ - names: append([]chat1.ChannelNameMention(nil), names...), - complete: complete, - mtime: gregor1.ToTime(time.Now()), - } -} - -func (i *topicNameMemCache) clearCache() { - i.Lock() - defer i.Unlock() - i.cache = make(map[string]topicNameCacheItem) -} - -func (i *topicNameMemCache) OnLogout(mctx libkb.MetaContext) error { - i.clearCache() - return nil -} - -func (i *topicNameMemCache) OnDbNuke(mctx libkb.MetaContext) error { - i.clearCache() - return nil -} - type TeamChannelSource struct { sync.Mutex globals.Contextified utils.DebugLabeler recentJoinsCache *recentJoinsMemCache lastActiveAtCache *lastActiveAtMemCache - topicNameCache *topicNameMemCache } var _ types.TeamChannelSource = (*TeamChannelSource)(nil) @@ -244,7 +145,6 @@ func NewTeamChannelSource(g *globals.Context) *TeamChannelSource { DebugLabeler: utils.NewDebugLabeler(g.ExternalG(), "TeamChannelSource", false), recentJoinsCache: newRecentJoinsMemCache(), lastActiveAtCache: newLastActiveAtMemCache(), - topicNameCache: newTopicNameMemCache(), } } @@ -252,7 +152,6 @@ func (c *TeamChannelSource) OnLogout(mctx libkb.MetaContext) error { epick := libkb.FirstErrorPicker{} epick.Push(c.recentJoinsCache.OnLogout(mctx)) epick.Push(c.lastActiveAtCache.OnLogout(mctx)) - epick.Push(c.topicNameCache.OnLogout(mctx)) return epick.Error() } @@ -260,7 +159,6 @@ func (c *TeamChannelSource) OnDbNuke(mctx libkb.MetaContext) error { epick := libkb.FirstErrorPicker{} epick.Push(c.recentJoinsCache.OnDbNuke(mctx)) epick.Push(c.lastActiveAtCache.OnDbNuke(mctx)) - epick.Push(c.topicNameCache.OnDbNuke(mctx)) return epick.Error() } @@ -358,56 +256,41 @@ func (c *TeamChannelSource) GetChannelsFull(ctx context.Context, uid gregor1.UID func (c *TeamChannelSource) GetChannelsTopicName(ctx context.Context, uid gregor1.UID, tlfID chat1.TLFID, topicType chat1.TopicType, ) (res []chat1.ChannelNameMention, err error) { - // Before the trace: this runs once per message body holding a `#token`, which reaches hundreds - // per second while paging a busy channel, and tracing a hit costs two log lines apiece. Safe - // because DebugLabeler.trace is pure logging - no context checks, no error handling. Note the - // misses below still fan out concurrently on a cold cache; this collapses the steady state, not - // the initial burst. - if cached, ok := c.topicNameCache.Get(tlfID, topicType, uid); ok { - return cached, nil - } ctx = globals.CtxModifyUnboxMode(ctx, types.UnboxModeQuick) defer c.Trace(ctx, &err, "GetChannelsTopicName: tlfID: %v, topicType: %v", tlfID, topicType)() - addValidMetadataMsg := func(convID chat1.ConversationID, msg chat1.MessageUnboxed) bool { + addValidMetadataMsg := func(convID chat1.ConversationID, msg chat1.MessageUnboxed) { if !msg.IsValid() { c.Debug(ctx, "GetChannelsTopicName: metadata message invalid: convID, %s", convID) - return false + return } body := msg.Valid().MessageBody typ, err := body.MessageType() if err != nil { c.Debug(ctx, "GetChannelsTopicName: error getting message type: convID, %s", convID, err) - return false + return } if typ != chat1.MessageType_METADATA { c.Debug(ctx, "GetChannelsTopicName: message not a real metadata message: convID, %s msgID: %d", convID, msg.GetMessageID()) - return false + return } res = append(res, chat1.ChannelNameMention{ ConvID: convID, TopicName: body.Metadata().ConversationTitle, }) - return true } convs, err := c.getTLFConversations(ctx, uid, tlfID, topicType) if err != nil { return nil, err } - // A channel we fail to resolve is left out of the result, and the result is then cached under a - // much shorter TTL (topicNameCacheIncompleteDuration) so the missing ones are retried soon. This - // matters most right after a db nuke, when local storage holds no METADATA messages yet and most - // of these fail. - complete := true for _, rc := range convs { conv := rc.Conv msg, err := conv.GetMaxMessage(chat1.MessageType_METADATA) if err != nil { - complete = false continue } unboxeds, err := c.G().ConvSource.GetMessages(ctx, conv.GetConvID(), uid, @@ -415,27 +298,13 @@ func (c *TeamChannelSource) GetChannelsTopicName(ctx context.Context, uid gregor if err != nil { c.Debug(ctx, "GetChannelsTopicName: failed to unbox metadata message for: convID: %s err: %s", conv.GetConvID(), err) - complete = false continue } if len(unboxeds) != 1 { c.Debug(ctx, "GetChannelsTopicName: empty result: convID: %s", conv.GetConvID()) - complete = false continue } - if !addValidMetadataMsg(conv.GetConvID(), unboxeds[0]) { - complete = false - } - } - // len(convs) == 0 is never a legitimate answer - a chat TLF always has at least #general - so it - // means the inbox read came back degraded, and caching it would pin "this team has no channels" - // for the whole window. - if len(convs) > 0 { - if !complete { - c.Debug(ctx, "GetChannelsTopicName: incomplete result (%d of %d channels), caching briefly", - len(res), len(convs)) - } - c.topicNameCache.Put(tlfID, topicType, uid, res, complete) + addValidMetadataMsg(conv.GetConvID(), unboxeds[0]) } return res, nil } diff --git a/go/chat/teamchannelsource_topicnamecache_test.go b/go/chat/teamchannelsource_topicnamecache_test.go deleted file mode 100644 index 2d2b7f54e150..000000000000 --- a/go/chat/teamchannelsource_topicnamecache_test.go +++ /dev/null @@ -1,182 +0,0 @@ -package chat - -import ( - "testing" - "time" - - "github.com/keybase/client/go/libkb" - "github.com/keybase/client/go/protocol/chat1" - "github.com/keybase/client/go/protocol/gregor1" - "github.com/stretchr/testify/require" -) - -func topicNameCacheFixture() (chat1.TLFID, chat1.TopicType, gregor1.UID, []chat1.ChannelNameMention) { - tlfID := chat1.TLFID([]byte{0x01, 0x02}) - uid := gregor1.UID([]byte{0x0a}) - names := []chat1.ChannelNameMention{ - {ConvID: chat1.ConversationID([]byte{0x10}), TopicName: "general"}, - {ConvID: chat1.ConversationID([]byte{0x11}), TopicName: "random"}, - } - return tlfID, chat1.TopicType_CHAT, uid, names -} - -func TestTopicNameMemCacheRoundTrip(t *testing.T) { - tlfID, topicType, uid, names := topicNameCacheFixture() - c := newTopicNameMemCache() - - _, ok := c.Get(tlfID, topicType, uid) - require.False(t, ok, "an empty cache must miss") - - c.Put(tlfID, topicType, uid, names, true) - got, ok := c.Get(tlfID, topicType, uid) - require.True(t, ok) - require.Equal(t, names, got) -} - -func TestTopicNameMemCacheKeysAreDistinct(t *testing.T) { - tlfID, topicType, uid, names := topicNameCacheFixture() - c := newTopicNameMemCache() - c.Put(tlfID, topicType, uid, names, true) - - otherTLF := chat1.TLFID([]byte{0x09, 0x09}) - otherUID := gregor1.UID([]byte{0xbb}) - - _, ok := c.Get(otherTLF, topicType, uid) - require.False(t, ok, "a different TLF must not share an entry") - _, ok = c.Get(tlfID, chat1.TopicType_DEV, uid) - require.False(t, ok, "a different topic type must not share an entry") - _, ok = c.Get(tlfID, topicType, otherUID) - require.False(t, ok, "a different uid must not share an entry") - - // The original is still there and untouched by the misses. - got, ok := c.Get(tlfID, topicType, uid) - require.True(t, ok) - require.Equal(t, names, got) -} - -// The cached slice is shared with every caller, so neither side may be able to reach into it. -func TestTopicNameMemCacheCopiesBothWays(t *testing.T) { - tlfID, topicType, uid, names := topicNameCacheFixture() - c := newTopicNameMemCache() - c.Put(tlfID, topicType, uid, names, true) - - // Mutating what the caller passed in must not reach the cache. - names[0].TopicName = "mutated-input" - got, ok := c.Get(tlfID, topicType, uid) - require.True(t, ok) - require.Equal(t, "general", got[0].TopicName) - - // Mutating what the caller got back must not reach the cache either. - got[0].TopicName = "mutated-output" - again, ok := c.Get(tlfID, topicType, uid) - require.True(t, ok) - require.Equal(t, "general", again[0].TopicName) -} - -func TestTopicNameMemCacheExpires(t *testing.T) { - tlfID, topicType, uid, names := topicNameCacheFixture() - c := newTopicNameMemCache() - c.Put(tlfID, topicType, uid, names, true) - - key := c.key(tlfID, topicType, uid) - - // Still inside the window. - c.Lock() - item := c.cache[key] - item.mtime = gregor1.ToTime(time.Now().Add(-topicNameCacheDuration + time.Second)) - c.cache[key] = item - c.Unlock() - _, ok := c.Get(tlfID, topicType, uid) - require.True(t, ok, "an entry inside the TTL must hit") - - // Past it. There is no explicit invalidation, so expiry is the only thing keeping a renamed - // channel from being served forever. - c.Lock() - item = c.cache[key] - item.mtime = gregor1.ToTime(time.Now().Add(-topicNameCacheDuration - time.Second)) - c.cache[key] = item - c.Unlock() - _, ok = c.Get(tlfID, topicType, uid) - require.False(t, ok, "an entry past the TTL must miss") -} - -// An empty conversation list is never a legitimate answer for a chat TLF, so it must not be cached. -// This pins the guard at the cache level; the caller-side guard lives in GetChannelsTopicName. -func TestTopicNameMemCacheEmptyIsStillAValue(t *testing.T) { - tlfID, topicType, uid, _ := topicNameCacheFixture() - c := newTopicNameMemCache() - - // The cache itself stores whatever it is given, including nothing - which is exactly why the - // caller must not hand it a degraded read. - c.Put(tlfID, topicType, uid, nil, true) - got, ok := c.Get(tlfID, topicType, uid) - require.True(t, ok, "an empty slice is a cached value, not a miss") - require.Empty(t, got) -} - -// A team almost always holds channels the user cannot resolve - ones they left or never joined - so -// an incomplete result is the steady state. It is cached anyway, or the fan-out this cache exists to -// collapse would never be collapsed, but only for the shorter window. -func TestTopicNameMemCacheIncompleteExpiresSooner(t *testing.T) { - tlfID, topicType, uid, names := topicNameCacheFixture() - c := newTopicNameMemCache() - c.Put(tlfID, topicType, uid, names, false) - - key := c.key(tlfID, topicType, uid) - got, ok := c.Get(tlfID, topicType, uid) - require.True(t, ok, "an incomplete result is still cached") - require.Equal(t, names, got) - - // Old enough that the complete TTL would still serve it, and the incomplete one does not. - c.Lock() - item := c.cache[key] - item.mtime = gregor1.ToTime(time.Now().Add(-topicNameCacheIncompleteDuration - time.Second)) - c.cache[key] = item - c.Unlock() - require.Less(t, topicNameCacheIncompleteDuration+time.Second, topicNameCacheDuration, - "the fixture only proves anything while the two windows differ by more than this") - _, ok = c.Get(tlfID, topicType, uid) - require.False(t, ok, "an incomplete entry must expire on the shorter window") - - // The same age under a complete entry still hits, so it is the flag doing the work. - c.Put(tlfID, topicType, uid, names, true) - c.Lock() - item = c.cache[key] - item.mtime = gregor1.ToTime(time.Now().Add(-topicNameCacheIncompleteDuration - time.Second)) - c.cache[key] = item - c.Unlock() - _, ok = c.Get(tlfID, topicType, uid) - require.True(t, ok, "a complete entry of the same age must still hit") -} - -// The TTL is the only bound on staleness - there is no invalidation hook - and the comment on the -// constant argues from it being short. Pin the value so widening it is a deliberate act. -func TestTopicNameCacheDurationStaysShort(t *testing.T) { - require.LessOrEqual(t, topicNameCacheDuration, 30*time.Second, - "a longer window widens the hole where a resolution is stored stale into a message") - require.Positive(t, topicNameCacheDuration) - require.Positive(t, topicNameCacheIncompleteDuration) - require.Less(t, topicNameCacheIncompleteDuration, topicNameCacheDuration, - "a result known to be missing channels must not be held as long as a whole one") -} - -func TestTopicNameMemCacheClear(t *testing.T) { - tlfID, topicType, uid, names := topicNameCacheFixture() - c := newTopicNameMemCache() - c.Put(tlfID, topicType, uid, names, true) - - c.clearCache() - _, ok := c.Get(tlfID, topicType, uid) - require.False(t, ok, "clearCache must drop everything") - - // Logout and db nuke both go through the same clear, and both must leave the cache usable. - c.Put(tlfID, topicType, uid, names, true) - require.NoError(t, c.OnLogout(libkb.MetaContext{})) - _, ok = c.Get(tlfID, topicType, uid) - require.False(t, ok, "OnLogout must drop everything") - - c.Put(tlfID, topicType, uid, names, true) - require.NoError(t, c.OnDbNuke(libkb.MetaContext{})) - _, ok = c.Get(tlfID, topicType, uid) - require.False(t, ok, "OnDbNuke must drop everything") -} From c7e884ff9b7d8a992680fb90b4bfdd013ed14c6b Mon Sep 17 00:00:00 2001 From: chrisnojima Date: Tue, 8 Sep 2026 11:13:15 -0400 Subject: [PATCH 16/21] fix(chat): close the holes review found around the window invariant - only the load that owns the window gate may refill it. clearVersion cannot separate two loads issued after the same clear, so a stale reload could fill the cleared window with the newest page and drop the gate under the reload the clear actually issued. - latch the orange line's read position when localization lands rather than reading the live one where it is used. A load finishing flips `loaded` and issues mark-read in the same breath, so the live value can already be the advanced one and the thread shows no unread divider. - keep the floor bound until a load has said otherwise. moreToLoadBack starts false and only a load ever sets it, while pushes reach the window before the first load answers. Only that edge: the ceiling has no load to make the drop temporary, so bounding it the same way would wedge a thread whose reload never applies. - let a pending send through a jump-to-recent gap. Its ordinal is the service's, so it belongs in the page that reload is fetching and nothing can open under it; a centered jump still drops it. The gate drop is judged on what the pass carried, so that row cannot stand in for a window the load filled. - prune against both passes of a warm-cache load. INCREMENTAL omits only what the cached pass already carried, so the two together are the window - judging the full pass alone left the stale-ordinal reconciliation running on cold caches only, which is not where ghost rows are. - the electron main window is main.html in dev and prod alike. --- .../conversation/normal/container.test.tsx | 48 ++++- shared/chat/conversation/normal/container.tsx | 28 ++- .../chat/conversation/thread-context.test.tsx | 203 +++++++++++++++++- shared/chat/conversation/thread-context.tsx | 84 ++++++-- shared/chat/conversation/thread-load.test.tsx | 160 +++++++++++++- shared/chat/conversation/thread-load.tsx | 108 +++++++--- .../thread-message-state.test.tsx | 75 +++++++ .../conversation/thread-message-state.tsx | 67 +++++- shared/tests/e2e/electron/helpers/connect.ts | 2 +- skill/playwright-cli/SKILL.md | 6 +- 10 files changed, 695 insertions(+), 86 deletions(-) diff --git a/shared/chat/conversation/normal/container.test.tsx b/shared/chat/conversation/normal/container.test.tsx index 699f5861d8bc..7a2622a60607 100644 --- a/shared/chat/conversation/normal/container.test.tsx +++ b/shared/chat/conversation/normal/container.test.tsx @@ -290,6 +290,45 @@ test('initial load uses the read message ID from mount even if meta changes befo expectOrangeLine(T.Chat.numberToOrdinal(15)) }) +test('the read position is latched when localization lands, not when the load finishes', async () => { + // The DB-nuke order: the conversation is unlocalized and the thread is still loading, so the + // loaded gate skips the fetch when localization lands. The load then finishing flips `loaded` + // and issues mark-read in the same breath, so by the next render the meta can already carry the + // advanced read position. Asking the service about that one puts the unreadline at the newest + // message and the thread shows no divider at all; the position from the moment localization + // landed is the only one that means anything here. + const unreadlineRpc = getUnreadlineRpc().mockResolvedValue({ + offline: false, + unreadlineID: T.Chat.numberToMessageID(6), + }) + mockLoaded = false + mockMeta = makeMeta(convID, -1) + + render() + await flushOrangeLine() + expect(unreadlineRpc).not.toHaveBeenCalled() + + // Localization lands while the thread load is still in flight. + mockMeta = makeMeta(convID, 5, 9) + act(() => { + useShellState.setState({mobileAppState: 'background'}) + }) + await flushOrangeLine() + expect(unreadlineRpc).not.toHaveBeenCalled() + + // The load finishes, and the mark-read it issues has already moved the read position. + mockMeta = makeMeta(convID, 9, 9) + act(() => { + mockLoaded = true + useShellState.setState({mobileAppState: 'active'}) + }) + await flushOrangeLine() + + expect(unreadlineRpc).toHaveBeenCalledTimes(1) + expectUnreadlineRpcReadMsgID(unreadlineRpc, 5) + expectOrangeLine(T.Chat.numberToOrdinal(6)) +}) + test('a thread reload does not refetch the orange line against the stale mount read position', async () => { const unreadlineRpc = getUnreadlineRpc().mockResolvedValue({ offline: false, @@ -323,10 +362,11 @@ test('a thread reload does not refetch the orange line against the stale mount r }) test('an unknown read position draws no orange line rather than one above everything', async () => { - // There is no valid message ID 0, so a non-positive read position means the conversation's meta - // has not landed yet (emptyConversationMeta reads -1), which a DB nuke makes the norm. Asking the - // service with 0 answers "everything is unread" and pins the line above the oldest message, and - // the state is set once, so that answer used to stick for the life of the mount. + // A negative read position means the conversation's meta has not landed yet + // (emptyConversationMeta reads -1), which a DB nuke makes the norm. The old code clamped it to 0, + // and the service answers 0 with "everything is unread", pinning the line above the oldest + // message - and since the state is set once, that answer used to stick for the life of the mount. + // 0 itself is a real read position and is still asked about; see the zero-value test below. const unreadlineRpc = getUnreadlineRpc().mockResolvedValue({ offline: false, unreadlineID: T.Chat.numberToMessageID(8), diff --git a/shared/chat/conversation/normal/container.tsx b/shared/chat/conversation/normal/container.tsx index ad977410c5b8..c1d97028da76 100644 --- a/shared/chat/conversation/normal/container.tsx +++ b/shared/chat/conversation/normal/container.tsx @@ -63,11 +63,18 @@ const useOrangeLine = ( // // An unlocalized conversation reads -1 ("not known yet"), which a DB nuke makes the norm, so // freezing on mount would pin that and the thread would never get an orange line for the life of - // the mount. Wait for the first real value instead. - const [mountReadMsgID] = React.useState(() => readMsgID) - // Fall back to the live value only while the mount-time one is unknown; once the latch below - // fires it stops mattering, so this cannot drift as mark-as-read moves readMsgID. - const initialReadMsgID = mountReadMsgID >= 0 ? mountReadMsgID : readMsgID + // the mount. Latch the first real value instead, on the commit it arrives in, rather than reading + // the live one where it is used. The two differ exactly when it matters: after a nuke the thread + // is still loading when localization lands, so the load below is skipped, and the load then + // finishing flips `loaded` and issues mark-read in the same breath. A live read on the next + // commit can already see the advanced position, and the thread then shows no unread divider at + // all - the failure this latch exists to prevent. + const latchedReadMsgIDRef = React.useRef(readMsgID) + React.useEffect(() => { + if (latchedReadMsgIDRef.current < 0 && readMsgID >= 0) { + latchedReadMsgIDRef.current = readMsgID + } + }, [readMsgID]) const loadOrangeLine = React.useEffectEvent( (conversationIDKey: T.Chat.ConversationIDKey, readMsgID: T.Chat.MessageID) => { @@ -119,12 +126,15 @@ const useOrangeLine = ( const initialOrangeLineLoadedRef = React.useRef(false) React.useEffect(() => { // Only claim the latch once there is a read position to ask about, so an unlocalized - // conversation gets its orange line when localization lands rather than never. - if (loaded && !initialOrangeLineLoadedRef.current && initialReadMsgID >= 0) { + // conversation gets its orange line when localization lands rather than never. readMsgID is a + // dep so that landing wakes this effect; the value asked about is the latched one, and the + // effect that sets it is declared above so it has already run for this commit. + const readMsgIDAtLocalization = latchedReadMsgIDRef.current + if (loaded && !initialOrangeLineLoadedRef.current && readMsgIDAtLocalization >= 0) { initialOrangeLineLoadedRef.current = true - loadOrangeLine(id, initialReadMsgID) + loadOrangeLine(id, readMsgIDAtLocalization) } - }, [id, loaded, initialReadMsgID]) + }, [id, loaded, readMsgID]) // just use the rpc for orange line if we're not active // if we are active we want to keep whatever state we had so it is maintained diff --git a/shared/chat/conversation/thread-context.test.tsx b/shared/chat/conversation/thread-context.test.tsx index dbda1edbe9a7..c7f5b6e761a9 100644 --- a/shared/chat/conversation/thread-context.test.tsx +++ b/shared/chat/conversation/thread-context.test.tsx @@ -1462,10 +1462,12 @@ test('mounted thread listener applies attachment download and upload progress', ).toBeUndefined() }) -test('a cached pass never prunes messages the incremental full pass no longer resends', async () => { +test('a warm-cache load prunes against both passes, not either one alone', async () => { // Regression: once the service has sent a cached thread it switches the full response to - // INCREMENTAL, so the full pass only carries the messages that changed. Treating either partial - // response as authoritative deleted real messages that were still in the thread. + // INCREMENTAL, so the full pass only carries what changed. Treating either pass on its own as + // authoritative deleted real messages that were still in the thread. The two together are the + // window - INCREMENTAL walks it and omits only what the cached pass already carried - so the + // range spans both, and everything inside it that either pass carried survives. useConfigState.setState({loggedIn: true}) jest.spyOn(Common, 'isUserActivelyLookingAtThisThread').mockReturnValue(true) jest.spyOn(T.RPCChat, 'localMarkAsReadLocalRpcPromise').mockResolvedValue({offline: false}) @@ -1476,15 +1478,13 @@ test('a cached pass never prunes messages the incremental full pass no longer re pagination: {last: true, next: '', num: 100, previous: ''}, }) - // A partial cached pass missing 302/303, then an incremental full pass that SPANS the same gap - - // it carries the oldest and newest but not the two in between. The span is what makes this - // dangerous: a validatedRange of [301..304] computed from a partial response covers 302 and 303, - // which are absent from that response and would be pruned. A full pass carrying only 304 gives a - // degenerate [304..304] range with nothing to prune, and would pass even without the fix. + // The cache holds the older three; only 304 changed, so that is all the full pass carries. The + // span is what makes this the dangerous shape: a range of [301..304] computed from the full pass + // alone covers 302 and 303, which are absent from it and would be pruned. jest.spyOn(T.RPCChat, 'localGetThreadNonblockRpcListener').mockImplementation(async p => { - p.incomingCallMap['chat.1.chatUi.chatThreadCached']?.({thread: threadJSON([ids[0]!, ids[3]!])}) + p.incomingCallMap['chat.1.chatUi.chatThreadCached']?.({thread: threadJSON(ids.slice(0, 3))}) await Promise.resolve() - p.incomingCallMap['chat.1.chatUi.chatThreadFull']?.({thread: threadJSON([ids[0]!, ids[3]!])}) + p.incomingCallMap['chat.1.chatUi.chatThreadFull']?.({thread: threadJSON([ids[3]!])}) await Promise.resolve() return {offline: false} }) @@ -1529,6 +1529,69 @@ test('a cached pass never prunes messages the incremental full pass no longer re expect(result.current.ordinals).toEqual([301, 302, 303, 304]) }) +test('a warm-cache load still prunes a row neither pass carries', async () => { + // The other half of the same rule: a row inside the range that neither pass returned is a ghost - + // a cache repair left it behind, or it was deleted while we were away - and reconciling it away + // is what the range is for. Gating on a full pass with no cached one before it would have given + // this up for every conversation the cache is warm for. + useConfigState.setState({loggedIn: true}) + jest.spyOn(Common, 'isUserActivelyLookingAtThisThread').mockReturnValue(true) + jest.spyOn(T.RPCChat, 'localMarkAsReadLocalRpcPromise').mockResolvedValue({offline: false}) + const ids = [301, 302, 303, 304].map(T.Chat.numberToMessageID) + const threadJSON = (msgIDs: ReadonlyArray) => + JSON.stringify({ + messages: msgIDs.map(id => makeValidTextUIMessage(id, `m${id}`)), + pagination: {last: true, next: '', num: 100, previous: ''}, + }) + + // 303 is in neither pass, and it sits inside the span the two of them cover. + jest.spyOn(T.RPCChat, 'localGetThreadNonblockRpcListener').mockImplementation(async p => { + p.incomingCallMap['chat.1.chatUi.chatThreadCached']?.({thread: threadJSON([ids[0]!, ids[1]!])}) + await Promise.resolve() + p.incomingCallMap['chat.1.chatUi.chatThreadFull']?.({thread: threadJSON([ids[3]!])}) + await Promise.resolve() + return {offline: false} + }) + const {result} = renderHook( + () => ({ + actions: useConversationThreadActions(), + loadMoreMessages: useConversationThreadLoadMoreMessages(), + ordinals: useConversationThreadSelector(s => s.messageOrdinals), + }), + {wrapper} + ) + + act(() => { + result.current.actions.applyThreadLoad({ + centered: false, + enableActiveMarkRead: false, + messages: ids.map(id => + Message.makeMessageText({ + author: 'alice', + conversationIDKey: convID, + id, + ordinal: T.Chat.numberToOrdinal(T.Chat.messageIDToNumber(id)), + outboxID: undefined, + text: new HiddenString(`m${id}`), + timestamp: 100, + }) + ), + moreToLoad: false, + scrollDirection: 'none', + }) + }) + expect(result.current.ordinals).toEqual([301, 302, 303, 304]) + + act(() => { + result.current.loadMoreMessages({reason: 'test'}) + }) + await act(async () => { + await flushPromises() + }) + + expect(result.current.ordinals).toEqual([301, 302, 304]) +}) + // The window invariant, at the callsite that enforces it. The four unit tests in // thread-message-state.test.tsx pass `dropNewBelowWindow` themselves; only this proves addMessages // actually sets it, and that thread loads are still allowed to extend the window downward. @@ -1630,6 +1693,126 @@ test('jumpToRecent drops the old window instead of merging a disjoint one into i expect(result.current.ordinals).toEqual([T.Chat.numberToOrdinal(9001)]) }) +test('only the load that claimed the window gate may drop it', () => { + // clearVersion cannot separate two loads of the same conversation - the load generation only + // moves on a conversation change or unmount, so both call themselves current. The reader taps a + // search result, messagesClear issues the centered reload, and a ChatThreadsStale notification + // then fires a second load at the same generation. If that one settles first - no thread, an + // error - it would take the gate down while the reload is still in flight, and a push landing in + // what is left of the gap strands exactly as it did before the gate existed. + const {result} = renderHook(() => ({actions: useConversationThreadActions()}), {wrapper}) + + act(() => { + result.current.actions.applyThreadLoad({ + centered: false, + enableActiveMarkRead: false, + messages: [textAt(7152), textAt(7153)], + moreToLoad: true, + scrollDirection: 'none', + }) + }) + act(() => { + result.current.actions.messagesClear() + }) + + // The reload the clear issued claims the gate; the stale-thread load that follows loses the race. + act(() => { + result.current.actions.claimWindowGate(1) + result.current.actions.claimWindowGate(2) + }) + act(() => { + result.current.actions.clearWindowGate(2) + }) + expect(result.current.actions.getSnapshot().windowCleared).toBe(true) + + act(() => { + result.current.actions.clearWindowGate(1) + }) + expect(result.current.actions.getSnapshot().windowCleared).toBe(false) +}) + +test('a window holding only a pending send is not a window the load filled', () => { + // The pending-send exemption lets our own outbox row in during a jump-to-recent gap, so the + // window is no longer empty. The gate must still be judged on what the load carried: an empty + // cached pass arriving behind that row would otherwise read as "the load filled the window" and + // drop the gate before the real page is anywhere. + const {result} = renderHook(() => ({actions: useConversationThreadActions()}), {wrapper}) + + act(() => { + result.current.actions.messagesClear({reloadsNewest: true}) + }) + act(() => { + result.current.actions.addMessages( + [ + Message.makeMessageText({ + conversationIDKey: convID, + ordinal: T.Chat.numberToOrdinal(7153.001), + outboxID: T.Chat.stringToOutboxID('sending-1'), + submitState: 'pending', + text: new HiddenString('hi'), + }), + ], + {liveUpdate: true} + ) + }) + expect(result.current.actions.getSnapshot().messageOrdinals).toEqual([7153.001]) + + act(() => { + result.current.actions.applyThreadLoad({ + centered: false, + enableActiveMarkRead: false, + messages: [], + moreToLoad: true, + scrollDirection: 'none', + }) + }) + expect(result.current.actions.getSnapshot().windowCleared).toBe(true) +}) + +test('an empty pass during a jump-to-recent gap leaves the gate up', () => { + // A cold cache sends a cached pass carrying no messages ahead of the full response, and it + // reaches applyThreadLoad like any other. Dropping the gate on it reopens the gap: a + // notification landing before the real page becomes the sole ordinal, and the page that follows + // is disjoint from it. + const {result} = renderHook(() => ({actions: useConversationThreadActions()}), {wrapper}) + + act(() => { + result.current.actions.applyThreadLoad({ + centered: false, + enableActiveMarkRead: false, + messages: [textAt(7152), textAt(7153)], + moreToLoad: true, + scrollDirection: 'none', + }) + }) + act(() => { + result.current.actions.messagesClear() + }) + act(() => { + result.current.actions.applyThreadLoad({ + centered: false, + enableActiveMarkRead: false, + messages: [], + moreToLoad: true, + scrollDirection: 'none', + }) + }) + act(() => { + result.current.actions.addMessages([textAt(7155)], {liveUpdate: true}) + }) + act(() => { + result.current.actions.applyThreadLoad({ + centered: false, + enableActiveMarkRead: false, + messages: [textAt(9001)], + moreToLoad: true, + scrollDirection: 'none', + }) + }) + + expect(result.current.actions.getSnapshot().messageOrdinals).toEqual([T.Chat.numberToOrdinal(9001)]) +}) + test('a notification during a jump-to-recent gap cannot become the new window', () => { // jumpToRecent and a centered jump both clear before reloading, so for one RPC round trip the // window is empty. The notification most likely to land in that gap is the post-send diff --git a/shared/chat/conversation/thread-context.tsx b/shared/chat/conversation/thread-context.tsx index 569d44a712ee..1455652dda95 100644 --- a/shared/chat/conversation/thread-context.tsx +++ b/shared/chat/conversation/thread-context.tsx @@ -20,6 +20,7 @@ import {useStore} from 'zustand' import {createStore, type StoreApi} from 'zustand/vanilla' import {useIsFocused} from '@react-navigation/core' import { + type ValidatedRange, addMessagesToThreadState, applyOptimisticReactionsToMessage, completeAttachmentDownloadInThreadState, @@ -111,6 +112,16 @@ export type ConversationThreadState = { // in that gap cannot install itself as the new window. Cleared once that load settles, however it // settles - see clearWindowGate. windowCleared?: boolean + // Whether the reload the clear issued fetches the newest page, which is the one region a message + // arriving during the gap can still be placed against. See the pending-send exemption in + // addMessagesToThreadState. + windowClearedForNewest?: boolean + // The load that owns the gate above: the first one to claim it after the clear, which is the + // reload the clear issued. Only that load may drop the gate. clearVersion alone cannot tell two + // loads of the same conversation apart, and a second load at the same generation - a + // ChatThreadsStale reload, say - would otherwise settle first and take down a gate the reload is + // still relying on. + windowGateOwner?: number messageTypeMap: Map moreToLoadBack: boolean moreToLoadForward: boolean @@ -214,7 +225,7 @@ type LoadNewerMessagesDueToScroll = ( options?: ThreadLoadStatusOptions ) => void type JumpToRecent = (options?: ThreadLoadStatusOptions) => void -type MessagesClear = () => void +type MessagesClear = (opts?: {reloadsNewest?: boolean}) => void type SelectedConversation = (options?: SelectedConversationOptions) => void export type ConversationThreadActions = { addMessages: ( @@ -222,7 +233,7 @@ export type ConversationThreadActions = { opt?: { liveUpdate?: boolean markAsRead?: boolean - validatedRange?: {from: T.Chat.Ordinal; to: T.Chat.Ordinal} + validatedRange?: ValidatedRange } ) => void applyThreadLoad: (p: { @@ -233,7 +244,7 @@ export type ConversationThreadActions = { messages: ReadonlyArray moreToLoad: boolean scrollDirection: ScrollDirection - validatedRange?: {from: T.Chat.Ordinal; to: T.Chat.Ordinal} + validatedRange?: ValidatedRange }) => void clearUnfurlPrompt: (messageID: T.Chat.MessageID, domain: string) => void deleteMessages: (p: { @@ -248,7 +259,8 @@ export type ConversationThreadActions = { explodedBy?: string, liveUpdate?: boolean ) => void - clearWindowGate: () => void + claimWindowGate: (loadID: number) => void + clearWindowGate: (loadID: number) => void getSnapshot: () => ConversationThreadState loadMoreMessages: LoadMoreMessages markThreadAsRead: () => void @@ -482,8 +494,9 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => // a read position, run it again now that there is one. Only on the transition, so an ordinary // mark-read moving readMsgID does not bounce back through here. // - // useOrangeLine latches in a child of this provider, and React runs child effects first, so it has - // already asked for the unreadline against the pre-mark position by the time this fires. + // Safe against the orange line: useOrangeLine latches the read position into state on the commit + // localization lands in, so the position it later asks the unreadline about does not depend on + // beating this mark-read to it. const metaReadMsgID = useInboxMetadataState( s => (s.metas.get(id) ?? emptyConversationMeta).readMsgID ) @@ -501,7 +514,7 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => opt: { liveUpdate?: boolean markAsRead?: boolean - validatedRange?: {from: T.Chat.Ordinal; to: T.Chat.Ordinal} + validatedRange?: ValidatedRange } = {} ) => { updateThreadState(s => { @@ -535,15 +548,30 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => messages: ReadonlyArray moreToLoad: boolean scrollDirection: ScrollDirection - validatedRange?: {from: T.Chat.Ordinal; to: T.Chat.Ordinal} + validatedRange?: ValidatedRange }) => { + // Judged on what this pass carried rather than on the window being non-empty: a pending send + // of our own is admitted during a jump-to-recent gap, and a window holding only that must not + // read as a window this load filled. + const carriedRenderedMessage = p.messages.some( + m => m.conversationMessage !== false && m.type !== 'deleted' + ) updateThreadState(s => { s.loaded = true - s.windowCleared = false if (p.messages.length) { addMessagesToThreadState(s, p.messages, {validatedRange: p.validatedRange}) clearOptimisticReactionsForMessagesInThreadState(s, p.messages) } + // Only a pass that actually rendered something drops the gate. A cold cache sends an empty + // cached pass ahead of the full response, and a page can be all tombstones: dropping the + // gate on either would let a notification arriving before the real page install itself as + // the whole window and strand once that page lands. A load that ends without ever producing + // an ordinal releases the gate in its own finally instead - see clearWindowGate. + if (carriedRenderedMessage) { + s.windowCleared = false + s.windowClearedForNewest = undefined + s.windowGateOwner = undefined + } switch (p.scrollDirection) { case 'forward': s.moreToLoadForward = p.moreToLoad @@ -908,18 +936,37 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => markThreadAsRead() } ) + // The reload a clear issues claims the gate, so a load that merely happens to be running at the + // same clear generation cannot drop it out from under that reload. First claim wins: the clear + // issues its reload synchronously, so that reload is the first to get here. + const claimWindowGate = React.useEffectEvent((loadID: number) => { + const s = threadStore.getState() + if (!s.windowCleared || s.windowGateOwner !== undefined) { + return + } + updateThreadState(d => { + d.windowGateOwner = loadID + }) + }) // applyThreadLoad drops the gate when a load refills the window, but a load can end without ever // applying: offline, scchatnotinteam, or a response that carries no thread. Left alone the gate // would keep dropping notifications for the life of the provider, with no window to correct it. - const clearWindowGate = React.useEffectEvent(() => { - if (!threadStore.getState().windowCleared) { + const clearWindowGate = React.useEffectEvent((loadID: number) => { + const s = threadStore.getState() + if (!s.windowCleared) { return } - updateThreadState(s => { - s.windowCleared = false + // An unclaimed gate is released by whoever settles first: nothing claimed it, so there is no + // reload in flight to protect, and leaving it up would strand the thread. + if (s.windowGateOwner !== undefined && s.windowGateOwner !== loadID) { + return + } + updateThreadState(d => { + d.windowCleared = false + d.windowGateOwner = undefined }) }) - const messagesClear = React.useEffectEvent(() => { + const messagesClear = React.useEffectEvent((opts?: {reloadsNewest?: boolean}) => { activeMarkReadEnabledRef.current = false shownUsernameCache.clear() updateThreadState(s => { @@ -932,6 +979,8 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => // arbitrary one, jumpToRecent the newest page - so nothing arriving first can be placed // against what is coming. s.windowCleared = true + s.windowClearedForNewest = opts?.reloadsNewest + s.windowGateOwner = undefined s.messageIDToOrdinal.clear() s.messageMap.clear() s.messageOrdinals = undefined @@ -1055,6 +1104,7 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => addOptimisticReaction, applyThreadLoad, clearUnfurlPrompt, + claimWindowGate, clearWindowGate, completeAttachmentDownload, deleteMessages, @@ -1245,8 +1295,10 @@ export const useConversationThreadJumpToRecent = () => { const jumpToRecent: JumpToRecent = options => { setMarkReadBlocked(false) // The newest window is disjoint from wherever the reader was, so merging the two would leave a - // gap in the ordinals. Drop the old window first, the way a centered jump does. - messagesClear() + // gap in the ordinals. Drop the old window first, the way a centered jump does - but say that + // the reload covers the newest page, so a send made in the same breath still shows its pending + // row (input-area/normal sends and then jumps here). + messagesClear({reloadsNewest: true}) loadMoreMessages({...(options ?? {}), reason: 'jump to recent'}) } return jumpToRecent diff --git a/shared/chat/conversation/thread-load.test.tsx b/shared/chat/conversation/thread-load.test.tsx index 30b46d51d8fd..0b1f725bf35b 100644 --- a/shared/chat/conversation/thread-load.test.tsx +++ b/shared/chat/conversation/thread-load.test.tsx @@ -14,6 +14,7 @@ import { import * as ThreadRpc from './thread-rpc' import {resetAllStores} from '@/util/zustand' import {useCurrentUserState} from '@/stores/current-user' +import type {ValidatedRange} from './thread-message-state' import type { ConversationThreadActions, ConversationThreadState, @@ -190,6 +191,7 @@ describe('a back page that adds no ordinals reloads itself', () => { } } }), + claimWindowGate: jest.fn(), clearWindowGate: jest.fn(), getSnapshot: () => ({ @@ -370,6 +372,7 @@ describe('a back page that adds no ordinals reloads itself', () => { let clearVersion = 0 const actions = { applyThreadLoad: jest.fn(), + claimWindowGate: jest.fn(), getSnapshot: () => ({ clearVersion, @@ -420,6 +423,7 @@ describe('a load releases the window gate it was issued under', () => { const gateActions = (clearVersion: () => number) => ({ applyThreadLoad: jest.fn(), + claimWindowGate: jest.fn(), clearWindowGate: jest.fn(), getSnapshot: () => ({ @@ -463,6 +467,132 @@ describe('a load releases the window gate it was issued under', () => { expect(actions.clearWindowGate).toHaveBeenCalledTimes(1) }) + test('does not apply a response that arrives after a clear', async () => { + // The back page is in flight when the reader taps jump-to-recent: messagesClear empties the + // window and starts its own load. Applying this one anyway repopulates the window the clear + // dropped and lowers the gate the new load is relying on, and the two disjoint pages then + // merge - the stranded-row bug the gate exists to prevent. + let clearVersion = 3 + jest.spyOn(ThreadRpc, 'loadThreadNonblock').mockImplementation(async p => { + await Promise.resolve() + clearVersion = 4 + p.onFullThread?.( + JSON.stringify({ + messages: [ + { + placeholder: {hidden: false, messageID: T.Chat.numberToMessageID(7152)}, + state: T.RPCChat.MessageUnboxedState.placeholder, + }, + ], + pagination: {last: false, num: 100}, + }) + ) + return undefined as never + }) + const actions = gateActions(() => clearVersion) + loadConversationThreadMessages(conversationIDKey, {reason: 'focused'}, actions) + await flushPromises() + + expect(actions.applyThreadLoad).not.toHaveBeenCalled() + expect(actions.clearWindowGate).not.toHaveBeenCalled() + }) + + test('does not apply a response while another load owns the gate', async () => { + // Two loads issued after the same clear: the reload the clear started owns the gate, and a + // ChatThreadsStale reload fired behind it answers first. clearVersion cannot tell them apart - + // it moved once, for the clear both of them started after. Applying this one would fill the + // cleared window with the newest page while the owner is still fetching a disjoint region, and + // the owner's page would then merge into it. + let claimed = -1 + const actions = { + applyThreadLoad: jest.fn(), + claimWindowGate: jest.fn((loadID: number) => { + claimed = loadID + }), + clearWindowGate: jest.fn(), + getSnapshot: () => + ({ + clearVersion: 3, + liveUpdateVersion: 0, + loaded: true, + messageIDToOrdinal: new Map(), + messageMap: new Map(), + messageOrdinals: undefined, + pendingOutboxToOrdinal: new Map(), + // Someone else got here first. + windowCleared: true, + windowGateOwner: claimed + 1, + }) as unknown as ConversationThreadState, + loadMoreMessages: jest.fn(), + markThreadAsRead: jest.fn(), + } as unknown as ConversationThreadActions + jest.spyOn(ThreadRpc, 'loadThreadNonblock').mockImplementation(async p => { + await Promise.resolve() + p.onFullThread?.( + JSON.stringify({ + messages: [ + { + placeholder: {hidden: false, messageID: T.Chat.numberToMessageID(7152)}, + state: T.RPCChat.MessageUnboxedState.placeholder, + }, + ], + pagination: {last: false, num: 100}, + }) + ) + return undefined as never + }) + loadConversationThreadMessages(conversationIDKey, {reason: 'focused'}, actions) + await flushPromises() + + expect(actions.applyThreadLoad).not.toHaveBeenCalled() + }) + + test('applies the response of the load that owns the gate', async () => { + // The other half: the gate is up and this is the reload that claimed it, so it is the one + // allowed to refill the window. + let claimed = -1 + const actions = { + applyThreadLoad: jest.fn(), + claimWindowGate: jest.fn((loadID: number) => { + claimed = loadID + }), + clearWindowGate: jest.fn(), + getSnapshot: () => + ({ + clearVersion: 3, + liveUpdateVersion: 0, + loaded: true, + messageIDToOrdinal: new Map(), + messageMap: new Map(), + messageOrdinals: undefined, + pendingOutboxToOrdinal: new Map(), + windowCleared: true, + windowGateOwner: claimed, + }) as unknown as ConversationThreadState, + loadMoreMessages: jest.fn(), + markThreadAsRead: jest.fn(), + } as unknown as ConversationThreadActions + jest.spyOn(ThreadRpc, 'loadThreadNonblock').mockImplementation(async p => { + await Promise.resolve() + p.onFullThread?.( + JSON.stringify({ + messages: [ + { + placeholder: {hidden: false, messageID: T.Chat.numberToMessageID(7152)}, + state: T.RPCChat.MessageUnboxedState.placeholder, + }, + ], + pagination: {last: false, num: 100}, + }) + ) + return undefined as never + }) + loadConversationThreadMessages(conversationIDKey, {reason: 'focused'}, actions) + await flushPromises() + + expect(actions.applyThreadLoad).toHaveBeenCalled() + }) + test('leaves a newer clear’s gate alone', async () => { // The load is in flight when the user taps a search result: messagesClear bumps clearVersion and // starts its own load. This one must not pull down the gate that one is relying on - the load @@ -481,7 +611,7 @@ describe('a load releases the window gate it was issued under', () => { expect(actions.clearWindowGate).not.toHaveBeenCalled() }) }) -describe('a cached pass that carries nothing leaves the full pass a whole window', () => { +describe('the prune range is judged against a whole window', () => { const flushPromises = async () => { for (let i = 0; i < 200; i++) { await Promise.resolve() @@ -497,6 +627,7 @@ describe('a cached pass that carries nothing leaves the full pass a whole window const recordingActions = () => ({ applyThreadLoad: jest.fn(), + claimWindowGate: jest.fn(), clearWindowGate: jest.fn(), getSnapshot: () => ({ @@ -522,7 +653,13 @@ describe('a cached pass that carries nothing leaves the full pass a whole window const validatedRangeOfLastLoad = (actions: ConversationThreadActions) => { const calls = (actions.applyThreadLoad as unknown as jest.Mock).mock.calls - return (calls.at(-1)?.[0] as {validatedRange?: {from: number; to: number}} | undefined)?.validatedRange + return (calls.at(-1)?.[0] as {validatedRange?: ValidatedRange} | undefined)?.validatedRange + } + // The span alone. What the range also carries - the ordinals the cached pass delivered - is + // asserted where it matters rather than in every expectation. + const validatedSpanOfLastLoad = (actions: ConversationThreadActions) => { + const range = validatedRangeOfLastLoad(actions) + return range && {from: range.from, to: range.to} } beforeEach(() => { @@ -552,15 +689,19 @@ describe('a cached pass that carries nothing leaves the full pass a whole window loadConversationThreadMessages(conversationIDKey, {reason: 'focused'}, actions) await flushPromises() - expect(validatedRangeOfLastLoad(actions)).toEqual({ + expect(validatedSpanOfLastLoad(actions)).toEqual({ from: T.Chat.numberToOrdinal(7152), to: T.Chat.numberToOrdinal(7153), }) }) - test('does not prune against a full pass that followed a cached page', async () => { - // The warm-cache sequence: the cached pass carried the page, so the full pass is INCREMENTAL - - // only what changed. Pruning against that deletes messages that are still in the thread. + test('prunes against both passes of a warm-cache load, not the full one alone', async () => { + // The warm-cache sequence: the cached pass carries the page and the full pass behind it is + // INCREMENTAL, only what changed. Neither is a window on its own - but INCREMENTAL walks the + // authoritative window and omits only what the cached pass already carried unchanged, so the + // two together are that window, and the range spans both. Judging the full pass alone would + // give up pruning on every conversation the cache is warm for, which is all of them after the + // first open. const actions = recordingActions() mockPasses( JSON.stringify({messages: page(7153, 7052), pagination: {last: false, num: 100}}), @@ -569,6 +710,11 @@ describe('a cached pass that carries nothing leaves the full pass a whole window loadConversationThreadMessages(conversationIDKey, {reason: 'focused'}, actions) await flushPromises() - expect(validatedRangeOfLastLoad(actions)).toBeUndefined() + expect(validatedSpanOfLastLoad(actions)).toEqual({ + from: T.Chat.numberToOrdinal(7052), + to: T.Chat.numberToOrdinal(7153), + }) + // ...and the rows only the cached pass carried count as present, or the prune would take them. + expect(validatedRangeOfLastLoad(actions)?.alsoPresent?.has(T.Chat.numberToOrdinal(7052))).toBe(true) }) }) diff --git a/shared/chat/conversation/thread-load.tsx b/shared/chat/conversation/thread-load.tsx index 66dcbab37b18..b0d1545a66d5 100644 --- a/shared/chat/conversation/thread-load.tsx +++ b/shared/chat/conversation/thread-load.tsx @@ -12,7 +12,7 @@ import {persistRoute} from '@/util/storeless-actions' import {uint8ArrayToString} from '@/util/uint8array' import {useCurrentUserState} from '@/stores/current-user' import {useConfigState} from '@/stores/config' -import {getOrdinalForMessageID} from './thread-message-state' +import {type ValidatedRange, getOrdinalForMessageID} from './thread-message-state' import {getInboxConversationMeta, updateInboxConversationMeta} from '@/chat/inbox/metadata' import {loadThreadNonblock, threadLoadReasonToRPCReason} from './thread-rpc' import type { @@ -22,6 +22,10 @@ import type { ScrollDirection, } from './thread-context' +// Identifies one load, so the window gate can tell two loads of the same conversation apart. +// Only ever compared for equality, never ordered. +let nextLoadID = 0 + export const numMessagesOnInitialLoad = isMobile ? 20 : 100 // How far the no-new-ordinals back-page chain will walk on its own before handing the thread back to // the reader. See the reload block in loadConversationThreadMessages. @@ -205,9 +209,18 @@ export const loadConversationThreadMessages = ( // conversation changes or the thread unmounts, so two loads of the same conversation both call // themselves current. A load that started before the clear would otherwise pull down the gate // belonging to the load that started after it, while that one is still in flight. + // + // clearVersion alone still cannot separate two loads issued after the same clear, so the gate + // is also owned: first claim wins, and only the owner may drop it. Claimed here, before the + // first await, rather than when a response arrives - both clear paths bypass the load throttle + // (see loadMoreMessages in thread-context) and call in synchronously, so the reload the clear + // issued is always the first to get here, and a load that ends without ever applying still has + // to be the one that releases. + const loadID = nextLoadID++ + actions.claimWindowGate(loadID) const releaseWindowGate = () => { if (actions.getSnapshot().clearVersion === clearVersionAtLoadStart) { - actions.clearWindowGate() + actions.clearWindowGate(loadID) } } const currentMeta = getMeta(conversationIDKey) @@ -228,14 +241,13 @@ export const loadConversationThreadMessages = ( ) const loadingKey = Strings.waitingKeyChatThreadLoad(conversationIDKey) - // Set once a cached response arrives carrying messages. Once the service has sent a cached - // thread it switches the full response to INCREMENTAL, filtering it down to only the messages - // that changed (chat/uithreadloader.go mergeLocalRemoteThread). From that point neither response - // is a complete window. Judged on the messages, not on the response: a cold cache still sends a - // pass, because PullLocalOnly's collector suppresses the miss, and that pass carries no - // messages - INCREMENTAL against an empty local thread filters nothing out, so the full pass - // that follows is a whole window after all. - let sawCachedPass = false + // The ordinals the cached pass carried. Once the service has sent a cached thread it switches + // the full response to INCREMENTAL, which walks the authoritative window and sends only the + // messages the cached pass did not already carry unchanged (mergeLocalRemoteThread in + // go/chat/uithreadloader.go, where localSentThread is that exact cached pass). Neither pass is + // a whole window on its own, but together they cover every message in the window - which is + // what the prune below needs, and why it unions them rather than gating on the full pass alone. + const cachedPassOrdinals = new Set() // The reload below is judged against the whole load, not one pass of it. A warm-cache load // delivers the page on the cached pass and then an INCREMENTAL full pass carrying only what // changed, so measuring the full pass alone says "added nothing" for a perfectly good page. @@ -251,10 +263,33 @@ export const loadConversationThreadMessages = ( logger.info(`loadMoreMessages: stale response ignored: ${why}`) return } + // A clear under us - jump to recent, a centered jump - dropped the window this load was + // paging against, and the reload that follows fetches a disjoint region. isCurrentThreadLoad + // does not catch it: the load generation only moves when the conversation changes or the + // thread unmounts, so a load that started before the clear still calls itself current. + // Applying it anyway would repopulate the cleared window and lower the gate belonging to the + // reload, which then merges its own page into the leftovers. + const snapshotAtResponse = actions.getSnapshot() + if (snapshotAtResponse.clearVersion !== clearVersionAtLoadStart) { + logger.info(`loadMoreMessages: response ignored after clear: ${why}`) + return + } + // clearVersion cannot separate two loads issued after the same clear, and the second one is + // not hypothetical: a ChatThreadsStale or ChatInboxSynced reload fires with scrollDirection + // 'none' and fetches the newest page, not the region the clear asked for. If it answers + // first it would fill the cleared window with that disjoint page and drop the gate, and the + // reload the clear issued would then merge its own page into the leftovers - exactly the + // ordinal gap the gate exists to prevent. While the gate is up only its owner may refill the + // window; once the owner settles the gate is down and everyone applies normally again. if ( - protectLoadedFocusRefresh && - actions.getSnapshot().liveUpdateVersion !== loadStartedLiveUpdateVersion + snapshotAtResponse.windowCleared && + snapshotAtResponse.windowGateOwner !== undefined && + snapshotAtResponse.windowGateOwner !== loadID ) { + logger.info(`loadMoreMessages: response ignored, another load owns the window: ${why}`) + return + } + if (protectLoadedFocusRefresh && snapshotAtResponse.liveUpdateVersion !== loadStartedLiveUpdateVersion) { logger.info( `loadMoreMessages: stale response ignored after live update: ${why} reason=${reason} convID=${conversationIDKey}` ) @@ -269,9 +304,6 @@ export const loadConversationThreadMessages = ( devicename, () => getLastOrdinalFromSnapshot(actions.getSnapshot()) ) - if (why === 'cached' && messages.length) { - sawCachedPass = true - } const moreToLoad = pagination ? !pagination.last : true const canMarkReadForThreadWindow = allowMarkAsRead && @@ -280,16 +312,29 @@ export const loadConversationThreadMessages = ( scrollDirection !== 'back' && reason !== 'findNewestConversation' && reason !== 'findNewestConversationFromLayout' - // Pruning is only safe against a response that is a whole window: a full pass with no cached - // pass before it. Anything else is partial, and pruning against it deletes messages that are - // still in the thread. - let validatedRange: {from: T.Chat.Ordinal; to: T.Chat.Ordinal} | undefined - if (messages.length && scrollDirection === 'none' && why === 'full' && !sawCachedPass) { - const ords = messages - .filter(m => m.conversationMessage !== false && m.type !== 'deleted') - .map(m => m.ordinal) + // Pruning is only safe against a whole window, and a single pass is not one: the cached pass + // is whatever the local cache holds, gaps included, and the full pass behind it carries only + // what changed. The two together are the window, so the range is computed on the full pass + // from the union of both. Waiting for a pass with no cached one before it would leave the + // stale-ordinal cleanup running on cold caches only, which is where ghost rows are least + // likely to be - a reopened conversation is warm every time. + const renderedOrdinals = messages + .filter(m => m.conversationMessage !== false && m.type !== 'deleted') + .map(m => m.ordinal) + if (why === 'cached') { + for (const o of renderedOrdinals) { + cachedPassOrdinals.add(o) + } + } + let validatedRange: ValidatedRange | undefined + if (scrollDirection === 'none' && why === 'full') { + const ords = [...renderedOrdinals, ...cachedPassOrdinals] if (ords.length > 0) { validatedRange = { + // The cached pass was applied in its own call, so what it delivered is not among the + // messages this one carries. Without it every row only that pass mentioned would read + // as missing from the window and be pruned. + alsoPresent: cachedPassOrdinals, from: Math.min(...ords) as T.Chat.Ordinal, to: Math.max(...ords) as T.Chat.Ordinal, } @@ -336,10 +381,7 @@ export const loadConversationThreadMessages = ( // remove more from the window, which nets negative on a count but is real progress. !windowGrewDownward && oldestSeenThisLoad < (retryBelowMessageID ?? Number.MAX_SAFE_INTEGER) && - retryCount < maxBackPageReloads && - // A clear under us - jump to recent, a centered jump - means this chain is walking back - // from a window that no longer exists, and would prepend pages the reader never asked for. - after.clearVersion === clearVersionAtLoadStart + retryCount < maxBackPageReloads ) { logger.info( `loadMoreMessages: back page added no ordinals, reloading below ${oldestSeenThisLoad} (${ @@ -350,6 +392,18 @@ export const loadConversationThreadMessages = ( // 500ms throttle and the unmount cancel(), and a long run of tombstones would otherwise // issue these back to back with no pacing. The throttle only ever drops a call that a // later load supersedes, and that load extends the window or retries in turn. + // + // The delay has a cost: the next page comes from a cursor the daemon holds, not one we + // send. pgmode is SERVER (see thread-rpc), so `next` resolves against convPageStatus in the + // service, and any first-page request resets it (applyPagerModeOutgoing in + // go/chat/uithreadloader.go) - which every scrollDirection 'none' load is, stale and focus + // reloads included. One landing inside the throttle window makes this retry fetch near the + // top of the thread instead of the next page back. It fails closed rather than looping: + // oldestSeenThisLoad is then no lower than retryBelowMessageID, so the chain stops and the + // reader is left where another scroll gesture starts a fresh one. + // + // Sizing, for the same reason the chain is bounded at all: a full run is 11 sequential + // 100-message RPCs off one gesture, several seconds of paging with nothing visible moving. actions.loadMoreMessages({ ...p, retryBelowMessageID: oldestSeenThisLoad, diff --git a/shared/chat/conversation/thread-message-state.test.tsx b/shared/chat/conversation/thread-message-state.test.tsx index fb81f7b87b66..d1829b1f1ab6 100644 --- a/shared/chat/conversation/thread-message-state.test.tsx +++ b/shared/chat/conversation/thread-message-state.test.tsx @@ -80,6 +80,10 @@ const makeThreadState = ( messageMap, messageOrdinals, messageTypeMap, + // A thread that has loaded at least once, so the two flags below mean what they say. + loaded: true, + // A partial window by default: the drop rules only bound an edge that still has more past it. + moreToLoadBack: true, moreToLoadForward: false, pendingOutboxToOrdinal, ...extra, @@ -531,6 +535,22 @@ describe('addMessagesToThreadState', () => { expect(state.messageMap.has(T.Chat.numberToOrdinal(9001))).toBe(false) }) + test('a message older than the window prepends once the window reaches the oldest message', () => { + // The mirror of the ceiling rule. A small channel pages back to its start, but message 1 - the + // setChannelname system message - came back as a hidden placeholder and was dropped, so the + // floor is 2. The ResolveSkippedUnboxeds push then delivers the real message 1. With no more to + // load back there is no hole under the floor for it to strand against, and nothing else will + // ever fetch it: loadOlderMessagesDueToScroll bails outright once moreToLoadBack is false. + const state = makeThreadState([textAt(2), textAt(3)], {moreToLoadBack: false}) + addMessagesToThreadState(state, [textAt(1)], {dropNewBelowWindow: true}) + + expect(state.messageOrdinals).toEqual([ + T.Chat.numberToOrdinal(1), + T.Chat.numberToOrdinal(2), + T.Chat.numberToOrdinal(3), + ]) + }) + test('a message newer than the window appends once the window reaches the newest message', () => { // The ordinary live path: the window contains the latest message, so there is no hole to open // above it and an incoming message must land. @@ -569,6 +589,61 @@ describe('addMessagesToThreadState', () => { expect(state.messageOrdinals).toEqual([T.Chat.numberToOrdinal(1)]) }) + test('the floor is still bound while no load has landed to say otherwise', () => { + // moreToLoadBack is initialized false and only a thread load ever sets it. Pushes reach the + // window before the first load answers, so a second push older than the one they installed + // would read that false as "the window reaches the oldest message" and prepend. The load then + // fills the region between, and the pushed row is left stranded over the hole. + const state = makeThreadState([textAt(7153)], {loaded: false, moreToLoadBack: false}) + addMessagesToThreadState(state, [textAt(1)], {dropNewBelowWindow: true}) + expect(state.messageOrdinals).toEqual([T.Chat.numberToOrdinal(7153)]) + + // The ceiling is deliberately not bound the same way. A clear whose reload never applies leaves + // `loaded` false with no load coming, and dropping everything newer than the window would then + // be permanent - the thread would stop receiving messages for good. + addMessagesToThreadState(state, [textAt(9001)], {dropNewBelowWindow: true}) + expect(state.messageOrdinals).toEqual([T.Chat.numberToOrdinal(7153), T.Chat.numberToOrdinal(9001)]) + }) + + test('a pending send lands during a jump-to-recent gap', () => { + // The reader sends from a search-jumped thread: input-area posts and jumps to recent in the + // same tick, so the clear happens first and the outbox notification arrives into the gap. Its + // ordinal is the service's - the outbox record's, above the newest message - so it belongs in + // the very page the reload is fetching, and dropping it leaves the composer empty with no + // "sending..." row for as long as that reload takes. + const pending = makeTextMessage({ + id: T.Chat.numberToMessageID(0), + ordinal: T.Chat.numberToOrdinal(7153.001), + outboxID: T.Chat.stringToOutboxID('sending-1'), + submitState: 'pending', + }) + const state = makeThreadState([]) + state.windowCleared = true + state.windowClearedForNewest = true + addMessagesToThreadState(state, [pending], {dropNewBelowWindow: true}) + expect(state.messageOrdinals).toEqual([T.Chat.numberToOrdinal(7153.001)]) + + // Nothing else gets in on its coattails. + addMessagesToThreadState(state, [textAt(9001)], {dropNewBelowWindow: true}) + expect(state.messageOrdinals).toEqual([T.Chat.numberToOrdinal(7153.001)]) + }) + + test('a pending send is still dropped during a centered-jump gap', () => { + // The exemption is only sound because jump-to-recent reloads the newest page. A centered jump + // lands on an arbitrary older region, and a pending row sitting at the bottom of the thread + // would strand above it once that page arrives. + const pending = makeTextMessage({ + id: T.Chat.numberToMessageID(0), + ordinal: T.Chat.numberToOrdinal(7153.001), + outboxID: T.Chat.stringToOutboxID('sending-1'), + submitState: 'pending', + }) + const state = makeThreadState([]) + state.windowCleared = true + addMessagesToThreadState(state, [pending], {dropNewBelowWindow: true}) + expect(state.messageOrdinals ?? []).toEqual([]) + }) + test('a message remapped out of the window is dropped, not stranded', () => { // The window is judged on the ordinal the message will occupy, which an outbox or messageID // match can move. Here messageIDToOrdinal still points at an ancient ordinal the thread no diff --git a/shared/chat/conversation/thread-message-state.tsx b/shared/chat/conversation/thread-message-state.tsx index d0eef0d2f786..18779914e9a1 100644 --- a/shared/chat/conversation/thread-message-state.tsx +++ b/shared/chat/conversation/thread-message-state.tsx @@ -3,8 +3,22 @@ import * as T from '@/constants/types' import HiddenString from '@/util/hidden-string' import type {WritableDraft} from '@/util/zustand' +// A message we are posting: it exists only in the outbox, so it has no server ID yet. +const isPendingSend = (m: T.Chat.Message) => + !m.id && 'submitState' in m && m.submitState === 'pending' + type MessageLookup = Pick +// The span a thread load is authoritative over, and what it holds. Ordinals inside the span that +// the load did not carry are stale and get pruned. `alsoPresent` is the rest of what the load +// carried: a warm load answers in two passes, and a message the earlier one delivered is still +// present even though the pass doing the pruning no longer mentions it. +export type ValidatedRange = { + from: T.Chat.Ordinal + to: T.Chat.Ordinal + alsoPresent?: ReadonlySet +} + type WritableConversationThreadMessageState = { messageIDToOrdinal: Map messageMap: Map> @@ -13,7 +27,16 @@ type WritableConversationThreadMessageState = { // there is no window to place an arriving message against. See the drop rules in // addMessagesToThreadState. windowCleared?: boolean + // Whether the reload that clear issued fetches the newest page. Only jump-to-recent does; a + // centered jump lands on an arbitrary older region. It is the one case where something arriving + // during the gap can be placed after all - see the pending-send exemption below. + windowClearedForNewest?: boolean messageTypeMap: Map + // Set by a thread load, cleared by messagesClear: whether either flag below means anything yet. + loaded: boolean + // False once the window reaches the oldest message, which is what makes a push older than the + // floor a prepend rather than a stranded row. + moreToLoadBack: boolean // False once the window reaches the newest message, which is what makes a push newer than the // ceiling an append rather than a stranded row. moreToLoadForward: boolean @@ -161,7 +184,7 @@ export const addMessagesToThreadState = ( messages: ReadonlyArray, opt: { dropNewBelowWindow?: boolean - validatedRange?: {from: T.Chat.Ordinal; to: T.Chat.Ordinal} + validatedRange?: ValidatedRange } ) => { const {dropNewBelowWindow, validatedRange} = opt @@ -202,16 +225,18 @@ export const addMessagesToThreadState = ( // // Both edges matter. A centered jump - a search result - leaves a contiguous window with more to // load above and below it, and the reader can page either way from there, so a push newer than - // the ceiling strands exactly as one older than the floor does. The ceiling is only a bound while - // moreToLoadForward: once the window reaches the newest message there is no hole to open above - // it, and a live message must append. + // the ceiling strands exactly as one older than the floor does. Each edge is only a bound while + // there is still more to load past it: once the window reaches the end of the thread on that + // side there is no hole to open, and the message must simply join the window. A fully paged-back + // thread is the case that matters below - the ResolveSkippedUnboxeds push carrying the real + // message 1 has nowhere else to come from, and paging cannot fetch it again. // // Decided before anything is written for the message, so it is skipped whole. Dropping only the // ordinal later would leave messageMap and messageIDToOrdinal holding a message the thread does // not render, and getOrdinalForMessageID would then hand out an ordinal with no row. Nothing is // lost either way: paging to it loads it in the ordinary way. const windowCeiling = ords?.[ords.length - 1] - const isOutsideWindow = (o: T.Chat.Ordinal) => { + const isOutsideWindow = (o: T.Chat.Ordinal, m: T.Chat.Message) => { if (!dropNewBelowWindow || existing.has(o)) { return false } @@ -221,9 +246,28 @@ export const addMessagesToThreadState = ( // reader was. A message landing in the gap that the reload does not carry waits for the next // load or push; a stranded ordinal, by contrast, breaks paging for the life of the thread. if (state.windowCleared) { + // A send of our own is the exception, and only while the reload is fetching the newest page. + // Its ordinal comes from the service (the outbox record's, not our window's), so it sits at + // the bottom of the thread, which is exactly the region that reload is going to cover - + // nothing can open under it. Dropping it instead shows the composer emptying with no + // "sending..." row behind it, for as long as the reload takes. + if (state.windowClearedForNewest && isPendingSend(m)) { + return false + } return true } - const below = windowFloor !== undefined && o < windowFloor + // moreToLoadBack starts false and only a thread load ever sets it, so until one has landed a + // false reads as "the window reaches the oldest message" when it only means "nothing has said + // yet" - and pushes do reach the window before the first load answers. An older push admitted + // on that reading lands under a floor the load is about to fill, stranded over the hole. + // + // Only this edge. The same reasoning would wedge the other one: after a clear whose reload + // never applies there is no load coming at all, and bounding the ceiling on a flag that can no + // longer change would drop every incoming message for the life of the thread. Below is the + // edge with a backstop - the load that fills the hole is what makes the drop temporary, and + // paging back reaches those messages again in the ordinary way. + const below = + windowFloor !== undefined && o < windowFloor && (!state.loaded || state.moreToLoadBack) const above = windowCeiling !== undefined && o > windowCeiling && state.moreToLoadForward return below || above } @@ -235,7 +279,7 @@ export const addMessagesToThreadState = ( // Judged on mapOrdinal, the ordinal the message will actually occupy: an outbox or messageID // match can move it out of the window, or onto a row already inside it. Deletions and // non-conversation messages are not rows, so the window does not bound them. - if (regularMessage && _m.type !== 'deleted' && isOutsideWindow(mapOrdinal)) { + if (regularMessage && _m.type !== 'deleted' && isOutsideWindow(mapOrdinal, _m)) { incomingOrdinals.delete(_m.ordinal) incomingOrdinals.delete(mapOrdinal) continue @@ -323,7 +367,12 @@ export const addMessagesToThreadState = ( if (validatedRange) { // The service response is authoritative within this range; prune stale local ordinals. for (const o of existing) { - if (o >= validatedRange.from && o <= validatedRange.to && !incomingOrdinals.has(o)) { + if ( + o >= validatedRange.from && + o <= validatedRange.to && + !incomingOrdinals.has(o) && + !validatedRange.alsoPresent?.has(o) + ) { clearMessageIDIndexForOrdinal(state, o) existing.delete(o) state.messageMap.delete(o) @@ -337,7 +386,7 @@ export const addMessagesToThreadState = ( from: Math.min(prev.from, validatedRange.from) as T.Chat.Ordinal, to: Math.max(prev.to, validatedRange.to) as T.Chat.Ordinal, } - : validatedRange + : {from: validatedRange.from, to: validatedRange.to} } if (changed || !state.messageOrdinals) { state.messageOrdinals = [...existing].sort((a, b) => a - b) diff --git a/shared/tests/e2e/electron/helpers/connect.ts b/shared/tests/e2e/electron/helpers/connect.ts index a0225f35d3f8..02f9f23bebf1 100644 --- a/shared/tests/e2e/electron/helpers/connect.ts +++ b/shared/tests/e2e/electron/helpers/connect.ts @@ -19,7 +19,7 @@ export async function connectToElectron(): Promise<{browser: Browser; page: Page // KB_E2E_TEST=1 suppresses the menubar widget and devtools windows, so // pages()[0] is always the main app. Keep the URL check as a safety net. const allPages = browser.contexts().flatMap(ctx => ctx.pages()) - const mainPage = allPages.find(p => p.url().includes('main.dev.html')) ?? allPages[0] + const mainPage = allPages.find(p => p.url().includes('main.html')) ?? allPages[0] if (!mainPage) { throw new Error('Could not find main app page. Is the app running with KB_ENABLE_REMOTE_DEBUG=1 KB_E2E_TEST=1?') diff --git a/skill/playwright-cli/SKILL.md b/skill/playwright-cli/SKILL.md index 409e859cfaa9..2659ff077d07 100644 --- a/skill/playwright-cli/SKILL.md +++ b/skill/playwright-cli/SKILL.md @@ -308,8 +308,8 @@ playwright — ask them to paste it, or have them re-trigger while you are attac Use the page URL, not the title — the title stays `"Keybase DEV"` until the router navigates and can't be relied on: -- Main app: URL contains `main.dev.html` -- Menubar: URL contains `menubar.dev.html` +- Main app: URL contains `main.html` +- Menubar: URL contains `remote.html?component=menubar` (the menubar is a remote window, not its own shell) - Avoid: `devtools://` pages and the `"Keybase DEV"` standalone DevTools window ```js @@ -317,7 +317,7 @@ Use the page URL, not the title — the title stays `"Keybase DEV"` until the ro let mainPage for (const ctx of browser.contexts()) { for (const p of ctx.pages()) { - if (p.url().includes('main.dev.html')) { mainPage = p; break } + if (p.url().includes('main.html')) { mainPage = p; break } } if (mainPage) break } From 9e742666e6e6d5823f955b2d5ce5affc9c89cd2b Mon Sep 17 00:00:00 2001 From: chrisnojima Date: Tue, 8 Sep 2026 11:25:14 -0400 Subject: [PATCH 17/21] fix(chat): record what a cached pass delivered in the window's terms The prune walks the window, so what the passes delivered has to be recorded the same way. A message you sent keeps the fractional ordinal it had in the outbox, so the ordinal it parses with - the server one - is not the ordinal it occupies: recording the parsed one left the row unprotected and the next warm reload deleted the message you had just sent. Also apply a range that arrives with no messages behind it. The warm reload where nothing changed answers with an empty full pass, which is still an authoritative statement about the span, and skipping the prune for want of messages to add leaves a ghost row up until the conversation is reopened. validatedOrdinalRange goes with them: nothing has read it since its last reader was removed, and a union kept only for a future reader is one that can only mislead. --- .../chat/conversation/thread-context.test.tsx | 164 +++++++++++++++++- shared/chat/conversation/thread-context.tsx | 8 +- shared/chat/conversation/thread-load.tsx | 26 ++- .../thread-message-state.test.tsx | 7 +- .../conversation/thread-message-state.tsx | 8 - 5 files changed, 185 insertions(+), 28 deletions(-) diff --git a/shared/chat/conversation/thread-context.test.tsx b/shared/chat/conversation/thread-context.test.tsx index c7f5b6e761a9..43f9c5fbf63c 100644 --- a/shared/chat/conversation/thread-context.test.tsx +++ b/shared/chat/conversation/thread-context.test.tsx @@ -73,7 +73,11 @@ const makeAttachmentMessage = (override?: Partial) => ...override, }) -const makeValidTextUIMessage = (serverMsgID: T.Chat.MessageID, text: string): T.RPCChat.UIMessage => ({ +const makeValidTextUIMessage = ( + serverMsgID: T.Chat.MessageID, + text: string, + outboxID = '' +): T.RPCChat.UIMessage => ({ state: T.RPCChat.MessageUnboxedState.valid, valid: { atMentions: null, @@ -103,7 +107,7 @@ const makeValidTextUIMessage = (serverMsgID: T.Chat.MessageID, text: string): T. }, }, messageID: T.Chat.messageIDToNumber(serverMsgID), - outboxID: '', + outboxID, paymentInfos: null, pinnedMessageID: null, reactions: {}, @@ -1529,6 +1533,162 @@ test('a warm-cache load prunes against both passes, not either one alone', async expect(result.current.ordinals).toEqual([301, 302, 303, 304]) }) +test('a warm-cache load does not prune a message sitting on its outbox ordinal', async () => { + // A message you sent keeps the fractional ordinal it had in the outbox, so the ordinal it parses + // with - its server one - is not the ordinal it occupies. The prune walks the window, so what + // the passes delivered has to be recorded in the window's terms too; recording the parsed + // ordinal deletes the row it was meant to protect. + useConfigState.setState({loggedIn: true}) + jest.spyOn(Common, 'isUserActivelyLookingAtThisThread').mockReturnValue(true) + jest.spyOn(T.RPCChat, 'localMarkAsReadLocalRpcPromise').mockResolvedValue({offline: false}) + const outboxID = T.Chat.stringToOutboxID('sent-1') + const sentOrdinal = T.Chat.numberToOrdinal(302.001) + + jest.spyOn(T.RPCChat, 'localGetThreadNonblockRpcListener').mockImplementation(async p => { + p.incomingCallMap['chat.1.chatUi.chatThreadCached']?.({ + thread: JSON.stringify({ + messages: [ + makeValidTextUIMessage(T.Chat.numberToMessageID(301), 'm301'), + makeValidTextUIMessage(T.Chat.numberToMessageID(302), 'm302'), + makeValidTextUIMessage(T.Chat.numberToMessageID(303), 'mine', 'sent-1'), + ], + pagination: {last: true, next: '', num: 100, previous: ''}, + }), + }) + await Promise.resolve() + p.incomingCallMap['chat.1.chatUi.chatThreadFull']?.({ + thread: JSON.stringify({ + messages: [makeValidTextUIMessage(T.Chat.numberToMessageID(301), 'm301 edited')], + pagination: {last: true, next: '', num: 100, previous: ''}, + }), + }) + await Promise.resolve() + return {offline: false} + }) + const {result} = renderHook( + () => ({ + actions: useConversationThreadActions(), + loadMoreMessages: useConversationThreadLoadMoreMessages(), + ordinals: useConversationThreadSelector(s => s.messageOrdinals), + }), + {wrapper} + ) + + // The window as it stands after the send settled: the message is at its outbox ordinal, indexed + // under the server ID the service will send it back as. + act(() => { + result.current.actions.applyThreadLoad({ + centered: false, + enableActiveMarkRead: false, + messages: [ + Message.makeMessageText({ + author: 'alice', + conversationIDKey: convID, + id: T.Chat.numberToMessageID(301), + ordinal: T.Chat.numberToOrdinal(301), + outboxID: undefined, + text: new HiddenString('m301'), + timestamp: 100, + }), + Message.makeMessageText({ + author: 'alice', + conversationIDKey: convID, + id: T.Chat.numberToMessageID(302), + ordinal: T.Chat.numberToOrdinal(302), + outboxID: undefined, + text: new HiddenString('m302'), + timestamp: 100, + }), + Message.makeMessageText({ + author: 'testuser', + conversationIDKey: convID, + id: T.Chat.numberToMessageID(303), + ordinal: sentOrdinal, + outboxID, + text: new HiddenString('mine'), + timestamp: 100, + }), + ], + moreToLoad: false, + scrollDirection: 'none', + }) + }) + expect(result.current.ordinals).toEqual([301, 302, sentOrdinal]) + + act(() => { + result.current.loadMoreMessages({reason: 'test'}) + }) + await act(async () => { + await flushPromises() + }) + + expect(result.current.ordinals).toEqual([301, 302, sentOrdinal]) +}) + +test('a full pass that changed nothing still reconciles the window', async () => { + // The ordinary warm reload: the cached pass is the window and the INCREMENTAL full pass behind it + // carries nothing at all, because nothing changed. That is still an authoritative answer about + // the span, so a row the service no longer has is still a ghost - skipping the prune for want of + // messages to add leaves it on screen until the conversation is reopened. + useConfigState.setState({loggedIn: true}) + jest.spyOn(Common, 'isUserActivelyLookingAtThisThread').mockReturnValue(true) + jest.spyOn(T.RPCChat, 'localMarkAsReadLocalRpcPromise').mockResolvedValue({offline: false}) + const ids = [301, 302, 303].map(T.Chat.numberToMessageID) + + jest.spyOn(T.RPCChat, 'localGetThreadNonblockRpcListener').mockImplementation(async p => { + p.incomingCallMap['chat.1.chatUi.chatThreadCached']?.({ + thread: JSON.stringify({ + messages: [ids[0]!, ids[2]!].map(id => makeValidTextUIMessage(id, `m${id}`)), + pagination: {last: true, next: '', num: 100, previous: ''}, + }), + }) + await Promise.resolve() + p.incomingCallMap['chat.1.chatUi.chatThreadFull']?.({ + thread: JSON.stringify({messages: null, pagination: {last: true, next: '', num: 100, previous: ''}}), + }) + await Promise.resolve() + return {offline: false} + }) + const {result} = renderHook( + () => ({ + actions: useConversationThreadActions(), + loadMoreMessages: useConversationThreadLoadMoreMessages(), + ordinals: useConversationThreadSelector(s => s.messageOrdinals), + }), + {wrapper} + ) + + act(() => { + result.current.actions.applyThreadLoad({ + centered: false, + enableActiveMarkRead: false, + messages: ids.map(id => + Message.makeMessageText({ + author: 'alice', + conversationIDKey: convID, + id, + ordinal: T.Chat.numberToOrdinal(T.Chat.messageIDToNumber(id)), + outboxID: undefined, + text: new HiddenString(`m${id}`), + timestamp: 100, + }) + ), + moreToLoad: false, + scrollDirection: 'none', + }) + }) + expect(result.current.ordinals).toEqual([301, 302, 303]) + + act(() => { + result.current.loadMoreMessages({reason: 'test'}) + }) + await act(async () => { + await flushPromises() + }) + + expect(result.current.ordinals).toEqual([301, 303]) +}) + test('a warm-cache load still prunes a row neither pass carries', async () => { // The other half of the same rule: a row inside the range that neither pass returned is a ghost - // a cache repair left it behind, or it was deleted while we were away - and reconciling it away diff --git a/shared/chat/conversation/thread-context.tsx b/shared/chat/conversation/thread-context.tsx index 1455652dda95..84c22409ad48 100644 --- a/shared/chat/conversation/thread-context.tsx +++ b/shared/chat/conversation/thread-context.tsx @@ -130,7 +130,6 @@ export type ConversationThreadState = { pendingOutboxToOrdinal: Map typing: Set unfurlPrompt: Map> - validatedOrdinalRange?: {from: T.Chat.Ordinal; to: T.Chat.Ordinal} } type ConversationThreadStore = StoreApi @@ -165,7 +164,6 @@ const makeEmptyThreadState = (): ConversationThreadState => pendingOutboxToOrdinal: new Map(), typing: new Set(), unfurlPrompt: new Map>(), - validatedOrdinalRange: undefined as {from: T.Chat.Ordinal; to: T.Chat.Ordinal} | undefined, }, () => {} ) @@ -558,7 +556,10 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => ) updateThreadState(s => { s.loaded = true - if (p.messages.length) { + // A range with no messages behind it is still worth applying: the warm reload where nothing + // changed answers with an empty full pass, and that is an authoritative statement about the + // span - the stale rows inside it are exactly what the prune is for. + if (p.messages.length || p.validatedRange) { addMessagesToThreadState(s, p.messages, {validatedRange: p.validatedRange}) clearOptimisticReactionsForMessagesInThreadState(s, p.messages) } @@ -986,7 +987,6 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => s.messageOrdinals = undefined s.messageTypeMap.clear() s.optimisticReactionMap.clear() - s.validatedOrdinalRange = undefined }) }) const setTyping = React.useEffectEvent((typing: ReadonlySet) => { diff --git a/shared/chat/conversation/thread-load.tsx b/shared/chat/conversation/thread-load.tsx index b0d1545a66d5..a9684114a592 100644 --- a/shared/chat/conversation/thread-load.tsx +++ b/shared/chat/conversation/thread-load.tsx @@ -318,14 +318,10 @@ export const loadConversationThreadMessages = ( // from the union of both. Waiting for a pass with no cached one before it would leave the // stale-ordinal cleanup running on cold caches only, which is where ghost rows are least // likely to be - a reopened conversation is warm every time. - const renderedOrdinals = messages - .filter(m => m.conversationMessage !== false && m.type !== 'deleted') - .map(m => m.ordinal) - if (why === 'cached') { - for (const o of renderedOrdinals) { - cachedPassOrdinals.add(o) - } - } + const renderedMessages = messages.filter( + m => m.conversationMessage !== false && m.type !== 'deleted' + ) + const renderedOrdinals = renderedMessages.map(m => m.ordinal) let validatedRange: ValidatedRange | undefined if (scrollDirection === 'none' && why === 'full') { const ords = [...renderedOrdinals, ...cachedPassOrdinals] @@ -356,6 +352,20 @@ export const loadConversationThreadMessages = ( validatedRange, }) const after = actions.getSnapshot() + if (why === 'cached') { + // Recorded once the pass has landed, and in the window's terms rather than the response's. + // A message you sent keeps the fractional ordinal it had in the outbox, so the ordinal it + // parsed with - the server one - is not the ordinal it occupies. The prune walks the + // window, so an entry under the parsed ordinal protects nothing and the row goes. Both are + // recorded: whichever one the row ends up under, it counts as delivered. + for (const m of renderedMessages) { + cachedPassOrdinals.add(m.ordinal) + const occupied = m.id ? getOrdinalForMessageIDInSnapshot(after, m.id) : undefined + if (occupied) { + cachedPassOrdinals.add(occupied) + } + } + } // A back page can be composed entirely of messages the thread will never render: a message // superseded by a DELETE arrives as a hidden placeholder, becomes `deleted`, and addMessages // drops it. The ordinal list is then identical to what it was, so the list never fires diff --git a/shared/chat/conversation/thread-message-state.test.tsx b/shared/chat/conversation/thread-message-state.test.tsx index d1829b1f1ab6..726572401213 100644 --- a/shared/chat/conversation/thread-message-state.test.tsx +++ b/shared/chat/conversation/thread-message-state.test.tsx @@ -387,20 +387,15 @@ describe('addMessagesToThreadState', () => { }) expect(state.messageOrdinals).toEqual([10, 30]) expect(state.messageMap.has(T.Chat.numberToOrdinal(20))).toBe(false) - expect(state.validatedOrdinalRange).toEqual({from: 10, to: 30}) }) - test('a validated range leaves ordinals outside of it alone and widens the known range', () => { + test('a validated range leaves ordinals outside of it alone', () => { const state = makeThreadState([]) addMessagesToThreadState(state, [textAt(10), textAt(50)], {}) addMessagesToThreadState(state, [textAt(50)], { validatedRange: {from: T.Chat.numberToOrdinal(40), to: T.Chat.numberToOrdinal(60)}, }) expect(state.messageOrdinals).toEqual([10, 50]) - addMessagesToThreadState(state, [textAt(10)], { - validatedRange: {from: T.Chat.numberToOrdinal(5), to: T.Chat.numberToOrdinal(15)}, - }) - expect(state.validatedOrdinalRange).toEqual({from: 5, to: 60}) }) test('a notification may not strand a new ordinal below the loaded window', () => { diff --git a/shared/chat/conversation/thread-message-state.tsx b/shared/chat/conversation/thread-message-state.tsx index 18779914e9a1..571445374bda 100644 --- a/shared/chat/conversation/thread-message-state.tsx +++ b/shared/chat/conversation/thread-message-state.tsx @@ -41,7 +41,6 @@ type WritableConversationThreadMessageState = { // ceiling an append rather than a stranded row. moreToLoadForward: boolean pendingOutboxToOrdinal: Map - validatedOrdinalRange?: {from: T.Chat.Ordinal; to: T.Chat.Ordinal} } type ThreadMessagesDeleteParams = { @@ -380,13 +379,6 @@ export const addMessagesToThreadState = ( changed = true } } - const prev = state.validatedOrdinalRange - state.validatedOrdinalRange = prev - ? { - from: Math.min(prev.from, validatedRange.from) as T.Chat.Ordinal, - to: Math.max(prev.to, validatedRange.to) as T.Chat.Ordinal, - } - : {from: validatedRange.from, to: validatedRange.to} } if (changed || !state.messageOrdinals) { state.messageOrdinals = [...existing].sort((a, b) => a - b) From d6df26886357a989a8b998f529695d74997a4646 Mon Sep 17 00:00:00 2001 From: chrisnojima Date: Tue, 8 Sep 2026 11:38:17 -0400 Subject: [PATCH 18/21] fix(chat): a refused cached pass leaves nothing to prune against The gate-owner guard is the one guard that can turn a cached pass away and still admit the full pass behind it: the owner drops the gate in between. The service counts that cached pass as sent either way, so what follows is INCREMENTAL - only the messages that changed - and a span built from those alone covers every row between them with nothing recorded as present. That deletes the thread rather than reconciling it. Track whether a cached pass arrived and whether it landed, and skip the range when the two disagree. Also reset windowClearedForNewest where the gate is released in the load's finally, so both release paths leave the same state behind. --- shared/chat/conversation/thread-context.tsx | 1 + shared/chat/conversation/thread-load.test.tsx | 47 +++++++++++++++++++ shared/chat/conversation/thread-load.tsx | 16 ++++++- 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/shared/chat/conversation/thread-context.tsx b/shared/chat/conversation/thread-context.tsx index 84c22409ad48..09eb3cc9ea92 100644 --- a/shared/chat/conversation/thread-context.tsx +++ b/shared/chat/conversation/thread-context.tsx @@ -964,6 +964,7 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => } updateThreadState(d => { d.windowCleared = false + d.windowClearedForNewest = undefined d.windowGateOwner = undefined }) }) diff --git a/shared/chat/conversation/thread-load.test.tsx b/shared/chat/conversation/thread-load.test.tsx index 0b1f725bf35b..255cca5f7906 100644 --- a/shared/chat/conversation/thread-load.test.tsx +++ b/shared/chat/conversation/thread-load.test.tsx @@ -695,6 +695,53 @@ describe('the prune range is judged against a whole window', () => { }) }) + test('does not prune when a cached pass arrived but another load owned the window', async () => { + // The gate-owner guard is the one guard that can turn a cached pass away and still let the full + // pass behind it through: the owner drops the gate in between. The service counts that cached + // pass as sent either way, so the full pass is INCREMENTAL - a handful of changed messages - and + // a span built from those alone covers every row between them with nothing recorded as present. + // That is not a stale-row cleanup, it is deleting the thread. + let claimed = -1 + let ownedByAnother = true + const actions = { + applyThreadLoad: jest.fn(), + claimWindowGate: jest.fn((loadID: number) => { + claimed = loadID + }), + clearWindowGate: jest.fn(), + getSnapshot: () => + ({ + clearVersion: 0, + liveUpdateVersion: 0, + loaded: true, + messageIDToOrdinal: new Map(), + messageMap: new Map(), + messageOrdinals: undefined, + pendingOutboxToOrdinal: new Map(), + windowCleared: ownedByAnother, + windowGateOwner: claimed + 1, + }) as unknown as ConversationThreadState, + loadMoreMessages: jest.fn(), + markThreadAsRead: jest.fn(), + } as unknown as ConversationThreadActions + jest.spyOn(ThreadRpc, 'loadThreadNonblock').mockImplementation(async p => { + await Promise.resolve() + p.onCachedThread?.( + JSON.stringify({messages: page(7153, 7052), pagination: {last: false, num: 100}}) + ) + // The load that owned the gate settles here, so the full pass is no longer refused. + ownedByAnother = false + p.onFullThread?.( + JSON.stringify({messages: page(7153, 7150), pagination: {last: false, num: 100}}) + ) + return undefined as never + }) + loadConversationThreadMessages(conversationIDKey, {reason: 'focused'}, actions) + await flushPromises() + + expect(validatedRangeOfLastLoad(actions)).toBeUndefined() + }) + test('prunes against both passes of a warm-cache load, not the full one alone', async () => { // The warm-cache sequence: the cached pass carries the page and the full pass behind it is // INCREMENTAL, only what changed. Neither is a window on its own - but INCREMENTAL walks the diff --git a/shared/chat/conversation/thread-load.tsx b/shared/chat/conversation/thread-load.tsx index a9684114a592..74feff4dfcd1 100644 --- a/shared/chat/conversation/thread-load.tsx +++ b/shared/chat/conversation/thread-load.tsx @@ -248,6 +248,14 @@ export const loadConversationThreadMessages = ( // a whole window on its own, but together they cover every message in the window - which is // what the prune below needs, and why it unions them rather than gating on the full pass alone. const cachedPassOrdinals = new Set() + // Whether a cached pass reached us at all, and whether it made it into the window. They come + // apart: the gate-owner guard turns a pass away, and the owner can drop the gate before the + // full pass arrives, so a load can have its cached pass refused and its full pass admitted. The + // service counts that cached pass as sent either way, so what follows is still INCREMENTAL - + // only the messages that changed - and a span built from those alone would prune every row + // between them. Recorded before the guards, because the guards are what turn a pass away. + let sawCachedResponse = false + let appliedCachedPass = false // The reload below is judged against the whole load, not one pass of it. A warm-cache load // delivers the page on the cached pass and then an INCREMENTAL full pass carrying only what // changed, so measuring the full pass alone says "added nothing" for a perfectly good page. @@ -257,8 +265,13 @@ export const loadConversationThreadMessages = ( let oldestSeenThisLoad = Number.MAX_SAFE_INTEGER as T.Chat.MessageID const onGotThread = (thread: string, why: string) => { if (!thread) { + // No cached thread was sent, so the service has nothing to filter the full pass against and + // it stays a whole window. Deliberately not counted as a cached response. return } + if (why === 'cached') { + sawCachedResponse = true + } if (!isCurrentThreadLoad()) { logger.info(`loadMoreMessages: stale response ignored: ${why}`) return @@ -323,7 +336,7 @@ export const loadConversationThreadMessages = ( ) const renderedOrdinals = renderedMessages.map(m => m.ordinal) let validatedRange: ValidatedRange | undefined - if (scrollDirection === 'none' && why === 'full') { + if (scrollDirection === 'none' && why === 'full' && !(sawCachedResponse && !appliedCachedPass)) { const ords = [...renderedOrdinals, ...cachedPassOrdinals] if (ords.length > 0) { validatedRange = { @@ -353,6 +366,7 @@ export const loadConversationThreadMessages = ( }) const after = actions.getSnapshot() if (why === 'cached') { + appliedCachedPass = true // Recorded once the pass has landed, and in the window's terms rather than the response's. // A message you sent keeps the fractional ordinal it had in the outbox, so the ordinal it // parsed with - the server one - is not the ordinal it occupies. The prune walks the From a77fc1497b599aaec4ba291ba5f998892dfa1771 Mon Sep 17 00:00:00 2001 From: chrisnojima Date: Tue, 8 Sep 2026 11:44:34 -0400 Subject: [PATCH 19/21] refactor(chat): let the store say what a load put in the window The prune walked the window while the range describing it was assembled out in thread-load, from the response. The two are not in the same terms - a message you sent lives at its outbox ordinal, not the server one it arrives under - and every pass at reconciling them cost a bug: first the sent row deleted, then a range dropped for want of messages to add. So hand a set to addMessagesToThreadState and let it fill the set itself, in the same breath as it decides where each message goes. Each pass adds what it delivered, the last one prunes against everything gathered, and the two ways of naming a row cannot drift apart because only one of them is ever written down. The load still says whether it can account for a whole window, which is the one thing it knows and the store does not. No behavior change intended - the outbox row, the empty final pass, and the refused cached pass are all still covered, now by construction. --- shared/chat/conversation/thread-context.tsx | 19 +++--- shared/chat/conversation/thread-load.test.tsx | 37 ++++------ shared/chat/conversation/thread-load.tsx | 66 ++++++------------ .../thread-message-state.test.tsx | 41 +++++++++--- .../conversation/thread-message-state.tsx | 67 +++++++++++-------- 5 files changed, 112 insertions(+), 118 deletions(-) diff --git a/shared/chat/conversation/thread-context.tsx b/shared/chat/conversation/thread-context.tsx index 09eb3cc9ea92..10704c8be0d3 100644 --- a/shared/chat/conversation/thread-context.tsx +++ b/shared/chat/conversation/thread-context.tsx @@ -20,7 +20,7 @@ import {useStore} from 'zustand' import {createStore, type StoreApi} from 'zustand/vanilla' import {useIsFocused} from '@react-navigation/core' import { - type ValidatedRange, + type ThreadLoadReconcile, addMessagesToThreadState, applyOptimisticReactionsToMessage, completeAttachmentDownloadInThreadState, @@ -231,7 +231,6 @@ export type ConversationThreadActions = { opt?: { liveUpdate?: boolean markAsRead?: boolean - validatedRange?: ValidatedRange } ) => void applyThreadLoad: (p: { @@ -241,8 +240,8 @@ export type ConversationThreadActions = { forceContainsLatestCalc?: boolean messages: ReadonlyArray moreToLoad: boolean + reconcile?: ThreadLoadReconcile scrollDirection: ScrollDirection - validatedRange?: ValidatedRange }) => void clearUnfurlPrompt: (messageID: T.Chat.MessageID, domain: string) => void deleteMessages: (p: { @@ -512,7 +511,6 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => opt: { liveUpdate?: boolean markAsRead?: boolean - validatedRange?: ValidatedRange } = {} ) => { updateThreadState(s => { @@ -522,7 +520,6 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => addMessagesToThreadState(s, messages, { // Only thread loads may extend the window downward; a notification must not. dropNewBelowWindow: true, - validatedRange: opt.validatedRange, }) clearOptimisticReactionsForMessagesInThreadState(s, messages) }) @@ -545,8 +542,8 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => forceContainsLatestCalc?: boolean messages: ReadonlyArray moreToLoad: boolean + reconcile?: ThreadLoadReconcile scrollDirection: ScrollDirection - validatedRange?: ValidatedRange }) => { // Judged on what this pass carried rather than on the window being non-empty: a pending send // of our own is admitted during a jump-to-recent gap, and a window holding only that must not @@ -556,11 +553,11 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => ) updateThreadState(s => { s.loaded = true - // A range with no messages behind it is still worth applying: the warm reload where nothing - // changed answers with an empty full pass, and that is an authoritative statement about the - // span - the stale rows inside it are exactly what the prune is for. - if (p.messages.length || p.validatedRange) { - addMessagesToThreadState(s, p.messages, {validatedRange: p.validatedRange}) + // The reconciling pass runs even with nothing to add: the warm reload where nothing changed + // answers with an empty full pass, and the span its earlier pass covered is authoritative + // all the same - the stale rows inside it are exactly what the prune is for. + if (p.messages.length || p.reconcile) { + addMessagesToThreadState(s, p.messages, {reconcile: p.reconcile}) clearOptimisticReactionsForMessagesInThreadState(s, p.messages) } // Only a pass that actually rendered something drops the gate. A cold cache sends an empty diff --git a/shared/chat/conversation/thread-load.test.tsx b/shared/chat/conversation/thread-load.test.tsx index 255cca5f7906..2d9c7e8a0708 100644 --- a/shared/chat/conversation/thread-load.test.tsx +++ b/shared/chat/conversation/thread-load.test.tsx @@ -14,7 +14,7 @@ import { import * as ThreadRpc from './thread-rpc' import {resetAllStores} from '@/util/zustand' import {useCurrentUserState} from '@/stores/current-user' -import type {ValidatedRange} from './thread-message-state' +import type {ThreadLoadReconcile} from './thread-message-state' import type { ConversationThreadActions, ConversationThreadState, @@ -611,7 +611,7 @@ describe('a load releases the window gate it was issued under', () => { expect(actions.clearWindowGate).not.toHaveBeenCalled() }) }) -describe('the prune range is judged against a whole window', () => { +describe('only a pass that can account for a whole window reconciles', () => { const flushPromises = async () => { for (let i = 0; i < 200; i++) { await Promise.resolve() @@ -651,15 +651,12 @@ describe('the prune range is judged against a whole window', () => { return undefined as never }) - const validatedRangeOfLastLoad = (actions: ConversationThreadActions) => { + // Whether the last pass of a load was the one that reconciles. What gets pruned is the store's + // business - addMessagesToThreadState fills the carried set itself - so these tests only check + // which passes are allowed to ask for it; the thread-context suite covers the pruning. + const prunedOnLastPass = (actions: ConversationThreadActions) => { const calls = (actions.applyThreadLoad as unknown as jest.Mock).mock.calls - return (calls.at(-1)?.[0] as {validatedRange?: ValidatedRange} | undefined)?.validatedRange - } - // The span alone. What the range also carries - the ordinals the cached pass delivered - is - // asserted where it matters rather than in every expectation. - const validatedSpanOfLastLoad = (actions: ConversationThreadActions) => { - const range = validatedRangeOfLastLoad(actions) - return range && {from: range.from, to: range.to} + return (calls.at(-1)?.[0] as {reconcile?: ThreadLoadReconcile} | undefined)?.reconcile?.prune } beforeEach(() => { @@ -676,7 +673,7 @@ describe('the prune range is judged against a whole window', () => { resetAllStores() }) - test('prunes against a full pass that followed an empty cached one', async () => { + test('reconciles on a full pass that followed an empty cached one', async () => { // First open after a db nuke: PullLocalOnly finds nothing, but its collector suppresses the miss // and a cached pass is sent anyway, carrying no messages. INCREMENTAL against an empty local // thread filters nothing out, so the full pass really is the whole window - and only a whole @@ -689,13 +686,10 @@ describe('the prune range is judged against a whole window', () => { loadConversationThreadMessages(conversationIDKey, {reason: 'focused'}, actions) await flushPromises() - expect(validatedSpanOfLastLoad(actions)).toEqual({ - from: T.Chat.numberToOrdinal(7152), - to: T.Chat.numberToOrdinal(7153), - }) + expect(prunedOnLastPass(actions)).toBe(true) }) - test('does not prune when a cached pass arrived but another load owned the window', async () => { + test('does not reconcile when a cached pass arrived but another load owned the window', async () => { // The gate-owner guard is the one guard that can turn a cached pass away and still let the full // pass behind it through: the owner drops the gate in between. The service counts that cached // pass as sent either way, so the full pass is INCREMENTAL - a handful of changed messages - and @@ -739,10 +733,10 @@ describe('the prune range is judged against a whole window', () => { loadConversationThreadMessages(conversationIDKey, {reason: 'focused'}, actions) await flushPromises() - expect(validatedRangeOfLastLoad(actions)).toBeUndefined() + expect(prunedOnLastPass(actions)).toBe(false) }) - test('prunes against both passes of a warm-cache load, not the full one alone', async () => { + test('reconciles on the full pass of a warm-cache load, against both passes', async () => { // The warm-cache sequence: the cached pass carries the page and the full pass behind it is // INCREMENTAL, only what changed. Neither is a window on its own - but INCREMENTAL walks the // authoritative window and omits only what the cached pass already carried unchanged, so the @@ -757,11 +751,6 @@ describe('the prune range is judged against a whole window', () => { loadConversationThreadMessages(conversationIDKey, {reason: 'focused'}, actions) await flushPromises() - expect(validatedSpanOfLastLoad(actions)).toEqual({ - from: T.Chat.numberToOrdinal(7052), - to: T.Chat.numberToOrdinal(7153), - }) - // ...and the rows only the cached pass carried count as present, or the prune would take them. - expect(validatedRangeOfLastLoad(actions)?.alsoPresent?.has(T.Chat.numberToOrdinal(7052))).toBe(true) + expect(prunedOnLastPass(actions)).toBe(true) }) }) diff --git a/shared/chat/conversation/thread-load.tsx b/shared/chat/conversation/thread-load.tsx index 74feff4dfcd1..723816246e4f 100644 --- a/shared/chat/conversation/thread-load.tsx +++ b/shared/chat/conversation/thread-load.tsx @@ -12,7 +12,7 @@ import {persistRoute} from '@/util/storeless-actions' import {uint8ArrayToString} from '@/util/uint8array' import {useCurrentUserState} from '@/stores/current-user' import {useConfigState} from '@/stores/config' -import {type ValidatedRange, getOrdinalForMessageID} from './thread-message-state' +import {type ThreadLoadReconcile, getOrdinalForMessageID} from './thread-message-state' import {getInboxConversationMeta, updateInboxConversationMeta} from '@/chat/inbox/metadata' import {loadThreadNonblock, threadLoadReasonToRPCReason} from './thread-rpc' import type { @@ -241,18 +241,18 @@ export const loadConversationThreadMessages = ( ) const loadingKey = Strings.waitingKeyChatThreadLoad(conversationIDKey) - // The ordinals the cached pass carried. Once the service has sent a cached thread it switches - // the full response to INCREMENTAL, which walks the authoritative window and sends only the - // messages the cached pass did not already carry unchanged (mergeLocalRemoteThread in - // go/chat/uithreadloader.go, where localSentThread is that exact cached pass). Neither pass is - // a whole window on its own, but together they cover every message in the window - which is - // what the prune below needs, and why it unions them rather than gating on the full pass alone. - const cachedPassOrdinals = new Set() + // What this load has put in the window, filled in by addMessagesToThreadState as each pass + // applies. Once the service has sent a cached thread it switches the full response to + // INCREMENTAL, which walks the authoritative window and sends only the messages that cached + // pass did not already carry unchanged (mergeLocalRemoteThread in go/chat/uithreadloader.go, + // where localSentThread is that exact pass). Neither pass is a whole window on its own, so the + // two are gathered here and the last one reconciles against the both of them. + const carried = new Set() // Whether a cached pass reached us at all, and whether it made it into the window. They come // apart: the gate-owner guard turns a pass away, and the owner can drop the gate before the // full pass arrives, so a load can have its cached pass refused and its full pass admitted. The // service counts that cached pass as sent either way, so what follows is still INCREMENTAL - - // only the messages that changed - and a span built from those alone would prune every row + // only the messages that changed - and reconciling against those alone would take out every row // between them. Recorded before the guards, because the guards are what turn a pass away. let sawCachedResponse = false let appliedCachedPass = false @@ -325,30 +325,16 @@ export const loadConversationThreadMessages = ( scrollDirection !== 'back' && reason !== 'findNewestConversation' && reason !== 'findNewestConversationFromLayout' - // Pruning is only safe against a whole window, and a single pass is not one: the cached pass - // is whatever the local cache holds, gaps included, and the full pass behind it carries only - // what changed. The two together are the window, so the range is computed on the full pass - // from the union of both. Waiting for a pass with no cached one before it would leave the - // stale-ordinal cleanup running on cold caches only, which is where ghost rows are least - // likely to be - a reopened conversation is warm every time. - const renderedMessages = messages.filter( - m => m.conversationMessage !== false && m.type !== 'deleted' - ) - const renderedOrdinals = renderedMessages.map(m => m.ordinal) - let validatedRange: ValidatedRange | undefined - if (scrollDirection === 'none' && why === 'full' && !(sawCachedResponse && !appliedCachedPass)) { - const ords = [...renderedOrdinals, ...cachedPassOrdinals] - if (ords.length > 0) { - validatedRange = { - // The cached pass was applied in its own call, so what it delivered is not among the - // messages this one carries. Without it every row only that pass mentioned would read - // as missing from the window and be pruned. - alsoPresent: cachedPassOrdinals, - from: Math.min(...ords) as T.Chat.Ordinal, - to: Math.max(...ords) as T.Chat.Ordinal, - } - } - } + // Reconciling is only safe against a whole window, and a single pass is not one: the cached + // pass is whatever the local cache holds, gaps included, and the full pass behind it carries + // only what changed. The full pass is the last one, so it is the one that prunes - against + // everything both passes delivered. Waiting instead for a pass with no cached one before it + // would leave the stale-row cleanup running on cold caches only, which is where ghost rows + // are least likely to be: a reopened conversation is warm every time. + const reconcile: ThreadLoadReconcile | undefined = + scrollDirection === 'none' + ? {carried, prune: why === 'full' && !(sawCachedResponse && !appliedCachedPass)} + : undefined for (const m of messages) { if (m.id > 0 && m.id < oldestSeenThisLoad) { oldestSeenThisLoad = m.id @@ -361,24 +347,12 @@ export const loadConversationThreadMessages = ( forceContainsLatestCalc, messages, moreToLoad, + reconcile, scrollDirection, - validatedRange, }) const after = actions.getSnapshot() if (why === 'cached') { appliedCachedPass = true - // Recorded once the pass has landed, and in the window's terms rather than the response's. - // A message you sent keeps the fractional ordinal it had in the outbox, so the ordinal it - // parsed with - the server one - is not the ordinal it occupies. The prune walks the - // window, so an entry under the parsed ordinal protects nothing and the row goes. Both are - // recorded: whichever one the row ends up under, it counts as delivered. - for (const m of renderedMessages) { - cachedPassOrdinals.add(m.ordinal) - const occupied = m.id ? getOrdinalForMessageIDInSnapshot(after, m.id) : undefined - if (occupied) { - cachedPassOrdinals.add(occupied) - } - } } // A back page can be composed entirely of messages the thread will never render: a message // superseded by a DELETE arrives as a hidden placeholder, becomes `deleted`, and addMessages diff --git a/shared/chat/conversation/thread-message-state.test.tsx b/shared/chat/conversation/thread-message-state.test.tsx index 726572401213..92ed4867c2d6 100644 --- a/shared/chat/conversation/thread-message-state.test.tsx +++ b/shared/chat/conversation/thread-message-state.test.tsx @@ -379,23 +379,44 @@ describe('addMessagesToThreadState', () => { expect(state.messageMap.has(T.Chat.numberToOrdinal(20))).toBe(true) }) - test('a validated range prunes local ordinals the service did not send back', () => { + test('a reconciling pass prunes local ordinals the service did not send back', () => { const state = makeThreadState([]) addMessagesToThreadState(state, [textAt(10), textAt(20), textAt(30)], {}) addMessagesToThreadState(state, [textAt(10), textAt(30)], { - validatedRange: {from: T.Chat.numberToOrdinal(10), to: T.Chat.numberToOrdinal(30)}, + reconcile: {carried: new Set(), prune: true}, }) expect(state.messageOrdinals).toEqual([10, 30]) expect(state.messageMap.has(T.Chat.numberToOrdinal(20))).toBe(false) }) - test('a validated range leaves ordinals outside of it alone', () => { + test('a reconciling pass counts what an earlier pass of the same load carried', () => { + // The warm shape, at this level: the cached pass delivers the page, the full pass behind it + // only what changed. The span covers both, and a row the first pass delivered is present even + // though the second never mentions it. const state = makeThreadState([]) - addMessagesToThreadState(state, [textAt(10), textAt(50)], {}) - addMessagesToThreadState(state, [textAt(50)], { - validatedRange: {from: T.Chat.numberToOrdinal(40), to: T.Chat.numberToOrdinal(60)}, + const carried = new Set() + addMessagesToThreadState(state, [textAt(10), textAt(20), textAt(30)], { + reconcile: {carried, prune: false}, }) - expect(state.messageOrdinals).toEqual([10, 50]) + addMessagesToThreadState(state, [textAt(30)], {reconcile: {carried, prune: true}}) + expect(state.messageOrdinals).toEqual([10, 20, 30]) + + // ...and a row neither pass carried is stale, so it goes. + const withGhost = makeThreadState([textAt(10), textAt(20), textAt(30)]) + const carriedAgain = new Set() + addMessagesToThreadState(withGhost, [textAt(10)], {reconcile: {carried: carriedAgain, prune: false}}) + addMessagesToThreadState(withGhost, [textAt(30)], {reconcile: {carried: carriedAgain, prune: true}}) + expect(withGhost.messageOrdinals).toEqual([10, 30]) + }) + + test('a reconciling pass leaves ordinals outside its span alone', () => { + const state = makeThreadState([]) + addMessagesToThreadState(state, [textAt(10), textAt(50), textAt(60)], {}) + addMessagesToThreadState(state, [textAt(50), textAt(60)], { + reconcile: {carried: new Set(), prune: true}, + }) + // 10 is below everything the load covered, so nothing is known about it. + expect(state.messageOrdinals).toEqual([10, 50, 60]) }) test('a notification may not strand a new ordinal below the loaded window', () => { @@ -470,15 +491,15 @@ describe('addMessagesToThreadState', () => { }) test('a placeholder for a message we already hold does not get it pruned', () => { - // Regression: the placeholder bailed out of incomingOrdinals bookkeeping, so the validatedRange - // prune saw its ordinal as absent from the response and deleted the real message underneath. + // Regression: the placeholder bailed out of incomingOrdinals bookkeeping, so the prune saw its + // ordinal as absent from the response and deleted the real message underneath. // A quick-mode Pull returns a placeholder for anything it could not unbox, so this is the // ordinary shape of a focused refresh, not an edge case. const state = makeThreadState([textAt(49), textAt(50), textAt(51)]) addMessagesToThreadState( state, [textAt(49), Message.makeMessagePlaceholder({ordinal: T.Chat.numberToOrdinal(50)}), textAt(51)], - {validatedRange: {from: T.Chat.numberToOrdinal(49), to: T.Chat.numberToOrdinal(51)}} + {reconcile: {carried: new Set(), prune: true}} ) expect(state.messageOrdinals).toEqual([ T.Chat.numberToOrdinal(49), diff --git a/shared/chat/conversation/thread-message-state.tsx b/shared/chat/conversation/thread-message-state.tsx index 571445374bda..23cfb6084771 100644 --- a/shared/chat/conversation/thread-message-state.tsx +++ b/shared/chat/conversation/thread-message-state.tsx @@ -9,14 +9,20 @@ const isPendingSend = (m: T.Chat.Message) => type MessageLookup = Pick -// The span a thread load is authoritative over, and what it holds. Ordinals inside the span that -// the load did not carry are stale and get pruned. `alsoPresent` is the rest of what the load -// carried: a warm load answers in two passes, and a message the earlier one delivered is still -// present even though the pass doing the pruning no longer mentions it. -export type ValidatedRange = { - from: T.Chat.Ordinal - to: T.Chat.Ordinal - alsoPresent?: ReadonlySet +// How a thread load reconciles the window against what the service returned. +// +// A load answers in passes - a cached one off the local database, then a full one the service has +// filtered down to what changed - and only the passes together are a whole window. So each pass +// adds what it delivered to `carried`, and the last one prunes: rows inside the span of `carried` +// that no pass delivered are stale, and go. +// +// The set is filled in here rather than by the caller, and that is the point of it: it holds the +// ordinals the messages actually occupy, after the outbox and message-ID remaps below. A message +// you sent keeps the fractional ordinal it had in the outbox, so the ordinal it arrives under is +// not the one it lives at, and a set built from the response would leave that row unprotected. +export type ThreadLoadReconcile = { + carried: Set + prune: boolean } type WritableConversationThreadMessageState = { @@ -183,10 +189,10 @@ export const addMessagesToThreadState = ( messages: ReadonlyArray, opt: { dropNewBelowWindow?: boolean - validatedRange?: ValidatedRange + reconcile?: ThreadLoadReconcile } ) => { - const {dropNewBelowWindow, validatedRange} = opt + const {dropNewBelowWindow, reconcile} = opt // The bounds of the loaded window before this batch is merged in. const ords = state.messageOrdinals const windowFloor = ords?.[0] @@ -300,9 +306,9 @@ export const addMessagesToThreadState = ( // below would strand _m.ordinal in the list with nothing stored under it. // // Do the remap anyway rather than just forgetting _m.ordinal. `incomingOrdinals` is what - // the validatedRange prune treats as "still present", so an ordinal missing from it - // inside the range gets the real message deleted - including when mapOrdinal and - // _m.ordinal are the same, where the delete below would otherwise be a plain loss. + // the prune treats as "still present", so an ordinal missing from it inside the span + // gets the real message deleted - including when mapOrdinal and _m.ordinal are the same, + // where the delete below would otherwise be a plain loss. incomingOrdinals.delete(_m.ordinal) incomingOrdinals.add(mapOrdinal) continue @@ -363,20 +369,27 @@ export const addMessagesToThreadState = ( changed = true } } - if (validatedRange) { - // The service response is authoritative within this range; prune stale local ordinals. - for (const o of existing) { - if ( - o >= validatedRange.from && - o <= validatedRange.to && - !incomingOrdinals.has(o) && - !validatedRange.alsoPresent?.has(o) - ) { - clearMessageIDIndexForOrdinal(state, o) - existing.delete(o) - state.messageMap.delete(o) - state.messageTypeMap.delete(o) - changed = true + if (reconcile) { + for (const o of incomingOrdinals) { + reconcile.carried.add(o) + } + if (reconcile.prune) { + // The load is authoritative over the span it covered, so a row inside it that no pass of the + // load delivered is stale. Outside the span nothing is known and nothing is touched. + let from = Number.MAX_SAFE_INTEGER as T.Chat.Ordinal + let to = Number.MIN_SAFE_INTEGER as T.Chat.Ordinal + for (const o of reconcile.carried) { + from = Math.min(from, o) as T.Chat.Ordinal + to = Math.max(to, o) as T.Chat.Ordinal + } + for (const o of existing) { + if (o >= from && o <= to && !reconcile.carried.has(o)) { + clearMessageIDIndexForOrdinal(state, o) + existing.delete(o) + state.messageMap.delete(o) + state.messageTypeMap.delete(o) + changed = true + } } } } From 8530d2edde0673970ebfe68023de0cbf7c0d9947 Mon Sep 17 00:00:00 2001 From: chrisnojima Date: Tue, 8 Sep 2026 12:05:26 -0400 Subject: [PATCH 20/21] fix(chat): a load the window turned away is over, all of it The owner guard refuses a pass only while the gate is up, and the owner drops that gate as soon as its own page lands - so the refused load's next pass walked straight into the window the owner had just built. That is the disjoint window this invariant exists to prevent, and it arrived through the guard meant to prevent it. A load is now all or nothing: refuse one pass and the rest are ignored too, which is also the honest reading, since the service filters each pass against what it already sent that load. Reconciling now needs the service to have reported a cached pass at all - with a thread, or with the nil a cold cache sends. It records that thread as sent before it marshals it, so a failure there leaves the full pass filtered against a pass we were never shown, and pruning against that takes out every row between the few messages it carries. A pass with nothing to add and nothing to reconcile no longer reaches the store at all: it leaves a messageOrdinals array behind either way, and an empty one reads as a loaded, empty thread, so the top of the conversation rendered against it and swapped when the page arrived. The window gate is claimed before the two bails that used to sit above it. The clear issues its reload synchronously, so a reload bailing there left the gate up with nothing coming to take it down. Drops the pending-send exemption. It could not fire in the flow it was written for - the engine drops incoming messages while moreToLoadForward is set, which is what a centered load leaves behind, and that is the only path that sends and then jumps. The row comes back with the reload either way: the service appends the outbox to both passes. --- .../chat/conversation/thread-context.test.tsx | 32 ++------ shared/chat/conversation/thread-context.tsx | 29 +++---- shared/chat/conversation/thread-load.test.tsx | 35 +++++++++ shared/chat/conversation/thread-load.tsx | 78 +++++++++++-------- .../thread-message-state.test.tsx | 39 ---------- .../conversation/thread-message-state.tsx | 20 +---- 6 files changed, 100 insertions(+), 133 deletions(-) diff --git a/shared/chat/conversation/thread-context.test.tsx b/shared/chat/conversation/thread-context.test.tsx index 43f9c5fbf63c..7213d4cd0630 100644 --- a/shared/chat/conversation/thread-context.test.tsx +++ b/shared/chat/conversation/thread-context.test.tsx @@ -1891,42 +1891,24 @@ test('only the load that claimed the window gate may drop it', () => { expect(result.current.actions.getSnapshot().windowCleared).toBe(false) }) -test('a window holding only a pending send is not a window the load filled', () => { - // The pending-send exemption lets our own outbox row in during a jump-to-recent gap, so the - // window is no longer empty. The gate must still be judged on what the load carried: an empty - // cached pass arriving behind that row would otherwise read as "the load filled the window" and - // drop the gate before the real page is anywhere. +test('an empty pass leaves the thread unloaded rather than loaded and empty', () => { + // addMessagesToThreadState always leaves a messageOrdinals array behind, and the top-of-thread + // block reads `messageOrdinals !== undefined` as "this conversation has loaded at least once". + // A cold open answers with an empty cached pass first, so handing that pass to the store renders + // the top of the conversation against an empty thread, which then swaps when the page lands. const {result} = renderHook(() => ({actions: useConversationThreadActions()}), {wrapper}) - act(() => { - result.current.actions.messagesClear({reloadsNewest: true}) - }) - act(() => { - result.current.actions.addMessages( - [ - Message.makeMessageText({ - conversationIDKey: convID, - ordinal: T.Chat.numberToOrdinal(7153.001), - outboxID: T.Chat.stringToOutboxID('sending-1'), - submitState: 'pending', - text: new HiddenString('hi'), - }), - ], - {liveUpdate: true} - ) - }) - expect(result.current.actions.getSnapshot().messageOrdinals).toEqual([7153.001]) - act(() => { result.current.actions.applyThreadLoad({ centered: false, enableActiveMarkRead: false, messages: [], moreToLoad: true, + reconcile: {carried: new Set(), prune: false}, scrollDirection: 'none', }) }) - expect(result.current.actions.getSnapshot().windowCleared).toBe(true) + expect(result.current.actions.getSnapshot().messageOrdinals).toBeUndefined() }) test('an empty pass during a jump-to-recent gap leaves the gate up', () => { diff --git a/shared/chat/conversation/thread-context.tsx b/shared/chat/conversation/thread-context.tsx index 10704c8be0d3..066fa1b2c8ce 100644 --- a/shared/chat/conversation/thread-context.tsx +++ b/shared/chat/conversation/thread-context.tsx @@ -112,10 +112,6 @@ export type ConversationThreadState = { // in that gap cannot install itself as the new window. Cleared once that load settles, however it // settles - see clearWindowGate. windowCleared?: boolean - // Whether the reload the clear issued fetches the newest page, which is the one region a message - // arriving during the gap can still be placed against. See the pending-send exemption in - // addMessagesToThreadState. - windowClearedForNewest?: boolean // The load that owns the gate above: the first one to claim it after the clear, which is the // reload the clear issued. Only that load may drop the gate. clearVersion alone cannot tell two // loads of the same conversation apart, and a second load at the same generation - a @@ -223,7 +219,7 @@ type LoadNewerMessagesDueToScroll = ( options?: ThreadLoadStatusOptions ) => void type JumpToRecent = (options?: ThreadLoadStatusOptions) => void -type MessagesClear = (opts?: {reloadsNewest?: boolean}) => void +type MessagesClear = () => void type SelectedConversation = (options?: SelectedConversationOptions) => void export type ConversationThreadActions = { addMessages: ( @@ -545,9 +541,8 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => reconcile?: ThreadLoadReconcile scrollDirection: ScrollDirection }) => { - // Judged on what this pass carried rather than on the window being non-empty: a pending send - // of our own is admitted during a jump-to-recent gap, and a window holding only that must not - // read as a window this load filled. + // Judged on what this pass carried rather than on the state of the window, so the gate turns + // on the one thing that decides it: whether this pass put a row on screen. const carriedRenderedMessage = p.messages.some( m => m.conversationMessage !== false && m.type !== 'deleted' ) @@ -555,8 +550,11 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => s.loaded = true // The reconciling pass runs even with nothing to add: the warm reload where nothing changed // answers with an empty full pass, and the span its earlier pass covered is authoritative - // all the same - the stale rows inside it are exactly what the prune is for. - if (p.messages.length || p.reconcile) { + // all the same - the stale rows inside it are exactly what the prune is for. A pass with + // neither is skipped rather than passed through: addMessagesToThreadState always leaves a + // messageOrdinals array behind, and an empty one reads as a loaded, empty thread - the top + // of the conversation renders against it and then swaps when the real page arrives. + if (p.messages.length || p.reconcile?.prune) { addMessagesToThreadState(s, p.messages, {reconcile: p.reconcile}) clearOptimisticReactionsForMessagesInThreadState(s, p.messages) } @@ -567,7 +565,6 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => // an ordinal releases the gate in its own finally instead - see clearWindowGate. if (carriedRenderedMessage) { s.windowCleared = false - s.windowClearedForNewest = undefined s.windowGateOwner = undefined } switch (p.scrollDirection) { @@ -961,11 +958,10 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => } updateThreadState(d => { d.windowCleared = false - d.windowClearedForNewest = undefined d.windowGateOwner = undefined }) }) - const messagesClear = React.useEffectEvent((opts?: {reloadsNewest?: boolean}) => { + const messagesClear = React.useEffectEvent(() => { activeMarkReadEnabledRef.current = false shownUsernameCache.clear() updateThreadState(s => { @@ -978,7 +974,6 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => // arbitrary one, jumpToRecent the newest page - so nothing arriving first can be placed // against what is coming. s.windowCleared = true - s.windowClearedForNewest = opts?.reloadsNewest s.windowGateOwner = undefined s.messageIDToOrdinal.clear() s.messageMap.clear() @@ -1293,10 +1288,8 @@ export const useConversationThreadJumpToRecent = () => { const jumpToRecent: JumpToRecent = options => { setMarkReadBlocked(false) // The newest window is disjoint from wherever the reader was, so merging the two would leave a - // gap in the ordinals. Drop the old window first, the way a centered jump does - but say that - // the reload covers the newest page, so a send made in the same breath still shows its pending - // row (input-area/normal sends and then jumps here). - messagesClear({reloadsNewest: true}) + // gap in the ordinals. Drop the old window first, the way a centered jump does. + messagesClear() loadMoreMessages({...(options ?? {}), reason: 'jump to recent'}) } return jumpToRecent diff --git a/shared/chat/conversation/thread-load.test.tsx b/shared/chat/conversation/thread-load.test.tsx index 2d9c7e8a0708..993a38961ce2 100644 --- a/shared/chat/conversation/thread-load.test.tsx +++ b/shared/chat/conversation/thread-load.test.tsx @@ -453,6 +453,22 @@ describe('a load releases the window gate it was issued under', () => { resetAllStores() }) + test('releases it when the load bails before the rpc is even made', async () => { + // The clear issues its reload synchronously, so if that reload is the one bailing there is + // nothing else coming to take the gate down and the thread stops receiving messages for good. + const rpc = jest.spyOn(ThreadRpc, 'loadThreadNonblock') + const actions = gateActions(() => 3) + loadConversationThreadMessages( + conversationIDKey, + {isThreadLoadCurrent: () => false, reason: 'focused'}, + actions + ) + await flushPromises() + + expect(rpc).not.toHaveBeenCalled() + expect(actions.clearWindowGate).toHaveBeenCalledTimes(1) + }) + test('releases it when the load ends without ever applying', async () => { // A response that carries no thread: applyThreadLoad never runs, so nothing else would take the // gate down. Left up it drops every notification for the life of the provider. @@ -733,6 +749,25 @@ describe('only a pass that can account for a whole window reconciles', () => { loadConversationThreadMessages(conversationIDKey, {reason: 'focused'}, actions) await flushPromises() + expect(actions.applyThreadLoad).not.toHaveBeenCalled() + }) + + test('does not reconcile when the service never reported a cached pass', async () => { + // The service records the cached thread as sent before it marshals it, so a failure there + // leaves the full pass INCREMENTAL against a pass we were never shown. The cached callback + // firing - with a thread, or with the nil a cold cache sends - is the only sign we get that + // this did not happen. + const actions = recordingActions() + jest.spyOn(ThreadRpc, 'loadThreadNonblock').mockImplementation(async p => { + await Promise.resolve() + p.onFullThread?.( + JSON.stringify({messages: page(7153, 7150), pagination: {last: false, num: 100}}) + ) + return undefined as never + }) + loadConversationThreadMessages(conversationIDKey, {reason: 'focused'}, actions) + await flushPromises() + expect(prunedOnLastPass(actions)).toBe(false) }) diff --git a/shared/chat/conversation/thread-load.tsx b/shared/chat/conversation/thread-load.tsx index 723816246e4f..1798882eb82f 100644 --- a/shared/chat/conversation/thread-load.tsx +++ b/shared/chat/conversation/thread-load.tsx @@ -188,16 +188,6 @@ export const loadConversationThreadMessages = ( const isCurrentThreadLoad = () => isThreadLoadCurrent?.() ?? true const f = async () => { - if (!isCurrentThreadLoad()) { - logger.info('loadMoreMessages: bail: stale mounted thread load') - return - } - - if (!conversationIDKey || !T.Chat.isValidConversationIDKey(conversationIDKey)) { - logger.info('loadMoreMessages: bail: no conversationIDKey') - return - } - const loadStartedSnapshot = actions.getSnapshot() const clearVersionAtLoadStart = loadStartedSnapshot.clearVersion // applyThreadLoad drops the window gate when a load refills the window, but a load can end @@ -223,6 +213,21 @@ export const loadConversationThreadMessages = ( actions.clearWindowGate(loadID) } } + // Every bail from here on releases, including the two that used to sit above the claim: the + // clear issues its reload synchronously, so if that reload is the one bailing there is nothing + // else coming to take the gate down, and the thread stops receiving messages for good. + if (!isCurrentThreadLoad()) { + logger.info('loadMoreMessages: bail: stale mounted thread load') + releaseWindowGate() + return + } + + if (!conversationIDKey || !T.Chat.isValidConversationIDKey(conversationIDKey)) { + logger.info('loadMoreMessages: bail: no conversationIDKey') + releaseWindowGate() + return + } + const currentMeta = getMeta(conversationIDKey) if (currentMeta.membershipType === 'youAreReset' || currentMeta.rekeyers.size > 0) { logger.info('loadMoreMessages: bail: we are reset') @@ -248,14 +253,20 @@ export const loadConversationThreadMessages = ( // where localSentThread is that exact pass). Neither pass is a whole window on its own, so the // two are gathered here and the last one reconciles against the both of them. const carried = new Set() - // Whether a cached pass reached us at all, and whether it made it into the window. They come - // apart: the gate-owner guard turns a pass away, and the owner can drop the gate before the - // full pass arrives, so a load can have its cached pass refused and its full pass admitted. The - // service counts that cached pass as sent either way, so what follows is still INCREMENTAL - - // only the messages that changed - and reconciling against those alone would take out every row - // between them. Recorded before the guards, because the guards are what turn a pass away. - let sawCachedResponse = false - let appliedCachedPass = false + // A load is all or nothing. Once one of its passes is turned away, the rest of them are too: + // the service filters each pass against what it has already sent this load, so the ones that + // follow a refused pass are a subset of a window we never took, and both ways of using them + // are wrong. Merging one into whatever refilled the window in the meantime is the disjoint + // window this whole invariant exists to prevent; reconciling against one takes out every row + // between the few messages it happens to carry. + let refusedAPass = false + // Whether the service's cached goroutine reported at all - with a thread, or with the nil it + // sends when the local cache had nothing. It is the only evidence the client gets that the + // full pass was not filtered behind our back: the service records the cached thread as sent + // before it marshals it, so a marshal failure there leaves us with an INCREMENTAL full pass + // and no sign of the pass it was filtered against (LoadNonblock in + // go/chat/uithreadloader.go). No report, no reconciling. + let sawCachedReport = false // The reload below is judged against the whole load, not one pass of it. A warm-cache load // delivers the page on the cached pass and then an INCREMENTAL full pass carrying only what // changed, so measuring the full pass alone says "added nothing" for a perfectly good page. @@ -265,15 +276,18 @@ export const loadConversationThreadMessages = ( let oldestSeenThisLoad = Number.MAX_SAFE_INTEGER as T.Chat.MessageID const onGotThread = (thread: string, why: string) => { if (!thread) { - // No cached thread was sent, so the service has nothing to filter the full pass against and - // it stays a whole window. Deliberately not counted as a cached response. return } - if (why === 'cached') { - sawCachedResponse = true + if (refusedAPass) { + logger.info(`loadMoreMessages: pass ignored, an earlier one of this load was: ${why}`) + return + } + const refuse = (msg: string) => { + refusedAPass = true + logger.info(msg) } if (!isCurrentThreadLoad()) { - logger.info(`loadMoreMessages: stale response ignored: ${why}`) + refuse(`loadMoreMessages: stale response ignored: ${why}`) return } // A clear under us - jump to recent, a centered jump - dropped the window this load was @@ -284,7 +298,7 @@ export const loadConversationThreadMessages = ( // reload, which then merges its own page into the leftovers. const snapshotAtResponse = actions.getSnapshot() if (snapshotAtResponse.clearVersion !== clearVersionAtLoadStart) { - logger.info(`loadMoreMessages: response ignored after clear: ${why}`) + refuse(`loadMoreMessages: response ignored after clear: ${why}`) return } // clearVersion cannot separate two loads issued after the same clear, and the second one is @@ -299,11 +313,11 @@ export const loadConversationThreadMessages = ( snapshotAtResponse.windowGateOwner !== undefined && snapshotAtResponse.windowGateOwner !== loadID ) { - logger.info(`loadMoreMessages: response ignored, another load owns the window: ${why}`) + refuse(`loadMoreMessages: response ignored, another load owns the window: ${why}`) return } if (protectLoadedFocusRefresh && snapshotAtResponse.liveUpdateVersion !== loadStartedLiveUpdateVersion) { - logger.info( + refuse( `loadMoreMessages: stale response ignored after live update: ${why} reason=${reason} convID=${conversationIDKey}` ) return @@ -332,9 +346,7 @@ export const loadConversationThreadMessages = ( // would leave the stale-row cleanup running on cold caches only, which is where ghost rows // are least likely to be: a reopened conversation is warm every time. const reconcile: ThreadLoadReconcile | undefined = - scrollDirection === 'none' - ? {carried, prune: why === 'full' && !(sawCachedResponse && !appliedCachedPass)} - : undefined + scrollDirection === 'none' ? {carried, prune: why === 'full' && sawCachedReport} : undefined for (const m of messages) { if (m.id > 0 && m.id < oldestSeenThisLoad) { oldestSeenThisLoad = m.id @@ -351,9 +363,6 @@ export const loadConversationThreadMessages = ( scrollDirection, }) const after = actions.getSnapshot() - if (why === 'cached') { - appliedCachedPass = true - } // A back page can be composed entirely of messages the thread will never render: a message // superseded by a DELETE arrives as a hidden placeholder, becomes `deleted`, and addMessages // drops it. The ordinal list is then identical to what it was, so the list never fires @@ -422,7 +431,10 @@ export const loadConversationThreadMessages = ( conversationIDKey, knownRemotes, messageIDControl, - onCachedThread: thread => onGotThread(thread, 'cached'), + onCachedThread: thread => { + sawCachedReport = true + onGotThread(thread, 'cached') + }, onFullThread: thread => onGotThread(thread, 'full'), onThreadStatus: status => { logger.info( diff --git a/shared/chat/conversation/thread-message-state.test.tsx b/shared/chat/conversation/thread-message-state.test.tsx index 92ed4867c2d6..720b68fdca85 100644 --- a/shared/chat/conversation/thread-message-state.test.tsx +++ b/shared/chat/conversation/thread-message-state.test.tsx @@ -621,45 +621,6 @@ describe('addMessagesToThreadState', () => { expect(state.messageOrdinals).toEqual([T.Chat.numberToOrdinal(7153), T.Chat.numberToOrdinal(9001)]) }) - test('a pending send lands during a jump-to-recent gap', () => { - // The reader sends from a search-jumped thread: input-area posts and jumps to recent in the - // same tick, so the clear happens first and the outbox notification arrives into the gap. Its - // ordinal is the service's - the outbox record's, above the newest message - so it belongs in - // the very page the reload is fetching, and dropping it leaves the composer empty with no - // "sending..." row for as long as that reload takes. - const pending = makeTextMessage({ - id: T.Chat.numberToMessageID(0), - ordinal: T.Chat.numberToOrdinal(7153.001), - outboxID: T.Chat.stringToOutboxID('sending-1'), - submitState: 'pending', - }) - const state = makeThreadState([]) - state.windowCleared = true - state.windowClearedForNewest = true - addMessagesToThreadState(state, [pending], {dropNewBelowWindow: true}) - expect(state.messageOrdinals).toEqual([T.Chat.numberToOrdinal(7153.001)]) - - // Nothing else gets in on its coattails. - addMessagesToThreadState(state, [textAt(9001)], {dropNewBelowWindow: true}) - expect(state.messageOrdinals).toEqual([T.Chat.numberToOrdinal(7153.001)]) - }) - - test('a pending send is still dropped during a centered-jump gap', () => { - // The exemption is only sound because jump-to-recent reloads the newest page. A centered jump - // lands on an arbitrary older region, and a pending row sitting at the bottom of the thread - // would strand above it once that page arrives. - const pending = makeTextMessage({ - id: T.Chat.numberToMessageID(0), - ordinal: T.Chat.numberToOrdinal(7153.001), - outboxID: T.Chat.stringToOutboxID('sending-1'), - submitState: 'pending', - }) - const state = makeThreadState([]) - state.windowCleared = true - addMessagesToThreadState(state, [pending], {dropNewBelowWindow: true}) - expect(state.messageOrdinals ?? []).toEqual([]) - }) - test('a message remapped out of the window is dropped, not stranded', () => { // The window is judged on the ordinal the message will occupy, which an outbox or messageID // match can move. Here messageIDToOrdinal still points at an ancient ordinal the thread no diff --git a/shared/chat/conversation/thread-message-state.tsx b/shared/chat/conversation/thread-message-state.tsx index 23cfb6084771..4a7b17fe207e 100644 --- a/shared/chat/conversation/thread-message-state.tsx +++ b/shared/chat/conversation/thread-message-state.tsx @@ -3,10 +3,6 @@ import * as T from '@/constants/types' import HiddenString from '@/util/hidden-string' import type {WritableDraft} from '@/util/zustand' -// A message we are posting: it exists only in the outbox, so it has no server ID yet. -const isPendingSend = (m: T.Chat.Message) => - !m.id && 'submitState' in m && m.submitState === 'pending' - type MessageLookup = Pick // How a thread load reconciles the window against what the service returned. @@ -33,10 +29,6 @@ type WritableConversationThreadMessageState = { // there is no window to place an arriving message against. See the drop rules in // addMessagesToThreadState. windowCleared?: boolean - // Whether the reload that clear issued fetches the newest page. Only jump-to-recent does; a - // centered jump lands on an arbitrary older region. It is the one case where something arriving - // during the gap can be placed after all - see the pending-send exemption below. - windowClearedForNewest?: boolean messageTypeMap: Map // Set by a thread load, cleared by messagesClear: whether either flag below means anything yet. loaded: boolean @@ -241,7 +233,7 @@ export const addMessagesToThreadState = ( // not render, and getOrdinalForMessageID would then hand out an ordinal with no row. Nothing is // lost either way: paging to it loads it in the ordinary way. const windowCeiling = ords?.[ords.length - 1] - const isOutsideWindow = (o: T.Chat.Ordinal, m: T.Chat.Message) => { + const isOutsideWindow = (o: T.Chat.Ordinal) => { if (!dropNewBelowWindow || existing.has(o)) { return false } @@ -251,14 +243,6 @@ export const addMessagesToThreadState = ( // reader was. A message landing in the gap that the reload does not carry waits for the next // load or push; a stranded ordinal, by contrast, breaks paging for the life of the thread. if (state.windowCleared) { - // A send of our own is the exception, and only while the reload is fetching the newest page. - // Its ordinal comes from the service (the outbox record's, not our window's), so it sits at - // the bottom of the thread, which is exactly the region that reload is going to cover - - // nothing can open under it. Dropping it instead shows the composer emptying with no - // "sending..." row behind it, for as long as the reload takes. - if (state.windowClearedForNewest && isPendingSend(m)) { - return false - } return true } // moreToLoadBack starts false and only a thread load ever sets it, so until one has landed a @@ -284,7 +268,7 @@ export const addMessagesToThreadState = ( // Judged on mapOrdinal, the ordinal the message will actually occupy: an outbox or messageID // match can move it out of the window, or onto a row already inside it. Deletions and // non-conversation messages are not rows, so the window does not bound them. - if (regularMessage && _m.type !== 'deleted' && isOutsideWindow(mapOrdinal, _m)) { + if (regularMessage && _m.type !== 'deleted' && isOutsideWindow(mapOrdinal)) { incomingOrdinals.delete(_m.ordinal) incomingOrdinals.delete(mapOrdinal) continue From 7bbf8004e0c3a8419afc584eedde59e9e49488da Mon Sep 17 00:00:00 2001 From: chrisnojima Date: Tue, 8 Sep 2026 12:24:27 -0400 Subject: [PATCH 21/21] fix(chat): a page that cannot reach the window is not a refresh A 'none' load fetches the newest page, and a window with more to load forward does not reach it. Merging the two leaves ordinals with a hole through the middle and then reports the result as containing the latest message - the gap this invariant is about, arriving through a ChatThreadsStale reload while the reader sits on a search result. Both conditions have to hold before a page is turned away: a window that already reaches the newest message merges fine, and so does a page that overlaps what we hold, however far back the reader is. The prune also leaves rows with no server ID alone. Now that the span covers both passes of a load it reaches above the window the first one carried, so a message sent between the passes lands inside a span neither pass could have carried. The service has never been told about that row, so its absence says nothing, and deleting it takes the row out from under a send in flight. --- .../chat/conversation/thread-context.test.tsx | 95 +++++++++++++++++++ shared/chat/conversation/thread-context.tsx | 37 +++++++- .../thread-message-state.test.tsx | 22 +++++ .../conversation/thread-message-state.tsx | 6 +- 4 files changed, 156 insertions(+), 4 deletions(-) diff --git a/shared/chat/conversation/thread-context.test.tsx b/shared/chat/conversation/thread-context.test.tsx index 7213d4cd0630..abcf188d74e0 100644 --- a/shared/chat/conversation/thread-context.test.tsx +++ b/shared/chat/conversation/thread-context.test.tsx @@ -1891,6 +1891,101 @@ test('only the load that claimed the window gate may drop it', () => { expect(result.current.actions.getSnapshot().windowCleared).toBe(false) }) +test('a stale reload does not merge the newest page into a centered window', () => { + // The reader taps a search result and sits on the window around it, with more to load forward. + // A ChatThreadsStale reload fetches the newest page, which is nowhere near that window: merging + // the two leaves ordinals with a hole through the middle and then calls the result the latest + // message, which is the gap this invariant is about. + const {result} = renderHook( + () => ({ + actions: useConversationThreadActions(), + ordinals: useConversationThreadSelector(s => s.messageOrdinals), + }), + {wrapper} + ) + + const textAt = (ord: number) => + Message.makeMessageText({ + author: 'alice', + conversationIDKey: convID, + id: T.Chat.numberToMessageID(ord), + ordinal: T.Chat.numberToOrdinal(ord), + outboxID: undefined, + text: new HiddenString(`m${ord}`), + timestamp: 100, + }) + + act(() => { + result.current.actions.applyThreadLoad({ + centered: true, + enableActiveMarkRead: false, + messages: [textAt(7000), textAt(7001)], + moreToLoad: true, + scrollDirection: 'none', + }) + }) + expect(result.current.actions.getSnapshot().moreToLoadForward).toBe(true) + + act(() => { + result.current.actions.applyThreadLoad({ + centered: false, + enableActiveMarkRead: false, + messages: [textAt(9900), textAt(9901)], + moreToLoad: true, + scrollDirection: 'none', + }) + }) + + expect(result.current.ordinals).toEqual([7000, 7001]) + // ...and the window still knows it has not reached the latest message. + expect(result.current.actions.getSnapshot().moreToLoadForward).toBe(true) +}) + +test('a newest page that reaches the window is still merged', () => { + // The other side of the rule. A reader near the bottom gets a page that overlaps what they hold, + // so there is no hole to open and the refresh must land. + const {result} = renderHook( + () => ({ + actions: useConversationThreadActions(), + ordinals: useConversationThreadSelector(s => s.messageOrdinals), + }), + {wrapper} + ) + + const textAt = (ord: number) => + Message.makeMessageText({ + author: 'alice', + conversationIDKey: convID, + id: T.Chat.numberToMessageID(ord), + ordinal: T.Chat.numberToOrdinal(ord), + outboxID: undefined, + text: new HiddenString(`m${ord}`), + timestamp: 100, + }) + + act(() => { + result.current.actions.applyThreadLoad({ + centered: true, + enableActiveMarkRead: false, + messages: [textAt(9900), textAt(9901)], + moreToLoad: true, + scrollDirection: 'none', + }) + }) + + act(() => { + result.current.actions.applyThreadLoad({ + centered: false, + enableActiveMarkRead: false, + messages: [textAt(9901), textAt(9902)], + moreToLoad: true, + scrollDirection: 'none', + }) + }) + + expect(result.current.ordinals).toEqual([9900, 9901, 9902]) +}) + test('an empty pass leaves the thread unloaded rather than loaded and empty', () => { // addMessagesToThreadState always leaves a messageOrdinals array behind, and the top-of-thread // block reads `messageOrdinals !== undefined` as "this conversation has loaded at least once". diff --git a/shared/chat/conversation/thread-context.tsx b/shared/chat/conversation/thread-context.tsx index 066fa1b2c8ce..0a7abbe52b08 100644 --- a/shared/chat/conversation/thread-context.tsx +++ b/shared/chat/conversation/thread-context.tsx @@ -541,11 +541,42 @@ const ConversationThreadProviderInner = (p: ConversationThreadProviderProps) => reconcile?: ThreadLoadReconcile scrollDirection: ScrollDirection }) => { + const rendered = p.messages.filter(m => m.conversationMessage !== false && m.type !== 'deleted') // Judged on what this pass carried rather than on the state of the window, so the gate turns // on the one thing that decides it: whether this pass put a row on screen. - const carriedRenderedMessage = p.messages.some( - m => m.conversationMessage !== false && m.type !== 'deleted' - ) + const carriedRenderedMessage = rendered.length > 0 + // A 'none' load fetches the newest page, and a window with more to load forward does not + // reach it. Merging the two leaves ordinals with a hole through the middle, and the branch + // below then reports that window as containing the latest message - which is the gap this + // whole invariant is about, arriving through a ChatThreadsStale reload while the reader sits + // on a search result. Both conditions are needed: a window that already reaches the newest + // message merges fine, and so does a page that overlaps what we hold, however far back the + // reader is. Neither holds here, so the page is left alone rather than applied - the reader + // keeps their window, and jumping to recent (which empties it first) is what replaces it. + const beforeApply = threadStore.getState() + const windowOrdinals = beforeApply.messageOrdinals + const floor = windowOrdinals?.[0] + const ceiling = windowOrdinals?.[windowOrdinals.length - 1] + if ( + p.scrollDirection === 'none' && + rendered.length && + beforeApply.moreToLoadForward && + floor !== undefined && + ceiling !== undefined + ) { + let lowest = Number.MAX_SAFE_INTEGER + let highest = Number.MIN_SAFE_INTEGER + for (const m of rendered) { + lowest = Math.min(lowest, m.ordinal) + highest = Math.max(highest, m.ordinal) + } + if (lowest > ceiling || highest < floor) { + logger.info( + `applyThreadLoad: page ${lowest}-${highest} does not reach window ${floor}-${ceiling}, ignoring` + ) + return + } + } updateThreadState(s => { s.loaded = true // The reconciling pass runs even with nothing to add: the warm reload where nothing changed diff --git a/shared/chat/conversation/thread-message-state.test.tsx b/shared/chat/conversation/thread-message-state.test.tsx index 720b68fdca85..8ba2bfb62629 100644 --- a/shared/chat/conversation/thread-message-state.test.tsx +++ b/shared/chat/conversation/thread-message-state.test.tsx @@ -621,6 +621,28 @@ describe('addMessagesToThreadState', () => { expect(state.messageOrdinals).toEqual([T.Chat.numberToOrdinal(7153), T.Chat.numberToOrdinal(9001)]) }) + test('a reconciling pass leaves a send that is still in the outbox alone', () => { + // The span covers both passes now, so it reaches above the window the first one carried: send a + // message between the passes and its fractional ordinal falls inside a span that neither pass + // could have carried. The service has never been told about that row, so its absence says + // nothing - deleting it takes the row out from under a send in flight. + const state = makeThreadState([textAt(6900), textAt(7000)]) + const carried = new Set() + addMessagesToThreadState(state, [textAt(6900), textAt(7000)], {reconcile: {carried, prune: false}}) + + const pending = makeTextMessage({ + id: T.Chat.numberToMessageID(0), + ordinal: T.Chat.numberToOrdinal(7000.001), + outboxID: T.Chat.stringToOutboxID('sending-1'), + submitState: 'pending', + }) + addMessagesToThreadState(state, [pending], {}) + expect(state.messageOrdinals).toEqual([6900, 7000, 7000.001]) + + addMessagesToThreadState(state, [textAt(7001)], {reconcile: {carried, prune: true}}) + expect(state.messageOrdinals).toEqual([6900, 7000, 7000.001, 7001]) + }) + test('a message remapped out of the window is dropped, not stranded', () => { // The window is judged on the ordinal the message will occupy, which an outbox or messageID // match can move. Here messageIDToOrdinal still points at an ancient ordinal the thread no diff --git a/shared/chat/conversation/thread-message-state.tsx b/shared/chat/conversation/thread-message-state.tsx index 4a7b17fe207e..a2c575bb253b 100644 --- a/shared/chat/conversation/thread-message-state.tsx +++ b/shared/chat/conversation/thread-message-state.tsx @@ -367,7 +367,11 @@ export const addMessagesToThreadState = ( to = Math.max(to, o) as T.Chat.Ordinal } for (const o of existing) { - if (o >= from && o <= to && !reconcile.carried.has(o)) { + // A row with no server ID is one of ours, still in the outbox: the service cannot have + // failed to return what it has never been told about. It sits on a fractional ordinal just + // above the message it was composed after, so the span reaches it as soon as anything + // newer arrives - and deleting it takes the row out from under a send in flight. + if (o >= from && o <= to && !reconcile.carried.has(o) && state.messageMap.get(o)?.id) { clearMessageIDIndexForOrdinal(state, o) existing.delete(o) state.messageMap.delete(o)